diff --git a/.env b/.env index 3bb32c35..4ffc4215 100644 --- a/.env +++ b/.env @@ -12,6 +12,7 @@ SHUFFLE_APP_DOWNLOAD_LOCATION=https://github.com/frikky/shuffle-apps SHUFFLE_DOWNLOAD_AUTH_USERNAME= SHUFFLE_DOWNLOAD_AUTH_PASSWORD= SHUFFLE_DOWNLOAD_AUTH_BRANCH= +SHUFFLE_APP_FORCE_UPDATE=false # User config for first load. Username & PW: min length 3 SHUFFLE_DEFAULT_USERNAME= diff --git a/.gitignore b/.gitignore index 5bebe49f..ca9d2090 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,7 @@ functions/generated_apps backend/onprem/app_sdk/apps *test.py + +shuffle-database +*.exe +*debug* diff --git a/README.md b/README.md index 895bd472..c09bd090 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ These are the main areas to contribute in: * Workflow creation (GUI & Conceptualizing) * Content Creation (Blogs, videos etc) -Contributing guidelines for Github are outlined [here](https://github.com/frikky/Shuffle/blob/master/.github/CONTRIBUTING.md). +Contributing guidelines are outlined [here](https://github.com/frikky/Shuffle/blob/master/.github/CONTRIBUTING.md). ## Contributors ![ICPL logo](https://github.com/frikky/Shuffle/blob/launch/frontend/src/assets/img/icpl_logo.png) diff --git a/backend/Dockerfile b/backend/Dockerfile index 1856d84f..5d7c484d 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,4 +1,4 @@ -from golang as builder +FROM golang:1.16.0-buster as builder # Add files RUN mkdir /app @@ -7,14 +7,12 @@ WORKDIR /app ADD ./go-app/main.go /app ADD ./go-app/walkoff.go /app ADD ./go-app/docker.go /app -ADD ./go-app/codegen.go /app -ADD ./go-app/files.go /app +ADD ./go-app/oauth2.go /app ADD ./go-app/go.mod /app # Required files for code generation ADD ./app_sdk/app_base.py /app_sdk -ADD ./app_sdk/static_baseline.py /app_sdk ADD ./app_sdk_kali/app_base.py /app_sdk_kali ADD ./app_sdk_kali/static_baseline.py /app_sdk_kali ADD ./app_sdk_blackarch/app_base.py /app_sdk_blackarch diff --git a/backend/app_sdk/README.md b/backend/app_sdk/README.md index 754392ff..170781f3 100644 --- a/backend/app_sdk/README.md +++ b/backend/app_sdk/README.md @@ -1,9 +1,5 @@ # app_sdk.py This is the SDK used for apps to behave like they should. -To change it in the backend, upload it to Buckets/shuffler.appspot.com/generated_apps/baseline. - -# static_baseline.py -It's used for python code generation and should be under MIT. Has to be located here because it's used by the backend. ## If you want to update apps.. PS: downloads from docker hub do overrides.. :) 1. Write your code & check if runtime works diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index e428da9f..c3bc15c4 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -9,6 +9,7 @@ import requests import urllib.parse import http.client import urllib3 +import hashlib class AppBase: __version__ = None @@ -27,6 +28,7 @@ class AppBase: self.authorization = os.getenv("AUTHORIZATION", "") self.current_execution_id = os.getenv("EXECUTIONID", "") self.full_execution = os.getenv("FULL_EXECUTION", "") + self.start_time = int(time.time()) self.result_wrapper_count = 0 if isinstance(self.action, str): @@ -91,6 +93,135 @@ class AppBase: else: return {()} + # Handles unique fields by negoiating with the backend + def validate_unique_fields(self, params): + #print("IN THE UNIQUE FIELDS PLACE!") + + newlist = [params] + if isinstance(params, list): + #print("ITS A LIST!") + newlist = params + + #self.full_execution = os.getenv("FULL_EXECUTION", "") + #print(len(params)) + #print(params.items()) + #print(list(params.items())) + #print(f"PARAM: {params}") + #print(f"NEWLIST: {newlist}") + + # FIXME: Also handle MULTI PARAM + values = [] + param_names = [] + all_values = {} + index = 0 + for outerparam in newlist: + + #print(f"INNERTYPE: {type(outerparam)}") + #print(f"HANDLING PARAM {key}") + param_value = "" + for key, value in outerparam.items(): + #print("KEY: %s" % key) + #value = params[key] + for param in self.action["parameters"]: + try: + if param["name"] == key and param["unique_toggled"]: + print(f"FOUND: {key} with param {param}!") + if isinstance(value, dict) or isinstance(value, list): + try: + value = json.dumps(value) + except json.decoder.JSONDecodeError as e: + print(f"Error in json decode for param {value}: {e}") + continue + elif isinstance(value, int) or isinstance(value, float): + value = str(value) + elif value == False: + value = "False" + elif value == True: + value = "True" + + print(f"VALUE APPEND: {value}") + param_value += value + if param["name"] not in param_names: + param_names.append(param["name"]) + + except (KeyError, NameError) as e: + print(f"""Key/NameError in param handler for {param["name"]}: {e}""") + + print(f"OUTER VALUE: {param_value}") + if len(param_value) > 0: + md5 = hashlib.md5(param_value.encode('utf-8')).hexdigest() + values.append(md5) + all_values[md5] = { + "index": index, + } + + index += 1 + + # When in here, it means it should be unique + # Should this be done by the backend? E.g. ask it if the value is valid? + # 1. Check if it's unique towards key:value store in org for action + # 2. Check if COMBINATION is unique towards key:value store of action for org + # 3. Have a workflow configuration for unique ID's in unison or per field? E.g. if toggled, then send a hash of all fields together alphabetically, but if not, send one field at a time + + # org_id = full_execution["workflow"]["execution_org"]["id"] + + # USE ARRAY? + + new_params = [] + if len(values) > 0: + org_id = self.full_execution["workflow"]["execution_org"]["id"] + data = { + "append": True, + "workflow_check": False, + "authorization": self.authorization, + "execution_ref": self.current_execution_id, + "org_id": org_id, + "values": [{ + "app": self.action["app_name"], + "action": self.action["name"], + "parameternames": param_names, + "parametervalues": values, + }] + } + + #print(f"DATA: {data}") + # 1594869a676630b397bc34f7dc0951a3 + + #print(f"VALUE URL: {url}") + #print(f"RET: {ret.text}") + #print(f"ID: {ret.status_code}") + url = f"{self.url}/api/v1/orgs/{org_id}/validate_app_values" + ret = requests.post(url, json=data) + if ret.status_code == 200: + json_value = ret.json() + if len(json_value["found"]) > 0: + modifier = 0 + for item in json_value["found"]: + print(f"Should remove {item}") + + try: + print(f"FOUND: {all_values[item]}") + print(f"SHOULD REMOVE INDEX: {all_values[item]['index']}") + + try: + newlist.pop(all_values[item]["index"]-modifier) + modifier += 1 + except IndexError as e: + print(f"Error popping value from array: {e}") + except (NameError, KeyError) as e: + print(f"Failed removal: {e}") + + + #return False + else: + print("None of the items were found!") + return newlist + else: + print(f"[WARNING] Failed checking values with status code {ret.status_code}!") + + #return True + return newlist + # Returns a list of all the executions to be done in the inner loop # FIXME: Doesn't take into account whether you actually WANT to loop or not # Check if the last part of the value is #? @@ -147,6 +278,7 @@ class AppBase: #self.action = action loopnames = [] + print(f"Baseparams to check!!: {baseparams}") for key, value in baseparams.items(): check_value = "" for param in self.action["parameters"]: @@ -160,6 +292,7 @@ class AppBase: self.result_wrapper_count = octothorpe_count print("[INFO] NEW OCTOTHORPE WRAPPER: %d" % octothorpe_count) + # This whole thing is hard. # item = [{"data": "1.2.3.4", "dataType": "ip"}] # $item = DONT loop items. @@ -178,12 +311,40 @@ class AppBase: # FIXME: Check the above, and fix so that nested looped items can be # Skipped if wanted - print("\nCHECK: %s" % check_value) + #print("\nCHECK: %s" % check_value) + #try: + # values = parameter["value_replace"] + # if values != None: + # print(values) + # for val in values: + # print(val) + #except: + # pass + should_merge = False if "#" in check_value: should_merge = True + # Specific for OpenAPI body replacement + print("\n\n\nDOING STUFF BELOW HERE") + if not should_merge: + for parameter in self.action["parameters"]: + if parameter["name"] == key: + print("CHECKING BODY FOR VALUE REPLACE DATA!") + try: + values = parameter["value_replace"] + if values != None: + print(values) + for val in values: + if "#" in val["value"]: + should_merge = True + break + except: + pass + + print(f"MERGE: {should_merge}") if isinstance(value, list): + print("Item {value} is a list.") if len(value) <= 1: if len(value) == 1: baseparams[key] = value[0] @@ -206,7 +367,7 @@ class AppBase: all_list_keys.append(key) all_lists.append(baseparams[key]) else: - print("%s is not a list: " % value) + print(f"{value} is not a list") print("Listlengths: %s" % listlengths) if len(listlengths) == 0: @@ -271,20 +432,25 @@ class AppBase: # Runs recursed versions with inner loops and such async def run_recursed_items(self, func, baseparams, loop_wrapper): + print(f"RECURSED ITEMS: {baseparams}") has_loop = False newparams = {} for key, value in baseparams.items(): if isinstance(value, list) and len(value) > 0: - print("In list check") + print(f"In list check for {key}") + try: - value[0] = json.loads(value[0]) + # Added skip for body (OpenAPI) which uses data= in requests + # Can be screwed up if they name theirs body too + if key != "body": + value[0] = json.loads(value[0]) except json.decoder.JSONDecodeError as e: print("JSON casting error: %s" % e) except TypeError as e: print("TypeError: %s" % e) - print("POST list check") + print("POST initial list check") if isinstance(value, list) and len(value) == 1 and isinstance(value[0], list): try: @@ -294,12 +460,11 @@ class AppBase: except KeyError: loop_wrapper[key] = 1 - print("Key %s is a list: %s" % (key, value)) + print(f"Key {key} is a list: {value}") newparams[key] = value[0] has_loop = True else: - print("Key %s is NOT a list within a list" % (key)) - + print(f"Key {key} is NOT a list within a list. Value: {value}") newparams[key] = value results = [] @@ -313,8 +478,43 @@ class AppBase: ret = [] param_multiplier = await self.get_param_multipliers(newparams) + # FIXME: This does a deduplication of the data + new_params = self.validate_unique_fields(param_multiplier) + print(f"NEW PARAMS: {new_params}") + if len(new_params) == 0: + print("[WARNING] SHOULD STOP MULTI-EXECUTION BECAUSE FIELDS AREN'T UNIQUE") + action_result = { + "action": self.action, + "authorization": self.authorization, + "execution_id": self.current_execution_id, + "result": f"All {len(param_multiplier)} values were non-unique", + "started_at": self.start_time, + "status": "SKIPPED", + "completed_at": int(time.time()), + } + + self.send_result(action_result, {"Content-Type": "application/json", "Authorization": "Bearer %s" % self.authorization}, "/api/v1/streams") + exit() + #return + else: + #subparams = new_params + print(f"NEW PARAMS: {new_params}") + param_multiplier = new_params + + #print("Returned with newparams of length %d", len(new_params)) + #if isinstance(new_params, list) and len(new_params) == 1: + # params = new_params[0] + #else: + # print("[WARNING] SHOULD STOP EXECUTION BECAUSE FIELDS AREN'T UNIQUE") + # action_result["status"] = "SKIPPED" + # action_result["result"] = f"A non-unique value was found" + # action_result["completed_at"] = int(time.time()) + # self.send_result(action_result, headers, stream_path) + # return + print("[INFO] Multiplier length: %d" % len(param_multiplier)) for subparams in param_multiplier: + print(f"SUBPARAMS IN MULTI: {subparams}") try: tmp = await func(**subparams) except: @@ -356,7 +556,8 @@ class AppBase: print("Ret length: %d" % len(ret)) if len(ret) == 1: - ret = ret[0] + #ret = ret[0] + print("DONT make list of 1 into 0!!") print("Return from execution: %s" % ret) if ret == None: @@ -383,7 +584,8 @@ class AppBase: results.append(ret) if len(results) == 1: - results = results[0] + #results = results[0] + print("DONT MAKE LIST FROM 1 TO 0!!") print("\nLOOP: %s\nRESULTS: %s" % (loop_wrapper, results)) return results @@ -486,11 +688,11 @@ class AppBase: data["filename"] = curfile["filename"] filename = curfile["filename"] except KeyError as e: - print("KeyError in file setup: %s" % e) + print(f"KeyError in file setup: {e}") pass ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data) - print("Ret CREATE: %s" % ret.text) + print(f"Ret CREATE: {ret.text}") cur_id = "" if ret.status_code == 200: print("RET: %s" % ret.text) @@ -511,7 +713,7 @@ class AppBase: continue new_headers = { - "Authorization": "Bearer %s" % self.authorization, + "Authorization": f"Bearer {self.authorization}", } upload_path = "/api/v1/files/%s/upload?execution_id=%s" % (cur_id, full_execution["execution_id"]) @@ -712,6 +914,41 @@ class AppBase: return data.strip() if "split" in thistype: return data.split() + if "join" in thistype: + print(f"SHOULD JOIN: {data}") + try: + splitvalues = data.split(",") + if "," not in data: + return f"join({data})" + + if len(splitvalues) >= 2: + print(f"SPLITVALUE: {splitvalues[-1]}") + + # 1. Take the list and parse it from string + # 2. Take all the items and join them + # 3. Parse them back as string and return + values = ",".join(splitvalues[0:-1]) + print(f"VALUES: {values}") + tmp = json.loads(values) + print(f"TMP: {tmp}") + #tmp = tmp[1:-1] + #print(f"TMP2: {tmp}") + try: + newvalues = splitvalues[-1].join(str(item).strip() for item in tmp) + except TypeError: + newvalues = splitvalues[-1].join(json.dumps(item).strip() for item in tmp) + + print(f"new: {newvalues}") + return newvalues + else: + print("Returning default") + return f"join({data})" + + except (KeyError, IndexError) as e: + print(f"ERROR in join(): {e}") + except json.decoder.JSONDecodeError as e: + print(f"JSON ERROR in join(): {e}") + if "len" in thistype or "length" in thistype or "lenght" in thistype: tmp = "" try: @@ -725,9 +962,9 @@ class AppBase: pass if isinstance(tmp, list): - return len(tmp) + return str(len(tmp)) elif isinstance(tmp, object): - return len(tmp) + return str(len(tmp)) return str(len(data)) if "parse" in thistype: @@ -773,7 +1010,7 @@ class AppBase: #print("Running %s" % data) # Look for the INNER wrapper first, then move out - wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length", "lenght"] + wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length", "lenght", "join"] found = False for wrapper in wrappers: if wrapper not in data.lower(): @@ -788,8 +1025,8 @@ class AppBase: # Do stuff here. innervalue = parse_nested_param(data, maxDepth(data)-0) outervalue = parse_nested_param(data, maxDepth(data)-1) - #print("INNER: ", innervalue) - #print("OUTER: ", outervalue) + print("INNER: ", innervalue) + print("OUTER: ", outervalue) if outervalue != innervalue: #print("Outer: ", outervalue, " inner: ", innervalue) @@ -971,7 +1208,9 @@ class AppBase: outercnt += 1 except KeyError as e: - print("Lower keyerror: %s" % e) + print("[INFO] Lower keyerror: %s" % e) + return "", False + #return basejson #return "KeyError: Couldn't find key: %s" % e @@ -1030,10 +1269,10 @@ class AppBase: baseresult = variable["value"] break except KeyError as e: - print("KeyError wf variables: %s" % e) + print("[INFO] KeyError wf variables: %s" % e) pass except TypeError as e: - print("TypeError wf variables: %s" % e) + print("[INFO] TypeError wf variables: %s" % e) pass print("BEFORE EXECUTION VAR") @@ -1046,10 +1285,10 @@ class AppBase: baseresult = variable["value"] break except KeyError as e: - print("KeyError exec variables: %s" % e) + print("[INFO] KeyError exec variables: %s" % e) pass except TypeError as e: - print("TypeError exec variables: %s" % e) + print("[INFO] TypeError exec variables: %s" % e) pass except KeyError as error: @@ -1105,6 +1344,7 @@ class AppBase: # Matches with space in the first part, but not in subsequent parts. # JSON / yaml etc shouldn't have spaces in their fields anyway. + #match = ".*?([$]{1}([a-zA-Z0-9 _-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})[$/, ]?" match = ".*?([$]{1}([a-zA-Z0-9 _-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})" # Regex to find all the things @@ -1220,16 +1460,16 @@ class AppBase: self.logger.info("Checking %s %s %s" % (sourcevalue, check, destinationvalue)) if check == "=" or check.lower() == "equals": - if sourcevalue.lower() == destinationvalue.lower(): + if str(sourcevalue).lower() == str(destinationvalue).lower(): return True elif check == "!=" or check.lower() == "does not equal": - if sourcevalue.lower() != destinationvalue.lower(): + if str(sourcevalue).lower() != str(destinationvalue).lower(): return True elif check.lower() == "startswith": - if sourcevalue.lower().startswith(destinationvalue.lower()): + if str(sourcevalue).lower().startswith(str(destinationvalue).lower()): return True elif check.lower() == "endswith": - if sourcevalue.lower().endswith(destinationvalue.lower()): + if str(sourcevalue).lower().endswith(str(destinationvalue).lower()): return True elif check.lower() == "contains": if destinationvalue.lower() in sourcevalue.lower(): @@ -1303,7 +1543,7 @@ class AppBase: sourcevalue = condition["source"]["value"] check, sourcevalue, is_loop = parse_params(action, fullexecution, condition["source"]) if check: - return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check) + return False, {"success": False, "reason": "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)} #sourcevalue = sourcevalue.encode("utf-8") @@ -1312,7 +1552,7 @@ class AppBase: check, destinationvalue, is_loop = parse_params(action, fullexecution, condition["destination"]) if check: - return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check) + return False, {"success": False, "reason": "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)} #destinationvalue = destinationvalue.encode("utf-8") destinationvalue = parse_wrapper_start(destinationvalue) @@ -1353,7 +1593,7 @@ class AppBase: if not validation: self.logger.info("Failed condition check for %s %s %s." % (sourcevalue, condition["condition"]["value"], destinationvalue)) - return False, "Failed condition: %s %s %s" % (sourcevalue, condition["condition"]["value"], destinationvalue) + return False, {"success": False, "reason": "Failed condition: %s %s %s" % (sourcevalue, condition["condition"]["value"], destinationvalue)} # Make a general parser here, at least to get param["name"] = param["value"] in maparameter[string]string @@ -1361,12 +1601,14 @@ class AppBase: return True, "" + + # THE START IS ACTUALLY RIGHT HERE :O # Checks whether conditions are met, otherwise set branchcheck, tmpresult = check_branch_conditions(action, fullexecution) if not branchcheck: self.logger.info("Failed one or more branch conditions.") action_result["result"] = tmpresult - action_result["status"] = "FAILURE" + action_result["status"] = "SKIPPED" try: ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) self.logger.info("Result: %d" % ret.status_code) @@ -1392,7 +1634,7 @@ class AppBase: try: func = getattr(self, actionname, None) if func == None: - self.logger.debug("Failed executing %s because func is None." % actionname) + self.logger.debug(f"Failed executing {actionname} because func is None.") action_result["status"] = "FAILURE" action_result["result"] = "Function %s doesn't exist." % actionname elif callable(func): @@ -1409,7 +1651,7 @@ class AppBase: params = {} try: for item in action["authentication"]: - print("AUTH: ", key, value) + #print("AUTH: ", key, value) params[item["key"]] = item["value"] except KeyError: print("No authentication specified!") @@ -1431,12 +1673,16 @@ class AppBase: if "||" in parameter["value"]: splitvalue = parameter["value"].split("||") if len(splitvalue) > 1: - print(f'[INFO] Parsed split || options of actions["parameters"]["name"]') + #print(f'[INFO] Parsed split || options of actions["parameters"]["name"]') action["parameters"][counter]["value"] = splitvalue[1] except (IndexError, KeyError, TypeError) as e: - print("Options err: {e}") + print("[WARNING] Options err: {e}") + # This part is purely for OpenAPI accessibility. + # It replaces the data back into the main item + # Earlier, we handled each of the items and did later string replacement, + # but this has changed to do lists within items and such if parameter["name"] == "body": bodyindex = counter #print("PARAM: %s" % parameter) @@ -1445,16 +1691,28 @@ class AppBase: if values != None: added = 0 for val in values: - newparams.append({ - "name": val["key"], - "value": val["value"], - "variant": "STATIC_VALUE", - "id": "body_replacement", - }) + #print(f"VAL: {val}") + #parameter["value"].replace(val["key"], val["value"], -1) + #print(f'PARAM1: {action["parameters"][counter]["value"]}') + action["parameters"][counter]["value"] = action["parameters"][counter]["value"].replace(val["key"], val["value"], 1) + #action["parameters"][counter]["value"].replace(r"${url}", r"$Find_URLs.valid.#.data", 1) + #print(f'PARAM2: {action["parameters"][counter]["value"]}') + #newparams.append({ + # "name": val["key"], + # "value": val["value"], + # "variant": "STATIC_VALUE", + # "id": "body_replacement", + # "schema": { + # "type": "string", + # }, + #}) - print("Added param %s for body" % val["key"]) + #print(f'[INFO] Added param {val["key"]} for body with value {val["value"]} (using OpenAPI)') + print(f'[INFO] Added param {val["key"]} for body (using OpenAPI)') added += 1 + #action["parameters"]["body"] + print("ADDED %d parameters for body" % added) except KeyError as e: print("KeyError body OpenAPI: %s" % e) @@ -1462,6 +1720,7 @@ class AppBase: break + #print(action["parameters"]) for parameter in newparams: action["parameters"].append(parameter) @@ -1478,6 +1737,7 @@ class AppBase: multi_parameters = json.loads(json.dumps(params)) multiexecution = False multi_execution_lists = [] + remove_params = [] for parameter in action["parameters"]: check, value, is_loop = parse_params(action, fullexecution, parameter) if check: @@ -1485,7 +1745,10 @@ class AppBase: # Custom format for ${name[0,1,2,...]}$ #submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" - submatch = "([${]{2}#?([0-9a-zA-Z_-]+)#?(\[.*\])[}$]{2})" + #print(f"Returnedvalue: {value}") + # OLD: Used until 13.03.2021: submatch = "([${]{2}#?([0-9a-zA-Z_-]+)#?(\[.*\])[}$]{2})" + # \${[0-9a-zA-Z_-]+#?(\[.*?]}\$) + submatch = "([${]{2}#?([0-9a-zA-Z_-]+)#?(\[.*?]}\$))" actualitem = re.findall(submatch, value, re.MULTILINE) try: if action["skip_multicheck"]: @@ -1505,17 +1768,24 @@ class AppBase: # Loop WITH variables go in else. print("Before first part in multiexec!") handled = False + + # Has a loop without a variable used inside if len(actualitem[0]) > 2 and actualitem[0][1] == "SHUFFLE_NO_SPLITTER": + print("(1) Pre replacement: %s" % actualitem[0][2]) tmpitem = value - replacement = actualitem[0][2] + index = 0 + replacement = actualitem[index][2] + if replacement.endswith("}$"): + replacement = replacement[:-2] + if replacement.startswith("\"") and replacement.endswith("\""): replacement = replacement[1:len(replacement)-1] print("POST replacement: %s" % replacement) - #json_replacement = tmpitem.replace(actualitem[0][0], replacement, 1) + #json_replacement = tmpitem.replace(actualitem[index][0], replacement, 1) #print("AFTER POST replacement: %s" % json_replacement) #json_replacement = replacement try: @@ -1537,9 +1807,9 @@ class AppBase: for i in range(len(json_replacement)): if isinstance(json_replacement[i], dict) or isinstance(json_replacement[i], list): tmp_replacer = json.dumps(json_replacement[i]) - newvalue = tmpitem.replace(actualitem[0][0], tmp_replacer, 1) + newvalue = tmpitem.replace(actualitem[index][0], tmp_replacer, 1) else: - newvalue = tmpitem.replace(actualitem[0][0], json_replacement[i], 1) + newvalue = tmpitem.replace(actualitem[index][0], json_replacement[i], 1) try: newvalue = json.loads(newvalue) @@ -1552,13 +1822,13 @@ class AppBase: print("New replacement: %s" % new_replacement) # New - tmpitem = tmpitem.replace(actualitem[0][0], replacement, 1) + tmpitem = tmpitem.replace(actualitem[index][0], replacement, 1) # This code handles files. - print("(1) ------------ PARAM: %s" % parameter["schema"]["type"]) resultarray = [] isfile = False try: + print("(1) ------------ PARAM: %s" % parameter["schema"]["type"]) if parameter["schema"]["type"] == "file" and len(value) > 0: print("(1) SHOULD HANDLE FILE IN MULTI. Get based on value %s" % tmpitem) # This is silly :) @@ -1572,8 +1842,10 @@ class AppBase: print("(1) FILE VALUE FOR VAL %s: %s" % (tmp_file_split, file_value)) isfile = True + except NameError as e: + print("(1) SCHEMA NAMEERROR IN FILE HANDLING: %s" % e) except KeyError as e: - print("(1) SCHEMA ERROR IN FILE HANDLING: %s" % e) + print("(1) SCHEMA KEYERROR IN FILE HANDLING: %s" % e) except json.decoder.JSONDecodeError as e: print("(1) JSON ERROR IN FILE HANDLING: %s" % e) @@ -1589,7 +1861,7 @@ class AppBase: multi_execution_lists.append(new_replacement) #print("MULTI finished: %s" % json_replacement) else: - print("(2) Pre replacement. ") #% actualitem) + print(f"(2) Pre replacement (loop with variables). Variables: {actualitem}") #% actualitem) # This is here to handle for loops within variables.. kindof # 1. Find the length of the longest array # 2. Build an array with the base values based on parameter["value"] @@ -1601,9 +1873,15 @@ class AppBase: try: to_be_replaced = replace[0] actualitem = replace[2] + if actualitem.endswith("}$"): + actualitem = actualitem[:-2] except IndexError: continue + #print(f"\n\nTMPITEM: {actualitem}\n\n") + #actualitem = parse_wrapper_start(actualitem) + #print(f"\n\nTMPITEM2: {actualitem}\n\n") + try: itemlist = json.loads(actualitem) if len(itemlist) > minlength: @@ -1611,14 +1889,19 @@ class AppBase: if len(itemlist) > curminlength: curminlength = len(itemlist) + except json.decoder.JSONDecodeError as e: - print("JSON Error: %s in %s" % (e, actualitem)) + print("JSON Error (replace): %s in %s" % (e, actualitem)) replacements[to_be_replaced] = actualitem + + # Parses the data as string with length, split etc. before moving on. + + + #print("In second part of else: %s" % (len(itemlist))) # This is a result array for JUST this value.. # What if there are more? - print("LENGTH: %d. In second part of else: %s" % (len(itemlist), replacements)) resultarray = [] for i in range(0, curminlength): tmpitem = json.loads(json.dumps(parameter["value"])) @@ -1664,22 +1947,56 @@ class AppBase: multi_execution_lists.append(resultarray) multi_parameters[parameter["name"]] = resultarray + + #if parameter["id"] == "body_replacement": + # print("Should run body MULTI replacement in index %d with %s" % (bodyindex, parameter)) + # try: + # print("PREBODY: %s" % params["body"]) + + # parsedarray = str(resultarray) + # try: + # parsedarray = json.dumps(resultarray) + # except: + # pass + + # if f'\"{parameter["name"]}\"' in params["body"]: + # params["body"] = params["body"].replace(f'\"{parameter["name"]}\"' , parsedarray, -1) + # multi_parameters["body"] = multi_parameters["body"].replace(f'\"{parameter["name"]}\"' , parsedarray, -1) + # else: + # params["body"] = params["body"].replace(parameter["name"], parsedarray, -1) + # multi_parameters["body"] = multi_parameters["body"].replace(parameter["name"], parsedarray, -1) + + # #print("POSTBODY: %s" % params["body"]) + # #if isinstance(multi_parameters, list): + # # print("MULTIPARAM AS LIST (NOT REPLACING!!)!") + # # for multiparam in multi_parameters: + # # print(f"MULTIPARAM: {multiparam}") + # # #multi_parameters["body"] = multi_parameters["body"].replace(parameter["name"], str(parameter["value"]), -1) + # #else: + + # except KeyError as e: + # print("KEYERROR: %s" % e) + + # remove_params.append(parameter["name"]) + # #bodyindex = counter + # continue + else: # Parses things like int(value) print("Normal parsing (not looping)")#with data %s" % value) value = parse_wrapper_start(value) - if parameter["id"] == "body_replacement": - print("Should run body replacement in index %d with %s" % (bodyindex, parameter)) - try: - print("PREBODY: %s" % params["body"]) - params["body"] = params["body"].replace(parameter["name"], parameter["value"], -1) - print("POSTBODY: %s" % params["body"]) - except KeyError as e: - print("KEYERROR: %s" % e) + #if parameter["id"] == "body_replacement": + # print("Should run body replacement in index %d with %s" % (bodyindex, parameter)) + # try: + # print("PREBODY: %s" % params["body"]) + # params["body"] = params["body"].replace(parameter["name"], parameter["value"], -1) + # print("POSTBODY: %s" % params["body"]) + # except KeyError as e: + # print("KEYERROR: %s" % e) - #bodyindex = counter - continue + # #bodyindex = counter + # continue #for parameter in action["parameters"]: #if parameter["name"] == "body": @@ -1702,6 +2019,8 @@ class AppBase: except KeyError as e: print("SCHEMA ERROR IN FILE HANDLING: %s" % e) + + #remove_params.append(parameter["name"]) # Fix lists here # FIXME: This doesn't really do anything anymore print("CHECKING multi execution list!") @@ -1723,7 +2042,7 @@ class AppBase: #print("New list length: %d" % len(filteredlist)) if len(filteredlist) > 1: - print("Calculating new multi-loop length with %d lists" % len(filteredlist)) + print(f"Calculating new multi-loop length with {len(filteredlist)} lists") tmplength = 1 for innerlist in filteredlist: tmplength = len(innerlist)*tmplength @@ -1732,19 +2051,43 @@ class AppBase: minlength = tmplength print("New multi execution length: %d\n" % tmplength) + + # Cleaning up extra list params + for subparam in remove_params: + #print(f"DELETING {subparam}") + try: + del params[subparam] + except: + pass + #print(f"Error with subparam deletion of {subparam} in {params}") + try: + del multi_parameters[subparam] + except: + #print(f"Error with subparam deletion of {subparam} in {multi_parameters} (2)") + pass + + #print() + #print(f"Param: {params}") + #print(f"Multiparams: {multi_parameters}") + #print() if not multiexecution: - #newparams.append({ - # "name": val["key"], - # "value": val["value"], - # "variant": "STATIC_VALUE", - # "id": "body_replacement", - #}) + # Runs a single iteration here + new_params = self.validate_unique_fields(params) + print(f"Returned with newparams of length {len(new_params)}") + if isinstance(new_params, list) and len(new_params) == 1: + params = new_params[0] + else: + print("[WARNING] SHOULD STOP EXECUTION BECAUSE FIELDS AREN'T UNIQUE") + action_result["status"] = "SKIPPED" + action_result["result"] = f"A non-unique value was found" + action_result["completed_at"] = int(time.time()) + self.send_result(action_result, headers, stream_path) + return - #print("[INFO] APP_SDK DONE: Starting NORMAL execution of function") print("[INFO] Running normal execution\n") newres = await func(**params) - print("\n[INFO] Returned from execution with datalength!")#, newres) + print("\n[INFO] Returned from execution!")#, newres) if isinstance(newres, tuple): print("[INFO] Handling return as tuple") # Handles files. @@ -1772,6 +2115,17 @@ class AppBase: elif isinstance(newres, str): print("[INFO] Handling return as string of length %d" % len(newres)) result += newres + elif isinstance(newres, dict) or isinstance(newres, list): + try: + result += json.dumps(newres, indent=4) + except json.JSONDecodeError as e: + print("Failed decoding result: %s" % e) + + try: + result += str(newres) + except ValueError: + result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) + print("Can't handle type %s value from function" % (type(newres))) else: try: result += str(newres) @@ -1898,10 +2252,16 @@ class AppBase: # Dump the result as a string of a list #print("RESULTS: %s" % results) - if isinstance(results, list): + if isinstance(results, list) or isinstance(results, dict): print("JSON OBJECT? ", json_object) + + # This part is weird lol if json_object: - result = json.dumps(results) + try: + result = json.dumps(results) + except json.JSONDecodeError as e: + print(f"Failed to decode: {e}") + result = results else: result = "[" for item in results: @@ -1926,14 +2286,13 @@ class AppBase: else: print("Normal result - no list?") result = results - - print("RESULT: %s" % result) + action_result["status"] = "SUCCESS" action_result["result"] = str(result) if action_result["result"] == "": action_result["result"] = result - self.logger.debug(f"Executed {action['label']}-{action['id']} with result: {result}") + self.logger.debug(f"Executed {action['label']}-{action['id']}")#with result: {result}") #self.logger.debug(f"Data: %s" % action_result) except TypeError as e: print("TypeError issue: %s" % e) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 25fa59d6..eaee980c 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.60 +VERSION=0.8.64 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/app_sdk/requirements.txt b/backend/app_sdk/requirements.txt index 0da82340..acde3f06 100644 --- a/backend/app_sdk/requirements.txt +++ b/backend/app_sdk/requirements.txt @@ -1,2 +1,2 @@ -urllib3 -requests +urllib3=1.25.9 +requests=2.25.1 diff --git a/backend/app_sdk/static_baseline.py b/backend/app_sdk/static_baseline.py deleted file mode 100644 index 152131cd..00000000 --- a/backend/app_sdk/static_baseline.py +++ /dev/null @@ -1,76 +0,0 @@ -import os -import sys -import time -import logging -import requests - -# Goal here: -# * Make an app from WALKOFF able to run without app_base.py from WALKOFF -# # How: -# * Make it rely 100% on INPUT throug HTTP invocations instead of redis READS -# # But really, how? -# * Make a WORKER that reads the queue, and reuses a function - -# Here to get it global -apikey = "" -try: - apikey = os.environ["FUNCTION_APIKEY"] -except KeyError: - pass - -# Authorize the execution -def authorization(request): - # This is basically my issue, but it enforces the use of an internal API key for execution - try: - apikey = os.environ["FUNCTION_APIKEY"] - except KeyError: - return f"Internal server error", 500 - - - # Check API key from ENV authentication - authentication = request.headers.get("Authorization") - if authentication == None: return f"Unauthorized", 401 - - apikey_split = authentication.split(" ") - if apikey_split[0] != "Bearer" or len(apikey_split) != 2: - return f"Apikey error", 401 - - if apikey != apikey_split[1]: - return f"Unauthorized", 401 - - return run(request) - -class AppBase: - """ The base class for Python-based Walkoff applications, handles Redis and logging configurations. """ - __version__ = None - app_name = None - - def __init__(self, redis=None, logger=None, console_logger=None):#, docker_client=None): - self.logger = logger if logger is not None else logging.getLogger("AppBaseLogger") - self.redis=redis - self.console_logger=console_logger - self.current_execution_id = None - self.url = "https://shuffler.io" - self.apikey = apikey - - @classmethod - async def run(cls, action): - """ Connect to Redis and HTTP session, await actions """ - logging.basicConfig(format="{asctime} - {name} - {levelname}:{message}", style='{') - logger = logging.getLogger(f"{cls.__name__}") - logger.setLevel(logging.DEBUG) - - app = cls(redis=None, logger=logger, console_logger=logger) - - # Authorization for the app/function to control the workflow - # Function will crash if its wrong, which it probably should. - - await app.execute_action(action) - - async def execute_action(self, action): - # FIXME - add request for the function STARTING here. Use "results stream" or something - # PAUSED, AWAITING_DATA, PENDING, COMPLETED, ABORTED, EXECUTING, SUCCESS, FAILURE - - self.authorization = action["authorization"] - self.execution_id = action["execution_id"] - self.current_execution_id = action["execution_id"] diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go deleted file mode 100644 index 74776e88..00000000 --- a/backend/go-app/codegen.go +++ /dev/null @@ -1,1962 +0,0 @@ -package main - -import ( - "archive/zip" - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "log" - "os" - "strconv" - "strings" - - "cloud.google.com/go/storage" - "github.com/getkin/kin-openapi/openapi3" - //"github.com/satori/go.uuid" - "gopkg.in/yaml.v2" -) - -func copyFile(fromfile, tofile string) error { - from, err := os.Open(fromfile) - if err != nil { - return err - } - defer from.Close() - - to, err := os.OpenFile(tofile, os.O_RDWR|os.O_CREATE, 0666) - if err != nil { - return err - } - defer to.Close() - - _, err = io.Copy(to, from) - if err != nil { - return err - } - - return nil -} - -func formatAppfile(filedata string) (string, string) { - lines := strings.Split(filedata, "\n") - - newfile := []string{} - classname := "" - for _, line := range lines { - if strings.Contains(line, "walkoff_app_sdk") { - continue - } - - // Remap logging. CBA this right now - // This issue also persists in onprem apps because of await thingies.. :( - // FIXME - if strings.Contains(line, "console_logger") && strings.Contains(line, "await") { - continue - //line = strings.Replace(line, "console_logger", "logger", -1) - //log.Println(line) - } - - // Might not work with different import names - // Could be fucked up with spaces everywhere? Idk - if strings.Contains(line, "class") && strings.Contains(line, "(AppBase)") { - items := strings.Split(line, " ") - if len(items) > 0 && strings.Contains(items[1], "(AppBase)") { - classname = strings.Split(items[1], "(")[0] - } else { - // This could break something.. - classname = "TMP" - } - } - - if strings.Contains(line, "if __name__ ==") { - break - } - - // asyncio.run(HelloWorld.run(), debug=True) - - newfile = append(newfile, line) - } - - filedata = strings.Join(newfile, "\n") - return classname, filedata -} - -// Streams the data into a zip to be used for a cloud function -func streamZipdata(ctx context.Context, identifier, pythoncode, requirements string) (string, error) { - filename := fmt.Sprintf("generated_cloudfunctions/%s.zip", identifier) - - buf := new(bytes.Buffer) - zipWriter := zip.NewWriter(buf) - - zipFile, err := zipWriter.Create("main.py") - if err != nil { - log.Printf("Packing failed to create zip file from bucket: %v", err) - return filename, err - } - - // Have to use Fprintln otherwise it tries to parse all strings etc. - if _, err := fmt.Fprintln(zipFile, pythoncode); err != nil { - return filename, err - } - - zipFile, err = zipWriter.Create("requirements.txt") - if err != nil { - log.Printf("Packing failed to create zip file from bucket: %v", err) - return filename, err - } - if _, err := fmt.Fprintln(zipFile, requirements); err != nil { - return filename, err - } - - err = zipWriter.Close() - if err != nil { - log.Printf("Packing failed to close zip file writer from bucket: %v", err) - return filename, err - } - - return filename, nil -} - -func getAppbase() ([]byte, []byte, error) { - // 1. Have baseline in bucket/generated_apps/baseline - // 2. Copy the baseline to a new folder with identifier name - static := "../app_sdk/static_baseline.py" - appbase := "../app_sdk/app_base.py" - - staticData, err := ioutil.ReadFile(static) - if err != nil { - return []byte{}, []byte{}, err - } - - appbaseData, err := ioutil.ReadFile(appbase) - if err != nil { - return []byte{}, []byte{}, err - } - - return appbaseData, staticData, nil -} - -// Builds the structure for the new generated app in storage (copying baseline files) -func getAppbaseGCP(ctx context.Context, client *storage.Client) ([]byte, []byte, error) { - // 1. Have baseline in bucket/generated_apps/baseline - // 2. Copy the baseline to a new folder with identifier name - basePath := "generated_apps/baseline" - static, err := client.Bucket(bucketName).Object(fmt.Sprintf("%s/static_baseline.py", basePath)).NewReader(ctx) - if err != nil { - return []byte{}, []byte{}, err - } - appbase, err := client.Bucket(bucketName).Object(fmt.Sprintf("%s/app_base.py", basePath)).NewReader(ctx) - if err != nil { - return []byte{}, []byte{}, err - } - - defer static.Close() - defer appbase.Close() - - staticData, err := ioutil.ReadAll(static) - if err != nil { - return []byte{}, []byte{}, err - } - - appbaseData, err := ioutil.ReadAll(appbase) - if err != nil { - return []byte{}, []byte{}, err - } - - return appbaseData, staticData, nil -} - -func fixAppbase(appbase []byte) []string { - record := false - validLines := []string{} - for _, line := range strings.Split(string(appbase), "\n") { - if strings.Contains(line, "#STOPCOPY") { - //log.Println("Stopping copy") - break - } - - if record { - validLines = append(validLines, line) - } - - if strings.Contains(line, "#STARTCOPY") { - //log.Println("Starting copy") - record = true - } - } - - return validLines -} - -// Builds the structure for the new generated app in storage (copying baseline files) -func buildStructureGCP(ctx context.Context, client *storage.Client, swagger *openapi3.Swagger, curHash string) (string, error) { - // 1. Have baseline in bucket/generated_apps/baseline - // 2. Copy the baseline to a new folder with identifier name - - basePath := "generated_apps" - identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash) - appPath := fmt.Sprintf("%s/%s", basePath, identifier) - fileNames := []string{"Dockerfile", "requirements.txt"} - for _, file := range fileNames { - src := client.Bucket(bucketName).Object(fmt.Sprintf("%s/baseline/%s", basePath, file)) - dst := client.Bucket(bucketName).Object(fmt.Sprintf("%s/%s", appPath, file)) - if _, err := dst.CopierFrom(src).Run(ctx); err != nil { - return "", err - } - } - - return appPath, nil -} - -// Builds the base structure for the app that we're making -// Returns error if anything goes wrong. This has to work if -// the python code is supposed to be generated -func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) { - //log.Printf("%#v", swagger) - - // adding md5 based on input data to not overwrite earlier data. - generatedPath := "generated" - subpath := "../app_gen/openapi/" - identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash) - appPath := fmt.Sprintf("%s/%s", generatedPath, identifier) - - os.MkdirAll(appPath, os.ModePerm) - os.Mkdir(fmt.Sprintf("%s/src", appPath), os.ModePerm) - - err := copyFile(fmt.Sprintf("%sbaseline/Dockerfile", subpath), fmt.Sprintf("%s/%s", appPath, "Dockerfile")) - if err != nil { - log.Println("Failed to move Dockerfile") - return appPath, err - } - - err = copyFile(fmt.Sprintf("%sbaseline/requirements.txt", subpath), fmt.Sprintf("%s/%s", appPath, "requirements.txt")) - if err != nil { - log.Println("Failed to move requrements.txt") - return appPath, err - } - - return appPath, nil -} - -// This function generates the python code that's being used. -// This is really meta when you program it. Handling parameters is hard here. -func makePythoncode(swagger *openapi3.Swagger, name, url, method string, parameters, optionalQueries, headers []string, fileField string) (string, string) { - method = strings.ToLower(method) - queryString := "" - queryData := "" - - // FIXME - this might break - need to check if ? or & should be set as query - parameterData := "" - if len(optionalQueries) > 0 { - queryString += ", " - for index, query := range optionalQueries { - // Check if it's a part of the URL already - queryString += fmt.Sprintf("%s=\"\"", query) - if index != len(optionalQueries)-1 { - queryString += ", " - } - - queryData += fmt.Sprintf(` - if %s: - url += f"&%s={%s}"`, query, query, query) - } - } else { - //log.Printf("No optional queries?") - } - - // api.Authentication.Parameters[0].Value = "BearerAuth" - authenticationParameter := "" - authenticationSetup := "" - authenticationAddin := "" - // Python configuration code that should work :) - if swagger.Components.SecuritySchemes != nil { - if swagger.Components.SecuritySchemes["BearerAuth"] != nil { - authenticationParameter = ", apikey" - authenticationSetup = "if apikey != \" \": headers[\"Authorization\"] = f\"Bearer {apikey}\"" - } else if swagger.Components.SecuritySchemes["BasicAuth"] != nil { - authenticationParameter = ", username_basic, password_basic" - authenticationAddin = ", auth=(username_basic, password_basic)" - } else if swagger.Components.SecuritySchemes["ApiKeyAuth"] != nil { - authenticationParameter = ", apikey" - if swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.In == "header" { - // This is a way to bypass apikeys by passing " " - authenticationSetup = fmt.Sprintf(`if apikey != " ": headers["%s"] = apikey`, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name) - } else if swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.In == "query" { - // This might suck lol - key := "?" - if strings.Contains(url, "?") { - key = "&" - } - - authenticationSetup = fmt.Sprintf("if apikey != \" \": url+=f\"%s%s={apikey}\"", key, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name) - } - } - } - - //baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - // This is a quickfix for onpremises stuff. Does work, but should really be - // part of the authentication scheme from openapi3 - urlParameter := "" - urlInline := "" - //log.Printf("URL: %s", url) - if !strings.HasPrefix(strings.ToLower(url), "http") { - urlParameter = ", url" - urlInline = "{url}" - } - - // Specific check for SSL verification - // This is critical for onprem stuff. - //verifyParam := "" - //verifyWrapper := "" - //verifyAddin := "" - verifyParam := ", ssl_verify=False" - verifyWrapper := `if type(ssl_verify) == str: ssl_verify = False if ssl_verify.lower() == "false" or ssl_verify == "0" else True` - verifyAddin := ", verify=ssl_verify" - - if len(parameters) > 0 { - parameterData = fmt.Sprintf(", %s", strings.Join(parameters, ", ")) - } - - // FIXME - add checks for query data etc - - functionname := strings.ToLower(fmt.Sprintf("%s_%s", method, name)) - if strings.Contains(strings.ToLower(name), strings.ToLower(method)) { - functionname = strings.ToLower(name) - } - - bodyParameter := "" - bodyAddin := "" - bodyFormatter := "" - postParameters := []string{"post", "patch", "put"} - for _, item := range postParameters { - if method == item { - bodyParameter = ", body=\"\"" - bodyAddin = ", data=body" - - // FIXME: Does JSON data work? - bodyFormatter = `body = " ".join(body.strip().split()).encode("utf-8")` - } - } - - preparedHeaders := "headers={}" - if len(headers) > 0 { - preparedHeaders = "headers={" - for count, header := range headers { - headerSplit := strings.Split(header, "=") - added := false - if len(headerSplit) == 2 { - if strings.Contains(preparedHeaders, headerSplit[0]) { - continue - } - - preparedHeaders += fmt.Sprintf(`"%s": "%s"`, headerSplit[0], headerSplit[1]) - added = true - } - - if count != len(headers)-1 && added { - preparedHeaders += "," - } - } - - preparedHeaders += "}" - } - - fileBalance := "" - fileAdder := `` - fileGrabber := `` - fileParameter := `` - if method == "post" && len(fileField) > 0 { - fileParameter = ", file_id" - fileGrabber = `filedata = self.get_file(file_id)` - - // This indentation is confusing (but correct) ROFL - fileAdder = fmt.Sprintf(`if not filedata["success"]: - return file_id+" is not a valid File ID" - files = {"%s": (filedata["filename"], filedata["data"])}`, fileField) - fileBalance = ", files=files" - } - - // Extra param for url if it's changeable - // Extra param for authentication scheme(s) - // The last weird one is the body.. Tabs & spaces sucks. - data := fmt.Sprintf(` async def %s(self%s%s%s%s%s%s%s): - %s - url=f"%s%s" - %s - %s - %s - %s - %s - %s - return requests.%s(url, headers=headers%s%s%s%s).text - `, - functionname, - authenticationParameter, - urlParameter, - fileParameter, - parameterData, - queryString, - bodyParameter, - verifyParam, - preparedHeaders, - urlInline, - url, - verifyWrapper, - authenticationSetup, - queryData, - bodyFormatter, - fileGrabber, - fileAdder, - method, - authenticationAddin, - bodyAddin, - verifyAddin, - fileBalance, - ) - - // Use lowercase when checking - /* - if strings.Contains(functionname, "filter") { - //log.Printf("FUNCTION: %s", data) - log.Println(data) - log.Printf("Queries: %s", queryString) - } - */ - - //log.Printf(data) - return functionname, data -} - -func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, WorkflowApp, []string, error) { - api := WorkflowApp{} - //log.Printf("%#v", swagger.Info) - - if len(swagger.Info.Title) == 0 { - return swagger, WorkflowApp{}, []string{}, errors.New("Swagger.Info.Title can't be empty.") - } - - if len(swagger.Servers) == 0 { - //return swagger, WorkflowApp{}, []string{}, errors.New("Swagger.Servers can't be empty. Add 'servers':[{'url':'hostname.com'}'") - //return swagger, WorkflowApp{}, []string{}, errors.New("Swagger.Servers can't be empty. Add 'servers':[{'url':'hostname.com'}'") - swagger.Servers = openapi3.Servers{ - &openapi3.Server{ - URL: "https://hostname.com", - }, - } - } - - api.Name = swagger.Info.Title - api.Description = swagger.Info.Description - - // FIXME: Versioning issue? - api.ID = newmd5 - //uuid.NewV4().String() - - api.IsValid = true - api.Link = swagger.Servers[0].URL // host does not exist lol - if strings.HasSuffix(api.Link, "/") { - api.Link = api.Link[:len(api.Link)-1] - } - - api.AppVersion = "1.0.0" - api.Environment = "Shuffle" - api.SmallImage = "" - api.LargeImage = "" - api.Sharing = false - api.Verified = false - api.Tested = false - api.Invalid = false - api.PrivateID = newmd5 - api.Generated = true - api.Activated = true - // Setting up security schemes - extraParameters := []WorkflowAppActionParameter{} - - if val, ok := swagger.Info.ExtensionProps.Extensions["x-logo"]; ok { - j, err := json.Marshal(&val) - if err == nil { - if j[0] == 0x22 && j[len(j)-1] == 0x22 { - j = j[1 : len(j)-1] - } - - //log.Printf("%s", j) - api.SmallImage = string(j) - api.LargeImage = string(j) - } - } - - // Jesus what a clusterfuck. - // Handles parsing of categories from OpenApi3 custom field - if val, ok := swagger.Info.ExtensionProps.Extensions["x-categories"]; ok { - //log.Printf("Categories: %#v", val) - j, err := json.Marshal(&val) - if err == nil { - if j[0] == 0x22 && j[len(j)-1] == 0x22 { - j = j[1 : len(j)-1] - } - - parsedCategories := fmt.Sprintf(`{"categories": %s}`, string(j)) - type parsed struct { - Categories []string `json:"categories"` - } - - var parse parsed - err := json.Unmarshal([]byte(parsedCategories), &parse) - if err != nil { - log.Printf("Failed unmarshaling categories: %v", err) - } else { - api.Categories = parse.Categories - } - } - } - - if len(swagger.Tags) > 0 { - newTags := []string{} - for _, tag := range swagger.Tags { - newTags = append(newTags, tag.Name) - } - - api.Tags = newTags - } - - securitySchemes := swagger.Components.SecuritySchemes - if securitySchemes != nil { - //log.Printf("%#v", securitySchemes) - - api.Authentication = Authentication{ - Required: true, - Parameters: []AuthenticationParams{}, - } - - // Used for python code generation lol - // Not sure how this should work with oauth - if securitySchemes["BearerAuth"] != nil { - api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{ - Name: "apikey", - Value: "", - Example: "******", - Description: securitySchemes["BearerAuth"].Value.Description, - In: securitySchemes["BearerAuth"].Value.In, - Scheme: securitySchemes["BearerAuth"].Value.Scheme, - Schema: SchemaDefinition{ - Type: securitySchemes["BearerAuth"].Value.Scheme, - }, - }) - - //log.Printf("HANDLE BEARER AUTH") - extraParameters = append(extraParameters, WorkflowAppActionParameter{ - Name: "apikey", - Description: "The apikey to use", - Multiline: false, - Required: true, - Example: "The API key to use. Space = skip", - Configuration: true, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } else if securitySchemes["ApiKeyAuth"] != nil { - api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{ - Name: "apikey", - Value: "", - Example: "******", - Description: securitySchemes["ApiKeyAuth"].Value.Description, - In: securitySchemes["ApiKeyAuth"].Value.In, - Scheme: securitySchemes["ApiKeyAuth"].Value.Scheme, - Schema: SchemaDefinition{ - Type: securitySchemes["ApiKeyAuth"].Value.Scheme, - }, - }) - - //log.Printf("HANDLE APIKEY AUTH") - extraParameters = append(extraParameters, WorkflowAppActionParameter{ - Name: "apikey", - Description: "The apikey to use", - Multiline: false, - Required: true, - Example: "**********", - Configuration: true, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } else if securitySchemes["BasicAuth"] != nil { - api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{ - Name: "username_basic", - Value: "", - Example: "username", - Description: securitySchemes["BasicAuth"].Value.Description, - In: securitySchemes["BasicAuth"].Value.In, - Scheme: securitySchemes["BasicAuth"].Value.Scheme, - Schema: SchemaDefinition{ - Type: securitySchemes["BasicAuth"].Value.Scheme, - }, - }) - - api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{ - Name: "password_basic", - Value: "", - Example: "*****", - Description: securitySchemes["BasicAuth"].Value.Description, - In: securitySchemes["BasicAuth"].Value.In, - Scheme: securitySchemes["BasicAuth"].Value.Scheme, - Schema: SchemaDefinition{ - Type: securitySchemes["BasicAuth"].Value.Scheme, - }, - }) - - extraParameters = append(extraParameters, WorkflowAppActionParameter{ - Name: "username_basic", - Description: "The username to use", - Multiline: false, - Required: true, - Example: "The username to use", - Configuration: true, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - extraParameters = append(extraParameters, WorkflowAppActionParameter{ - Name: "password_basic", - Description: "The password to use", - Multiline: false, - Required: true, - Example: "***********", - Configuration: true, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } - } - - // Adds a link parameter if it's not already defined - if len(api.Link) == 0 { - api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{ - Name: "url", - Description: "The URL of the app", - Multiline: false, - Required: true, - Example: "https://shuffler.io", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - extraParameters = append(extraParameters, WorkflowAppActionParameter{ - Name: "url", - Description: "The URL of the app", - Multiline: false, - Required: true, - Configuration: true, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } - - // This is the python code to be generated - // Could just as well be go at this point lol - pythonFunctions := []string{} - //Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` - for actualPath, path := range swagger.Paths { - actualPath = strings.Replace(actualPath, " ", "_", -1) - //actualPath = strings.Replace(actualPath, ".", "", -1) - actualPath = strings.Replace(actualPath, "\\", "", -1) - if !api.Invalid && strings.HasPrefix(actualPath, "tmp") { - log.Printf("[WARNING] Set api %s to invalid because of path %s", swagger.Info.Title, actualPath) - api.Invalid = true - } - - // FIXME: Handle everything behind questionmark (?) with dots as well. - // https://godoc.org/github.com/getkin/kin-openapi/openapi3#PathItem - if path.Get != nil { - action, curCode := handleGet(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - if path.Connect != nil { - action, curCode := handleConnect(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - if path.Head != nil { - action, curCode := handleHead(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - if path.Delete != nil { - action, curCode := handleDelete(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - if path.Post != nil { - action, curCode := handlePost(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - if path.Patch != nil { - action, curCode := handlePatch(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - if path.Put != nil { - action, curCode := handlePut(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - - // Has to be here because its used differently above. - // FIXING this is done during export instead? - //log.Printf("OLDPATH: %s", actualPath) - //if strings.Contains(actualPath, "?") { - // actualPath = strings.Split(actualPath, "?")[0] - //} - - //log.Printf("NEWPATH: %s", actualPath) - //newPaths[actualPath] = path - } - - return swagger, api, pythonFunctions, nil -} - -// FIXME - have this give a real version? -func verifyApi(api WorkflowApp) WorkflowApp { - if api.AppVersion == "" { - api.AppVersion = "1.0.0" - } - - return api -} - -func getBasePython() string { - baseString := `import requests -import asyncio -import json -import urllib3 - -from walkoff_app_sdk.app_base import AppBase - -class %s(AppBase): - """ - Autogenerated class by Shuffler - """ - - __version__ = "%s" - app_name = "%s" - - def __init__(self, redis, logger, console_logger=None): - self.verify = False - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - super().__init__(redis, logger, console_logger) - -%s - -if __name__ == "__main__": - asyncio.run(%s.run(), debug=True) -` - return baseString -} - -func dumpPythonGCP(ctx context.Context, client *storage.Client, basePath, name, version string, pythonFunctions []string) (string, error) { - parsedCode := fmt.Sprintf(getBasePython(), name, version, name, strings.Join(pythonFunctions, "\n"), name) - - // Create bucket handle - bucket := client.Bucket(bucketName) - obj := bucket.Object(fmt.Sprintf("%s/src/app.py", basePath)) - w := obj.NewWriter(ctx) - if _, err := fmt.Fprintf(w, parsedCode); err != nil { - return "", err - } - // Close, just like writing a file. - if err := w.Close(); err != nil { - return "", err - } - - return parsedCode, nil -} - -func dumpPython(basePath, name, version string, pythonFunctions []string) (string, error) { - //log.Printf("%#v", api) - //log.Printf(strings.Join(pythonFunctions, "\n")) - - parsedCode := fmt.Sprintf(getBasePython(), name, version, name, strings.Join(pythonFunctions, "\n"), name) - - err := ioutil.WriteFile(fmt.Sprintf("%s/src/app.py", basePath), []byte(parsedCode), os.ModePerm) - if err != nil { - return "", err - } - //fmt.Println(parsedCode) - //log.Println(string(data)) - return parsedCode, nil -} - -func dumpApiGCP(ctx context.Context, client *storage.Client, swagger *openapi3.Swagger, basePath string, api WorkflowApp) error { - //log.Printf("%#v", api) - data, err := yaml.Marshal(api) - if err != nil { - log.Printf("Error with yaml marshal: %s", err) - return err - } - - // Create bucket handle - bucket := client.Bucket(bucketName) - obj := bucket.Object(fmt.Sprintf("%s/app.yaml", basePath)) - w := obj.NewWriter(ctx) - if _, err := fmt.Fprintln(w, string(data)); err != nil { - return err - } - // Close, just like writing a file. - if err := w.Close(); err != nil { - return err - } - - openapidata, err := yaml.Marshal(swagger) - if err != nil { - log.Printf("Error with yaml marshal: %s", err) - return err - } - obj = bucket.Object(fmt.Sprintf("%s/openapi.yaml", basePath)) - //log.Println(string(openapidata)) - w = obj.NewWriter(ctx) - if _, err := fmt.Fprintln(w, string(openapidata)); err != nil { - return err - } - // Close, just like writing a file. - if err := w.Close(); err != nil { - return err - } - - //log.Println(string(data)) - return nil -} - -func dumpApi(basePath string, api WorkflowApp) error { - //log.Printf("%#v", api) - data, err := yaml.Marshal(api) - if err != nil { - log.Printf("Error with yaml marshal: %s", err) - return err - } - - err = ioutil.WriteFile(fmt.Sprintf("%s/api.yaml", basePath), []byte(data), os.ModePerm) - if err != nil { - return err - } - - //log.Println(string(data)) - return nil -} - -func getRunner(classname string) string { - return fmt.Sprintf(` -# Run the actual thing after we've checked params -def run(request): - print("Started execution!") - action = request.get_json() - print(action) - print(type(action)) - authorization_key = action.get("authorization") - current_execution_id = action.get("execution_id") - - if action and "name" in action and "app_name" in action: - asyncio.run(%s.run(action), debug=True) - return f'Attempting to execute function {action["name"]} in app {action["app_name"]}' - else: - return f'Invalid action' - - `, classname) -} - -func deployAppToDatastore(ctx context.Context, workflowapp WorkflowApp) error { - err := setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) - if err != nil { - log.Printf("[ERROR] Failed setting workflowapp: %s", err) - return err - } else { - log.Printf("[INFO] Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion) - } - - return nil -} - -// FIXME: -// https://docs.python.org/3.2/reference/lexical_analysis.html#identifiers -// This is used to build the python functions. -func fixFunctionName(functionName, actualPath string) string { - if len(functionName) == 0 { - functionName = actualPath - } - - // REGEX THIS SHIT - // ROFL - - //log.Printf("Fixing function name for %s", functionName) - functionName = strings.Replace(functionName, ".", "", -1) - functionName = strings.Replace(functionName, ",", "", -1) - functionName = strings.Replace(functionName, ".", "", -1) - functionName = strings.Replace(functionName, "&", "", -1) - functionName = strings.Replace(functionName, "/", "", -1) - functionName = strings.Replace(functionName, "\\", "", -1) - - functionName = strings.Replace(functionName, "!", "", -1) - functionName = strings.Replace(functionName, "?", "", -1) - functionName = strings.Replace(functionName, "@", "", -1) - functionName = strings.Replace(functionName, "#", "", -1) - functionName = strings.Replace(functionName, "$", "", -1) - functionName = strings.Replace(functionName, "&", "", -1) - functionName = strings.Replace(functionName, "*", "", -1) - functionName = strings.Replace(functionName, "(", "", -1) - functionName = strings.Replace(functionName, ")", "", -1) - functionName = strings.Replace(functionName, "[", "", -1) - functionName = strings.Replace(functionName, "]", "", -1) - functionName = strings.Replace(functionName, "{", "", -1) - functionName = strings.Replace(functionName, "}", "", -1) - functionName = strings.Replace(functionName, `"`, "", -1) - functionName = strings.Replace(functionName, `'`, "", -1) - functionName = strings.Replace(functionName, `|`, "", -1) - functionName = strings.Replace(functionName, `~`, "", -1) - - functionName = strings.Replace(functionName, " ", "_", -1) - functionName = strings.Replace(functionName, "-", "_", -1) - - functionName = strings.ToLower(functionName) - - return functionName -} - -// Returns a valid param name -func validateParameterName(name string) string { - invalid := []string{"False", - "await", - "else", - "import", - "pass", - "None", - "break", - "except", - "in", - "raise", - "True", - "class", - "finally", - "is", - "return", - "and", - "continue", - "for", - "lambda", - "try", - "as", - "def", - "from", - "nonlocal", - "while", - "assert", - "del", - "global", - "not", - "with", - "async", - "elif", - "if", - "or", - "yield", - } - - newname := name - for _, item := range invalid { - if item == name { - //log.Printf("%s is NOT a valid parameter name!", item) - newname = fmt.Sprintf("%s_shuffle", item) - break - } - } - - newname = strings.ReplaceAll(newname, " ", "_") - newname = strings.ReplaceAll(newname, ",", "_") - newname = strings.ReplaceAll(newname, ".", "_") - newname = strings.ReplaceAll(newname, "|", "_") - newname = strings.ReplaceAll(newname, "-", "_") - - return newname -} - -func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - functionName := fixFunctionName(path.Connect.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Connect.Description, - Name: fmt.Sprintf("%s %s", "Connect", path.Connect.Summary), - Label: fmt.Sprintf(path.Connect.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - //log.Println(path.Parameters) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - headersFound := []string{} - if len(path.Connect.Parameters) > 0 { - for counter, param := range path.Connect.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = strings.ReplaceAll(parsedName, "-", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Connect.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if param.Value.Name == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, curParam.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - - } - } - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "connect", parameters, optionalQueries, headersFound, "") - - if len(functionname) > 0 { - action.Name = functionname - } - - return action, curCode -} - -func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - functionName := fixFunctionName(path.Get.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Get.Description, - Name: fmt.Sprintf("%s %s", "Get", path.Get.Summary), - Label: fmt.Sprintf(path.Get.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - - // FIXME - remove this when authentication is properly introduced - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify the SSL certificate request", - Multiline: false, - Required: false, - Example: "False - default=True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - headersFound := []string{} - if len(path.Get.Parameters) > 0 { - for counter, param := range path.Get.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Get.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if param.Value.Name == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, curParam.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - // Skipping simial - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - - } - } - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "get", parameters, optionalQueries, headersFound, "") - - if len(functionname) > 0 { - action.Name = functionname - } - - return action, curCode -} - -func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - functionName := fixFunctionName(path.Head.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Head.Description, - Name: fmt.Sprintf("%s %s", "Head", path.Head.Summary), - Label: fmt.Sprintf(path.Head.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - //log.Println(path.Parameters) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - headersFound := []string{} - if len(path.Head.Parameters) > 0 { - for counter, param := range path.Head.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Head.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if param.Value.Name == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, curParam.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - } - } - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "head", parameters, optionalQueries, headersFound, "") - - if len(functionname) > 0 { - action.Name = functionname - } - - return action, curCode -} - -func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - functionName := fixFunctionName(path.Delete.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Delete.Description, - Name: fmt.Sprintf("%s %s", "Delete", path.Delete.Summary), - Label: fmt.Sprintf(path.Delete.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - //log.Println(path.Parameters) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - headersFound := []string{} - if len(path.Delete.Parameters) > 0 { - for counter, param := range path.Delete.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Delete.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if param.Value.Name == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, curParam.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - - } - } - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound, "") - - if len(functionname) > 0 { - action.Name = functionname - } - - return action, curCode -} - -func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - //log.Printf("PATH: %s", actualPath) - functionName := fixFunctionName(path.Post.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Post.Description, - Name: fmt.Sprintf("%s %s", "Post", path.Post.Summary), - Label: fmt.Sprintf(path.Post.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - fileField := "" - if path.Post.RequestBody != nil { - //log.Printf("DATA: %#v", - value := path.Post.RequestBody.Value - //log.Printf("VAL: %#v", value.Content) - if val, ok := value.Content["multipart/form-data"]; ok { - if val.Schema.Value != nil { - if innerval, ok := val.Schema.Value.Properties["fieldname"]; ok { - if extensionvalue, ok := innerval.Value.ExtensionProps.Extensions["value"]; ok { - fieldname := extensionvalue.(json.RawMessage) - newName := string(fmt.Sprintf("%s", string(fieldname))) - if newName[0] == 0x22 && newName[len(newName)-1] == 0x22 { - parsedName := newName[1 : len(newName)-1] - log.Printf("Parse name: %s", parsedName) - fileField = parsedName - - curParam := WorkflowAppActionParameter{ - Name: "file_id", - Description: "Files to be uploaded", - Multiline: false, - Required: true, - Schema: SchemaDefinition{ - Type: "string", - }, - } - - action.Parameters = append(action.Parameters, curParam) - } - } - } - } - } - } - - headersFound := []string{} - if len(path.Post.Parameters) > 0 { - for counter, param := range path.Post.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Post.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if parsedName == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, curParam.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - } - } - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "post", parameters, optionalQueries, headersFound, fileField) - - if len(functionname) > 0 { - action.Name = functionname - } - - //log.Printf("PARAMS: %d", len(action.Parameters)) - //for _, param := range action.Parameters { - // log.Printf("%#v", param) - //} - - return action, curCode -} - -func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - functionName := fixFunctionName(path.Patch.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Patch.Description, - Name: fmt.Sprintf("%s %s", "Patch", path.Patch.Summary), - Label: fmt.Sprintf(path.Patch.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - //log.Println(path.Parameters) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - headersFound := []string{} - if len(path.Patch.Parameters) > 0 { - for counter, param := range path.Patch.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Patch.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if param.Value.Name == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, curParam.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - } - } - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "patch", parameters, optionalQueries, headersFound, "") - - if len(functionname) > 0 { - action.Name = functionname - } - - return action, curCode -} - -func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - functionName := fixFunctionName(path.Put.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Put.Description, - Name: fmt.Sprintf("%s %s", "Put", path.Put.Summary), - Label: fmt.Sprintf(path.Put.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - //log.Println(path.Parameters) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - headersFound := []string{} - if len(path.Put.Parameters) > 0 { - for counter, param := range path.Put.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Put.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if param.Value.Name == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, param.Value.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - - } - } - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "put", parameters, optionalQueries, headersFound, "") - - if len(functionname) > 0 { - action.Name = functionname - } - - return action, curCode -} diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 776dbea0..214f81c9 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -2,6 +2,8 @@ package main // Docker import ( + "github.com/frikky/shuffle-shared" + "archive/tar" "path/filepath" @@ -24,7 +26,6 @@ import ( "net/http" "os" "strings" - //"google.golang.org/appengine" ) // Parses a directory with a Dockerfile into a tar for Docker images.. @@ -798,7 +799,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { } // Just here to verify that the user is logged in - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in validate swagger: %s", err) resp.WriteHeader(401) @@ -862,7 +863,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { tagFound := "" for _, image := range images { for _, tag := range image.RepoTags { - log.Printf("Image: %s", tag) + log.Printf("[INFO] Docker Image: %s", tag) if strings.ToLower(tag) == strings.ToLower(version.Name) { img = image diff --git a/backend/go-app/files.go b/backend/go-app/files.go deleted file mode 100644 index 419d834d..00000000 --- a/backend/go-app/files.go +++ /dev/null @@ -1,882 +0,0 @@ -package main - -/* - Handles files within Workflows.of Shuffle -*/ - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "log" - "net/http" - "os" - "strconv" - "strings" - "time" - - "cloud.google.com/go/datastore" - "github.com/satori/go.uuid" -) - -type File struct { - Id string `json:"id" datastore:"id"` - Type string `json:"type" datastore:"type"` - CreatedAt int64 `json:"created_at" datastore:"created_at"` - UpdatedAt int64 `json:"updated_at" datastore:"updated_at"` - MetaAccessAt int64 `json:"meta_access_at" datastore:"meta_access_at"` - DownloadAt int64 `json:"last_downloaded" datastore:"last_downloaded"` - Description string `json:"description" datastore:"description"` - ExpiresAt string `json:"expires_at" datastore:"expires_at"` - Status string `json:"status" datastore:"status"` - Filename string `json:"filename" datastore:"filename"` - URL string `json:"url" datastore:"org"` - OrgId string `json:"org_id" datastore:"org_id"` - WorkflowId string `json:"workflow_id" datastore:"workflow_id"` - Workflows []string `json:"workflows" datastore:"workflows"` - DownloadPath string `json:"download_path" datastore:"download_path"` - Md5sum string `json:"md5_sum" datastore:"md5_sum"` - Sha256sum string `json:"sha256_sum" datastore:"sha256_sum"` - FileSize int64 `json:"filesize" datastore:"filesize"` - Duplicate bool `json:"duplicate" datastore:"duplicate"` - Subflows []string `json:"subflows" datastore:"subflows"` -} - -var basepath = os.Getenv("SHUFFLE_FILE_LOCATION") - -func fileAuthentication(request *http.Request) (string, error) { - executionId, ok := request.URL.Query()["execution_id"] - if ok && len(executionId) > 0 { - ctx := context.Background() - workflowExecution, err := getWorkflowExecution(ctx, executionId[0]) - if err != nil { - log.Printf("[ERROR] Couldn't find execution ID %s", executionId[0]) - return "", err - } - - apikey := request.Header.Get("Authorization") - if !strings.HasPrefix(apikey, "Bearer ") { - log.Printf("[ERROR} Apikey doesn't start with bearer (2)") - return "", errors.New("No auth key found") - } - - apikeyCheck := strings.Split(apikey, " ") - if len(apikeyCheck) != 2 { - log.Printf("[ERROR] Invalid format for apikey (2)") - return "", errors.New("No space in authkey") - } - - // This is annoying af and is done because of maxlength lol - newApikey := apikeyCheck[1] - if newApikey != workflowExecution.Authorization { - //log.Printf("[ERROR] Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization) - log.Printf("[ERROR] Bad apikey for execution %s.", executionId[0]) - //%s vs %s", executionId[0], apikey, workflowExecution.Authorization) - return "", errors.New("Bad authorization key") - } - - log.Printf("[INFO] Authorization is correct for execution %s!", executionId[0]) - //%s vs %s. Setting Org", executionId, apikey, workflowExecution.Authorization) - if len(workflowExecution.ExecutionOrg) > 0 { - return workflowExecution.ExecutionOrg, nil - } else if len(workflowExecution.Workflow.ExecutingOrg.Id) > 0 { - return workflowExecution.ExecutionOrg, nil - } else { - log.Printf("[ERROR] Couldn't find org for workflow execution, but auth was correct.") - } - } - - return "", errors.New("No execution id specified") -} - -// https://golangcode.com/check-if-a-file-exists/ -func fileExists(filename string) bool { - info, err := os.Stat(filename) - if os.IsNotExist(err) { - return false - } - return !info.IsDir() -} - -func handleGetFiles(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - // 1. Check user directly - // 2. Check workflow execution authorization - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("[INFO] INITIAL Api authentication failed in file LIST: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if user.Role != "admin" { - log.Printf("[AUTH] User isn't admin") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Need to be admin"}`))) - return - } - - ctx := context.Background() - files, err := getAllFiles(ctx, user.ActiveOrg.Id) - if err != nil { - log.Printf("[ERROR] Failed to get files: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error getting files."}`))) - return - } - - log.Printf("[INFO] Got %d files for org %s", len(files), user.ActiveOrg.Id) - newBody, err := json.Marshal(files) - if err != nil { - log.Printf("[ERROR] Failed marshaling files: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed to marshal files"}`)) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(newBody)) -} - -func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - var fileId string - location := strings.Split(request.URL.String(), "/") - if location[1] == "api" { - if len(location) <= 4 { - log.Printf("[INFO] Path too short: %d", len(location)) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if strings.Contains(fileId, "?") { - fileId = strings.Split(fileId, "?")[0] - } - - if len(fileId) != 36 { - log.Printf("Bad format for fileId %s", fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) - return - } - - log.Printf("\n\n[INFO] User is trying to GET File Meta for %s\n\n", fileId) - - // 1. Check user directly - // 2. Check workflow execution authorization - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("[INFO] INITIAL Api authentication failed in file deletion: %s", err) - - orgId, err := fileAuthentication(request) - if err != nil { - log.Printf("[ERROR] Bad file authentication in get: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - user.ActiveOrg.Id = orgId - user.Username = "Execution File API" - } - - // 1. Verify if the user has access to the file: org_id and workflow - log.Printf("[INFO] Should GET FILE META for %s if user has access", fileId) - ctx := context.Background() - file, err := getFile(ctx, fileId) - if err != nil { - log.Printf("[INFO] File %s not found: %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - found := false - if file.OrgId == user.ActiveOrg.Id { - found = true - } else { - for _, item := range user.Orgs { - if item == file.OrgId { - found = true - break - } - } - } - - if !found { - log.Printf("[INFO] User %s doesn't have access to %s", user.Username, fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - newBody, err := json.Marshal(file) - if err != nil { - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed to marshal filedata"}`)) - return - } - - log.Printf("[INFO] Successfully got file meta for %s", fileId) - resp.WriteHeader(200) - resp.Write([]byte(newBody)) -} - -func handleDeleteFile(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - var fileId string - location := strings.Split(request.URL.String(), "/") - if location[1] == "api" { - if len(location) <= 4 { - log.Printf("[INFO] Path too short: %d", len(location)) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if strings.Contains(fileId, "?") { - fileId = strings.Split(fileId, "?")[0] - } - - if len(fileId) != 36 { - log.Printf("Bad format for fileId %s", fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) - return - } - - log.Printf("\n\n[INFO] User is trying to delete file %s\n\n", fileId) - - // 1. Check user directly - // 2. Check workflow execution authorization - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("[INFO] INITIAL Api authentication failed in file deletion: %s", err) - - orgId, err := fileAuthentication(request) - if err != nil { - log.Printf("[ERROR] Bad file authentication in get: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - user.ActiveOrg.Id = orgId - user.Username = "Execution File API" - } - - // 1. Verify if the user has access to the file: org_id and workflow - log.Printf("[INFO] Should DELETE file %s if user has access", fileId) - ctx := context.Background() - file, err := getFile(ctx, fileId) - if err != nil { - log.Printf("[INFO] File %s not found: %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - found := false - if file.OrgId == user.ActiveOrg.Id { - found = true - } else { - for _, item := range user.Orgs { - if item == file.OrgId { - found = true - break - } - } - } - - if !found { - log.Printf("[INFO] User %s doesn't have access to %s", user.Username, fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if file.Status == "deleted" { - log.Printf("[INFO] File with ID %s is already deleted.", fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if fileExists(file.DownloadPath) { - err = os.Remove(file.DownloadPath) - if err != nil { - log.Printf("[ERROR] Failed deleting file locally: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting filein path %s"}`, file.DownloadPath))) - return - } - - log.Printf("[INFO] Deleted file %s locally. Next is database.", file.DownloadPath) - } else { - log.Printf("[ERROR] File doesn't exist. Can't delete. Should maybe delete file anyway?") - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "File in location %s doesn't exist"}`, file.DownloadPath))) - return - } - - file.Status = "deleted" - err = setFile(ctx, *file) - if err != nil { - log.Printf("[ERROR] Failed setting file to deleted") - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed setting file to deleted"}`)) - return - } - - /* - //Actually delete it? - err = DeleteKey(ctx, "files", fileId) - if err != nil { - log.Printf("Failed deleting file with ID %s: %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - */ - - log.Printf("[INFO] Successfully deleted file %s", fileId) - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) -} - -func handleGetFileContent(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - var fileId string - location := strings.Split(request.URL.String(), "/") - if location[1] == "api" { - if len(location) <= 4 { - log.Printf("Path too short: %d", len(location)) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if len(fileId) != 36 { - log.Printf("Bad format for fileId %s", fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) - return - } - - log.Printf("\n\n[INFO] User is trying to download file %s\n\n", fileId) - - // 1. Check user directly - // 2. Check workflow execution authorization - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("INITIAL Api authentication failed in file download: %s", err) - - orgId, err := fileAuthentication(request) - if err != nil { - log.Printf("Bad file authentication in get: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - user.ActiveOrg.Id = orgId - user.Username = "Execution File API" - /* - } else { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - */ - } - - // 1. Verify if the user has access to the file: org_id and workflow - log.Printf("[INFO] Should get file %s", fileId) - ctx := context.Background() - file, err := getFile(ctx, fileId) - if err != nil { - log.Printf("[ERROR] File %s not found: %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - found := false - if file.OrgId == user.ActiveOrg.Id { - found = true - } else { - for _, item := range user.Orgs { - if item == file.OrgId { - found = true - break - } - } - } - - if !found { - log.Printf("User %s doesn't have access to %s", user.Username, fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if file.Status != "active" { - log.Printf("[ERROR] File status isn't active, but %s. Can't continue.", file.Status) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "The file isn't ready to be downloaded yet. Status required: active"}`)) - return - } - - // Fixme: More auth: org and workflow! - downloadPath := file.DownloadPath - log.Printf("[INFO] Downloadpath: %s", downloadPath) - Openfile, err := os.Open(downloadPath) - defer Openfile.Close() //Close after function return - if err != nil { - file.Status = "deleted" - err = setFile(ctx, *file) - if err != nil { - log.Printf("Failed setting file to uploading") - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed setting file to uploading"}`)) - return - } - - //File not found, send 404 - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "File doesn't exist locally"}`)) - return - } - - //File is found, create and send the correct headers - //Get the Content-Type of the file - //Create a buffer to store the header of the file in - FileHeader := make([]byte, 512) - //Copy the headers into the FileHeader buffer - Openfile.Read(FileHeader) - //Get content type of file - FileContentType := http.DetectContentType(FileHeader) - - //Get the file size - FileStat, _ := Openfile.Stat() //Get info from file - FileSize := strconv.FormatInt(FileStat.Size(), 10) //Get file size as a string - - //Send the headers - resp.Header().Set("Content-Disposition", "attachment; filename="+fileId) - resp.Header().Set("Content-Type", FileContentType) - resp.Header().Set("Content-Length", FileSize) - - //Send the file - //We read 512 bytes from the file already, so we reset the offset back to 0 - Openfile.Seek(0, 0) - io.Copy(resp, Openfile) //'Copy' the file to the client - return - - //log.Printf("Should download file %s", downloadPath) -} -func handleUploadFile(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - var fileId string - location := strings.Split(request.URL.String(), "/") - if location[1] == "api" { - if len(location) <= 4 { - log.Printf("Path too short: %d", len(location)) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if len(fileId) != 36 { - log.Printf("Bad format for fileId %s", fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) - return - } - - // 1. Check user directly - // 2. Check workflow execution authorization - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("INITIAL Api authentication failed in file upload: %s", err) - - orgId, err := fileAuthentication(request) - if err != nil { - log.Printf("Bad file authentication in create file: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - user.ActiveOrg.Id = orgId - user.Username = "Execution File API" - } - - log.Printf("[INFO] Should UPLOAD file %s if user has access", fileId) - ctx := context.Background() - file, err := getFile(ctx, fileId) - if err != nil { - log.Printf("File %s not found: %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - found := false - if file.OrgId == user.ActiveOrg.Id { - found = true - } else { - for _, item := range user.Orgs { - if item == file.OrgId { - found = true - break - } - } - } - - if !found { - log.Printf("User %s doesn't have access to %s", user.Username, fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Printf("[INFO] STATUS: %s", file.Status) - if file.Status != "created" { - log.Printf("File status isn't created. Can't upload.") - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "This file already has data."}`)) - return - } - - request.ParseMultipartForm(32 << 20) - parsedFile, _, err := request.FormFile("shuffle_file") - if err != nil { - log.Printf("[ERROR] Couldn't upload file: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed uploading file"}`)) - return - } - defer parsedFile.Close() - - file.Status = "uploading" - err = setFile(ctx, *file) - if err != nil { - log.Printf("Failed setting file to uploading") - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed setting file to uploading"}`)) - return - } - - // Can be used for validation files for change - var buf bytes.Buffer - io.Copy(&buf, parsedFile) - contents := buf.Bytes() - file.FileSize = int64(len(contents)) - md5 := md5sum(contents) - buf.Reset() - - sha256Sum := sha256.Sum256(contents) - //parsedFile.Reset() - - f, err := os.OpenFile(file.DownloadPath, os.O_WRONLY|os.O_CREATE, os.ModePerm) - if err != nil { - // Rolling back file - file.Status = "created" - setFile(ctx, *file) - - log.Printf("[ERROR] Failed uploading and creating file: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false}`)) - return - } - - defer f.Close() - parsedFile.Seek(0, io.SeekStart) - io.Copy(f, parsedFile) - - // FIXME: Set this one to 200 anyway? Can't download file then tho.. - file.Status = "active" - file.Md5sum = md5 - file.Sha256sum = fmt.Sprintf("%x", sha256Sum) - log.Printf("[INFO] MD5 for file %s (%s) is %s and SHA256 is %s", file.Filename, file.Id, file.Md5sum, file.Sha256sum) - - err = setFile(ctx, *file) - if err != nil { - log.Printf("[ERROR] Failed setting file back to active") - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed setting file to active"}`)) - return - } - - log.Printf("[INFO] Successfully uploaded file ID %s", file.Id) - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) -} - -func handleCreateFile(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - // 1. Check user directly - // 2. Check workflow execution authorization - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("[INFO] INITIAL Api authentication failed in file creation: %s", err) - - orgId, err := fileAuthentication(request) - if err != nil { - log.Printf("[ERROR] Bad file authentication in create file: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - user.ActiveOrg.Id = orgId - user.Username = "Execution File API" - } - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Println("Failed reading body") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to read data"}`))) - return - } - - type FileStructure struct { - Filename string `json:"filename"` - OrgId string `json:"org_id"` - WorkflowId string `json:"workflow_id"` - } - - var curfile FileStructure - err = json.Unmarshal(body, &curfile) - if err != nil { - log.Printf("[ERROR] Failed unmarshaling: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to unmarshal data"}`))) - return - } - - // Loads of validation below - if len(curfile.Filename) == 0 || len(curfile.OrgId) == 0 || len(curfile.WorkflowId) == 0 { - log.Printf("[ERROR] Missing field during fileupload. Required: filename, org_id, workflow_id") - log.Printf("INPUT: %s", string(body)) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Missing field. Required: filename, org_id, workflow_id"}`))) - return - } - - ctx := context.Background() - if user.ActiveOrg.Id != curfile.OrgId { - log.Printf("[ERROR] User can't access org %s", curfile.OrgId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Error with organization"}`)) - return - } - - var workflow *Workflow - if curfile.WorkflowId == "global" { - // PS: Not a security issue. - // Files are global anyway, but the workflow_id is used to identify origin - log.Printf("[INFO] Uploading filename %s for org %s as global file.", curfile.Filename, curfile.OrgId) - } else { - // Try to get the org and workflow in case they don't exist - workflow, err = getWorkflow(ctx, curfile.WorkflowId) - if err != nil { - log.Printf("[ERROR] Workflow %s doesn't exist.", curfile.WorkflowId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) - return - } - - _, err = getOrg(ctx, curfile.OrgId) - if err != nil { - log.Printf("[ERROR] Org %s doesn't exist.", curfile.OrgId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) - return - } - - if workflow.ExecutingOrg.Id != curfile.OrgId { - found := false - for _, curorg := range workflow.Org { - if curorg.Id == curfile.OrgId { - found = true - break - } - } - - if !found { - log.Printf("[ERROR] Org %s doesn't have access to %s.", curfile.OrgId, curfile.WorkflowId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) - return - } - } - } - - if strings.Contains(curfile.Filename, "/") || strings.Contains(curfile.Filename, `"`) || strings.Contains(curfile.Filename, "..") || strings.Contains(curfile.Filename, "~") { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Invalid characters in filename"}`)) - return - } - - // 1. Create the file object. - if len(basepath) == 0 { - basepath = "shuffle-files" - } - folderPath := fmt.Sprintf("%s/%s/%s", basepath, curfile.OrgId, curfile.WorkflowId) - - // Try to make the full file location - err = os.MkdirAll(folderPath, os.ModePerm) - if err != nil { - log.Printf("[ERROR] Writing issue for file location creation: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed creating upload location"}`)) - return - } - - filename := curfile.Filename - fileId := uuid.NewV4().String() - downloadPath := fmt.Sprintf("%s/%s", folderPath, fileId) - - duplicateWorkflows := []string{} - if curfile.WorkflowId != "global" { - for _, trigger := range workflow.Triggers { - if trigger.AppName == "Shuffle Workflow" && trigger.TriggerType == "SUBFLOW" { - for _, parameter := range trigger.Parameters { - if parameter.Name == "workflow" && len(parameter.Value) > 0 { - - found := false - for _, workflow := range duplicateWorkflows { - if workflow == parameter.Value { - found = true - break - } - } - - if !found { - duplicateWorkflows = append(duplicateWorkflows, parameter.Value) - } - - break - } - } - } - } - } - - timeNow := time.Now().Unix() - newFile := File{ - Id: fileId, - CreatedAt: timeNow, - UpdatedAt: timeNow, - Description: "", - Status: "created", - Filename: filename, - OrgId: curfile.OrgId, - WorkflowId: curfile.WorkflowId, - DownloadPath: downloadPath, - Subflows: duplicateWorkflows, - } - - err = setFile(ctx, newFile) - if err != nil { - log.Printf("[ERROR] Failed setting file: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed setting file reference"}`)) - return - } else { - log.Printf("[INFO] Created file %s", newFile.DownloadPath) - } - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, fileId))) - -} - -func getFile(ctx context.Context, id string) (*File, error) { - key := datastore.NameKey("Files", id, nil) - curFile := &File{} - if err := dbclient.Get(ctx, key, curFile); err != nil { - return &File{}, err - } - - return curFile, nil -} - -func setFile(ctx context.Context, file File) error { - // clear session_token and API_token for user - timeNow := time.Now().Unix() - file.UpdatedAt = timeNow - - k := datastore.NameKey("Files", file.Id, nil) - if _, err := dbclient.Put(ctx, k, &file); err != nil { - log.Println(err) - return err - } - - return nil -} - -func getAllFiles(ctx context.Context, orgId string) ([]File, error) { - var files []File - q := datastore.NewQuery("Files").Filter("org_id =", orgId).Order("-updated_at").Limit(100) - - _, err := dbclient.GetAll(ctx, q, &files) - if err != nil { - if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { - q = q.Limit(50) - _, err := dbclient.GetAll(ctx, q, &files) - if err != nil { - return []File{}, err - } - } else { - return []File{}, err - } - } - - return files, nil -} diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 0bf0ccce..4e64fc86 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,36 +2,40 @@ module shuffle go 1.13 +//replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared +//replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi + require ( - cloud.google.com/go v0.57.0 - cloud.google.com/go/datastore v1.1.0 + cloud.google.com/go v0.75.0 + cloud.google.com/go/datastore v1.4.0 cloud.google.com/go/pubsub v1.3.1 - cloud.google.com/go/storage v1.7.0 + cloud.google.com/go/storage v1.12.0 github.com/Microsoft/go-winio v0.4.14 // indirect github.com/basgys/goxml2json v1.1.0 + github.com/frikky/kin-openapi v0.38.0 github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v1.13.1 github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 // indirect - github.com/getkin/kin-openapi v0.8.0 + github.com/frikky/shuffle-shared v0.0.23 github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.0.0 github.com/google/go-github/v28 v28.1.1 github.com/gorilla/handlers v1.4.2 // indirect - github.com/gorilla/mux v1.7.4 + github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.0.12 github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible github.com/satori/go.uuid v1.2.0 - golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 - golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d - google.golang.org/api v0.23.0 - google.golang.org/appengine v1.6.6 - google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 - google.golang.org/grpc v1.29.1 + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423 + google.golang.org/api v0.36.0 + google.golang.org/appengine v1.6.7 + google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595 + google.golang.org/grpc v1.34.1 gopkg.in/src-d/go-git.v4 v4.13.1 - gopkg.in/yaml.v2 v2.2.8 - gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86 + gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 2d63be91..21fd33ff 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -12,14 +12,24 @@ cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bP cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0 h1:EpMNVUorLiZIELdMZbCYX/ByTFCdoYopYAGxaGVz9ms= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.75.0 h1:XgtDnVJRCPEUG21gjFiRPz4zI1Mjg16R+NYQjfmU4XY= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.6.0/go.mod h1:hyFDG0qSGdHNz8Q6nDN8rYIkld0q/+5uBZaelxiDLfE= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastore v1.4.0 h1:CFDJm15RpYXeEblQ0TMDUrYtqmBmbAWTy536nA8JIc8= +cloud.google.com/go/datastore v1.4.0/go.mod h1:d18825/a9bICdAIJy2EkHs9joU4RlIZ1t6l8WDdbdY0= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -30,6 +40,10 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.7.0 h1:DzdLPI8Em+DEk7IzA2a10ivq3mxIEASC9GeNJ6FFt5Q= cloud.google.com/go/storage v1.7.0/go.mod h1:jGMIBwF+L/tL6WN/W5InNgYYu4HP0DvGB6rQ1mufWfs= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lUX4= +cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -49,6 +63,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -66,10 +81,27 @@ github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/frikky/kin-openapi v0.38.0 h1:V7ttwIJS8Vks4KL+mZVj1ZSqhIcQtgaG8akeqXEQgsE= +github.com/frikky/kin-openapi v0.38.0/go.mod h1:Fr28TtCHL4K0kIqtqui8HWxN1LG5uAh3z/tDfFyiA1s= +github.com/frikky/shuffle-shared v0.0.12 h1:+0EIfThmK47Po+LogPYZR4XjbS4Ds19WNMFu2YUSjhw= +github.com/frikky/shuffle-shared v0.0.12/go.mod h1:SEY432/xs4oBkOUnGwxiYKCZWZLRaheGInhAoW7N8ww= +github.com/frikky/shuffle-shared v0.0.15 h1:508ceeEHfPBMCC8/K4Zve3kwRQqiXNJSw6+BDoq9X4E= +github.com/frikky/shuffle-shared v0.0.15/go.mod h1:SEY432/xs4oBkOUnGwxiYKCZWZLRaheGInhAoW7N8ww= +github.com/frikky/shuffle-shared v0.0.20 h1:y6JlPnQDq//elICWvVfUfJyU9gH3fSpQmPy+agqZ5sA= +github.com/frikky/shuffle-shared v0.0.20/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= +github.com/frikky/shuffle-shared v0.0.21 h1:xj/XPsXTa2rx41mm4nUc7+2K9RGkq2/mpjSPtIfpjE4= +github.com/frikky/shuffle-shared v0.0.21/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= +github.com/frikky/shuffle-shared v0.0.22 h1:TFMcJCNmOOSneMMWbg5dNzp2z6m0aZLROuL+bzVToRE= +github.com/frikky/shuffle-shared v0.0.22/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= +github.com/frikky/shuffle-shared v0.0.23 h1:Pnlc2M6fHnFRLFd5K1iLTVv4/t4P04Ri1GJ5CMxzq0U= +github.com/frikky/shuffle-shared v0.0.23/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= github.com/getkin/kin-openapi v0.8.0/go.mod h1:zZQMFkVgRHCdhgb6ihCTIo9dyDZFvX0k/xAKqw1FhPw= +github.com/getkin/kin-openapi v0.52.0 h1:6WqsF5d6PfJ8AscdD+9Rtb2RP2iBWyC7V6GcjssWg7M= +github.com/getkin/kin-openapi v0.52.0/go.mod h1:fRpo2Nw4Czgy0QnrIesRrEXs5+15N1F9mGZLP/aIomE= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= @@ -85,6 +117,10 @@ github.com/go-git/go-git/v5 v5.0.0/go.mod h1:oYD8y9kWsGINPFJoLdaScGCN6dlKg23blmC github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -96,6 +132,7 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -108,6 +145,10 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -115,19 +156,32 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github/v28 v28.1.1 h1:kORf5ekX5qwXO2mGzXXOjMe/g6ap8ahVe0sBEulhSxo= github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -135,11 +189,14 @@ github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YAR github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/h2non/filetype v1.0.12 h1:yHCsIe0y2cvbDARtJhGBTD2ecvqMSTvlIcph9En/Zao= github.com/h2non/filetype v1.0.12/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -155,6 +212,9 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= @@ -183,15 +243,21 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -201,6 +267,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 h1:IaQbIIB2X/Mp/DKctl6ROxz1KyMlKp4uyvL6+kQ7C88= golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -224,6 +292,7 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -232,6 +301,9 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -242,6 +314,7 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -252,12 +325,28 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5 h1:WQ8q63x+f/zpC8Ac1s9wLElVoHhm32p6tudrU72n1QA= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423 h1:/hEknzWkMPCjTo7StMHRrBRa8YBbXuBWfck8680k3RE= +golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -266,6 +355,9 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -291,11 +383,25 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200409092240-59c9f1ba88fa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e h1:hq86ru83GdWTlfQFZGO4nZJTU4Bs2wfHl8oFHRaXsfc= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3 h1:kzM6+9dur93BcC2kVlYl34cHU+TYZLanmpSJHVMmL64= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -336,10 +442,25 @@ golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWc golang.org/x/tools v0.0.0-20200409170454-77362c5149f0/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d h1:lzLdP95xJmMpwQ6LUHwrc5V7js93hTiY7gkznu0BgmY= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -355,6 +476,15 @@ google.golang.org/api v0.21.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/ google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.23.0 h1:YlvGEOq2NA2my8cZ/9V8BcEO9okD48FlJcdqN0xJL3s= google.golang.org/api v0.23.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQWYWo= +google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0 h1:l2Nfbl2GPXdWorv+dT2XfinX2jOOw4zv1VhLstx+6rE= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -362,6 +492,8 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -387,6 +519,22 @@ google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 h1:Bz1qTn2YRWV+9OKJtxHJiQKCiXIdf+kwuKXdt9cBxyU= google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595 h1:x7nk+/4+SvuTDI4wnzQUlhvi+DTpyfncXBo3QWTFs7U= +google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -399,12 +547,26 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.34.1 h1:ugq+9++ZQPFzM2pKUMCIK8gj9M0pFyuUWO9Q8kwEDQw= +google.golang.org/grpc v1.34.1/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -421,8 +583,14 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86 h1:OfFoIUYv/me30yv7XlMy4F9RJw8DEm8WQ6QG1Ph4bH0= gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -430,6 +598,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 3031b0b5..097bc190 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1,6 +1,8 @@ package main import ( + "github.com/frikky/shuffle-shared" + "bufio" "bytes" @@ -29,12 +31,11 @@ import ( "cloud.google.com/go/datastore" "cloud.google.com/go/pubsub" "cloud.google.com/go/storage" - "google.golang.org/api/option" "google.golang.org/appengine/mail" - "github.com/getkin/kin-openapi/openapi2" - "github.com/getkin/kin-openapi/openapi2conv" - "github.com/getkin/kin-openapi/openapi3" + "github.com/frikky/kin-openapi/openapi2" + "github.com/frikky/kin-openapi/openapi2conv" + "github.com/frikky/kin-openapi/openapi3" /* "github.com/frikky/kin-openapi/openapi2" "github.com/frikky/kin-openapi/openapi2conv" @@ -42,7 +43,6 @@ import ( */ "github.com/google/go-github/v28/github" - "golang.org/x/oauth2" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" @@ -62,16 +62,10 @@ import ( // githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http" // Web - // "github.com/gorilla/handlers" "github.com/gorilla/mux" + "google.golang.org/api/option" "google.golang.org/grpc" http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" - // Old items (cloud) - // "google.golang.org/appengine" - // "google.golang.org/appengine/memcache" - // applog "google.golang.org/appengine/log" - //cloudrun "google.golang.org/api/run/v1" - "github.com/patrickmn/go-cache" ) // This is used to handle onprem vs offprem databases etc @@ -80,14 +74,17 @@ var bucketName = "shuffler.appspot.com" var baseAppPath = "/home/frikky/git/shaffuru/tmp/apps" var baseDockerName = "frikky/shuffle" var registryName = "registry.hub.docker.com" +var runningEnvironment = "onprem" -//var syncUrl = "http://192.168.102.54:5002" var syncUrl = "https://shuffler.io" //var syncUrl = "http://localhost:5002" +var syncSubUrl = "https://shuffler.io" + +//var syncUrl = "http://localhost:5002" +//var syncSubUrl = "https://050196912a9d.ngrok.io" var dbclient *datastore.Client -var requestCache *cache.Cache type Userapi struct { Username string `datastore:"username"` @@ -155,11 +152,11 @@ type UserLimits struct { } type retStruct struct { - Success bool `json:"success"` - SyncFeatures SyncFeatures `json:"sync_features"` - SessionKey string `json:"session_key"` - IntervalSeconds int64 `json:"interval_seconds"` - Reason string `json:"reason"` + Success bool `json:"success"` + SyncFeatures shuffle.SyncFeatures `json:"sync_features"` + SessionKey string `json:"session_key"` + IntervalSeconds int64 `json:"interval_seconds"` + Reason string `json:"reason"` } // Saves some data, not sure what to have here lol @@ -177,37 +174,15 @@ type UserAuthField struct { } // Not environment, but execution environment -type Environment struct { - Name string `datastore:"name"` - Type string `datastore:"type"` - Registered bool `datastore:"registered"` - Default bool `datastore:"default" json:"default"` - Archived bool `datastore:"archived" json:"archived"` - Id string `datastore:"id" json:"id"` - OrgId string `datastore:"org_id" json:"org_id"` -} - -type User struct { - Username string `datastore:"Username" json:"username"` - Password string `datastore:"password,noindex" password:"password,omitempty"` - Session string `datastore:"session,noindex" json:"session"` - Verified bool `datastore:"verified,noindex" json:"verified"` - PrivateApps []WorkflowApp `datastore:"privateapps" json:"privateapps":` - Role string `datastore:"role" json:"role"` - Roles []string `datastore:"roles" json:"roles"` - VerificationToken string `datastore:"verification_token" json:"verification_token"` - ApiKey string `datastore:"apikey" json:"apikey"` - ResetReference string `datastore:"reset_reference" json:"reset_reference"` - Executions ExecutionInfo `datastore:"executions" json:"executions"` - Limits UserLimits `datastore:"limits" json:"limits"` - Authentication []UserAuth `datastore:"authentication,noindex" json:"authentication"` - ResetTimeout int64 `datastore:"reset_timeout,noindex" json:"reset_timeout"` - Id string `datastore:"id" json:"id"` - Orgs []string `datastore:"orgs" json:"orgs"` - CreationTime int64 `datastore:"creation_time" json:"creation_time"` - ActiveOrg Org `json:"active_org" datastore:"active_org"` - Active bool `datastore:"active" json:"active"` -} +//type Environment struct { +// Name string `datastore:"name"` +// Type string `datastore:"type"` +// Registered bool `datastore:"registered"` +// Default bool `datastore:"default" json:"default"` +// Archived bool `datastore:"archived" json:"archived"` +// Id string `datastore:"id" json:"id"` +// OrgId string `datastore:"org_id" json:"org_id"` +//} // timeout maybe? idk type session struct { @@ -616,83 +591,6 @@ func checkFileExistsLocal(basepath string, filepath string) bool { return true } -func handleApiAuthentication(resp http.ResponseWriter, request *http.Request) (User, error) { - apikey := request.Header.Get("Authorization") - if len(apikey) > 0 { - if !strings.HasPrefix(apikey, "Bearer ") { - log.Printf("[WARNING] Apikey doesn't start with bearer") - return User{}, errors.New("No bearer token for authorization header") - } - - apikeyCheck := strings.Split(apikey, " ") - if len(apikeyCheck) != 2 { - log.Printf("[WARNING] Invalid format for apikey.") - return User{}, errors.New("Invalid format for apikey") - } - - // This is annoying af and is done because of maxlength lol - newApikey := apikeyCheck[1] - if len(newApikey) > 249 { - newApikey = newApikey[0:248] - } - - ctx := context.Background() - - // Make specific check for just service user? - // Get the user based on APIkey here - Userdata, err := getApikey(ctx, apikeyCheck[1]) - if err != nil { - log.Printf("Apikey %s doesn't exist: %s", apikey, err) - return User{}, err - } - - if len(Userdata.Username) > 0 { - return Userdata, nil - } else { - return Userdata, errors.New(fmt.Sprintf("[WARNING] User is invalid - no username found")) - } - } - - // One time API keys - authorizationArr, ok := request.URL.Query()["authorization"] - ctx := context.Background() - if ok { - authorization := "" - if len(authorizationArr) > 0 { - authorization = authorizationArr[0] - } - _ = authorization - } - - c, err := request.Cookie("session_token") - if err == nil { - sessionToken := c.Value - session, err := getSession(ctx, sessionToken) - if err != nil { - log.Printf("Session %s doesn't exist (session auth): %s", sessionToken, err) - return User{}, err - } - - // Get session first - // Should basically never happen - Userdata, err := getUser(ctx, session.Id) - if err != nil { - log.Printf("Username %s doesn't exist (authcheck): %s", session.Username, err) - return User{}, err - } - - if Userdata.Session != sessionToken { - return User{}, errors.New("Wrong session token") - } - - // Means session exists, but - return *Userdata, nil - } - - // Key = apikey - return User{}, errors.New("Missing authentication") -} - func handleGetallSchedules(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -769,120 +667,6 @@ func parseLoginParameters(resp http.ResponseWriter, request *http.Request) (logi return t, nil } -// Can check against HIBP etc? -// Removed for localhost -func checkPasswordStrength(password string) error { - // Check password strength here - if len(password) < 3 { - return errors.New("Minimum password length is 3.") - } - - //if len(password) > 128 { - // return errors.New("Maximum password length is 128.") - //} - - //re := regexp.MustCompile("[0-9]+") - //if len(re.FindAllString(password, -1)) == 0 { - // return errors.New("Password must contain a number") - //} - - //re = regexp.MustCompile("[a-z]+") - //if len(re.FindAllString(password, -1)) == 0 { - // return errors.New("Password must contain a lower case char") - //} - - //re = regexp.MustCompile("[A-Z]+") - //if len(re.FindAllString(password, -1)) == 0 { - // return errors.New("Password must contain an upper case char") - //} - - return nil -} - -func deleteUser(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - userInfo, userErr := handleApiAuthentication(resp, request) - if userErr != nil { - log.Printf("Api authentication failed in edit workflow: %s", userErr) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if userInfo.Role != "admin" { - log.Printf("Wrong user (%s) when deleting - must be admin", userInfo.Username) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Must be admin"}`)) - return - } - - location := strings.Split(request.URL.String(), "/") - var userId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - userId = location[4] - } - - ctx := context.Background() - foundUser, err := getUser(ctx, userId) - if err != nil { - log.Printf("Can't find user %s (delete user): %s", userId, err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) - return - } - - orgFound := false - if userInfo.ActiveOrg.Id == foundUser.ActiveOrg.Id { - orgFound = true - } else { - log.Printf("FoundUser: %#v", foundUser.Orgs) - for _, item := range foundUser.Orgs { - if item == userInfo.ActiveOrg.Id { - orgFound = true - break - } - } - } - - if !orgFound { - log.Printf("User %s is admin, but can't delete users outside their own org.", userInfo.Id) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't change users outside your org."}`))) - return - } - - // Invert. No user deletion. - if foundUser.Active { - - foundUser.Active = false - } else { - foundUser.Active = true - } - - err = setUser(ctx, foundUser) - if err != nil { - log.Printf("Failed swapping active for user %s (%s)", foundUser.Username, foundUser.Id) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) - return - } - - log.Printf("Successfully inverted %s", foundUser.Username) - - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) -} - // No more emails :) func checkUsername(Username string) error { // Stupid first check of email loool @@ -925,7 +709,7 @@ func handleRegisterVerification(resp http.ResponseWriter, request *http.Request) // With user, do a search for workflows with user or user's org attached // Only giving 200 to not give any suspicion whether they're onto an actual user or not q := datastore.NewQuery("Users").Filter("verification_token =", reference) - var users []User + var users []shuffle.User _, err := dbclient.GetAll(ctx, q, &users) if err != nil { log.Printf("Failed getting users for verification token: %s", err) @@ -946,7 +730,7 @@ func handleRegisterVerification(resp http.ResponseWriter, request *http.Request) // FIXME: Not for cloud! Userdata.Verified = true - err = setUser(ctx, &Userdata) + err = shuffle.SetUser(ctx, &Userdata) if err != nil { log.Printf("Failed adding verification for user %s: %s", Userdata.Username, err) resp.WriteHeader(401) @@ -967,7 +751,7 @@ func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) { // FIXME: Overhaul the top part. // Only admin can change environments, but if there are no users, anyone can make (first) - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Can't handle set env auth"}`)) @@ -981,7 +765,7 @@ func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - var environments []Environment + var environments []shuffle.Environment q := datastore.NewQuery("Environments").Filter("org_id =", user.ActiveOrg.Id) _, err = dbclient.GetAll(ctx, q, &environments) if err != nil { @@ -998,7 +782,7 @@ func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) { return } - var newEnvironments []Environment + var newEnvironments []shuffle.Environment err = json.Unmarshal(body, &newEnvironments) if err != nil { log.Printf("Failed unmarshaling: %s", err) @@ -1058,10 +842,10 @@ func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": true}`)) } -func createNewUser(username, password, role, apikey string, org Org) error { +func createNewUser(username, password, role, apikey string, org shuffle.Org) error { // Returns false if there is an issue // Use this for register - err := checkPasswordStrength(password) + err := shuffle.CheckPasswordStrength(password) if err != nil { log.Printf("Bad password strength: %s", err) return err @@ -1075,7 +859,7 @@ func createNewUser(username, password, role, apikey string, org Org) error { ctx := context.Background() q := datastore.NewQuery("Users").Filter("Username =", username) - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err != nil { log.Printf("Failed getting user for registration: %s", err) @@ -1092,7 +876,7 @@ func createNewUser(username, password, role, apikey string, org Org) error { return err } - newUser := new(User) + newUser := new(shuffle.User) newUser.Username = username newUser.Password = string(hashedPassword) newUser.Verified = false @@ -1141,16 +925,16 @@ func createNewUser(username, password, role, apikey string, org Org) error { newUser.Id = ID.String() newUser.VerificationToken = verifyToken.String() - err = setUser(ctx, newUser) + err = shuffle.SetUser(ctx, newUser) if err != nil { log.Printf("Error adding User %s: %s", username, err) return err } - neworg, err := getOrg(ctx, org.Id) + neworg, err := shuffle.GetOrg(ctx, org.Id) if err == nil { //neworg.Users = append(neworg.Users, *newUser) - err = setOrg(ctx, *neworg, neworg.Id) + err = shuffle.SetOrg(ctx, *neworg, neworg.Id) if err != nil { log.Printf("Failed updating org with user %s", newUser.Username) } else { @@ -1175,7 +959,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { // FIXME: Overhaul the top part. // Only admin can CREATE users, but if there are no users, anyone can make (first) count, countErr := getUserCount() - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { if (countErr == nil && count > 0) || countErr != nil { resp.WriteHeader(401) @@ -1211,7 +995,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { if user.ActiveOrg.Id == "" { log.Printf("There's no active org for the user. Checking if there's a single one to assing it to.") - var orgs []Org + var orgs []shuffle.Org q := datastore.NewQuery("Organizations") _, err = dbclient.GetAll(ctx, q, &orgs) if err == nil && len(orgs) == 1 { @@ -1247,133 +1031,13 @@ func handleCookie(request *http.Request) bool { return true } -func handleLogout(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - http.SetCookie(resp, &http.Cookie{ - Name: "session_token", - Value: "", - Path: "/", - Expires: time.Unix(0, 0), - }) - - userInfo, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in handleLogout: %s", err) - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true, "reason": "Not logged in"}`)) - return - } - - ctx := context.Background() - session, err := getSession(ctx, userInfo.Session) - if err != nil { - log.Printf("Session %#v doesn't exist: %s", session, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "No session"}`)) - return - } - - // Check cookie - //c, err := request.Cookie("session_token") - //if err != nil { - // resp.WriteHeader(200) - // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - // return - //} else { - // log.Printf("Session cookie is set to %s!", c.Value) - //} - - //var Userdata User - //ctx := context.Background() - //sessionToken = c.Value - //session, err := getSession(ctx, sessionToken) - //if err != nil { - // log.Printf("Session %s doesn't exist (logout): %s", sessionToken, err) - // resp.WriteHeader(401) - // resp.Write([]byte(`{"success": false, "reason": "Couldn't find your session"}`)) - // return - //} - - // Get session first - // Should basically never happen - //_, err = getUser(ctx, session.Id) - //if err != nil { - // log.Printf("Username %s doesn't exist (logout): %s", session.Username, err) - // resp.WriteHeader(401) - // resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) - // return - //} - - // Userdata = *tmpdata - //} - - // FIXME - // Session might delete someone elses here? - // No need to think about before possible scale..? - err = SetSession(ctx, userInfo, "") - if err != nil { - log.Printf("Error removing session for: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) - return - } - - err = DeleteKey(ctx, "sessions", userInfo.Session) - if err != nil { - log.Printf("Error deleting key %s for %s: %s", userInfo.Session, userInfo.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) - return - } - - userInfo.Session = "" - err = setUser(ctx, &userInfo) - if err != nil { - log.Printf("Failed updating user: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed updating apikey"}`)) - return - } - - //memcache.Delete(request.Context(), sessionToken) - - resp.WriteHeader(200) - resp.Write([]byte(`{"success": false, "reason": "Successfully logged out"}`)) -} - -func generateApikey(ctx context.Context, userInfo User) (User, error) { - // Generate UUID - // Set uuid to apikey in backend (update) - apikey := uuid.NewV4() - userInfo.ApiKey = apikey.String() - - err := SetApikey(ctx, userInfo) - if err != nil { - log.Printf("Failed updating apikey: %s", err) - return userInfo, err - } - - // Updating user - err = setUser(ctx, &userInfo) - if err != nil { - log.Printf("Failed updating user: %s", err) - return userInfo, err - } - - return userInfo, nil -} - func handleUpdateUser(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } - userInfo, err := handleApiAuthentication(resp, request) + userInfo, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in apigen: %s", err) resp.WriteHeader(401) @@ -1414,7 +1078,7 @@ func handleUpdateUser(resp http.ResponseWriter, request *http.Request) { return } - foundUser, err := getUser(ctx, t.UserId) + foundUser, err := shuffle.GetUser(ctx, t.UserId) if err != nil { log.Printf("Can't find user %s (update user): %s", t.UserId, err) resp.WriteHeader(401) @@ -1457,7 +1121,7 @@ func handleUpdateUser(resp http.ResponseWriter, request *http.Request) { if len(t.Username) > 0 { q := datastore.NewQuery("Users").Filter("username =", t.Username) - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err != nil { resp.WriteHeader(401) @@ -1482,7 +1146,7 @@ func handleUpdateUser(resp http.ResponseWriter, request *http.Request) { foundUser.Username = t.Username } - err = setUser(ctx, foundUser) + err = shuffle.SetUser(ctx, foundUser) if err != nil { log.Printf("Error patching user %s: %s", foundUser.Username, err) resp.WriteHeader(401) @@ -1494,114 +1158,15 @@ func handleUpdateUser(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } -func handleApiGeneration(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - userInfo, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in apigen: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - ctx := context.Background() - if request.Method == "GET" { - newUserInfo, err := generateApikey(ctx, userInfo) - if err != nil { - log.Printf("Failed to generate apikey for user %s: %s", userInfo.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": ""}`)) - return - } - userInfo = newUserInfo - log.Printf("Updated apikey for user %s", userInfo.Username) - } else if request.Method == "POST" { - log.Printf("Handling post!") - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Println("Failed reading body") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Missing field: user_id"}`))) - return - } - - type userId struct { - UserId string `json:"user_id"` - } - - var t userId - err = json.Unmarshal(body, &t) - if err != nil { - log.Printf("Failed unmarshaling userId: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unmarshaling. Missing field: user_id"}`))) - return - } - - if userInfo.Role != "admin" { - log.Printf("%s tried and failed to change apikey for %s", userInfo.Username, t.UserId) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "You need to be admin to change others' apikey"}`))) - return - } - - foundUser, err := getUser(ctx, t.UserId) - if err != nil { - log.Printf("Can't find user %s (apikey gen): %s", t.UserId, err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) - return - } - - newUserInfo, err := generateApikey(ctx, *foundUser) - if err != nil { - log.Printf("Failed to generate apikey for user %s: %s", foundUser.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - foundUser = &newUserInfo - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "username": "%s", "verified": %t, "apikey": "%s"}`, foundUser.Username, foundUser.Verified, foundUser.ApiKey))) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "username": "%s", "verified": %t, "apikey": "%s"}`, userInfo.Username, userInfo.Verified, userInfo.ApiKey))) -} - -func handleSettings(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - userInfo, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in apigen: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "username": "%s", "verified": %t, "apikey": "%s"}`, userInfo.Username, userInfo.Verified, userInfo.ApiKey))) -} - func handleInfo(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } - userInfo, err := handleApiAuthentication(resp, request) + userInfo, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { - log.Printf("Api authentication failed in handleInfo: %s", err) + log.Printf("[WARNING] Api authentication failed in handleInfo: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -1628,7 +1193,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() q := datastore.NewQuery("Users") - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err != nil { resp.WriteHeader(401) @@ -1693,14 +1258,14 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { // Updating user info if there's something wrong if (len(userInfo.ActiveOrg.Name) == 0 || len(userInfo.ActiveOrg.Id) == 0) && len(userInfo.Orgs) > 0 { - _, err := getOrg(ctx, userInfo.Orgs[0]) + _, err := shuffle.GetOrg(ctx, userInfo.Orgs[0]) if err != nil { - var orgs []Org + var orgs []shuffle.Org q := datastore.NewQuery("Organizations") _, err = dbclient.GetAll(ctx, q, &orgs) if err == nil { newStringOrgs := []string{} - newOrgs := []Org{} + newOrgs := []shuffle.Org{} for _, org := range orgs { if strings.ToLower(org.Name) == strings.ToLower(userInfo.Orgs[0]) { newOrgs = append(newOrgs, org) @@ -1712,7 +1277,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { userInfo.ActiveOrg = newOrgs[0] userInfo.Orgs = newStringOrgs - err = setUser(ctx, &userInfo) + err = shuffle.SetUser(ctx, &userInfo) if err != nil { log.Printf("Error patching User for activeOrg: %s", err) } else { @@ -1726,10 +1291,10 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { } else { // 1. Check if the org exists by ID // 2. if it does, overwrite user - userInfo.ActiveOrg = Org{ + userInfo.ActiveOrg = shuffle.Org{ Id: userInfo.Orgs[0], } - err = setUser(ctx, &userInfo) + err = shuffle.SetUser(ctx, &userInfo) if err != nil { log.Printf("Error patching User for activeOrg: %s", err) } @@ -1737,12 +1302,14 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { } // FIXME: Remove this dependency by updating users' orgs when org itself is updated - org, err := getOrg(ctx, userInfo.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, userInfo.ActiveOrg.Id) if err == nil { userInfo.ActiveOrg = *org - userInfo.ActiveOrg.Users = []User{} + userInfo.ActiveOrg.Users = []shuffle.User{} } + userInfo.ActiveOrg.Users = []shuffle.User{} + userInfo.ActiveOrg.SyncConfig = shuffle.SyncConfig{} currentOrg, err := json.Marshal(userInfo.ActiveOrg) if err != nil { currentOrg = []byte("{}") @@ -1769,97 +1336,6 @@ type passwordReset struct { Reference string `json:"reference"` } -type passwordChange struct { - Username string `json:"username"` - Newpassword string `json:"newpassword"` - Newpassword2 string `json:"newpassword2"` - Currentpassword string `json:"currentpassword"` -} - -func handlePasswordResetMail(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - log.Println("Handling password reset mail") - defaultMessage := "We have sent you an email :)" - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Println("Failed reading body") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, defaultMessage))) - return - } - - type passwordReset struct { - Username string `json:"username"` - } - - var t passwordReset - err = json.Unmarshal(body, &t) - if err != nil { - log.Printf("Failed unmarshaling: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, defaultMessage))) - return - } - - ctx := context.Background() - Userdata, err := getUser(ctx, t.Username) - if err != nil { - log.Printf("Username %s doesn't exist (pw reset mail): %s", t.Username, err) - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) - return - } - - resetToken := uuid.NewV4() - // FIXME: - // Weakness with this system is that you can spam someone with password resets, - // and they would never be able to reset, as a new token is always generated - url := fmt.Sprintf("https://shuffler.io/passwordreset/%s", resetToken.String()) - - Userdata.ResetReference = resetToken.String() - Userdata.ResetTimeout = 0 - err = setUser(ctx, Userdata) - if err != nil { - log.Printf("Error patching User for mail %s: %s", Userdata.Username, err) - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) - return - } - - log.Printf("%#v", Userdata) - addr := t.Username - const confirmMessage = ` -Reset URL :) - -%s - ` - - msg := &mail.Message{ - Sender: "Shuffle ", - To: []string{addr}, - Subject: "Reset your password - Shuffle", - Body: fmt.Sprintf(confirmMessage, url), - } - - log.Println(msg.Body) - if err := mail.Send(ctx, msg); err != nil { - log.Printf("Couldn't send email: %v", err) - } - - // FIXME - // Generate an email to send - // Generate a reset code with a reset link - // Build frontend to handle reset link with "new password" etc. - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) -} - func handlePasswordReset(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -1905,7 +1381,7 @@ func handlePasswordReset(resp http.ResponseWriter, request *http.Request) { // With user, do a search for workflows with user or user's org attached // Only giving 200 to not give any suspicion whether they're onto an actual user or not q := datastore.NewQuery("Users").Filter("reset_reference =", t.Reference) - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err != nil { log.Printf("Failed getting users: %s", err) @@ -1934,7 +1410,7 @@ func handlePasswordReset(resp http.ResponseWriter, request *http.Request) { Userdata.Password = string(hashedPassword) Userdata.ResetTimeout = 0 Userdata.ResetReference = "" - err = setUser(ctx, &Userdata) + err = shuffle.SetUser(ctx, &Userdata) if err != nil { log.Printf("Error adding User %s: %s", Userdata.Username, err) resp.WriteHeader(200) @@ -1948,159 +1424,6 @@ func handlePasswordReset(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) } -func handlePasswordChange(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - log.Println("Handling password change") - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Println("Failed reading body") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) - return - } - - // Get the current user - check if they're admin or the "username" user. - var t passwordChange - err = json.Unmarshal(body, &t) - if err != nil { - log.Println("Failed unmarshaling") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) - return - } - - userInfo, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - curUserFound := false - if t.Username != userInfo.Username && userInfo.Role != "admin" { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Admin required to change others' passwords"}`)) - return - } else if t.Username == userInfo.Username { - curUserFound = true - } - - if userInfo.Role != "admin" { - if t.Newpassword != t.Newpassword2 { - err := "Passwords don't match" - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - - if len(t.Newpassword) < 10 || len(t.Newpassword2) < 10 { - err := "Passwords too short - 2" - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - } else { - // Check ORG HERE? - } - - // Current password - err = checkPasswordStrength(t.Newpassword) - if err != nil { - log.Printf("Bad password strength: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - - ctx := context.Background() - foundUser := User{} - if !curUserFound { - log.Printf("Have to find a different user") - q := datastore.NewQuery("Users").Filter("Username =", strings.ToLower(t.Username)) - var users []User - _, err = dbclient.GetAll(ctx, q, &users) - if err != nil { - log.Printf("Failed getting user %s", t.Username) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) - return - } - - if len(users) != 1 { - log.Printf(`Found multiple or no users with the same username: %s: %d`, t.Username, len(users)) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Found %d users with the same username: %s"}`, len(users), t.Username))) - return - } - - foundUser = users[0] - orgFound := false - if userInfo.ActiveOrg.Id == foundUser.ActiveOrg.Id { - orgFound = true - } else { - log.Printf("FoundUser: %#v", foundUser.Orgs) - for _, item := range foundUser.Orgs { - if item == userInfo.ActiveOrg.Id { - orgFound = true - break - } - } - } - - if !orgFound { - log.Printf("User %s is admin, but can't change user's passowrd outside their own org.", userInfo.Id) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't change users outside your org."}`))) - return - } - } else { - // Admins can re-generate others' passwords as well. - if userInfo.Role != "admin" { - err = bcrypt.CompareHashAndPassword([]byte(userInfo.Password), []byte(t.Newpassword)) - if err != nil { - log.Printf("Bad password for %s: %s", userInfo.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) - return - } - } - } - - if len(foundUser.Id) == 0 { - log.Printf("Something went wrong in password reset: couldn't find user.") - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false}`)) - return - } - - hashedPassword, err := bcrypt.GenerateFromPassword([]byte(t.Newpassword), 8) - if err != nil { - log.Printf("New password failure for %s: %s", userInfo.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) - return - } - - userInfo.Password = string(hashedPassword) - err = setUser(ctx, &foundUser) - if err != nil { - log.Printf("Error fixing password for user %s: %s", userInfo.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) - return - } - - //memcache.Delete(ctx, sessionToken) - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) -} - // FIXME - forward this to emails or whatever CRM system in use func handleContact(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) @@ -2176,7 +1499,7 @@ func handleGetSchedules(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -2213,222 +1536,6 @@ func handleGetSchedules(resp http.ResponseWriter, request *http.Request) { resp.Write(newjson) } -func handleGetEnvironments(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - ctx := context.Background() - var environments []Environment - q := datastore.NewQuery("Environments").Filter("org_id =", user.ActiveOrg.Id) - _, err = dbclient.GetAll(ctx, q, &environments) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Can't get environments"}`)) - return - } - - newjson, err := json.Marshal(environments) - if err != nil { - log.Printf("Failed unmarshal: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking environments"}`))) - return - } - - //log.Printf("Existing environments: %s", string(newjson)) - - resp.WriteHeader(200) - resp.Write(newjson) -} - -func handleGetOrg(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - var fileId string - location := strings.Split(request.URL.String(), "/") - if location[1] == "api" { - if len(location) <= 4 { - log.Printf("Path too short: %d", len(location)) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - ctx := context.Background() - org, err := getOrg(ctx, fileId) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed getting org users"}`)) - return - } - - //FIXME : cleanup org before marshal - userFound := false - for _, foundUser := range org.Users { - if foundUser.Id == user.Id { - userFound = true - break - } - } - - if !userFound { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Use doesn't have access to org"}`)) - return - } - - org.Users = []User{} - org.SyncConfig.Apikey = "" - newjson, err := json.Marshal(org) - if err != nil { - log.Printf("Failed unmarshal of org: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking"}`))) - return - } - - resp.WriteHeader(200) - resp.Write(newjson) -} - -func handleGetOrgs(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if user.Role != "global_admin" { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Not admin"}`)) - return - } - - ctx := context.Background() - var orgs []Org - q := datastore.NewQuery("Organizations") - _, err = dbclient.GetAll(ctx, q, &orgs) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Can't get users"}`)) - return - } - - //newUsers := []User{} - //for _, item := range users { - // if len(item.Username) == 0 { - // continue - // } - - // item.Password = "" - // item.Session = "" - // item.VerificationToken = "" - - // newUsers = append(newUsers, item) - //} - - newjson, err := json.Marshal(orgs) - if err != nil { - log.Printf("Failed unmarshal: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking"}`))) - return - } - - resp.WriteHeader(200) - resp.Write(newjson) -} - -func handleGetUsers(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if user.Role != "admin" { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Not admin"}`)) - return - } - - // FIXME: Check by org. - ctx := context.Background() - org, err := getOrg(ctx, user.ActiveOrg.Id) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed getting org users"}`)) - return - } - - newUsers := []User{} - for _, item := range org.Users { - if len(item.Username) == 0 { - continue - } - - //for _, tmpUser := range newUsers { - // if tmpUser.Name - //} - - item.Password = "" - item.Session = "" - item.VerificationToken = "" - item.Orgs = []string{} - - newUsers = append(newUsers, item) - } - - newjson, err := json.Marshal(newUsers) - if err != nil { - log.Printf("Failed unmarshal: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking"}`))) - return - } - - resp.WriteHeader(200) - resp.Write(newjson) -} - func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -2467,7 +1574,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("Handling login of %s", data.Username) + log.Printf("[INFO] Handling login of %s", data.Username) err = checkUsername(data.Username) if err != nil { @@ -2477,9 +1584,9 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - log.Printf("Username: %s", data.Username) + log.Printf("[INFO] Login Username: %s", data.Username) q := datastore.NewQuery("Users").Filter("Username =", data.Username) - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err != nil { log.Printf("Failed getting user %s", data.Username) @@ -2515,7 +1622,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { // FIXME - have timeout here loginData := `{"success": true}` if len(Userdata.Session) != 0 { - log.Println("User session exists - resetting") + log.Println("[INFO] User session already exists - resetting it") expiration := time.Now().Add(3600 * time.Second) http.SetCookie(resp, &http.Cookie{ @@ -2527,7 +1634,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { loginData = fmt.Sprintf(`{"success": true, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}]}`, Userdata.Session, expiration.Unix()) //log.Printf("SESSION LENGTH MORE THAN 0 IN LOGIN: %s", Userdata.Session) - err = SetSession(ctx, Userdata, Userdata.Session) + err = shuffle.SetSession(ctx, Userdata, Userdata.Session) if err != nil { log.Printf("Error adding session to database: %s", err) } @@ -2536,7 +1643,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(loginData)) return } else { - log.Printf("User session is empty - create one!") + log.Printf("[INFO] User session is empty - create one!") sessionToken := uuid.NewV4().String() expiration := time.Now().Add(3600 * time.Second) @@ -2547,13 +1654,13 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { }) // ADD TO DATABASE - err = SetSession(ctx, Userdata, sessionToken) + err = shuffle.SetSession(ctx, Userdata, sessionToken) if err != nil { log.Printf("Error adding session to database: %s", err) } Userdata.Session = sessionToken - err = setUser(ctx, &Userdata) + err = shuffle.SetUser(ctx, &Userdata) if err != nil { log.Printf("Failed updating user when setting session: %s", err) resp.WriteHeader(500) @@ -2570,78 +1677,6 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(loginData)) } -func getApikey(ctx context.Context, apikey string) (User, error) { - // Query for the specifci workflowId - q := datastore.NewQuery("Users").Filter("apikey =", apikey) - var users []User - _, err := dbclient.GetAll(ctx, q, &users) - if err != nil { - log.Printf("[ERROR] Error getting users apikey (getapikey): %s", err) - return User{}, err - } - - if len(users) == 0 { - log.Printf("[WARNING] No users found for apikey %s", apikey) - return User{}, err - } - - return users[0], nil -} - -func getSession(ctx context.Context, thissession string) (*session, error) { - key := datastore.NameKey("sessions", thissession, nil) - curUser := &session{} - if err := dbclient.Get(ctx, key, curUser); err != nil { - return &session{}, err - } - - return curUser, nil -} - -// ListBooks returns a list of books, ordered by title. -func getOrg(ctx context.Context, id string) (*Org, error) { - key := datastore.NameKey("Organizations", id, nil) - curOrg := &Org{} - if err := dbclient.Get(ctx, key, curOrg); err != nil { - return &Org{}, err - } - - return curOrg, nil -} - -func setOrg(ctx context.Context, org Org, id string) error { - // clear session_token and API_token for user - timeNow := int64(time.Now().Unix()) - if org.Created == 0 { - org.Created = timeNow - } - - org.Edited = timeNow - - k := datastore.NameKey("Organizations", id, nil) - if _, err := dbclient.Put(ctx, k, &org); err != nil { - log.Printf("Failed setting org: %s", err) - return err - } - - // FIXME: Make this update every user to have the correct org data. - //org = fixOrgUser(ctx, &org) - //_ = org - - return nil -} - -// ListBooks returns a list of books, ordered by title. -func getUser(ctx context.Context, id string) (*User, error) { - key := datastore.NameKey("Users", id, nil) - curUser := &User{} - if err := dbclient.Get(ctx, key, curUser); err != nil { - return &User{}, err - } - - return curUser, nil -} - // Index = Username func DeleteKeys(ctx context.Context, entity string, value []string) error { // Non indexed User data @@ -2673,52 +1708,6 @@ func DeleteKey(ctx context.Context, entity string, value string) error { return nil } -// Index = Username -func SetApikey(ctx context.Context, Userdata User) error { - // Non indexed User data - newapiUser := new(Userapi) - newapiUser.ApiKey = Userdata.ApiKey - newapiUser.Username = Userdata.Username - key1 := datastore.NameKey("apikey", newapiUser.ApiKey, nil) - - // New struct, to not add body, author etc - if _, err := dbclient.Put(ctx, key1, newapiUser); err != nil { - log.Printf("Error adding apikey: %s", err) - return err - } - - return nil -} - -// Index = Username -func SetSession(ctx context.Context, Userdata User, value string) error { - // Non indexed User data - Userdata.Session = value - key1 := datastore.NameKey("Users", Userdata.Id, nil) - - // New struct, to not add body, author etc - if _, err := dbclient.Put(ctx, key1, &Userdata); err != nil { - log.Printf("rror adding Usersession: %s", err) - return err - } - - if len(Userdata.Session) > 0 { - // Indexed session data - sessiondata := new(session) - sessiondata.Username = Userdata.Username - sessiondata.Session = Userdata.Session - sessiondata.Id = Userdata.Id - key2 := datastore.NameKey("sessions", sessiondata.Session, nil) - - if _, err := dbclient.Put(ctx, key2, sessiondata); err != nil { - log.Printf("Error adding session: %s", err) - return err - } - } - - return nil -} - func setOpenApiDatastore(ctx context.Context, id string, data ParsedOpenApi) error { k := datastore.NameKey("openapi3", id, nil) if _, err := dbclient.Put(ctx, k, &data); err != nil { @@ -2738,7 +1727,7 @@ func getOpenApiDatastore(ctx context.Context, id string) (ParsedOpenApi, error) return *api, nil } -func setEnvironment(ctx context.Context, data *Environment) error { +func setEnvironment(ctx context.Context, data *shuffle.Environment) error { // clear session_token and API_token for user k := datastore.NameKey("Environments", strings.ToLower(data.Name), nil) @@ -2752,7 +1741,7 @@ func setEnvironment(ctx context.Context, data *Environment) error { return nil } -func fixOrgUser(ctx context.Context, org *Org) *Org { +func fixOrgUser(ctx context.Context, org *shuffle.Org) *shuffle.Org { //found := false //for _, id := range user.Orgs { // if user.ActiveOrg.Id == id { @@ -2771,7 +1760,7 @@ func fixOrgUser(ctx context.Context, org *Org) *Org { // continue // } - // org, err := getOrg(ctx, orgId) + // org, err := shuffle.GetOrg(ctx, orgId) // if err != nil { // log.Printf("Error getting org %s", orgId) // continue @@ -2798,7 +1787,7 @@ func fixOrgUser(ctx context.Context, org *Org) *Org { // org.Users = append(org.Users, *user) // } - // err = setOrg(ctx, *org, orgId) + // err = shuffle.SetOrg(ctx, *org, orgId) // if err != nil { // log.Printf("Failed setting org %s", orgId) // } @@ -2807,21 +1796,7 @@ func fixOrgUser(ctx context.Context, org *Org) *Org { return org } -// ListBooks returns a list of books, ordered by title. -func setUser(ctx context.Context, data *User) error { - data = fixUserOrg(ctx, data) - - // clear session_token and API_token for user - k := datastore.NameKey("Users", data.Id, nil) - if _, err := dbclient.Put(ctx, k, data); err != nil { - log.Println(err) - return err - } - - return nil -} - -func fixUserOrg(ctx context.Context, user *User) *User { +func fixUserOrg(ctx context.Context, user *shuffle.User) *shuffle.User { found := false for _, id := range user.Orgs { if user.ActiveOrg.Id == id { @@ -2840,7 +1815,7 @@ func fixUserOrg(ctx context.Context, user *User) *User { continue } - org, err := getOrg(ctx, orgId) + org, err := shuffle.GetOrg(ctx, orgId) if err != nil { log.Printf("Error getting org %s", orgId) continue @@ -2857,17 +1832,17 @@ func fixUserOrg(ctx context.Context, user *User) *User { } if userFound { - user.PrivateApps = []WorkflowApp{} - user.Executions = ExecutionInfo{} - user.Limits = UserLimits{} - user.Authentication = []UserAuth{} + user.PrivateApps = []shuffle.WorkflowApp{} + user.Executions = shuffle.ExecutionInfo{} + user.Limits = shuffle.UserLimits{} + user.Authentication = []shuffle.UserAuth{} org.Users[orgIndex] = *user } else { org.Users = append(org.Users, *user) } - err = setOrg(ctx, *org, orgId) + err = shuffle.SetOrg(ctx, *org, orgId) if err != nil { log.Printf("Failed setting org %s", orgId) } @@ -3044,9 +2019,9 @@ func handleSetHook(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) + log.Printf("[INFO] Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -3286,15 +2261,15 @@ func setSpecificSchedule(resp http.ResponseWriter, request *http.Request) { return } -func getSchedule(ctx context.Context, schedulename string) (*ScheduleOld, error) { - key := datastore.NameKey("schedules", strings.ToLower(schedulename), nil) - curUser := &ScheduleOld{} - if err := dbclient.Get(ctx, key, curUser); err != nil { - return &ScheduleOld{}, err - } - - return curUser, nil -} +//func GetSchedule(ctx context.Context, schedulename string) (*ScheduleOld, error) { +// key := datastore.NameKey("schedules", strings.ToLower(schedulename), nil) +// curUser := &ScheduleOld{} +// if err := dbclient.Get(ctx, key, curUser); err != nil { +// return &ScheduleOld{}, err +// } +// +// return curUser, nil +//} func getSpecificWebhook(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) @@ -3323,7 +2298,7 @@ func getSpecificWebhook(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() // FIXME: Schedule = trigger? - schedule, err := getSchedule(ctx, workflowId) + schedule, err := shuffle.GetSchedule(ctx, workflowId) if err != nil { log.Printf("Failed setting schedule: %s", err) resp.WriteHeader(401) @@ -3354,9 +2329,9 @@ func handleDeleteSchedule(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) + log.Printf("[WARNING] Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -3494,7 +2469,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { //resp.WriteHeader(200) //resp.Write([]byte(`{"success": true}`)) if hook.Status == "stopped" { - log.Printf("Not running %s because hook status is stopped", hook.Id) + log.Printf("[WARNING] Not running %s because hook status is stopped", hook.Id) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "The webhook isn't running. Click start to start it"}`))) return @@ -3525,10 +2500,34 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { return } + //log.Printf("BODY: %s", parsedBody) + + // This is a specific fix for MSteams and may fix other things as well + // Scared whether it may stop other things though, but that's a future problem + // (famous last words) + parsedBody := string(body) + if strings.Contains(parsedBody, "choice") { + if strings.Count(parsedBody, `\\n`) > 2 { + parsedBody = strings.Replace(parsedBody, `\\n`, "", -1) + } + if strings.Count(parsedBody, `\u0022`) > 2 { + parsedBody = strings.Replace(parsedBody, `\u0022`, `"`, -1) + } + if strings.Count(parsedBody, `\\"`) > 2 { + parsedBody = strings.Replace(parsedBody, `\\"`, `"`, -1) + } + + if strings.Contains(parsedBody, `"extra": "{`) { + parsedBody = strings.Replace(parsedBody, `"extra": "{`, `"extra": {`, 1) + parsedBody = strings.Replace(parsedBody, `}"}`, `}}`, 1) + } + } + + //log.Printf("\n\nPARSEDBODY: %s", parsedBody) newBody := ExecutionStruct{ Start: hook.Start, ExecutionSource: "webhook", - ExecutionArgument: string(body), + ExecutionArgument: parsedBody, } b, err := json.Marshal(newBody) @@ -3541,7 +2540,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { for _, item := range hook.Workflows { //log.Printf("Running webhook for workflow %s with startnode %s", item, hook.Start) - workflow := Workflow{ + workflow := shuffle.Workflow{ ID: "", } @@ -3559,9 +2558,8 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { // bodyWrapper = string(parsedBody) //} - url := &url.URL{} newRequest := &http.Request{ - URL: url, + URL: &url.URL{}, Method: "POST", Body: ioutil.NopCloser(bytes.NewReader(b)), } @@ -3570,10 +2568,12 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { // OrgId: activeOrgs[0].Id, workflowExecution, executionResp, err := handleExecution(item, workflow, newRequest) if err == nil { - err = increaseStatisticsField(ctx, "total_webhooks_ran", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) - if err != nil { - log.Printf("Failed to increase total apps loaded stats: %s", err) - } + /* + err = increaseStatisticsField(ctx, "total_webhooks_ran", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) + if err != nil { + log.Printf("Failed to increase total apps loaded stats: %s", err) + } + */ resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization))) @@ -3585,7 +2585,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { } } -func executeCloudAction(action CloudSyncJob, apikey string) error { +func executeCloudAction(action shuffle.CloudSyncJob, apikey string) error { data, err := json.Marshal(action) if err != nil { log.Printf("Failed cloud webhook action marshalling: %s", err) @@ -3630,295 +2630,6 @@ func executeCloudAction(action CloudSyncJob, apikey string) error { return nil } -// Starts a new webhook -func handleNewHook(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - type requestData struct { - Type string `json:"type"` - Description string `json:"description"` - Id string `json:"id"` - Name string `json:"name"` - Workflow string `json:"workflow"` - Start string `json:"start"` - Environment string `json:"environment"` - } - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Body data error: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - //log.Printf("Data: %s", string(body)) - - ctx := context.Background() - var requestdata requestData - err = yaml.Unmarshal([]byte(body), &requestdata) - if err != nil { - log.Printf("Failed unmarshaling inputdata: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - log.Printf("%#v", requestdata) - - // CBA making a real thing. Already had some code lol - newId := requestdata.Id - if len(newId) != 36 { - log.Printf("Bad ID") - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Invalid ID"}`)) - return - } - - if requestdata.Id == "" || requestdata.Name == "" { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Requires fields id and name can't be empty"}`)) - return - - } - - validTypes := []string{ - "webhook", - } - - isTypeValid := false - for _, thistype := range validTypes { - if requestdata.Type == thistype { - isTypeValid = true - break - } - } - - if !(isTypeValid) { - log.Printf("Type %s is not valid. Try any of these: %s", requestdata.Type, strings.Join(validTypes, ", ")) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // Let remote endpoint handle access checks (shuffler.io) - currentUrl := fmt.Sprintf("https://shuffler.io/api/v1/hooks/webhook_%s", newId) - startNode := requestdata.Start - if requestdata.Environment == "cloud" { - // https://shuffler.io/v1/hooks/webhook_80184973-3e82-4852-842e-0290f7f34d7c - log.Printf("[INFO] Should START a cloud webhook for url %s for startnode %s", currentUrl, startNode) - org, err := getOrg(ctx, user.ActiveOrg.Id) - if err != nil { - log.Printf("Failed finding org %s: %s", org.Id, err) - return - } - - action := CloudSyncJob{ - Type: "webhook", - Action: "start", - OrgId: org.Id, - PrimaryItemId: newId, - SecondaryItem: startNode, - ThirdItem: requestdata.Workflow, - } - - err = executeCloudAction(action, org.SyncConfig.Apikey) - if err != nil { - log.Printf("Failed cloud action START execution: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } else { - log.Printf("[INFO] Successfully set up cloud action schedule") - } - } - - hook := Hook{ - Id: newId, - Start: startNode, - Workflows: []string{requestdata.Workflow}, - Info: Info{ - Name: requestdata.Name, - Description: requestdata.Description, - Url: fmt.Sprintf("https://shuffler.io/api/v1/hooks/webhook_%s", newId), - }, - Type: "webhook", - Owner: user.Username, - Status: "uninitialized", - Actions: []HookAction{ - HookAction{ - Type: "workflow", - Name: requestdata.Name, - Id: requestdata.Workflow, - Field: "", - }, - }, - Running: false, - OrgId: user.ActiveOrg.Id, - Environment: requestdata.Environment, - } - - hook.Status = "running" - hook.Running = true - err = setHook(ctx, hook) - if err != nil { - log.Printf("Failed setting hook: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - err = increaseStatisticsField(ctx, "total_workflow_triggers", requestdata.Workflow, 1, user.ActiveOrg.Id) - if err != nil { - log.Printf("[INFO] Failed to increase total workflows: %s", err) - } - - log.Printf("Set up a new hook with ID %s and environment %s", newId, hook.Environment) - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) -} - -func sendHookResult(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - _ = user - - location := strings.Split(request.URL.String(), "/") - - var workflowId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - workflowId = location[4] - } - - if len(workflowId) != 32 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) - return - } - - ctx := context.Background() - hook, err := getHook(ctx, workflowId) - if err != nil { - log.Printf("Failed getting hook %s (send): %s", workflowId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Body data error: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Printf("SET the hook results for %s to %s", workflowId, body) - // FIXME - set the hook result in the DB somehow as interface{} - // FIXME - should the hook do the transform? Hmm - - b, err := json.Marshal(hook) - if err != nil { - log.Printf("Failed marshalling: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(b)) - return -} - -func handleGetHook(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - location := strings.Split(request.URL.String(), "/") - - var workflowId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - workflowId = location[4] - } - - if len(workflowId) != 36 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) - return - } - - ctx := context.Background() - hook, err := getHook(ctx, workflowId) - if err != nil { - log.Printf("Failed getting hook %s (get hook): %s", workflowId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if user.Id != hook.Owner && user.Role != "admin" && user.Role != "scheduler" { - log.Printf("Wrong user (%s) for hook %s", user.Username, hook.Id) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - b, err := json.Marshal(hook) - if err != nil { - log.Printf("Failed marshalling: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // FIXME - get some real data? - resp.WriteHeader(200) - resp.Write([]byte(b)) - return -} - func getSpecificSchedule(resp http.ResponseWriter, request *http.Request) { if request.Method != "GET" { setSpecificSchedule(resp, request) @@ -3950,7 +2661,7 @@ func getSpecificSchedule(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - schedule, err := getSchedule(ctx, workflowId) + schedule, err := shuffle.GetSchedule(ctx, workflowId) if err != nil { log.Printf("Failed getting schedule: %s", err) resp.WriteHeader(401) @@ -4017,7 +2728,7 @@ func executeSchedule(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() log.Printf("[INFO] EXECUTING %s!", workflowId) - idConfig, err := getSchedule(ctx, workflowId) + idConfig, err := shuffle.GetSchedule(ctx, workflowId) if err != nil { log.Printf("Error getting schedule: %s", err) resp.WriteHeader(401) @@ -4128,7 +2839,7 @@ func uploadWorkflowResult(resp http.ResponseWriter, request *http.Request) { // FIXME - validate ID as well ctx := context.Background() - schedule, err := getSchedule(ctx, workflowId) + schedule, err := shuffle.GetSchedule(ctx, workflowId) if err != nil { log.Printf("Failed setting schedule %s: %s", workflowId, err) resp.WriteHeader(401) @@ -4560,9 +3271,9 @@ func handleGetallHooks(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) + log.Printf("[WARNING] Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -4630,9 +3341,9 @@ func findAvailablePorts(startRange int64, endRange int64) string { } func handleSendalert(resp http.ResponseWriter, request *http.Request) { - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { - log.Printf("Api authentication failed in sendalert: %s", err) + log.Printf("[WARNING] Api authentication failed in sendalert: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -4961,405 +3672,13 @@ func getDocs(resp http.ResponseWriter, request *http.Request) { resp.Write(b) } -type OutlookProfile struct { - OdataContext string `json:"@odata.context"` - BusinessPhones []string `json:"businessPhones"` - DisplayName string `json:"displayName"` - GivenName string `json:"givenName"` - JobTitle interface{} `json:"jobTitle"` - Mail string `json:"mail"` - MobilePhone interface{} `json:"mobilePhone"` - OfficeLocation interface{} `json:"officeLocation"` - PreferredLanguage interface{} `json:"preferredLanguage"` - Surname string `json:"surname"` - UserPrincipalName string `json:"userPrincipalName"` - ID string `json:"id"` -} - -type OutlookFolder struct { - ID string `json:"id"` - DisplayName string `json:"displayName"` - ParentFolderID string `json:"parentFolderId"` - ChildFolderCount int `json:"childFolderCount"` - UnreadItemCount int `json:"unreadItemCount"` - TotalItemCount int `json:"totalItemCount"` -} - -type OutlookFolders struct { - OdataContext string `json:"@odata.context"` - OdataNextLink string `json:"@odata.nextLink"` - Value []OutlookFolder `json:"value"` -} - -func getOutlookFolders(client *http.Client) (OutlookFolders, error) { - requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/frikky@shuffletest.onmicrosoft.com/mailfolders") - - ret, err := client.Get(requestUrl) - if err != nil { - log.Printf("FolderErr: %s", err) - return OutlookFolders{}, err - } - - if ret.StatusCode != 200 { - log.Printf("Status folders: %d", ret.StatusCode) - return OutlookFolders{}, err - } - - body, err := ioutil.ReadAll(ret.Body) - if err != nil { - log.Printf("Body: %s", err) - return OutlookFolders{}, err - } - - //log.Printf("Body: %s", string(body)) - - mailfolders := OutlookFolders{} - err = json.Unmarshal(body, &mailfolders) - if err != nil { - log.Printf("Unmarshal: %s", err) - return OutlookFolders{}, err - } - - //fmt.Printf("%#v", mailfolders) - // FIXME - recursion for subfolders - // Recursive struct - // folderEndpoint := fmt.Sprintf("%s/%s/childfolders?$top=40", requestUrl, parentId) - //for _, folder := range mailfolders.Value { - // log.Println(folder.DisplayName) - //} - - return mailfolders, nil -} - -func getOutlookProfile(client *http.Client) (OutlookProfile, error) { - requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/me?$select=mail") - - ret, err := client.Get(requestUrl) - if err != nil { - log.Printf("FolderErr: %s", err) - return OutlookProfile{}, err - } - - log.Printf("Status folders: %d", ret.StatusCode) - body, err := ioutil.ReadAll(ret.Body) - if err != nil { - log.Printf("Body: %s", err) - return OutlookProfile{}, err - } - - profile := OutlookProfile{} - err = json.Unmarshal(body, &profile) - if err != nil { - log.Printf("Unmarshal: %s", err) - return OutlookProfile{}, err - } - - return profile, nil -} - -func handleNewOutlookRegister(resp http.ResponseWriter, request *http.Request) { - code := request.URL.Query().Get("code") - if len(code) == 0 { - log.Println("No code") - resp.WriteHeader(401) - return - } - - url := fmt.Sprintf("http://%s%s", request.Host, request.URL.EscapedPath()) - log.Println(url) - ctx := context.Background() - client, accessToken, err := getOutlookClient(ctx, code, OauthToken{}, url) - if err != nil { - log.Printf("Oauth client failure - outlook register: %s", err) - resp.WriteHeader(401) - return - } - // This should be possible, and will also give the actual username - profile, err := getOutlookProfile(client) - if err != nil { - log.Printf("Outlook profile failure: %s", err) - resp.WriteHeader(401) - return - } - - // This is a state workaround, which should really be for CSRF checks lol - state := request.URL.Query().Get("state") - if len(state) == 0 { - log.Println("No state") - resp.WriteHeader(401) - return - } - - stateitems := strings.Split(state, "%26") - if len(stateitems) == 1 { - stateitems = strings.Split(state, "&") - } - - // FIXME - trigger auth - senderUser := "" - trigger := TriggerAuth{} - for _, item := range stateitems { - itemsplit := strings.Split(item, "%3D") - if len(itemsplit) == 1 { - itemsplit = strings.Split(item, "=") - } - - if len(itemsplit) != 2 { - continue - } - - // Do something here - if itemsplit[0] == "workflow_id" { - trigger.WorkflowId = itemsplit[1] - } else if itemsplit[0] == "trigger_id" { - trigger.Id = itemsplit[1] - } else if itemsplit[0] == "type" { - trigger.Type = itemsplit[1] - } else if itemsplit[0] == "username" { - trigger.Username = itemsplit[1] - trigger.Owner = itemsplit[1] - senderUser = itemsplit[1] - } - } - - // THis is an override based on the user in oauth return - trigger.Username = profile.Mail - trigger.Code = code - trigger.OauthToken = OauthToken{ - AccessToken: accessToken.AccessToken, - TokenType: accessToken.TokenType, - RefreshToken: accessToken.RefreshToken, - Expiry: accessToken.Expiry, - } - - //log.Printf("%#v", trigger) - if trigger.WorkflowId == "" || trigger.Id == "" || trigger.Username == "" || trigger.Type == "" { - log.Printf("All oauth items need to contain data to register a new state") - resp.WriteHeader(401) - return - } - - // Should also update the user - Userdata, err := getUser(ctx, senderUser) - if err != nil { - log.Printf("Username %s doesn't exist (oauth2): %s", trigger.Username, err) - resp.WriteHeader(401) - return - } - - Userdata.Authentication = append(Userdata.Authentication, UserAuth{ - Name: "Outlook", - Description: "oauth2", - Workflows: []string{trigger.WorkflowId}, - Username: trigger.Username, - Fields: []UserAuthField{ - UserAuthField{ - Key: "trigger_id", - Value: trigger.Id, - }, - UserAuthField{ - Key: "username", - Value: trigger.Username, - }, - UserAuthField{ - Key: "code", - Value: code, - }, - UserAuthField{ - Key: "type", - Value: trigger.Type, - }, - }, - }) - - // Set apikey for the user if they don't have one - if len(Userdata.ApiKey) == 0 { - newUser, err := generateApikey(ctx, *Userdata) - Userdata = &newUser - if err != nil { - log.Printf("Failed to generate apikey for user %s when creating outlook sub: %s", Userdata.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": ""}`)) - return - } - } - - //err = setUser(Userdata) - //if err != nil { - // log.Printf("Failed setting user data for %s: %s", Userdata.Username, err) - // resp.WriteHeader(401) - // return - //} - - err = setTriggerAuth(ctx, trigger) - if err != nil { - log.Printf("Failed to set trigger auth for %s - %s", trigger.Username, err) - resp.WriteHeader(401) - return - } - - // FIXME - not sure if these are good at all :) - environmentVariables := map[string]string{ - "FUNCTION_APIKEY": Userdata.ApiKey, - "CALLBACKURL": "https://shuffler.io", - "WORKFLOW_ID": trigger.WorkflowId, - "TRIGGER_ID": trigger.Id, - } - - applocation := fmt.Sprintf("gs://%s/triggers/outlooktrigger.zip", bucketName) - hookname := fmt.Sprintf("outlooktrigger_%s", trigger.Id) - - err = deployCloudFunctionGo(ctx, hookname, defaultLocation, applocation, environmentVariables) - if err != nil { - log.Printf("Error deploying hook: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Issue with starting hook. Please wait a second and try again"}`))) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) -} - -type OauthToken struct { - AccessToken string `json:"AccessToken" datastore:"AccessToken,noindex"` - TokenType string `json:"TokenType" datastore:"TokenType,noindex"` - RefreshToken string `json:"RefreshToken" datastore:"RefreshToken,noindex"` - Expiry time.Time `json:"Expiry" datastore:"Expiry,noindex"` -} -type TriggerAuth struct { - Id string `json:"id" datastore:"id"` - SubscriptionId string `json:"subscriptionId" datastore:"subscriptionId"` - - Username string `json:"username" datastore:"username,noindex"` - WorkflowId string `json:"workflow_id" datastore:"workflow_id,noindex"` - Owner string `json:"owner" datastore:"owner"` - Type string `json:"type" datastore:"type"` - Code string `json:"code,omitempty" datastore:"code,noindex"` - OauthToken OauthToken `json:"oauth_token,omitempty" datastore:"oauth_token"` -} - -func getTriggerAuth(ctx context.Context, id string) (*TriggerAuth, error) { - key := datastore.NameKey("trigger_auth", strings.ToLower(id), nil) - triggerauth := &TriggerAuth{} - if err := dbclient.Get(ctx, key, triggerauth); err != nil { - return &TriggerAuth{}, err - } - - return triggerauth, nil -} - -func setTriggerAuth(ctx context.Context, trigger TriggerAuth) error { - key1 := datastore.NameKey("trigger_auth", strings.ToLower(trigger.Id), nil) - - // New struct, to not add body, author etc - if _, err := dbclient.Put(ctx, key1, &trigger); err != nil { - log.Printf("Error adding trigger auth: %s", err) - return err - } - - return nil -} - -// THis all of a sudden became really horrible.. fml -func getOutlookClient(ctx context.Context, code string, accessToken OauthToken, redirectUri string) (*http.Client, *oauth2.Token, error) { - - conf := &oauth2.Config{ - ClientID: "", - ClientSecret: "", - Scopes: []string{ - "Mail.Read", - "User.Read", - }, - RedirectURL: redirectUri, - Endpoint: oauth2.Endpoint{ - AuthURL: "https://login.microsoftonline.com/common/oauth2/authorize", - TokenURL: "https://login.microsoftonline.com/common/oauth2/token", - }, - } - - if len(code) > 0 { - access_token, err := conf.Exchange(ctx, code) - if err != nil { - log.Printf("Access_token issue: %s", err) - return &http.Client{}, access_token, err - } - - client := conf.Client(ctx, access_token) - return client, access_token, nil - } else { - // Manually recreate the oauthtoken - access_token := &oauth2.Token{ - AccessToken: accessToken.AccessToken, - TokenType: accessToken.TokenType, - RefreshToken: accessToken.RefreshToken, - Expiry: accessToken.Expiry, - } - - client := conf.Client(ctx, access_token) - return client, access_token, nil - } -} - -func handleGetOutlookFolders(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - // Exchange every time hmm - // FIXME - // Should really just get the code from the trigger that's being used OR the user - triggerId := request.URL.Query().Get("trigger_id") - if len(triggerId) == 0 { - log.Println("No trigger_id supplied") - resp.WriteHeader(401) - return - } - - ctx := context.Background() - trigger, err := getTriggerAuth(ctx, triggerId) - if err != nil { - log.Printf("Trigger %s doesn't exist - outlook folders.", triggerId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Trigger doesn't exist."}`)) - return - } - - // FIXME - should be shuffler in literally every case except testing lol - redirectDomain := "shuffler.io" - url := fmt.Sprintf("https://%s/functions/outlook/register", redirectDomain) - outlookClient, _, err := getOutlookClient(ctx, "", trigger.OauthToken, url) - if err != nil { - log.Printf("Oauth client failure - outlook folders: %s", err) - resp.WriteHeader(401) - return - } - - folders, err := getOutlookFolders(outlookClient) - if err != nil { - resp.WriteHeader(401) - return - } - - b, err := json.Marshal(folders.Value) - if err != nil { - log.Println("Failed to marshal folderdata") - resp.WriteHeader(401) - return - } - - resp.WriteHeader(200) - resp.Write(b) -} - func handleGetSpecificStats(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in getting specific workflow: %s", err) resp.WriteHeader(401) @@ -5402,473 +3721,6 @@ func handleGetSpecificStats(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(b)) } -func handleGetSpecificTrigger(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in getting specific workflow: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - location := strings.Split(request.URL.String(), "/") - - var workflowId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - workflowId = location[4] - } - - if strings.Contains(workflowId, "?") { - workflowId = strings.Split(workflowId, "?")[0] - } - - ctx := context.Background() - trigger, err := getTriggerAuth(ctx, workflowId) - if err != nil { - log.Printf("Trigger %s doesn't exist - specific trigger.", workflowId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": ""}`)) - return - } - - if user.Username != trigger.Owner && user.Role != "admin" { - log.Printf("Wrong user (%s) for trigger %s", user.Username, trigger.Id) - resp.WriteHeader(401) - return - } - - trigger.OauthToken = OauthToken{} - trigger.Code = "" - - b, err := json.Marshal(trigger) - if err != nil { - log.Println("Failed to marshal data") - resp.WriteHeader(401) - return - } - - resp.WriteHeader(200) - resp.Write(b) -} - -func handleDeleteOutlookSub(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - location := strings.Split(request.URL.String(), "/") - - var workflowId string - var triggerId string - if location[1] == "api" { - if len(location) <= 6 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - workflowId = location[4] - triggerId = location[6] - } - - if len(workflowId) == 0 || len(triggerId) == 0 { - log.Printf("Ids can't be zero when deleting %s", workflowId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - ctx := context.Background() - workflow, err := getWorkflow(ctx, workflowId) - if err != nil { - log.Printf("Failed getting the workflow locally (delete outlook): %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in outlook deploy: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // FIXME - have a check for org etc too.. - if user.Id != workflow.Owner && user.Role != "admin" { - log.Printf("Wrong user (%s) for workflow %s when deploying outlook", user.Username, workflow.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // Check what kind of sub it is - err = handleOutlookSubRemoval(ctx, workflowId, triggerId) - if err != nil { - log.Printf("Failed sub removal: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) -} - -func removeOutlookSubscription(outlookClient *http.Client, subscriptionId string) error { - // DELETE https://graph.microsoft.com/v1.0/subscriptions/{id} - fullUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/subscriptions/%s", subscriptionId) - req, err := http.NewRequest( - "DELETE", - fullUrl, - nil, - ) - req.Header.Add("Content-Type", "application/json") - res, err := outlookClient.Do(req) - if err != nil { - log.Printf("Client: %s", err) - return err - } - - if res.StatusCode != 200 && res.StatusCode != 201 && res.StatusCode != 204 { - return errors.New(fmt.Sprintf("Bad status code when deleting subscription: %d", res.StatusCode)) - } - - body, err := ioutil.ReadAll(res.Body) - if err != nil { - log.Printf("Body: %s", err) - return err - } - - _ = body - - return nil -} - -// Remove AUTH -// Remove function -// Remove subscription -func handleOutlookSubRemoval(ctx context.Context, workflowId, triggerId string) error { - // 1. Get the auth for trigger - // 2. Stop the subscription - // 3. Remove the function - // 4. Remove the database entry for auth - trigger, err := getTriggerAuth(ctx, triggerId) - if err != nil { - log.Printf("Trigger auth %s doesn't exist - outlook sub removal.", triggerId) - return err - } - - url := fmt.Sprintf("https://shuffler.io") - outlookClient, _, err := getOutlookClient(ctx, "", trigger.OauthToken, url) - if err != nil { - log.Printf("Oauth client failure - triggerauth sub removal: %s", err) - return err - } - - notificationURL := fmt.Sprintf("https://%s-%s.cloudfunctions.net/outlooktrigger_%s", defaultLocation, gceProject, trigger.Id) - curSubscriptions, err := getOutlookSubscriptions(outlookClient) - if err == nil { - for _, sub := range curSubscriptions.Value { - if sub.NotificationURL == notificationURL { - log.Printf("Removing existing subscription %s", sub.Id) - removeOutlookSubscription(outlookClient, sub.Id) - } - } - } else { - log.Printf("Failed to get subscriptions - need to overwrite") - } - - // FIXME - not removing the function, as the trigger still exists - //err = removeOutlookTriggerFunction(triggerId) - //if err != nil { - // return err - //} - - return nil -} - -// This sets up the sub with outlook itself -// Parses data from the workflow to see whether access is right to subscribe it -// Creates the cloud function for outlook return -// Wait for it to be available, then schedule a workflow to it -func createOutlookSub(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - location := strings.Split(request.URL.String(), "/") - - var workflowId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - workflowId = location[4] - } - - ctx := context.Background() - workflow, err := getWorkflow(ctx, workflowId) - if err != nil { - log.Printf("Failed getting the workflow locally (outlook sub): %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in outlook deploy: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // FIXME - have a check for org etc too.. - if user.Id != workflow.Owner && user.Role != "admin" { - log.Printf("Wrong user (%s) for workflow %s when deploying outlook", user.Username, workflow.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Println("Handle outlook subscription for trigger") - - // Should already be authorized at this point, as the workflow is shared - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Failed body read for workflow %s", workflow.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Println(string(body)) - - // Based on the input data from frontend - type CurTrigger struct { - Name string `json:"name"` - Folders []string `json:"folders"` - ID string `json:"id"` - } - - var curTrigger CurTrigger - err = json.Unmarshal(body, &curTrigger) - if err != nil { - log.Printf("Failed body read unmarshal for trigger %s", workflow.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if len(curTrigger.Folders) == 0 { - log.Printf("Error for %s. Choosing folders is required, currently 0", workflow.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // Now that it's deployed - wait a few seconds before generating: - // 1. Oauth2 token thingies for outlook.office.com - // 2. Set the url to have the right mailboxes (probably ID?) ("https://outlook.office.com/api/v2.0/me/mailfolders('inbox')/messages") - // 3. Set the callback URL to be the new trigger - // 4. Run subscription test - // 5. Set the subscriptionId to the trigger object - - // First - lets regenerate an oauth token for outlook.office.com from the original items - trigger, err := getTriggerAuth(ctx, curTrigger.ID) - if err != nil { - log.Printf("Trigger %s doesn't exist - outlook sub.", curTrigger.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": ""}`)) - return - } - - // url doesn't really matter here - url := fmt.Sprintf("https://shuffler.io") - outlookClient, _, err := getOutlookClient(ctx, "", trigger.OauthToken, url) - if err != nil { - log.Printf("Oauth client failure - triggerauth: %s", err) - resp.WriteHeader(401) - return - } - - // Location + - notificationURL := fmt.Sprintf("https://%s-%s.cloudfunctions.net/outlooktrigger_%s", defaultLocation, gceProject, curTrigger.ID) - log.Println(notificationURL) - - // This is here simply to let the function start - // Usually takes 10 attempts minimum :O - // 10 * 5 = 50 seconds. That's waaay too much :( - //notificationURL = "https://europe-west1-shuffler.cloudfunctions.net/outlooktrigger_e2ce43b0-997e-4980-9617-6eadbc68cf88" - //notificationURL = "https://de4fc12b.ngrok.io" - - curSubscriptions, err := getOutlookSubscriptions(outlookClient) - if err == nil { - for _, sub := range curSubscriptions.Value { - if sub.NotificationURL == notificationURL { - log.Printf("Removing existing subscription %s", sub.Id) - removeOutlookSubscription(outlookClient, sub.Id) - } - } - } else { - log.Printf("Failed to get subscriptions - need to overwrite") - } - - maxFails := 15 - failCnt := 0 - log.Println(curTrigger.Folders) - for { - subId, err := makeOutlookSubscription(outlookClient, curTrigger.Folders, notificationURL) - if err != nil { - failCnt += 1 - log.Printf("Failed making oauth subscription, retrying in 5 seconds: %s", err) - time.Sleep(5 * time.Second) - if failCnt == maxFails { - log.Printf("Failed to set up subscription %d times.", maxFails) - resp.WriteHeader(401) - return - } - - continue - } - - // Set the ID somewhere here - trigger.SubscriptionId = subId - err = setTriggerAuth(ctx, *trigger) - if err != nil { - log.Printf("Failed setting triggerauth: %s", err) - } - - break - } - - log.Printf("Successfully handled outlook subscription for trigger %s in workflow %s", curTrigger.ID, workflow.ID) - - //log.Printf("%#v", user) - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) -} - -// Lists the users current subscriptions -func getOutlookSubscriptions(outlookClient *http.Client) (SubscriptionsWrapper, error) { - fullUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/subscriptions") - req, err := http.NewRequest( - "GET", - fullUrl, - nil, - ) - req.Header.Add("Content-Type", "application/json") - res, err := outlookClient.Do(req) - if err != nil { - log.Printf("suberror Client: %s", err) - return SubscriptionsWrapper{}, err - } - - body, err := ioutil.ReadAll(res.Body) - if err != nil { - log.Printf("Suberror Body: %s", err) - return SubscriptionsWrapper{}, err - } - - newSubs := SubscriptionsWrapper{} - err = json.Unmarshal(body, &newSubs) - if err != nil { - return SubscriptionsWrapper{}, err - } - - return newSubs, nil -} - -type SubscriptionsWrapper struct { - OdataContext string `json:"@odata.context"` - Value []Subscription `json:"value"` -} - -type Subscription struct { - ChangeType string `json:"changeType"` - NotificationURL string `json:"notificationUrl"` - Resource string `json:"resource"` - ExpirationDateTime string `json:"expirationDateTime"` - ClientState string `json:"clientState"` - Id string `json:"id"` -} - -func makeOutlookSubscription(client *http.Client, folderIds []string, notificationURL string) (string, error) { - fullUrl := "https://graph.microsoft.com/v1.0/subscriptions" - - // FIXME - this expires rofl - t := time.Now().Local().Add(time.Minute * time.Duration(4300)) - timeFormat := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d.0000000Z", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) - log.Println(timeFormat) - - resource := fmt.Sprintf("me/mailfolders('%s')/messages", strings.Join(folderIds, "','")) - log.Println(resource) - sub := Subscription{ - ChangeType: "created", - NotificationURL: notificationURL, - ExpirationDateTime: timeFormat, - ClientState: "This is a test", - Resource: resource, - } - - data, err := json.Marshal(sub) - if err != nil { - log.Printf("Marshal: %s", err) - return "", err - } - - req, err := http.NewRequest( - "POST", - fullUrl, - bytes.NewBuffer(data), - ) - req.Header.Add("Content-Type", "application/json") - - res, err := client.Do(req) - if err != nil { - log.Printf("Client: %s", err) - return "", err - } - - log.Printf("Status: %d", res.StatusCode) - body, err := ioutil.ReadAll(res.Body) - if err != nil { - log.Printf("Body: %s", err) - return "", err - } - - if res.StatusCode != 200 && res.StatusCode != 201 { - return "", errors.New(fmt.Sprintf("Subscription failed: %s", string(body))) - } - - // Use data from body here to create thingy - newSub := Subscription{} - err = json.Unmarshal(body, &newSub) - if err != nil { - return "", err - } - - return newSub.Id, nil -} - func getOpenapi(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -5876,7 +3728,7 @@ func getOpenapi(resp http.ResponseWriter, request *http.Request) { } // Just here to verify that the user is logged in - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in validate swagger: %s", err) resp.WriteHeader(401) @@ -5904,7 +3756,7 @@ func getOpenapi(resp http.ResponseWriter, request *http.Request) { // FIXME - FIX AUTH WITH APP ctx := context.Background() - //_, err = getApp(ctx, id) + //_, err = shuffle.GetApp(ctx, id) //if err == nil { // log.Println("You're supposed to be able to continue now.") //} @@ -5916,7 +3768,7 @@ func getOpenapi(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("[INFO] API LENGTH GET: %d, ID: %s", len(parsedApi.Body), id) + log.Printf("[INFO] API LENGTH GET FOR OPENAPI %s: %d, ID: %s", id, len(parsedApi.Body), id) parsedApi.Success = true data, err := json.Marshal(parsedApi) @@ -5937,7 +3789,7 @@ func echoOpenapiData(resp http.ResponseWriter, request *http.Request) { } // Just here to verify that the user is logged in - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in validate swagger: %s", err) resp.WriteHeader(401) @@ -6124,7 +3976,7 @@ func validateSwagger(resp http.ResponseWriter, request *http.Request) { } // Just here to verify that the user is logged in - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in validate swagger: %s", err) resp.WriteHeader(401) @@ -6317,7 +4169,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { } log.Printf("[INFO] SETTING APP TO LIVE!!!") - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in verify swagger: %s", err) resp.WriteHeader(401) @@ -6354,7 +4206,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { if test.Editing { // Quick verification test ctx := context.Background() - app, err := getApp(ctx, test.Id) + app, err := shuffle.GetApp(ctx, test.Id, user) if err != nil { log.Printf("Error getting app when editing: %s", app.Name) resp.WriteHeader(401) @@ -6399,7 +4251,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { swagger.Info.Title = strings.Replace(swagger.Info.Title, " ", "_", -1) } - basePath, err := buildStructure(swagger, newmd5) + basePath, err := shuffle.BuildStructure(swagger, newmd5) if err != nil { log.Printf("Failed to build base structure: %s", err) resp.WriteHeader(500) @@ -6408,7 +4260,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { } //log.Printf("Should generate yaml") - swagger, api, pythonfunctions, err := generateYaml(swagger, newmd5) + swagger, api, pythonfunctions, err := shuffle.GenerateYaml(swagger, newmd5) if err != nil { log.Printf("Failed building and generating yaml: %s", err) resp.WriteHeader(500) @@ -6418,7 +4270,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { // FIXME: CHECK IF SAME NAME AS NORMAL APP // Can't overwrite existing normal app - workflowApps, err := getAllWorkflowApps(ctx, 500) + workflowApps, err := shuffle.GetPrioritizedApps(ctx, user) if err != nil { log.Printf("Failed getting all workflow apps from database to verify: %s", err) resp.WriteHeader(401) @@ -6438,7 +4290,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { api.Owner = user.Id - err = dumpApi(basePath, api) + err = shuffle.DumpApi(basePath, api) if err != nil { log.Printf("Failed dumping yaml: %s", err) resp.WriteHeader(500) @@ -6449,7 +4301,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, newmd5) classname := strings.Replace(identifier, " ", "", -1) classname = strings.Replace(classname, "-", "", -1) - parsedCode, err := dumpPython(basePath, classname, swagger.Info.Version, pythonfunctions) + parsedCode, err := shuffle.DumpPython(basePath, classname, swagger.Info.Version, pythonfunctions) if err != nil { log.Printf("Failed dumping python: %s", err) resp.WriteHeader(500) @@ -6469,7 +4321,8 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { // 5. Upload as cloud function // 1. Upload the API to datastore - err = deployAppToDatastore(ctx, api) + err = shuffle.DeployAppToDatastore(ctx, api) + //func DeployAppToDatastore(ctx context.Context, workflowapp WorkflowApp, bucketName string) error { if err != nil { log.Printf("Failed adding app to db: %s", err) resp.WriteHeader(500) @@ -6478,7 +4331,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { } // 2. Get all the required code - appbase, staticBaseline, err := getAppbase() + appbase, staticBaseline, err := shuffle.GetAppbase() if err != nil { log.Printf("Failed getting appbase: %s", err) resp.WriteHeader(500) @@ -6487,17 +4340,17 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { } // Have to do some quick checks of the python code (: - _, parsedCode = formatAppfile(parsedCode) + _, parsedCode = shuffle.FormatAppfile(parsedCode) - fixedAppbase := fixAppbase(appbase) - runner := getRunner(classname) + fixedAppbase := shuffle.FixAppbase(appbase) + runner := shuffle.GetRunnerOnprem(classname) // 2. Put it together stitched := string(staticBaseline) + strings.Join(fixedAppbase, "\n") + parsedCode + string(runner) //log.Println(stitched) // 3. Zip and stream it directly in the directory - _, err = streamZipdata(ctx, identifier, stitched, "requests\nurllib3") + _, err = shuffle.StreamZipdata(ctx, identifier, stitched, "requests\nurllib3", "") if err != nil { log.Printf("[ERROR] Zipfile error: %s", err) resp.WriteHeader(500) @@ -6566,7 +4419,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { user.PrivateApps[foundNumber] = api } - err = setUser(ctx, &user) + err = shuffle.SetUser(ctx, &user) if err != nil { log.Printf("[ERROR] Failed adding verification for user %s: %s", user.Username, err) resp.WriteHeader(500) @@ -6580,7 +4433,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { Body: string(body), } - log.Printf("[INFO] API LENGTH: %d, ID: %s", len(parsed.Body), newmd5) + log.Printf("[INFO] API LENGTH FOR %s: %d, ID: %s", api.Name, len(parsed.Body), newmd5) // FIXME: Might cause versioning issues if we re-use the same!! // FIXME: Need a way to track different versions of the same app properly. // Hint: Save API.id somewhere, and use newmd5 to save latest version @@ -6594,20 +4447,23 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { // Backup every single one setOpenApiDatastore(ctx, api.ID, parsed) - err = increaseStatisticsField(ctx, "total_apps_created", newmd5, 1, user.ActiveOrg.Id) - if err != nil { - log.Printf("Failed to increase success execution stats: %s", err) - } + /* + err = increaseStatisticsField(ctx, "total_apps_created", newmd5, 1, user.ActiveOrg.Id) + if err != nil { + log.Printf("Failed to increase success execution stats: %s", err) + } - err = increaseStatisticsField(ctx, "openapi_apps_created", newmd5, 1, user.ActiveOrg.Id) - if err != nil { - log.Printf("Failed to increase success execution stats: %s", err) - } + err = increaseStatisticsField(ctx, "openapi_apps_created", newmd5, 1, user.ActiveOrg.Id) + if err != nil { + log.Printf("Failed to increase success execution stats: %s", err) + } + */ cacheKey := fmt.Sprintf("workflowapps-sorted-100") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) + shuffle.DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id)) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, api.ID))) @@ -6673,7 +4529,8 @@ func createFs(basepath, pathname string) (billy.Filesystem, error) { } // Hotloads new apps from a folder -func handleAppHotload(location string, forceUpdate bool) error { +func handleAppHotload(ctx context.Context, location string, forceUpdate bool) error { + basepath := "base" fs, err := createFs(basepath, location) if err != nil { @@ -6697,32 +4554,21 @@ func handleAppHotload(location string, forceUpdate bool) error { } cacheKey := fmt.Sprintf("workflowapps-sorted") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-100") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") + shuffle.DeleteCache(ctx, cacheKey) + //shuffle.DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id)) return nil } -// Primary = usually an outer ID, e.g. workflow ID -// Secondary = something to specify what inside workflow to execute -// Third = Some data to add to it -type CloudSyncJob struct { - Id string `json:"id" datastore:"id"` - Type string `json:"type" datastore:"type"` - Action string `json:"action" datastore:"action"` - OrgId string `json:"org_id" datastore:"org_id"` - PrimaryItemId string `json:"primary_item_id" datastore:"primary_item_id"` - SecondaryItem string `json:"secondary_item" datastore:"secondary_item"` - ThirdItem string `json:"third_item" datastore:"third_item"` - FourthItem string `json:"fourth_item" datastore:"fourth_item"` - FifthItem string `json:"fifth_item" datastore:"fifth_item"` - Created string `json:"created" datastore:"created"` -} - func handleCloudExecutionOnprem(workflowId, startNode, executionSource, executionArgument string) error { ctx := context.Background() // 1. Get the workflow // 2. Execute it with the data - workflow, err := getWorkflow(ctx, workflowId) + workflow, err := shuffle.GetWorkflow(ctx, workflowId) if err != nil { return err } @@ -6731,15 +4577,15 @@ func handleCloudExecutionOnprem(workflowId, startNode, executionSource, executio _ = workflow parsedArgument := executionArgument - newExec := ExecutionRequest{ + newExec := shuffle.ExecutionRequest{ ExecutionSource: executionSource, ExecutionArgument: parsedArgument, } - var execution ExecutionRequest + var execution shuffle.ExecutionRequest err = json.Unmarshal([]byte(parsedArgument), &execution) if err == nil { - log.Printf("FOUND EXEC %#v", execution) + //log.Printf("[INFO] FOUND EXEC %#v", execution) if len(execution.ExecutionArgument) > 0 { parsedArgument := strings.Replace(string(execution.ExecutionArgument), "\\\"", "\"", -1) log.Printf("New exec argument: %s", execution.ExecutionArgument) @@ -6764,29 +4610,79 @@ func handleCloudExecutionOnprem(workflowId, startNode, executionSource, executio return err } - log.Println(string(b)) + //log.Println(string(b)) newRequest := &http.Request{ URL: &url.URL{}, Method: "POST", Body: ioutil.NopCloser(bytes.NewReader(b)), } - _, _, err = handleExecution(workflowId, Workflow{}, newRequest) + _, _, err = handleExecution(workflowId, shuffle.Workflow{}, newRequest) return err } -func handleCloudJob(job CloudSyncJob) error { +func handleCloudJob(job shuffle.CloudSyncJob) error { // May need authentication in all of these..? - log.Printf("Handle job with type %s and action %s", job.Type, job.Action) - if job.Type == "webhook" { + log.Printf("[INFO] Handle job with type %s and action %s", job.Type, job.Action) + if job.Type == "outlook" { if job.Action == "execute" { - log.Printf("Should handle webhook for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) + // FIXME: Get the email + ctx := context.Background() + maildata := MailData{} + err := json.Unmarshal([]byte(job.ThirdItem), &maildata) + if err != nil { + log.Printf("Maildata unmarshal error: %s", err) + return err + } + + hookId := job.Id + hook, err := getTriggerAuth(ctx, hookId) + if err != nil { + log.Printf("[INFO] Failed getting trigger %s (callback cloud): %s", hookId, err) + return err + } + + redirectDomain := "localhost:5001" + redirectUrl := fmt.Sprintf("http://%s/api/v1/triggers/outlook/register", redirectDomain) + outlookClient, _, err := getOutlookClient(ctx, "", hook.OauthToken, redirectUrl) + if err != nil { + log.Printf("Oauth client failure - triggerauth: %s", err) + return err + } + + emails, err := getOutlookEmail(outlookClient, maildata) + //log.Printf("EMAILS: %d", len(emails)) + //log.Printf("INSIDE GET OUTLOOK EMAIL!: %#v, %s", emails, err) + + //type FullEmail struct { + email := FullEmail{} + if len(emails) == 1 { + email = emails[0] + } + + emailBytes, err := json.Marshal(email) + if err != nil { + log.Printf("[INFO] Failed email marshaling: %s", err) + return err + } + + log.Printf("[INFO] Should handle outlook webhook for workflow %s with start node %s and data of length %d", job.PrimaryItemId, job.SecondaryItem, len(job.ThirdItem)) + err = handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "outlook", string(emailBytes)) + if err != nil { + log.Printf("[WARNING] Failed executing workflow from cloud outlook hook: %s", err) + } else { + log.Printf("[INFO] Successfully executed workflow from cloud outlook hook!") + } + } + } else if job.Type == "webhook" { + if job.Action == "execute" { + log.Printf("[INFO] Should handle normal webhook for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) err := handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "webhook", job.ThirdItem) if err != nil { - log.Printf("Failed executing workflow from cloud hook: %s", err) + log.Printf("[INFO] Failed executing workflow from cloud hook: %s", err) } else { - log.Printf("Successfully executed workflow from cloud hook!") + log.Printf("[INFO] Successfully executed workflow from cloud hook!") } } @@ -6795,9 +4691,9 @@ func handleCloudJob(job CloudSyncJob) error { log.Printf("Should handle schedule for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) err := handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "schedule", job.ThirdItem) if err != nil { - log.Printf("Failed executing workflow from cloud schedule: %s", err) + log.Printf("[INFO] Failed executing workflow from cloud schedule: %s", err) } else { - log.Printf("Successfully executed workflow from cloud schedule") + log.Printf("[INFO] Successfully executed workflow from cloud schedule") } } } else if job.Type == "email_trigger" { @@ -6816,7 +4712,7 @@ func handleCloudJob(job CloudSyncJob) error { log.Printf("Should handle user_input CONTINUE for workflow %s with start node %s and execution ID %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) // FIXME: Handle authorization ctx := context.Background() - workflowExecution, err := getWorkflowExecution(ctx, job.ThirdItem) + workflowExecution, err := shuffle.GetWorkflowExecution(ctx, job.ThirdItem) if err != nil { return err } @@ -6826,12 +4722,12 @@ func handleCloudJob(job CloudSyncJob) error { } workflowExecution.Status = "EXECUTING" - err = setWorkflowExecution(ctx, *workflowExecution, true) + err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) if err != nil { return err } - fullUrl := fmt.Sprintf("https://shuffler.io/api/v1/workflows/%s/execute?authorization=%s&start=%s&reference_execution=%s&answer=true", job.PrimaryItemId, job.FourthItem, job.SecondaryItem, job.ThirdItem) + fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute?authorization=%s&start=%s&reference_execution=%s&answer=true", syncUrl, job.PrimaryItemId, job.FourthItem, job.SecondaryItem, job.ThirdItem) newRequest, err := http.NewRequest( "GET", fullUrl, @@ -6842,7 +4738,7 @@ func handleCloudJob(job CloudSyncJob) error { return err } - _, _, err = handleExecution(job.PrimaryItemId, Workflow{}, newRequest) + _, _, err = handleExecution(job.PrimaryItemId, shuffle.Workflow{}, newRequest) if err != nil { log.Printf("Failed continuing workflow from cloud user_input: %s", err) return err @@ -6852,7 +4748,7 @@ func handleCloudJob(job CloudSyncJob) error { } else if job.Action == "stop" { log.Printf("Should handle user_input STOP for workflow %s with start node %s and execution ID %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) ctx := context.Background() - workflowExecution, err := getWorkflowExecution(ctx, job.ThirdItem) + workflowExecution, err := shuffle.GetWorkflowExecution(ctx, job.ThirdItem) if err != nil { return err } @@ -6867,7 +4763,7 @@ func handleCloudJob(job CloudSyncJob) error { } */ - newResults := []ActionResult{} + newResults := []shuffle.ActionResult{} for _, result := range workflowExecution.Results { if result.Action.AppName == "User Input" && result.Result == "Waiting for user feedback based on configuration" { result.Status = "ABORTED" @@ -6879,7 +4775,7 @@ func handleCloudJob(job CloudSyncJob) error { workflowExecution.Results = newResults workflowExecution.Status = "ABORTED" - err = setWorkflowExecution(ctx, *workflowExecution, true) + err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) if err != nil { return err } @@ -6894,11 +4790,11 @@ func handleCloudJob(job CloudSyncJob) error { } // Handles jobs from remote (cloud) -func remoteOrgJobController(org Org, body []byte) error { +func remoteOrgJobController(org shuffle.Org, body []byte) error { type retStruct struct { - Success bool `json:"success"` - Reason string `json:"reason"` - Jobs []CloudSyncJob `json:"jobs"` + Success bool `json:"success"` + Reason string `json:"reason"` + Jobs []shuffle.CloudSyncJob `json:"jobs"` } responseData := retStruct{} @@ -6909,9 +4805,9 @@ func remoteOrgJobController(org Org, body []byte) error { ctx := context.Background() if !responseData.Success { - log.Printf("Should stop org job controller because no success?") + log.Printf("[WARNING] Should stop org job controller because no success?") - if strings.Contains(responseData.Reason, "Bad apikey") || strings.Contains(responseData.Reason, "Error getting the organization") { + if strings.Contains(responseData.Reason, "Bad apikey") || strings.Contains(responseData.Reason, "Error getting the organization") || strings.Contains(responseData.Reason, "Organization isn't syncing") { log.Printf("[WARNING] Remote error; Bad apikey or org error. Stopping sync for org: %s", responseData.Reason) if value, exists := scheduledOrgs[org.Id]; exists { @@ -6919,7 +4815,7 @@ func remoteOrgJobController(org Org, body []byte) error { log.Printf("[WARNING] STOPPING ORG SCHEDULE for: %s", org.Id) value.Lock() - org, err := getOrg(ctx, org.Id) + org, err := shuffle.GetOrg(ctx, org.Id) if err != nil { log.Printf("[WARNING] Failed finding org %s: %s", org.Id, err) return err @@ -6928,7 +4824,20 @@ func remoteOrgJobController(org Org, body []byte) error { org.SyncConfig.Interval = 0 org.SyncConfig.Apikey = "" org.CloudSync = false - err = setOrg(ctx, *org, org.Id) + + // Just in case + org, err = handleStopCloudSync(syncUrl, *org) + + startDate := time.Now().Unix() + org.SyncFeatures.Webhook = shuffle.SyncData{Active: false, Type: "trigger", Name: "Webhook", StartDate: startDate} + org.SyncFeatures.UserInput = shuffle.SyncData{Active: false, Type: "trigger", Name: "User Input", StartDate: startDate} + org.SyncFeatures.EmailTrigger = shuffle.SyncData{Active: false, Type: "action", Name: "Email Trigger", StartDate: startDate} + org.SyncFeatures.Schedules = shuffle.SyncData{Active: false, Type: "trigger", Name: "Schedule", StartDate: startDate, Limit: 0} + org.SyncFeatures.SendMail = shuffle.SyncData{Active: false, Type: "action", Name: "Send Email", StartDate: startDate, Limit: 0} + org.SyncFeatures.SendSms = shuffle.SyncData{Active: false, Type: "action", Name: "Send SMS", StartDate: startDate, Limit: 0} + org.CloudSyncActive = false + + err = shuffle.SetOrg(ctx, *org, org.Id) if err != nil { log.Printf("[WARNING] Failed setting organization when stopping sync: %s", err) } else { @@ -6945,7 +4854,7 @@ func remoteOrgJobController(org Org, body []byte) error { } if len(responseData.Jobs) > 0 { - log.Printf("Remote JOB ret: %s", string(body)) + //log.Printf("[INFO] Remote JOB ret: %s", string(body)) log.Printf("Got job with reason %s and %d job(s)", responseData.Reason, len(responseData.Jobs)) } @@ -6959,7 +4868,7 @@ func remoteOrgJobController(org Org, body []byte) error { return nil } -func remoteOrgJobHandler(org Org, interval int) error { +func remoteOrgJobHandler(org shuffle.Org, interval int) error { client := &http.Client{} syncUrl := fmt.Sprintf("%s/api/v1/cloud/sync", syncUrl) req, err := http.NewRequest( @@ -6981,8 +4890,7 @@ func remoteOrgJobHandler(org Org, interval int) error { return err } - //log.Printf("Data: %s", respBody) - + //log.Printf("Remote Data: %s", respBody) err = remoteOrgJobController(org, respBody) if err != nil { log.Printf("[ERROR] Failed job controller run for %s: %s", respBody, err) @@ -7010,7 +4918,7 @@ func runInit(ctx context.Context) { log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy) } - requestCache = cache.New(5*time.Minute, 10*time.Minute) + //requestCache = cache.New(5*time.Minute, 10*time.Minute) /* proxyUrl, err := url.Parse(httpProxy) @@ -7025,7 +4933,7 @@ func runInit(ctx context.Context) { }, // 15 second timeout - Timeout: 15 * time.Second, + Timeout: 15 * 15time.Second, // don't follow redirect CheckRedirect: func(req *http.Request, via []*http.Request) error { @@ -7047,7 +4955,7 @@ func runInit(ctx context.Context) { setUsers := false orgQuery := datastore.NewQuery("Organizations") - var activeOrgs []Org + var activeOrgs []shuffle.Org _, err = dbclient.GetAll(ctx, orgQuery, &activeOrgs) if err != nil { log.Printf("Error getting organizations!") @@ -7062,16 +4970,16 @@ func runInit(ctx context.Context) { log.Printf(`No orgs. Setting org "default"`) orgSetupName := "default" orgId := uuid.NewV4().String() - newOrg := Org{ + newOrg := shuffle.Org{ Name: orgSetupName, Id: orgId, Org: orgSetupName, - Users: []User{}, + Users: []shuffle.User{}, Roles: []string{"admin", "user"}, CloudSync: false, } - err = setOrg(ctx, newOrg, orgId) + err = shuffle.SetOrg(ctx, newOrg, orgId) if err != nil { log.Printf("Failed setting organization: %s", err) } else { @@ -7088,15 +4996,15 @@ func runInit(ctx context.Context) { activeOrg := activeOrgs[0] q := datastore.NewQuery("Users") - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err == nil { setOrgBool := false for _, user := range users { - newUser := User{ + newUser := shuffle.User{ Username: user.Username, Id: user.Id, - ActiveOrg: Org{ + ActiveOrg: shuffle.Org{ Id: activeOrg.Id, }, Orgs: []string{activeOrg.Id}, @@ -7118,7 +5026,7 @@ func runInit(ctx context.Context) { } if setOrgBool { - err = setOrg(ctx, activeOrg, activeOrg.Id) + err = shuffle.SetOrg(ctx, activeOrg, activeOrg.Id) if err != nil { log.Printf("Failed setting org %s: %s!", activeOrg.Name, err) } else { @@ -7140,13 +5048,13 @@ func runInit(ctx context.Context) { // Fix active users etc q := datastore.NewQuery("Users").Filter("active =", true) - var activeusers []User + var activeusers []shuffle.User _, err = dbclient.GetAll(ctx, q, &activeusers) if err != nil { log.Printf("Error getting users during init: %s", err) } else { q := datastore.NewQuery("Users") - var users []User + var users []shuffle.User _, err := dbclient.GetAll(ctx, q, &users) if len(activeusers) == 0 && len(users) > 0 { @@ -7166,13 +5074,13 @@ func runInit(ctx context.Context) { if len(user.Orgs) == 0 { defaultName := "default" user.Orgs = []string{defaultName} - user.ActiveOrg = Org{ + user.ActiveOrg = shuffle.Org{ Name: defaultName, Role: "user", } } - err = setUser(ctx, &user) + err = shuffle.SetUser(ctx, &user) if err != nil { log.Printf("Failed to reset user") } else { @@ -7193,7 +5101,7 @@ func runInit(ctx context.Context) { } else { apikey := os.Getenv("SHUFFLE_DEFAULT_APIKEY") - tmpOrg := Org{ + tmpOrg := shuffle.Org{ Name: "default", } err = createNewUser(username, password, "admin", apikey, tmpOrg) @@ -7206,7 +5114,7 @@ func runInit(ctx context.Context) { } else { if len(users) < 5 && len(users) > 0 { for _, user := range users { - log.Printf("Username: %s, role: %s", user.Username, user.Role) + log.Printf("[INFO] Username: %s, role: %s", user.Username, user.Role) } } else { log.Printf("Found %d users.", len(users)) @@ -7216,7 +5124,7 @@ func runInit(ctx context.Context) { for _, user := range users { if user.ActiveOrg.Id == "" && len(user.Username) > 0 { user.ActiveOrg = activeOrgs[0] - err = setUser(ctx, &user) + err = shuffle.SetUser(ctx, &user) if err != nil { log.Printf("Failed updating user %s with org", user.Username) } else { @@ -7233,10 +5141,11 @@ func runInit(ctx context.Context) { count, err := getEnvironmentCount() if count == 0 && err == nil && len(activeOrgs) == 1 { log.Printf("Setting up environment with org %s", activeOrgs[0].Id) - item := Environment{ - Name: "Shuffle", - Type: "onprem", - OrgId: activeOrgs[0].Id, + item := shuffle.Environment{ + Name: "Shuffle", + Type: "onprem", + OrgId: activeOrgs[0].Id, + Default: true, } err = setEnvironment(ctx, &item) @@ -7245,7 +5154,7 @@ func runInit(ctx context.Context) { } } else if len(activeOrgs) == 1 { log.Printf("Setting up all environments with org %s", activeOrgs[0].Id) - var environments []Environment + var environments []shuffle.Environment q := datastore.NewQuery("Environments") _, err = dbclient.GetAll(ctx, q, &environments) if err == nil { @@ -7266,7 +5175,7 @@ func runInit(ctx context.Context) { // Fixing workflows to have real activeorg IDs if len(activeOrgs) == 1 { q := datastore.NewQuery("workflow").Limit(35) - var workflows []Workflow + var workflows []shuffle.Workflow _, err = dbclient.GetAll(ctx, q, &workflows) if err != nil { log.Printf("Error getting workflows in runinit: %s", err) @@ -7285,7 +5194,7 @@ func runInit(ctx context.Context) { } if setLocal { - err = setWorkflow(ctx, workflow, workflow.ID) + err = shuffle.SetWorkflow(ctx, workflow, workflow.ID) if err != nil { log.Printf("Failed setting workflow in init: %s", err) } else { @@ -7334,7 +5243,7 @@ func runInit(ctx context.Context) { } */ - var allworkflowapps []AppAuthenticationStorage + var allworkflowapps []shuffle.AppAuthenticationStorage q = datastore.NewQuery("workflowappauth") _, err = dbclient.GetAll(ctx, q, &allworkflowapps) if err == nil { @@ -7346,7 +5255,7 @@ func runInit(ctx context.Context) { //log.Printf("Should update auth for %#v!", item) item.OrgId = activeOrgs[0].Id - err = setWorkflowAppAuthDatastore(ctx, item, item.Id) + err = shuffle.SetWorkflowAppAuthDatastore(ctx, item, item.Id) if err != nil { log.Printf("Failed adding AUTH to org %s", activeOrgs[0].Id) } @@ -7419,6 +5328,7 @@ func runInit(ctx context.Context) { log.Printf("Failed getting schedules during service init: %s", err) } else { log.Printf("Setting up %d schedule(s)", len(schedules)) + url := &url.URL{} for _, schedule := range schedules { if schedule.Environment == "cloud" { log.Printf("Skipping cloud schedule") @@ -7428,11 +5338,12 @@ func runInit(ctx context.Context) { //log.Printf("Schedule: %#v", schedule) job := func() { request := &http.Request{ + URL: url, Method: "POST", Body: ioutil.NopCloser(strings.NewReader(schedule.WrappedArgument)), } - _, _, err := handleExecution(schedule.WorkflowId, Workflow{}, request) + _, _, err := handleExecution(schedule.WorkflowId, shuffle.Workflow{}, request) if err != nil { log.Printf("Failed to execute %s: %s", schedule.WorkflowId, err) } @@ -7448,20 +5359,27 @@ func runInit(ctx context.Context) { } } + // form force-flag to download workflow apps + forceUpdateEnv := os.Getenv("SHUFFLE_APP_FORCE_UPDATE") + forceUpdate := false + if len(forceUpdateEnv) > 0 && forceUpdateEnv == "true" { + log.Printf("Forcing to rebuild apps") + forceUpdate = true + } + // Getting apps to see if we should initialize a test log.Printf("Getting remote workflow apps") - workflowapps, err := getAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting apps (runInit): %s", err) } else if err == nil && len(workflowapps) > 0 { - //getAllWorkflowApps(ctx context.Context) ([]WorkflowApp, error) { - var allworkflowapps []WorkflowApp + var allworkflowapps []shuffle.WorkflowApp q := datastore.NewQuery("workflowapp") _, err := dbclient.GetAll(ctx, q, &allworkflowapps) if err == nil { for _, workflowapp := range allworkflowapps { if workflowapp.Edited == 0 { - err = setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) + err = shuffle.SetWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err == nil { log.Printf("Updating time for workflowapp %s:%s", workflowapp.Name, workflowapp.AppVersion) } @@ -7513,12 +5431,16 @@ func runInit(ctx context.Context) { //iterateAppGithubFolders(fs, dir, "", "testing") // FIXME: Get all the apps? - iterateAppGithubFolders(fs, dir, "", "", false) + _, _, err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate) + if err != nil { + log.Printf("[WARNING] Error from app load in init: %s", err) + } + //_, _, err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate) // Hotloads locally location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER") if len(location) != 0 { - handleAppHotload(location, false) + handleAppHotload(ctx, location, false) } } @@ -7536,21 +5458,21 @@ func runInit(ctx context.Context) { if err != nil { log.Printf("Failed loading repo %s into memory: %s", apis, err) } else { - log.Printf("Finished git clone. Looking for updates to the repo.") + log.Printf("[INFO] Finished git clone. Looking for updates to the repo.") dir, err := fs.ReadDir("") if err != nil { log.Printf("Failed reading folder: %s", err) } iterateOpenApiGithub(fs, dir, "", "") - log.Printf("Finished downloading extra API samples") + log.Printf("[INFO] Finished downloading extra API samples") } workflowLocation := os.Getenv("SHUFFLE_DOWNLOAD_WORKFLOW_LOCATION") if len(workflowLocation) > 0 { - log.Printf("Downloading WORKFLOWS from %s if no workflows - EXTRA workflows", workflowLocation) + log.Printf("[INFO] Downloading WORKFLOWS from %s if no workflows - EXTRA workflows", workflowLocation) q := datastore.NewQuery("workflow").Limit(35) - var workflows []Workflow + var workflows []shuffle.Workflow _, err = dbclient.GetAll(ctx, q, &workflows) if err != nil { log.Printf("Error getting workflows: %s", err) @@ -7579,11 +5501,11 @@ func runInit(ctx context.Context) { log.Printf("[INFO] Finished INIT") } -func handleVerifyCloudsync(orgId string) (SyncFeatures, error) { +func handleVerifyCloudsync(orgId string) (shuffle.SyncFeatures, error) { ctx := context.Background() - org, err := getOrg(ctx, orgId) + org, err := shuffle.GetOrg(ctx, orgId) if err != nil { - return SyncFeatures{}, err + return shuffle.SyncFeatures{}, err } //r.HandleFunc("/api/v1/getorgs", handleGetOrgs).Methods("GET", "OPTIONS") @@ -7599,26 +5521,26 @@ func handleVerifyCloudsync(orgId string) (SyncFeatures, error) { req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, org.SyncConfig.Apikey)) newresp, err := client.Do(req) if err != nil { - return SyncFeatures{}, err + return shuffle.SyncFeatures{}, err } respBody, err := ioutil.ReadAll(newresp.Body) if err != nil { - return SyncFeatures{}, err + return shuffle.SyncFeatures{}, err } responseData := retStruct{} err = json.Unmarshal(respBody, &responseData) if err != nil { - return SyncFeatures{}, err + return shuffle.SyncFeatures{}, err } if newresp.StatusCode != 200 { - return SyncFeatures{}, errors.New(fmt.Sprintf("Got status code %d when getting org remotely. Expected 200. Contact support.", newresp.StatusCode)) + return shuffle.SyncFeatures{}, errors.New(fmt.Sprintf("Got status code %d when getting org remotely. Expected 200. Contact support.", newresp.StatusCode)) } if !responseData.Success { - return SyncFeatures{}, errors.New(responseData.Reason) + return shuffle.SyncFeatures{}, errors.New(responseData.Reason) } return responseData.SyncFeatures, nil @@ -7626,9 +5548,9 @@ func handleVerifyCloudsync(orgId string) (SyncFeatures, error) { // Actually stops syncing with cloud for an org. // Disables potential schedules, removes environments, breaks workflows etc. -func handleStopCloudSync(syncUrl string, org Org) error { +func handleStopCloudSync(syncUrl string, org shuffle.Org) (*shuffle.Org, error) { if len(org.SyncConfig.Apikey) == 0 { - return errors.New(fmt.Sprintf("Couldn't find any sync key to disable org %s", org.Id)) + return &org, errors.New(fmt.Sprintf("Couldn't find any sync key to disable org %s", org.Id)) } log.Printf("Should run cloud sync disable for org %s with URL %s and sync key %s", org.Id, syncUrl, org.SyncConfig.Apikey) @@ -7643,49 +5565,49 @@ func handleStopCloudSync(syncUrl string, org Org) error { req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, org.SyncConfig.Apikey)) newresp, err := client.Do(req) if err != nil { - return err + return &org, err } respBody, err := ioutil.ReadAll(newresp.Body) if err != nil { - return err + return &org, err } log.Printf("Remote disable ret: %s", string(respBody)) responseData := retStruct{} err = json.Unmarshal(respBody, &responseData) if err != nil { - return err + return &org, err } if newresp.StatusCode != 200 { - return errors.New(fmt.Sprintf("Got status code %d when disabling org remotely. Expected 200. Contact support.", newresp.StatusCode)) + return &org, errors.New(fmt.Sprintf("Got status code %d when disabling org remotely. Expected 200. Contact support.", newresp.StatusCode)) } if !responseData.Success { //log.Printf("Success reason: %s", responseData.Reason) - return errors.New(responseData.Reason) + return &org, errors.New(responseData.Reason) } log.Printf("Everything is success. Should disable org sync for %s", org.Id) ctx := context.Background() org.CloudSync = false - org.SyncFeatures = SyncFeatures{} - org.SyncConfig = SyncConfig{} + org.SyncFeatures = shuffle.SyncFeatures{} + org.SyncConfig = shuffle.SyncConfig{} - err = setOrg(ctx, org, org.Id) + err = shuffle.SetOrg(ctx, org, org.Id) if err != nil { newerror := fmt.Sprintf("ERROR: Failed updating even though there was success: %s", err) log.Printf(newerror) - return errors.New(newerror) + return &org, errors.New(newerror) } - var environments []Environment + var environments []shuffle.Environment q := datastore.NewQuery("Environments").Filter("org_id =", org.Id) _, err = dbclient.GetAll(ctx, q, &environments) if err != nil { - return err + return &org, err } // Don't disable, this will be deleted entirely @@ -7695,9 +5617,9 @@ func handleStopCloudSync(syncUrl string, org Org) error { environment.Archived = true err = setEnvironment(ctx, &environment) if err == nil { - log.Printf("Updated cloud environment %s", environment.Name) + log.Printf("[INFO] Updated cloud environment %s", environment.Name) } else { - log.Printf("Failed to update cloud environment %s", environment.Name) + log.Printf("[INFO] Failed to update cloud environment %s", environment.Name) } } } @@ -7705,133 +5627,12 @@ func handleStopCloudSync(syncUrl string, org Org) error { // FIXME: This doesn't work? if value, exists := scheduledOrgs[org.Id]; exists { // Looks like this does the trick? Hurr - log.Printf("STOPPING ORG SCHEDULE for: %s", org.Id) + log.Printf("[WARNING] STOPPING ORG SCHEDULE for: %s", org.Id) value.Lock() } - return nil -} - -func handleEditOrg(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in cloud setup: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if user.Role != "admin" { - log.Printf("Not admin.") - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Not admin"}`)) - return - } - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`)) - return - } - - type ReturnData struct { - Image string `json:"image" datastore:"image"` - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` - OrgId string `json:"org_id" datastore:"org_id"` - Defaults Defaults `json:"defaults" datastore:"defaults"` - } - - var tmpData ReturnData - err = json.Unmarshal(body, &tmpData) - if err != nil { - log.Printf("Failed unmarshalling test: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - var fileId string - location := strings.Split(request.URL.String(), "/") - if location[1] == "api" { - if len(location) <= 4 { - log.Printf("Path too short: %d", len(location)) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if tmpData.OrgId != user.ActiveOrg.Id || fileId != user.ActiveOrg.Id { - log.Printf("User can't edit the org") - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "No permission to edit this org"}`)) - return - } - - ctx := context.Background() - org, err := getOrg(ctx, tmpData.OrgId) - if err != nil { - log.Printf("Organization doesn't exist: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - admin := false - userFound := false - for _, inneruser := range org.Users { - if inneruser.Id == user.Id { - userFound = true - if inneruser.Role == "admin" { - admin = true - } - - break - } - } - - if !userFound { - log.Printf("User %s doesn't exist in organization for edit %s", user.Id, org.Id) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if !admin { - log.Printf("User %s doesn't have edit rights to %s", user.Id, org.Id) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - org.Image = tmpData.Image - org.Name = tmpData.Name - org.Description = tmpData.Description - org.Defaults = tmpData.Defaults - - //log.Printf("Org: %#v", org) - err = setOrg(ctx, *org, org.Id) - if err != nil { - log.Printf("User %s doesn't have edit rights to %s", user.Id, org.Id) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Printf("SUCCESSFULLY UPDATED ORG") - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Successfully updated org"}`))) - + return &org, nil } // INFO: https://docs.google.com/drawings/d/1JJebpPeEVEbmH_qsAC6zf9Noygp7PytvesrkhE19QrY/edit @@ -7844,7 +5645,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in cloud setup: %s", err) resp.WriteHeader(401) @@ -7867,9 +5668,9 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { } type ReturnData struct { - Apikey string `datastore:"apikey"` - Organization Org `datastore:"organization"` - Disable bool `datastore:"disable"` + Apikey string `datastore:"apikey"` + Organization shuffle.Org `datastore:"organization"` + Disable bool `datastore:"disable"` } var tmpData ReturnData @@ -7882,7 +5683,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - org, err := getOrg(ctx, tmpData.Organization.Id) + org, err := shuffle.GetOrg(ctx, tmpData.Organization.Id) if err != nil { log.Printf("Organization doesn't exist: %s", err) resp.WriteHeader(401) @@ -7891,13 +5692,13 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { } // FIXME: Check if user is admin of this org - log.Printf("Checking org %s", org.Name) + //log.Printf("Checking org %s", org.Name) userFound := false admin := false for _, inneruser := range org.Users { if inneruser.Id == user.Id { userFound = true - log.Printf("Role: %s", inneruser.Role) + //log.Printf("[INFO] Role: %s", inneruser.Role) if inneruser.Role == "admin" { admin = true } @@ -7929,17 +5730,17 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { apiPath := "/api/v1/cloud/sync/setup" if tmpData.Disable { if !org.CloudSync { - log.Printf("Org %s isn't syncing. Can't stop.", org.Id) + log.Printf("[WARNING] Org %s isn't syncing. Can't stop.", org.Id) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Skipped cloud sync setup. Already syncing."}`))) return } - log.Printf("Should disable sync for org %s", org.Id) + log.Printf("[INFO] Should disable sync for org %s", org.Id) apiPath := "/api/v1/cloud/sync/stop" syncPath := fmt.Sprintf("%s%s", syncUrl, apiPath) - err = handleStopCloudSync(syncPath, *org) + _, err = handleStopCloudSync(syncPath, *org) if err != nil { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) @@ -8027,13 +5828,13 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { org.CloudSync = true org.SyncFeatures = responseData.SyncFeatures - org.SyncConfig = SyncConfig{ + org.SyncConfig = shuffle.SyncConfig{ Apikey: responseData.SessionKey, Interval: responseData.IntervalSeconds, } interval := int(responseData.IntervalSeconds) - log.Printf("Starting cloud sync on interval %d", interval) + log.Printf("[INFO] Starting cloud sync on interval %d", interval) job := func() { err := remoteOrgJobHandler(*org, interval) if err != nil { @@ -8045,18 +5846,18 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { if err != nil { log.Printf("[CRITICAL] Failed to schedule org: %s", err) } else { - log.Printf("Started sync on interval %d for org %s", interval, org.Name) + log.Printf("[INFO] Started sync on interval %d for org %s", interval, org.Name) scheduledOrgs[org.Id] = jobret } // FIXME: Add this for every feature if org.SyncFeatures.Workflows.Active { - log.Printf("Should activate cloud workflows for org %s!", org.Id) + log.Printf("[INFO] Should activate cloud workflows for org %s!", org.Id) // 1. Find environment // 2. If cloud env found, enable it (un-archive) // 3. If it doesn't create it - var environments []Environment + var environments []shuffle.Environment q := datastore.NewQuery("Environments").Filter("org_id =", org.Id) _, err = dbclient.GetAll(ctx, q, &environments) if err == nil { @@ -8069,9 +5870,9 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { environment.Archived = false err = setEnvironment(ctx, &environment) if err == nil { - log.Printf("Re-added cloud environment %s", environment.Name) + log.Printf("[INFO] Re-added cloud environment %s", environment.Name) } else { - log.Printf("Failed to re-enable cloud environment %s", environment.Name) + log.Printf("[INFO] Failed to re-enable cloud environment %s", environment.Name) } found = true @@ -8080,8 +5881,8 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { } if !found { - log.Printf("Env for cloud not found. Should add it!") - newEnv := Environment{ + log.Printf("[INFO] Env for cloud not found. Should add it!") + newEnv := shuffle.Environment{ Name: "Cloud", Type: "cloud", Archived: false, @@ -8102,7 +5903,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { } } - err = setOrg(ctx, *org, org.Id) + err = shuffle.SetOrg(ctx, *org, org.Id) if err != nil { log.Printf("ERROR: Failed updating org even though there was success: %s", err) resp.WriteHeader(400) @@ -8112,200 +5913,162 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { if responseData.IntervalSeconds > 0 { // FIXME: - log.Printf("Should set up interval for %d with session key %s for org %s", responseData.IntervalSeconds, responseData.SessionKey, org.Name) + log.Printf("[INFO] Should set up interval for %d with session key %s for org %s", responseData.IntervalSeconds, responseData.SessionKey, org.Name) } resp.WriteHeader(200) resp.Write(respBody) } -//func handleEditOrg(resp http.ResponseWriter, request *http.Request) { -// cors := handleCors(resp, request) -// if cors { -// return -// } -// -// user, err := handleApiAuthentication(resp, request) -// if err != nil { -// log.Printf("Api authentication failed in cloud setup: %s", err) -// resp.WriteHeader(401) -// resp.Write([]byte(`{"success": false}`)) -// return -// } -// -// ctx := context.Background() -// if user.Role != "admin" { -// /* -// log.Printf("User: %s", user.Role) -// dbclient, err := getDatastoreClient(ctx, gceProject) -// if err != nil { -// log.Printf("Err1: %s", err) -// } -// -// user.Role = "admin" -// key := datastore.NameKey("Users", strings.ToLower(user.Username), nil) -// if _, err := dbclient.Put(ctx, key, &user); err != nil { -// log.Printf("Err2: %s", err) -// } -// */ -// -// log.Printf("Not admin, can't edit org.") -// resp.WriteHeader(401) -// resp.Write([]byte(`{"success": false, "reason": "Not admin"}`)) -// return -// } -// -// body, err := ioutil.ReadAll(request.Body) -// if err != nil { -// resp.WriteHeader(401) -// resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`)) -// return -// } -// -// type ReturnData struct { -// Image string `json:"image" datastore:"image"` -// Name string `json:"name" datastore:"name"` -// Description string `json:"description" datastore:"description"` -// OrgId string `json:"org_id" datastore:"org_id"` -// SubscriptionId string `json:"subscription_id" datastore:"subscription_id"` -// Action string `json:"action" datastore:"action"` -// } -// -// var tmpData ReturnData -// err = json.Unmarshal(body, &tmpData) -// if err != nil { -// log.Printf("Failed unmarshalling test: %s", err) -// resp.WriteHeader(401) -// resp.Write([]byte(`{"success": false}`)) -// return -// } -// -// var fileId string -// location := strings.Split(request.URL.String(), "/") -// if location[1] == "api" { -// if len(location) <= 4 { -// log.Printf("Path too short: %d", len(location)) -// resp.WriteHeader(401) -// resp.Write([]byte(`{"success": false}`)) -// return -// } -// -// fileId = location[4] -// } -// -// if tmpData.OrgId != user.ActiveOrg.Id || fileId != user.ActiveOrg.Id { -// log.Printf("User can't edit the org. Not part of ORG: %s vs %s", tmpData.OrgId, user.ActiveOrg.Id) -// resp.WriteHeader(401) -// resp.Write([]byte(`{"success": false, "No permission to edit this org"}`)) -// return -// } -// -// org, err := getOrg(ctx, tmpData.OrgId) -// if err != nil { -// log.Printf("Organization doesn't exist: %s", err) -// resp.WriteHeader(401) -// resp.Write([]byte(`{"success": false}`)) -// return -// } -// -// admin := false -// userFound := false -// for _, inneruser := range org.Users { -// if inneruser.Id == user.Id { -// userFound = true -// if inneruser.Role == "admin" { -// admin = true -// } -// -// break -// } -// } -// -// if !userFound { -// log.Printf("User %s doesn't exist in organization for edit %s", user.Id, org.Id) -// resp.WriteHeader(401) -// resp.Write([]byte(`{"success": false}`)) -// return -// } -// -// if !admin { -// log.Printf("User %s doesn't have edit rights to %s", user.Id, org.Id) -// resp.WriteHeader(401) -// resp.Write([]byte(`{"success": false}`)) -// return -// } -// -// if tmpData.Image != org.Image { -// org.Image = tmpData.Image -// } -// -// if tmpData.Name != org.Name { -// org.Name = tmpData.Name -// } -// -// if tmpData.Description != org.Description { -// org.Description = tmpData.Description -// } -// -// if len(tmpData.SubscriptionId) > 0 { -// log.Printf("Should update subscription %s with action %s if it exists", tmpData.SubscriptionId, tmpData.Action) -// found := false -// foundIndex := 0 -// for index, sub := range org.Subscriptions { -// if tmpData.SubscriptionId == sub.Reference { -// found = true -// foundIndex = index -// } -// } -// -// if !found { -// log.Printf("Couldn't find sub %s in org %s", tmpData.SubscriptionId, tmpData.OrgId) -// resp.WriteHeader(401) -// resp.Write([]byte(`{"success": false}`)) -// return -// } -// -// if tmpData.Action == "cancel" { -// _, err := sub.Cancel(tmpData.SubscriptionId, nil) -// //log.Printf("Ret: %#v", subReturn) -// if err != nil { -// log.Printf("Failed canceling sub %s.", tmpData.SubscriptionId) -// resp.WriteHeader(401) -// resp.Write([]byte(`{"success": false}`)) -// return -// } else { -// log.Printf("Successfully canceled sub %s in org %s", tmpData.SubscriptionId, tmpData.OrgId) -// timeNow := time.Now().Unix() -// org.Subscriptions[foundIndex].Active = false -// org.Subscriptions[foundIndex].CancellationDate = timeNow -// } -// } -// } -// -// //log.Printf("Org: %#v", org) -// err = setOrg(ctx, *org, org.Id) -// if err != nil { -// log.Printf("User %s doesn't have edit rights to %s", user.Id, org.Id) -// resp.WriteHeader(401) -// resp.Write([]byte(`{"success": false}`)) -// return -// } -// -// resp.WriteHeader(200) -// resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Successfully updated org"}`))) -//} +func makeWorkflowPublic(resp http.ResponseWriter, request *http.Request) { + cors := shuffle.HandleCors(resp, request) + if cors { + return + } + + user, userErr := shuffle.HandleApiAuthentication(resp, request) + if userErr != nil { + log.Printf("[WARNING] Api authentication failed in make workflow public: %s", userErr) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + location := strings.Split(request.URL.String(), "/") + var fileId string + if location[1] == "api" { + if len(location) <= 4 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fileId = location[4] + } + + ctx := context.Background() + if strings.Contains(fileId, "?") { + fileId = strings.Split(fileId, "?")[0] + } + + if len(fileId) != 36 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Workflow ID when getting workflow is not valid"}`)) + return + } + + workflow, err := shuffle.GetWorkflow(ctx, fileId) + if err != nil { + log.Printf("[WARNING] Workflow %s doesn't exist in app publish. User: %s", fileId, user.Id) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // CHECK orgs of user, or if user is owner + // FIXME - add org check too, and not just owner + // Check workflow.Sharing == private / public / org too + if user.Id != workflow.Owner || len(user.Id) == 0 { + if workflow.OrgId == user.ActiveOrg.Id && user.Role == "admin" { + log.Printf("[INFO] User %s is accessing workflow %s as admin", user.Username, workflow.ID) + } else { + log.Printf("[WARNING] Wrong user (%s) for workflow %s (get workflow)", user.Username, workflow.ID) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + } + + if !workflow.IsValid || !workflow.PreviouslySaved { + log.Printf("[INFO] Failed uploading workflow because it's invalid or not saved") + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Invalid workflows are not sharable"}`)) + return + } + + // Starting validation of the POST workflow + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Printf("[WARNING] Body data error on mail: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + parsedWorkflow := shuffle.Workflow{} + err = json.Unmarshal(body, &parsedWorkflow) + if err != nil { + log.Printf("[WARNING] Unmarshal error on mail: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // Super basic validation. Doesn't really matter. + if parsedWorkflow.ID != workflow.ID || len(parsedWorkflow.Actions) != len(workflow.Actions) { + log.Printf("[WARNING] Bad ID during publish: %s vs %s", workflow.ID, parsedWorkflow.ID) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if !workflow.IsValid || !workflow.PreviouslySaved { + log.Printf("[INFO] Failed uploading new workflow because it's invalid or not saved") + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Invalid workflows are not sharable"}`)) + return + } + + workflowData, err := json.Marshal(parsedWorkflow) + if err != nil { + log.Printf("[WARNING] Failed marshalling workflow: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // Sanitization is done in the frontend as well + parsedWorkflow = shuffle.SanitizeWorkflow(parsedWorkflow) + parsedWorkflow.ID = uuid.NewV4().String() + action := shuffle.CloudSyncJob{ + Type: "workflow", + Action: "publish", + OrgId: user.ActiveOrg.Id, + PrimaryItemId: workflow.ID, + SecondaryItem: string(workflowData), + FifthItem: user.Id, + } + + err = executeCloudAction(action, user.ActiveOrg.SyncConfig.Apikey) + if err != nil { + log.Printf("[WARNING] Failed cloud PUBLISH: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + + log.Printf("[INFO] Successfully published workflow %s (%s) TO CLOUD", workflow.Name, workflow.ID) + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) +} func initHandlers() { var err error ctx := context.Background() log.Printf("Starting Shuffle backend - initializing database connection") - // option.WithoutAuthentication - + //requestCache = cache.New(5*time.Minute, 10*time.Minute) dbclient, err = datastore.NewClient(ctx, gceProject, option.WithGRPCDialOption(grpc.WithNoProxy())) if err != nil { panic(fmt.Sprintf("DBclient error during init: %s", err)) } + + //dbclient, err := shuffle.GetDatastoreClient(ctx, gceProject) + //if err != nil { + // panic(fmt.Sprintf("Error setting datastore connector: %s", err)) + //} + + _ = shuffle.RunInit(*dbclient, storage.Client{}, gceProject, "onprem", true) log.Printf("Finished Shuffle database init") go runInit(ctx) @@ -8315,32 +6078,33 @@ func initHandlers() { // Make user related locations // Fix user changes with org - r.HandleFunc("/api/v1/users/generateapikey", handleApiGeneration).Methods("GET", "POST", "OPTIONS") r.HandleFunc("/api/v1/users/login", handleLogin).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/users/logout", handleLogout).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/users/register", handleRegister).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/users/checkusers", checkAdminLogin).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/users/getinfo", handleInfo).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/users/getsettings", handleSettings).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/users/getusers", handleGetUsers).Methods("GET", "OPTIONS") + + r.HandleFunc("/api/v1/users/generateapikey", shuffle.HandleApiGeneration).Methods("GET", "POST", "OPTIONS") + r.HandleFunc("/api/v1/users/logout", shuffle.HandleLogout).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/users/getsettings", shuffle.HandleSettings).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/users/getusers", shuffle.HandleGetUsers).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/users/updateuser", handleUpdateUser).Methods("PUT", "OPTIONS") - r.HandleFunc("/api/v1/users/{user}", deleteUser).Methods("DELETE", "OPTIONS") - r.HandleFunc("/api/v1/users/passwordchange", handlePasswordChange).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/users", handleGetUsers).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/users/{user}", shuffle.DeleteUser).Methods("DELETE", "OPTIONS") + r.HandleFunc("/api/v1/users/passwordchange", shuffle.HandlePasswordChange).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/users", shuffle.HandleGetUsers).Methods("GET", "OPTIONS") // General - duplicates and old. + r.HandleFunc("/api/v1/getusers", shuffle.HandleGetUsers).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/login", handleLogin).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/logout", handleLogout).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/logout", shuffle.HandleLogout).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/register", handleRegister).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/checkusers", checkAdminLogin).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/getusers", handleGetUsers).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/getinfo", handleInfo).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/getsettings", handleSettings).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/generateapikey", handleApiGeneration).Methods("GET", "POST", "OPTIONS") - r.HandleFunc("/api/v1/passwordchange", handlePasswordChange).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/getsettings", shuffle.HandleSettings).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/generateapikey", shuffle.HandleApiGeneration).Methods("GET", "POST", "OPTIONS") + r.HandleFunc("/api/v1/passwordchange", shuffle.HandlePasswordChange).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/getenvironments", handleGetEnvironments).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/setenvironments", handleSetEnvironments).Methods("PUT", "OPTIONS") + r.HandleFunc("/api/v1/getenvironments", shuffle.HandleGetEnvironments).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/setenvironments", shuffle.HandleSetEnvironments).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/docs", getDocList).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/docs/{key}", getDocs).Methods("GET", "OPTIONS") @@ -8358,22 +6122,22 @@ func initHandlers() { // App specific // From here down isnt checked for org specific + r.HandleFunc("/api/v1/apps/{appId}", shuffle.UpdateWorkflowAppConfig).Methods("PATCH", "OPTIONS") + r.HandleFunc("/api/v1/apps/{appId}", shuffle.DeleteWorkflowApp).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps/get_existing", loadSpecificApps).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/download_remote", loadSpecificApps).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/apps/{appId}", updateWorkflowAppConfig).Methods("PATCH", "OPTIONS") r.HandleFunc("/api/v1/apps/validate", validateAppInput).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/apps/{appId}", deleteWorkflowApp).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}/config", getWorkflowAppConfig).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps", getWorkflowApps).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps", setNewWorkflowApp).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/apps/search", getSpecificApps).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/apps/authentication", getAppAuthentication).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/apps/authentication", addAppAuthentication).Methods("PUT", "OPTIONS") - r.HandleFunc("/api/v1/apps/authentication/{appauthId}/config", setAuthenticationConfig).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/apps/authentication", shuffle.GetAppAuthentication).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/apps/authentication", shuffle.AddAppAuthentication).Methods("PUT", "OPTIONS") + r.HandleFunc("/api/v1/apps/authentication/{appauthId}/config", shuffle.SetAuthenticationConfig).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/apps/authentication/{appauthId}", deleteAppAuthentication).Methods("DELETE", "OPTIONS") + r.HandleFunc("/api/v1/apps/authentication/{appauthId}", shuffle.DeleteAppAuthentication).Methods("DELETE", "OPTIONS") // Legacy app things r.HandleFunc("/api/v1/workflows/apps/validate", validateAppInput).Methods("POST", "OPTIONS") @@ -8383,8 +6147,10 @@ func initHandlers() { // Workflows // FIXME - implement the queue counter lol /* Everything below here increases the counters*/ - r.HandleFunc("/api/v1/workflows", getWorkflows).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/workflows", setNewWorkflow).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/workflows", shuffle.GetWorkflows).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/workflows", shuffle.SetNewWorkflow).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/workflows/{key}", shuffle.GetSpecificWorkflow).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/workflows/{key}", shuffle.SaveWorkflow).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/workflows/schedules", handleGetSchedules).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/schedule", scheduleWorkflow).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/download_remote", loadSpecificWorkflows).Methods("POST", "OPTIONS") @@ -8393,20 +6159,13 @@ func initHandlers() { r.HandleFunc("/api/v1/workflows/{key}/outlook", createOutlookSub).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/outlook/{triggerId}", handleDeleteOutlookSub).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/executions", getWorkflowExecutions).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/workflows/{key}/executions/{key}/abort", abortExecution).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/workflows/{key}", getSpecificWorkflow).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/workflows/{key}", saveWorkflow).Methods("PUT", "OPTIONS") + r.HandleFunc("/api/v1/workflows/{key}/executions/{key}/abort", shuffle.AbortExecution).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}", deleteWorkflow).Methods("DELETE", "OPTIONS") // Triggers - r.HandleFunc("/api/v1/hooks/new", handleNewHook).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/hooks/new", shuffle.HandleNewHook).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/hooks/{key}", handleWebhookCallback).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/hooks/{key}/delete", handleDeleteHook).Methods("DELETE", "OPTIONS") - - // Trigger hmm - //r.HandleFunc("/api/v1/triggers/{key}", handleGetSpecificTrigger).Methods("GET", "OPTIONS") - - r.HandleFunc("/api/v1/stats/{key}", handleGetSpecificStats).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/hooks/{key}/delete", shuffle.HandleDeleteHook).Methods("DELETE", "OPTIONS") // OpenAPI configuration r.HandleFunc("/api/v1/verify_swagger", verifySwagger).Methods("POST", "OPTIONS") @@ -8416,12 +6175,16 @@ func initHandlers() { r.HandleFunc("/api/v1/get_openapi/{key}", getOpenapi).Methods("GET", "OPTIONS") // NEW for 0.8.0 + r.HandleFunc("/api/v1/workflows/{key}/publish", makeWorkflowPublic).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/cloud/setup", handleCloudSetup).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/orgs", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/orgs/", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleGetOrg).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleEditOrg).Methods("POST", "OPTIONS") + // This is a new API that validates if a key has been seen before. + // Not sure what the best course of action is for it. + r.HandleFunc("/api/v1/orgs/{orgId}/validate_app_values", shuffle.HandleKeyValueCheck).Methods("POST", "OPTIONS") - // Orgs - r.HandleFunc("/api/v1/orgs", handleGetOrgs).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/orgs/{orgId}", handleGetOrg).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/orgs/{orgId}", handleEditOrg).Methods("POST", "OPTIONS") //r.HandleFunc("/api/v1/orgs/{orgId}", handleEditOrg).Methods("POST", "OPTIONS") // Docker orborus specific @@ -8431,12 +6194,19 @@ func initHandlers() { // PS: For cloud, this has to use cloud storage. // https://developer.box.com/reference/get-files-id-content/ // 1. Creating the "get file" option. Make it possible to run this in the frontend. - r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/files/create", handleCreateFile).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/files/{fileId}", handleGetFileMeta).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS") - r.HandleFunc("/api/v1/files", handleGetFiles).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}/content", shuffle.HandleGetFileContent).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/files/create", shuffle.HandleCreateFile).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}/upload", shuffle.HandleUploadFile).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}", shuffle.HandleGetFileMeta).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}", shuffle.HandleDeleteFile).Methods("DELETE", "OPTIONS") + r.HandleFunc("/api/v1/files", shuffle.HandleGetFiles).Methods("GET", "OPTIONS") + + // Trigger hmm + r.HandleFunc("/api/v1/triggers/outlook/register", handleNewOutlookRegister).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/triggers/outlook/getFolders", handleGetOutlookFolders).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/triggers/outlook/{key}", handleGetSpecificTrigger).Methods("GET", "OPTIONS") + //r.HandleFunc("/api/v1/triggers/outlook/{key}/callback", handleOutlookCallback).Methods("POST", "OPTIONS") + //r.HandleFunc("/api/v1/stats/{key}", handleGetSpecificStats).Methods("GET", "OPTIONS") http.Handle("/", r) } diff --git a/backend/go-app/oauth2.go b/backend/go-app/oauth2.go new file mode 100644 index 00000000..164c803d --- /dev/null +++ b/backend/go-app/oauth2.go @@ -0,0 +1,1329 @@ +package main + +import ( + "github.com/frikky/shuffle-shared" + + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "log" + "net/http" + "net/url" + "strings" + "time" + + "cloud.google.com/go/datastore" + "golang.org/x/oauth2" +) + +// This is what the structure should be when it's sent into a workflow +type ParsedShuffleMail struct { + Body struct { + URI []string `json:"uri"` + Email []string `json:"email"` + Domain []string `json:"domain"` + ContentHeader struct { + } `json:"content_header"` + Content string `json:"content"` + ContentType string `json:"content_type"` + Hash string `json:"hash"` + RawBody string `json:"raw_body"` + } `json:"body"` + Header struct { + Subject string `json:"subject"` + From string `json:"from"` + To []string `json:"to"` + Date string `json:"date"` + Received []struct { + Src string `json:"src"` + From []string `json:"from"` + By []string `json:"by"` + With string `json:"with"` + Date string `json:"date"` + } `json:"received"` + ReceivedDomain []string `json:"received_domain"` + ReceivedIP []string `json:"received_ip"` + Header struct { + } `json:"header"` + } `json:"header"` + MessageID string `json:"message_id"` + EmailFileid string `json:"email_fileid"` + AttachmentUids []string `json:"attachment_uids"` +} + +type FullEmail struct { + OdataContext string `json:"@odata.context"` + OdataEtag string `json:"@odata.etag"` + ID string `json:"id"` + Createddatetime time.Time `json:"createdDateTime"` + Lastmodifieddatetime time.Time `json:"lastModifiedDateTime"` + Changekey string `json:"changeKey"` + Categories []interface{} `json:"categories"` + Receiveddatetime time.Time `json:"receivedDateTime"` + Sentdatetime time.Time `json:"sentDateTime"` + Hasattachments bool `json:"hasAttachments"` + Internetmessageid string `json:"internetMessageId"` + Subject string `json:"subject"` + Bodypreview string `json:"bodyPreview"` + Importance string `json:"importance"` + Parentfolderid string `json:"parentFolderId"` + Conversationid string `json:"conversationId"` + Conversationindex string `json:"conversationIndex"` + Isdeliveryreceiptrequested interface{} `json:"isDeliveryReceiptRequested"` + Isreadreceiptrequested bool `json:"isReadReceiptRequested"` + Isread bool `json:"isRead"` + Isdraft bool `json:"isDraft"` + Weblink string `json:"webLink"` + Inferenceclassification string `json:"inferenceClassification"` + Body struct { + Contenttype string `json:"contentType"` + Content string `json:"content"` + } `json:"body"` + Sender struct { + Emailaddress struct { + Name string `json:"name"` + Address string `json:"address"` + } `json:"emailAddress"` + } `json:"sender"` + From struct { + Emailaddress struct { + Name string `json:"name"` + Address string `json:"address"` + } `json:"emailAddress"` + } `json:"from"` + Torecipients []struct { + Emailaddress struct { + Name string `json:"name"` + Address string `json:"address"` + } `json:"emailAddress"` + } `json:"toRecipients"` + Ccrecipients []interface{} `json:"ccRecipients"` + Bccrecipients []interface{} `json:"bccRecipients"` + Replyto []interface{} `json:"replyTo"` + Flag struct { + Flagstatus string `json:"flagStatus"` + } `json:"flag"` + Attachments []struct { + OdataType string `json:"@odata.type"` + OdataMediacontenttype string `json:"@odata.mediaContentType"` + ID string `json:"id"` + Lastmodifieddatetime time.Time `json:"lastModifiedDateTime"` + Name string `json:"name"` + Contenttype string `json:"contentType"` + Size int `json:"size"` + Isinline bool `json:"isInline"` + Contentid interface{} `json:"contentId"` + Contentlocation interface{} `json:"contentLocation"` + Contentbytes string `json:"contentBytes"` + } +} + +type MailData struct { + Value []struct { + Subscriptionid string `json:"subscriptionId"` + Subscriptionexpirationdatetime string `json:"subscriptionExpirationDateTime"` + Changetype string `json:"changeType"` + Resource string `json:"resource"` + Resourcedata struct { + OdataType string `json:"@odata.type"` + OdataID string `json:"@odata.id"` + OdataEtag string `json:"@odata.etag"` + ID string `json:"id"` + } `json:"resourceData"` + Clientstate string `json:"clientState"` + Tenantid string `json:"tenantId"` + } `json:"value"` +} + +type OutlookProfile struct { + OdataContext string `json:"@odata.context"` + BusinessPhones []string `json:"businessPhones"` + DisplayName string `json:"displayName"` + GivenName string `json:"givenName"` + JobTitle interface{} `json:"jobTitle"` + Mail string `json:"mail"` + MobilePhone interface{} `json:"mobilePhone"` + OfficeLocation interface{} `json:"officeLocation"` + PreferredLanguage interface{} `json:"preferredLanguage"` + Surname string `json:"surname"` + UserPrincipalName string `json:"userPrincipalName"` + ID string `json:"id"` +} + +type OutlookFolder struct { + ID string `json:"id"` + DisplayName string `json:"displayName"` + ParentFolderID string `json:"parentFolderId"` + ChildFolderCount int `json:"childFolderCount"` + UnreadItemCount int `json:"unreadItemCount"` + TotalItemCount int `json:"totalItemCount"` +} + +type OutlookFolders struct { + OdataContext string `json:"@odata.context"` + OdataNextLink string `json:"@odata.nextLink"` + Value []OutlookFolder `json:"value"` +} + +func getOutlookAttachment(client *http.Client, emailId, attachmentId string) ([]FullEmail, error) { + //requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/ec03b4f2-fccf-4c35-b0eb-be85a0f5dd43/mailFolders") + + requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/%s/attachments/%s", emailId, attachmentId) + //log.Printf("Outlook email URL: %#v", requestUrl) + + ret, err := client.Get(requestUrl) + if err != nil { + log.Printf("[INFO] OutlookErr: %s", err) + return []FullEmail{}, err + } + + body, err := ioutil.ReadAll(ret.Body) + if err != nil { + log.Printf("[WARNING] Failed body decoding from outlook email") + return []FullEmail{}, err + } + + //type FullEmail struct { + log.Printf("[INFO] Attachment Body: %s", string(body)) + log.Printf("[INFO] Status email: %d", ret.StatusCode) + if ret.StatusCode != 200 { + return []FullEmail{}, err + } + + //log.Printf("Body: %s", string(body)) + + /* + parsedmail := FullEmail{} + err = json.Unmarshal(body, &parsedmail) + if err != nil { + log.Printf("[INFO] Email unmarshal error: %s", err) + return []FullEmail{}, err + } + + emails = append(emails, parsedmail) + */ + + return []FullEmail{}, nil +} + +func getOutlookEmail(client *http.Client, maildata MailData) ([]FullEmail, error) { + //requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/ec03b4f2-fccf-4c35-b0eb-be85a0f5dd43/mailFolders") + + emails := []FullEmail{} + for _, email := range maildata.Value { + //messageId := email.Resourcedata.ID + //requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/me/%s", messageId) + requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/%s", email.Resource) + //log.Printf("Outlook email URL: %#v", requestUrl) + + ret, err := client.Get(requestUrl) + if err != nil { + log.Printf("[INFO] OutlookErr: %s", err) + return []FullEmail{}, err + } + + body, err := ioutil.ReadAll(ret.Body) + if err != nil { + log.Printf("[WARNING] Failed body decoding from outlook email") + return []FullEmail{}, err + } + + //type FullEmail struct { + //log.Printf("[INFO] EMAIL Body: %s", string(body)) + //log.Printf("[INFO] Status email: %d", ret.StatusCode) + if ret.StatusCode != 200 { + return []FullEmail{}, err + } + + //log.Printf("Body: %s", string(body)) + + parsedmail := FullEmail{} + err = json.Unmarshal(body, &parsedmail) + if err != nil { + log.Printf("[INFO] Email unmarshal error: %s", err) + return []FullEmail{}, err + } + + emails = append(emails, parsedmail) + } + + return emails, nil +} + +func getOutlookFolders(client *http.Client) (OutlookFolders, error) { + //requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/ec03b4f2-fccf-4c35-b0eb-be85a0f5dd43/mailFolders") + requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/me/mailFolders") + + ret, err := client.Get(requestUrl) + if err != nil { + log.Printf("[INFO] FolderErr: %s", err) + return OutlookFolders{}, err + } + + body, err := ioutil.ReadAll(ret.Body) + if err != nil { + log.Printf("[WARNING] Failed body decoding from mailfolders") + return OutlookFolders{}, err + } + + //log.Printf("[INFO] Folder Body: %s", string(body)) + log.Printf("[INFO] Status folders: %d", ret.StatusCode) + if ret.StatusCode != 200 { + return OutlookFolders{}, err + } + + //log.Printf("Body: %s", string(body)) + + mailfolders := OutlookFolders{} + err = json.Unmarshal(body, &mailfolders) + if err != nil { + log.Printf("Unmarshal: %s", err) + return OutlookFolders{}, err + } + + //fmt.Printf("%#v", mailfolders) + // FIXME - recursion for subfolders + // Recursive struct + // folderEndpoint := fmt.Sprintf("%s/%s/childfolders?$top=40", requestUrl, parentId) + //for _, folder := range mailfolders.Value { + // log.Println(folder.DisplayName) + //} + + return mailfolders, nil +} + +func getOutlookProfile(client *http.Client) (OutlookProfile, error) { + requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/me?$select=mail") + + ret, err := client.Get(requestUrl) + if err != nil { + log.Printf("[INFO] Folder error: %s", err) + return OutlookProfile{}, err + } + + log.Printf("[INFO] Status profile: %d", ret.StatusCode) + body, err := ioutil.ReadAll(ret.Body) + if err != nil { + log.Printf("[INFO] Body: %s", err) + return OutlookProfile{}, err + } + + log.Printf("[INFO] BODY: %s", string(body)) + + profile := OutlookProfile{} + err = json.Unmarshal(body, &profile) + if err != nil { + log.Printf("Unmarshal: %s", err) + return OutlookProfile{}, err + } + + return profile, nil +} + +func handleNewOutlookRegister(resp http.ResponseWriter, request *http.Request) { + code := request.URL.Query().Get("code") + if len(code) == 0 { + log.Println("No code") + resp.WriteHeader(401) + return + } + + url := fmt.Sprintf("http://%s%s", request.Host, request.URL.EscapedPath()) + log.Println(url) + ctx := context.Background() + _, accessToken, err := getOutlookClient(ctx, code, OauthToken{}, url) + if err != nil { + log.Printf("Oauth client failure - outlook register: %s", err) + resp.WriteHeader(401) + return + } + + // This should be possible, and will also give the actual username + + /* + profile, err := getOutlookProfile(client) + if err != nil { + log.Printf("Outlook profile failure: %s", err) + resp.WriteHeader(401) + return + } + */ + + // This is a state workaround, which should really be for CSRF checks lol + state := request.URL.Query().Get("state") + if len(state) == 0 { + log.Println("No state") + resp.WriteHeader(401) + return + } + + stateitems := strings.Split(state, "%26") + if len(stateitems) == 1 { + stateitems = strings.Split(state, "&") + } + + // FIXME - trigger auth + senderUser := "" + trigger := TriggerAuth{} + for _, item := range stateitems { + itemsplit := strings.Split(item, "%3D") + if len(itemsplit) == 1 { + itemsplit = strings.Split(item, "=") + } + + if len(itemsplit) != 2 { + continue + } + + //log.Printf("ITEM: %#v", itemsplit) + + // Do something here + if itemsplit[0] == "workflow_id" { + trigger.WorkflowId = itemsplit[1] + } else if itemsplit[0] == "trigger_id" { + trigger.Id = itemsplit[1] + } else if itemsplit[0] == "type" { + trigger.Type = itemsplit[1] + } else if itemsplit[0] == "start" { + trigger.Start = itemsplit[1] + } else if itemsplit[0] == "username" { + trigger.Username = itemsplit[1] + trigger.Owner = itemsplit[1] + senderUser = itemsplit[1] + } + } + + // THis is an override based on the user in oauth return + /* + if len(profile.Mail) > 0 { + trigger.Username = profile.Mail + } + */ + + trigger.Code = code + trigger.OauthToken = OauthToken{ + AccessToken: accessToken.AccessToken, + TokenType: accessToken.TokenType, + RefreshToken: accessToken.RefreshToken, + Expiry: accessToken.Expiry, + } + + //log.Printf("%#v", trigger) + log.Println(trigger.WorkflowId) + log.Println(trigger.Id) + log.Println(senderUser) + log.Println(trigger.Type) + log.Printf("STARTNODE: %s", trigger.Start) + log.Printf("[INFO] Attempting to set up outlook trigger for %s", senderUser) + if trigger.WorkflowId == "" || trigger.Id == "" || senderUser == "" || trigger.Type == "" { + log.Printf("[INFO] All oauth items need to contain data to register a new state") + resp.WriteHeader(401) + return + } + + // Should also update the user + Userdata, err := shuffle.GetUser(ctx, senderUser) + if err != nil { + log.Printf("[INFO] Username %s doesn't exist (oauth2): %s", trigger.Username, err) + resp.WriteHeader(401) + return + } + + Userdata.Authentication = append(Userdata.Authentication, shuffle.UserAuth{ + Name: "Outlook", + Description: "oauth2", + Workflows: []string{trigger.WorkflowId}, + Username: trigger.Username, + Fields: []shuffle.UserAuthField{ + shuffle.UserAuthField{ + Key: "trigger_id", + Value: trigger.Id, + }, + shuffle.UserAuthField{ + Key: "username", + Value: trigger.Username, + }, + shuffle.UserAuthField{ + Key: "code", + Value: code, + }, + shuffle.UserAuthField{ + Key: "type", + Value: trigger.Type, + }, + }, + }) + + // Set apikey for the user if they don't have one + err = shuffle.SetUser(ctx, Userdata) + if err != nil { + log.Printf("Failed setting user data for %s: %s", Userdata.Username, err) + resp.WriteHeader(401) + return + } + + err = setTriggerAuth(ctx, trigger) + if err != nil { + log.Printf("Failed to set trigger auth for %s - %s", trigger.Username, err) + resp.WriteHeader(401) + return + } + + resp.WriteHeader(200) + resp.Write([]byte(`{"success": true}`)) +} + +type OauthToken struct { + AccessToken string `json:"AccessToken" datastore:"AccessToken,noindex"` + TokenType string `json:"TokenType" datastore:"TokenType,noindex"` + RefreshToken string `json:"RefreshToken" datastore:"RefreshToken,noindex"` + Expiry time.Time `json:"Expiry" datastore:"Expiry,noindex"` +} + +type TriggerAuth struct { + Id string `json:"id" datastore:"id"` + SubscriptionId string `json:"subscriptionId" datastore:"subscriptionId"` + + Username string `json:"username" datastore:"username,noindex"` + WorkflowId string `json:"workflow_id" datastore:"workflow_id,noindex"` + Owner string `json:"owner" datastore:"owner"` + Type string `json:"type" datastore:"type"` + Code string `json:"code,omitempty" datastore:"code,noindex"` + Start string `json:"start" datastore:"start"` + OauthToken OauthToken `json:"oauth_token,omitempty" datastore:"oauth_token"` +} + +func getTriggerAuth(ctx context.Context, id string) (*TriggerAuth, error) { + key := datastore.NameKey("trigger_auth", strings.ToLower(id), nil) + triggerauth := &TriggerAuth{} + if err := dbclient.Get(ctx, key, triggerauth); err != nil { + return &TriggerAuth{}, err + } + + return triggerauth, nil +} + +func setTriggerAuth(ctx context.Context, trigger TriggerAuth) error { + key1 := datastore.NameKey("trigger_auth", strings.ToLower(trigger.Id), nil) + + // New struct, to not add body, author etc + if _, err := dbclient.Put(ctx, key1, &trigger); err != nil { + log.Printf("Error adding trigger auth: %s", err) + return err + } + + return nil +} + +// THis all of a sudden became really horrible.. fml +func getOutlookClient(ctx context.Context, code string, accessToken OauthToken, redirectUri string) (*http.Client, *oauth2.Token, error) { + + conf := &oauth2.Config{ + ClientID: "fd55c175-aa30-4fa6-b303-09a29fb3f750", + ClientSecret: "14OBKgUpov.D7fe0~hp0z-cIQdP~SlYm.8", + Scopes: []string{ + "Mail.Read", + }, + RedirectURL: redirectUri, + Endpoint: oauth2.Endpoint{ + AuthURL: "https://login.microsoftonline.com/common/oauth2/authorize", + TokenURL: "https://login.microsoftonline.com/common/oauth2/token", + }, + } + + if len(code) > 0 { + access_token, err := conf.Exchange(ctx, code) + if err != nil { + log.Printf("Access_token issue: %s", err) + return &http.Client{}, access_token, err + } + + client := conf.Client(ctx, access_token) + return client, access_token, nil + } + + // Manually recreate the oauthtoken + access_token := &oauth2.Token{ + AccessToken: accessToken.AccessToken, + TokenType: accessToken.TokenType, + RefreshToken: accessToken.RefreshToken, + Expiry: accessToken.Expiry, + } + + client := conf.Client(ctx, access_token) + return client, access_token, nil +} + +func handleGetOutlookFolders(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + // Exchange every time hmm + // FIXME + // Should really just get the code from the trigger that's being used OR the user + triggerId := request.URL.Query().Get("trigger_id") + if len(triggerId) == 0 { + log.Println("No trigger_id supplied") + resp.WriteHeader(401) + return + } + + ctx := context.Background() + trigger, err := getTriggerAuth(ctx, triggerId) + if err != nil { + log.Printf("[INFO] Trigger %s doesn't exist - outlook folders.", triggerId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Trigger doesn't exist."}`)) + return + } + + //client, accessToken, err := getOutlookClient(ctx, code, OauthToken{}, url) + //if err != nil { + // log.Printf("Oauth client failure - outlook register: %s", err) + // resp.WriteHeader(401) + // return + //} + + // FIXME - should be shuffler in literally every case except testing lol + //log.Printf("TRIGGER: %#v", trigger) + redirectDomain := "localhost:5001" + url := fmt.Sprintf("http://%s/api/v1/triggers/outlook/register", redirectDomain) + outlookClient, _, err := getOutlookClient(ctx, "", trigger.OauthToken, url) + if err != nil { + log.Printf("[WARNING] Oauth client failure - outlook folders: %s", err) + resp.Write([]byte(`{"success": false, "reason": "Failed creating outlook client"}`)) + resp.WriteHeader(401) + return + } + + // This should be possible, and will also give the actual username + /* + profile, err := getOutlookProfile(outlookClient) + if err != nil { + log.Printf("Outlook profile failure: %s", err) + resp.WriteHeader(401) + return + } + log.Printf("PROFILE: %#v", profile) + */ + + folders, err := getOutlookFolders(outlookClient) + if err != nil { + log.Printf("[WARNING] Failed setting outlook folders: %s", err) + resp.Write([]byte(`{"success": false, "reason": "Failed getting outlook folders"}`)) + resp.WriteHeader(401) + return + } + + b, err := json.Marshal(folders.Value) + if err != nil { + log.Println("[INFO] Failed to marshal folderdata") + resp.Write([]byte(`{"success": false, "reason": "Failed decoding JSON"}`)) + resp.WriteHeader(401) + return + } + + resp.WriteHeader(200) + resp.Write(b) +} + +func handleGetSpecificTrigger(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + user, err := shuffle.HandleApiAuthentication(resp, request) + if err != nil { + log.Printf("Api authentication failed in getting specific workflow: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + location := strings.Split(request.URL.String(), "/") + + var workflowId string + if location[1] == "api" { + if len(location) <= 4 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + workflowId = location[5] + } + + if strings.Contains(workflowId, "?") { + workflowId = strings.Split(workflowId, "?")[0] + } + + ctx := context.Background() + trigger, err := getTriggerAuth(ctx, workflowId) + if err != nil { + log.Printf("[INFO] Trigger %s doesn't exist - specific trigger.", workflowId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": ""}`)) + return + } + + if user.Username != trigger.Owner && user.Role != "admin" { + log.Printf("Wrong user (%s) for trigger %s", user.Username, trigger.Id) + resp.WriteHeader(401) + return + } + + trigger.OauthToken = OauthToken{} + trigger.Code = "" + + b, err := json.Marshal(trigger) + if err != nil { + log.Println("Failed to marshal data") + resp.WriteHeader(401) + return + } + + resp.WriteHeader(200) + resp.Write(b) +} + +// This sets up the sub with outlook itself +// Parses data from the workflow to see whether access is right to subscribe it +// Creates the cloud function for outlook return +// Wait for it to be available, then schedule a workflow to it +func createOutlookSub(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + location := strings.Split(request.URL.String(), "/") + + var workflowId string + if location[1] == "api" { + if len(location) <= 4 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + workflowId = location[4] + } + + ctx := context.Background() + workflow, err := shuffle.GetWorkflow(ctx, workflowId) + if err != nil { + log.Printf("Failed getting the workflow locally (outlook sub): %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + user, err := shuffle.HandleApiAuthentication(resp, request) + if err != nil { + log.Printf("Api authentication failed in outlook deploy: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // FIXME - have a check for org etc too.. + if user.Id != workflow.Owner && user.Role != "admin" { + log.Printf("Wrong user (%s) for workflow %s when deploying outlook", user.Username, workflow.ID) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + log.Println("[INFO] Handle outlook subscription for trigger") + + // Should already be authorized at this point, as the workflow is shared + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Printf("Failed body read for workflow %s", workflow.ID) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // Based on the input data from frontend + type CurTrigger struct { + Name string `json:"name"` + Folders []string `json:"folders"` + ID string `json:"id"` + } + + //log.Println(string(body)) + var curTrigger CurTrigger + err = json.Unmarshal(body, &curTrigger) + if err != nil { + log.Printf("Failed body read unmarshal for trigger %s", workflow.ID) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if len(curTrigger.Folders) == 0 { + log.Printf("Error for %s. Choosing folders is required, currently 0", workflow.ID) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // Now that it's deployed - wait a few seconds before generating: + // 1. Oauth2 token thingies for outlook.office.com + // 2. Set the url to have the right mailboxes (probably ID?) ("https://outlook.office.com/api/v2.0/me/mailfolders('inbox')/messages") + // 3. Set the callback URL to be the new trigger + // 4. Run subscription test + // 5. Set the subscriptionId to the trigger object + + // First - lets regenerate an oauth token for outlook.office.com from the original items + trigger, err := getTriggerAuth(ctx, curTrigger.ID) + if err != nil { + log.Printf("[INFO] Trigger %s doesn't exist - outlook sub.", curTrigger.ID) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": ""}`)) + return + } + + // url doesn't really matter here + //url := fmt.Sprintf("https://shuffler.io") + redirectDomain := "localhost:5001" + url := fmt.Sprintf("http://%s/api/v1/triggers/outlook/register", redirectDomain) + outlookClient, _, err := getOutlookClient(ctx, "", trigger.OauthToken, url) + if err != nil { + log.Printf("Oauth client failure - triggerauth: %s", err) + resp.Write([]byte(`{"success": false, "reason": ""}`)) + resp.WriteHeader(401) + return + } + + // Location + + + // This is here simply to let the function start + // Usually takes 10 attempts minimum :O + // 10 * 5 = 50 seconds. That's waaay too much :( + + if runningEnvironment != "cloud" { + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) + if err != nil { + log.Printf("Failed finding org %s: %s", org.Id, err) + return + } + log.Printf("[INFO] Starting cloud configuration TO START trigger %s in org %s for workflow %s", trigger.Id, org.Id, trigger.WorkflowId) + + action := shuffle.CloudSyncJob{ + Type: "outlook", + Action: "start", + OrgId: org.Id, + PrimaryItemId: trigger.Id, + SecondaryItem: trigger.Start, + ThirdItem: workflowId, + } + + err = executeCloudAction(action, org.SyncConfig.Apikey) + if err != nil { + log.Printf("[INFO] Failed cloud action START outlook execution: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } else { + log.Printf("[INFO] Successfully set up cloud action trigger") + } + } else { + log.Printf("Should configure a running environment for CLOUD") + } + + notificationURL := fmt.Sprintf("%s/api/v1/hooks/webhook_%s", syncSubUrl, trigger.Id) + curSubscriptions, err := getOutlookSubscriptions(outlookClient) + if err == nil { + for _, sub := range curSubscriptions.Value { + if sub.NotificationURL == notificationURL { + log.Printf("[INFO] Removing existing subscription %s", sub.Id) + removeOutlookSubscription(outlookClient, sub.Id) + } + } + } else { + log.Printf("[INFO] Failed to get subscriptions - need to overwrite") + } + + maxFails := 5 + failCnt := 0 + log.Println(curTrigger.Folders) + for { + subId, err := makeOutlookSubscription(outlookClient, curTrigger.Folders, notificationURL) + if err != nil { + failCnt += 1 + log.Printf("Failed making oauth subscription, retrying in 5 seconds: %s", err) + time.Sleep(5 * time.Second) + if failCnt == maxFails { + log.Printf("Failed to set up subscription %d times.", maxFails) + resp.WriteHeader(401) + return + } + + continue + } + + // Set the ID somewhere here + trigger.SubscriptionId = subId + err = setTriggerAuth(ctx, *trigger) + if err != nil { + log.Printf("Failed setting triggerauth: %s", err) + } + + break + } + + log.Printf("[INFO] Successfully handled outlook subscription for trigger %s in workflow %s", curTrigger.ID, workflow.ID) + + //log.Printf("%#v", user) + resp.WriteHeader(200) + resp.Write([]byte(`{"success": true}`)) +} + +// Lists the users current subscriptions +func getOutlookSubscriptions(outlookClient *http.Client) (SubscriptionsWrapper, error) { + fullUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/subscriptions") + req, err := http.NewRequest( + "GET", + fullUrl, + nil, + ) + req.Header.Add("Content-Type", "application/json") + res, err := outlookClient.Do(req) + if err != nil { + log.Printf("suberror Client: %s", err) + return SubscriptionsWrapper{}, err + } + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + log.Printf("Suberror Body: %s", err) + return SubscriptionsWrapper{}, err + } + + newSubs := SubscriptionsWrapper{} + err = json.Unmarshal(body, &newSubs) + if err != nil { + return SubscriptionsWrapper{}, err + } + + return newSubs, nil +} + +type SubscriptionsWrapper struct { + OdataContext string `json:"@odata.context"` + Value []Subscription `json:"value"` +} + +type Subscription struct { + ChangeType string `json:"changeType"` + NotificationURL string `json:"notificationUrl"` + Resource string `json:"resource"` + ExpirationDateTime string `json:"expirationDateTime"` + ClientState string `json:"clientState"` + Id string `json:"id"` +} + +func makeOutlookSubscription(client *http.Client, folderIds []string, notificationURL string) (string, error) { + fullUrl := "https://graph.microsoft.com/v1.0/subscriptions" + + // FIXME - this expires rofl + t := time.Now().Local().Add(time.Minute * time.Duration(4200)) + timeFormat := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d.0000000Z", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) + + resource := fmt.Sprintf("me/mailfolders('%s')/messages", strings.Join(folderIds, "','")) + log.Printf("[INFO] Subscription resource to get(s): %s", resource) + sub := Subscription{ + ChangeType: "created", + ClientState: "Shuffle subscription", + NotificationURL: notificationURL, + ExpirationDateTime: timeFormat, + Resource: resource, + } + //ClientState: "This is a test", + + data, err := json.Marshal(sub) + if err != nil { + log.Printf("Marshal: %s", err) + return "", err + } + + req, err := http.NewRequest( + "POST", + fullUrl, + bytes.NewBuffer(data), + ) + req.Header.Add("Content-Type", "application/json") + + res, err := client.Do(req) + if err != nil { + log.Printf("Client: %s", err) + return "", err + } + + log.Printf("[INFO] Subscription Status: %d", res.StatusCode) + body, err := ioutil.ReadAll(res.Body) + if err != nil { + log.Printf("Body: %s", err) + return "", err + } + + if res.StatusCode != 200 && res.StatusCode != 201 { + return "", errors.New(fmt.Sprintf("Subscription failed: %s", string(body))) + } + + // Use data from body here to create thingy + newSub := Subscription{} + err = json.Unmarshal(body, &newSub) + if err != nil { + return "", err + } + + return newSub.Id, nil +} + +// Basically the same as a webhook +func handleOutlookCallback(resp http.ResponseWriter, request *http.Request) { + path := strings.Split(request.URL.String(), "/") + if len(path) < 4 { + log.Printf("[INFO] Bad outlook callback URL: %s", path) + resp.WriteHeader(403) + resp.Write([]byte(`{"success": false}`)) + return + } + + // 1. Get config with hookId + //fmt.Sprintf("%s/api/v1/hooks/%s", callbackUrl, hookId) + ctx := context.Background() + location := strings.Split(request.URL.String(), "/") + + var hookId string + if location[1] == "api" { + if len(location) <= 4 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + hookId = location[5] + } + + // ID: webhook_ + if len(hookId) != 36 { + log.Printf("[WARNING] Bad hook ID: %s (%d)", hookId, len(hookId)) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) + return + } + + //func getTriggerAuth(ctx context.Context, id string) (*TriggerAuth, error) { + hook, err := getTriggerAuth(ctx, hookId) + if err != nil { + log.Printf("[INFO] Failed getting trigger %s (callback): %s", hookId, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Printf("[INFO] Body data error: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + //log.Printf("[INFO] BODY: %s. Len: %d", string(body), len(body)) + //key, ok := request.URL.Query()["validationToken"] + //if ok { + //} + token := request.URL.Query().Get("validationToken") + if len(body) == 0 && len(token) > 0 { + log.Printf("[INFO] Should handle trigger token %s", token) + resp.WriteHeader(200) + resp.Write([]byte(string(token))) + return + } + + // 1. Take the body and parse data -> Get the email itself + + maildata := MailData{} + err = json.Unmarshal(body, &maildata) + if err != nil { + log.Printf("Maildata unmarshal error: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + redirectDomain := "localhost:5001" + redirectUrl := fmt.Sprintf("http://%s/api/v1/triggers/outlook/register", redirectDomain) + outlookClient, _, err := getOutlookClient(ctx, "", hook.OauthToken, redirectUrl) + if err != nil { + log.Printf("Oauth client failure - triggerauth: %s", err) + resp.WriteHeader(401) + return + } + + emails, err := getOutlookEmail(outlookClient, maildata) + log.Printf("[INFO] EMAILS: %d. If this is more than 1, please contact frikky@shuffler.io", len(emails)) + //log.Printf("INSIDE GET OUTLOOK EMAIL!: %#v, %s", emails, err) + + //type FullEmail struct { + email := FullEmail{} + if len(emails) == 1 { + email = emails[0] + } + + // Parse indicators (domains, emails, ips, domains etc)! + newEmail := ParsedShuffleMail{} + newEmail.Body.ContentType = email.Body.Contenttype + newEmail.Body.Content = email.Body.Content + newEmail.Body.RawBody = email.Body.Content + + newEmail.Header.Subject = email.Subject + newEmail.Header.From = email.From.Emailaddress.Address + for _, to := range email.Torecipients { + newEmail.Header.To = append(newEmail.Header.To, to.Emailaddress.Address) + } + newEmail.Header.Date = email.Receiveddatetime.String() + + newEmail.MessageID = email.ID + + if email.Hasattachments { + log.Printf("SHOULD HANDLE ATTACHMENTS FOR EMAIL!") + + for _, attachment := range email.Attachments { + parsedAttachment, err := getOutlookAttachment(outlookClient, email.ID, attachment.ID) + if err != nil { + log.Printf("Failed attachment %s: %s", attachment.ID, err) + continue + } + + log.Printf("ATTACHMENT: %#v", parsedAttachment) + } + //log.Printf("%#v", attachments) + //log.Printf("%s", err) + //GET /users/{id | userPrincipalName}/events/{id}/attachments/{id} + } + + emailBytes, err := json.Marshal(email) + if err != nil { + log.Printf("[INFO] Failed email marshaling: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + type ExecutionStruct struct { + Start string `json:"start"` + ExecutionSource string `json:"execution_source"` + ExecutionArgument string `json:"execution_argument"` + } + + newBody := ExecutionStruct{ + Start: hook.Start, + ExecutionSource: "outlook", + ExecutionArgument: string(emailBytes), + } + + b, err := json.Marshal(newBody) + if err != nil { + log.Printf("[INFO] Failed newBody marshaling: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + baseUrl := &url.URL{} + newRequest := &http.Request{ + URL: baseUrl, + Method: "POST", + Body: ioutil.NopCloser(bytes.NewReader(b)), + } + + workflow := shuffle.Workflow{ + ID: "", + } + + // OrgId: activeOrgs[0].Id, + workflowExecution, executionResp, err := handleExecution(hook.WorkflowId, workflow, newRequest) + if err == nil { + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization))) + return + } + + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, executionResp))) +} + +func removeOutlookSubscription(outlookClient *http.Client, subscriptionId string) error { + // DELETE https://graph.microsoft.com/v1.0/subscriptions/{id} + fullUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/subscriptions/%s", subscriptionId) + req, err := http.NewRequest( + "DELETE", + fullUrl, + nil, + ) + req.Header.Add("Content-Type", "application/json") + res, err := outlookClient.Do(req) + if err != nil { + log.Printf("Client: %s", err) + return err + } + + if res.StatusCode != 200 && res.StatusCode != 201 && res.StatusCode != 204 { + return errors.New(fmt.Sprintf("Bad status code when deleting subscription: %d", res.StatusCode)) + } + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + log.Printf("Body: %s", err) + return err + } + + _ = body + + return nil +} + +// Remove AUTH +// Remove function +// Remove subscription +func handleOutlookSubRemoval(ctx context.Context, user shuffle.User, workflowId, triggerId string) error { + // 1. Get the auth for trigger + // 2. Stop the subscription + // 3. Remove the function + // 4. Remove the database entry for auth + trigger, err := getTriggerAuth(ctx, triggerId) + if err != nil { + log.Printf("Trigger auth %s doesn't exist - outlook sub removal.", triggerId) + return err + } + + if runningEnvironment != "cloud" { + log.Printf("[INFO] SHOULD STOP OUTLOOK SUB ONPREM SYNC WITH CLOUD for workflow ID %s", workflowId) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) + if err != nil { + log.Printf("[INFO] Failed finding org %s during outlook removal: %s", org.Id, err) + return err + } + + log.Printf("[INFO] Stopping cloud configuration for trigger %s in org %s for workflow %s", trigger.Id, org.Id, trigger.WorkflowId) + action := shuffle.CloudSyncJob{ + Type: "outlook", + Action: "stop", + OrgId: org.Id, + PrimaryItemId: trigger.Id, + SecondaryItem: trigger.Start, + ThirdItem: trigger.WorkflowId, + } + + err = executeCloudAction(action, org.SyncConfig.Apikey) + if err != nil { + log.Printf("[INFO] Failed cloud action STOP outlook execution: %s", err) + return err + } else { + log.Printf("[INFO] Successfully set STOPPED outlook execution trigger") + } + } else { + log.Printf("SHOULD STOP OUTLOOK SUB IN CLOUD") + } + + // Actually delete the thing + redirectDomain := "localhost:5001" + url := fmt.Sprintf("http://%s/api/v1/triggers/outlook/register", redirectDomain) + outlookClient, _, err := getOutlookClient(ctx, "", trigger.OauthToken, url) + if err != nil { + log.Printf("[WARNING] Oauth client failure - outlook folders: %s", err) + return err + } + notificationURL := fmt.Sprintf("%s/api/v1/hooks/webhook_%s", syncSubUrl, trigger.Id) + curSubscriptions, err := getOutlookSubscriptions(outlookClient) + if err == nil { + for _, sub := range curSubscriptions.Value { + if sub.NotificationURL == notificationURL { + log.Printf("[INFO] Removing subscription %s from o365 for workflow %s", sub.Id, workflowId) + removeOutlookSubscription(outlookClient, sub.Id) + } + } + } else { + log.Printf("Failed to get subscriptions - need to overwrite") + } + + return nil +} + +func handleDeleteOutlookSub(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + location := strings.Split(request.URL.String(), "/") + + var workflowId string + var triggerId string + if location[1] == "api" { + if len(location) <= 6 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + workflowId = location[4] + triggerId = location[6] + } + + if len(workflowId) == 0 || len(triggerId) == 0 { + log.Printf("Ids can't be zero when deleting %s", workflowId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + ctx := context.Background() + workflow, err := shuffle.GetWorkflow(ctx, workflowId) + if err != nil { + log.Printf("Failed getting the workflow locally (delete outlook): %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + user, err := shuffle.HandleApiAuthentication(resp, request) + if err != nil { + log.Printf("Api authentication failed in outlook deploy: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // FIXME - have a check for org etc too.. + if user.Id != workflow.Owner && user.Role != "admin" { + log.Printf("Wrong user (%s) for workflow %s when deploying outlook", user.Username, workflow.ID) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // Check what kind of sub it is + err = handleOutlookSubRemoval(ctx, user, workflowId, triggerId) + if err != nil { + log.Printf("Failed sub removal: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + resp.WriteHeader(200) + resp.Write([]byte(`{"success": true}`)) +} diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 5fbdb3e6..86f27be7 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1,6 +1,8 @@ package main import ( + "github.com/frikky/shuffle-shared" + "bytes" "context" "encoding/json" @@ -10,11 +12,16 @@ import ( "io/ioutil" "log" "net/http" + "net/url" "os" + "sort" "strconv" "strings" "time" + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" + "cloud.google.com/go/datastore" scheduler "cloud.google.com/go/scheduler/apiv1" gyaml "github.com/ghodss/yaml" @@ -24,7 +31,7 @@ import ( schedulerpb "google.golang.org/genproto/googleapis/cloud/scheduler/v1" newscheduler "github.com/carlescere/scheduler" - "github.com/getkin/kin-openapi/openapi3" + "github.com/frikky/kin-openapi/openapi3" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" @@ -37,7 +44,6 @@ import ( //"cloud.google.com/go/firestore" // "google.golang.org/api/option" - "github.com/patrickmn/go-cache" "google.golang.org/api/iterator" ) @@ -58,429 +64,416 @@ var scheduledOrgs = map[string]*newscheduler.Job{} // }, //} -type ExecutionRequest struct { - ExecutionId string `json:"execution_id,omitempty"` - ExecutionArgument string `json:"execution_argument,omitempty"` - ExecutionSource string `json:"execution_source,omitempty"` - WorkflowId string `json:"workflow_id,omitempty"` - Environments []string `json:"environments,omitempty"` - Authorization string `json:"authorization,omitempty"` - Status string `json:"status,omitempty"` - Start string `json:"start,omitempty"` - Type string `json:"type,omitempty"` -} - -type SyncFeatures struct { - Webhook SyncData `json:"webhook" datastore:"webhook"` - Schedules SyncData `json:"schedules" datastore:"schedules"` - UserInput SyncData `json:"user_input" datastore:"user_input"` - SendMail SyncData `json:"send_mail" datastore:"send_mail"` - SendSms SyncData `json:"send_sms" datastore:"send_sms"` - Updates SyncData `json:"updates" datastore:"updates"` - Notifications SyncData `json:"notifications" datastore:"notifications"` - EmailTrigger SyncData `json:"email_trigger" datastore:"email_trigger"` - AppExecutions SyncData `json:"app_executions" datastore:"app_executions"` - WorkflowExecutions SyncData `json:"workflow_executions" datastore:"workflow_executions"` - Apps SyncData `json:"apps" datastore:"apps"` - Workflows SyncData `json:"workflows" datastore:"workflows"` - Autocomplete SyncData `json:"autocomplete" datastore:"autocomplete"` - Authentication SyncData `json:"authentication" datastore:"authentication"` - Schedule SyncData `json:"schedule" datastore:"schedule"` -} - -type SyncData struct { - Active bool `json:"active" datastore:"active"` - Type string `json:"type,omitempty" datastore:"type"` - Name string `json:"name,omitempty" datastore:"name"` - Description string `json:"description,omitempty" datastore:"description"` - Limit int64 `json:"limit,omitempty" datastore:"limit"` - StartDate int64 `json:"start_date,omitempty" datastore:"start_date"` - EndDate int64 `json:"end_date,omitempty" datastore:"end_date"` - DataCollection int64 `json:"data_collection,omitempty" datastore:"data_collection"` -} - -type SyncConfig struct { - Interval int64 `json:"interval" datastore:"interval"` - Apikey string `json:"api_key" datastore:"api_key"` -} - -// Role is just used for feedback for a user -type Org struct { - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` - Image string `json:"image" datastore:"image,noindex"` - Id string `json:"id" datastore:"id"` - Org string `json:"org" datastore:"org"` - Users []User `json:"users" datastore:"users"` - Role string `json:"role" datastore:"role"` - Roles []string `json:"roles" datastore:"roles"` - CloudSync bool `json:"cloud_sync" datastore:"CloudSync"` - SyncConfig SyncConfig `json:"sync_config" datastore:"sync_config"` - SyncFeatures SyncFeatures `json:"sync_features" datastore:"sync_features"` - Subscriptions []PaymentSubscription `json:"subscriptions" datastore:"subscriptions"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` - Defaults Defaults `json:"defaults" datastore:"defaults"` -} - -type PaymentSubscription struct { - Active bool `json:"active" datastore:"active"` - Startdate int64 `json:"startdate" datastore:"startdate"` - CancellationDate int64 `json:"cancellationdate" datastore:"cancellationdate"` - Enddate int64 `json:"enddate" datastore:"enddate"` - Name string `json:"name" datastore:"name"` - Recurrence string `json:"recurrence" datastore:"recurrence"` - Reference string `json:"reference" datastore:"reference"` - Level string `json:"level" datastore:"level"` - Amount string `json:"amount" datastore:"amount"` - Currency string `json:"currency" datastore:"currency"` -} - -type Defaults struct { - AppDownloadRepo string `json:"app_download_repo" datastore:"app_download_repo"` - AppDownloadBranch string `json:"app_download_branch" datastore:"app_download_branch"` - WorkflowDownloadRepo string `json:"workflow_download_repo" datastore:"workflow_download_repo"` - WorkflowDownloadBranch string `json:"workflow_download_branch" datastore:"workflow_download_branch"` -} - -type AppAuthenticationStorage struct { - Active bool `json:"active" datastore:"active"` - Label string `json:"label" datastore:"label"` - Id string `json:"id" datastore:"id"` - App WorkflowApp `json:"app" datastore:"app,noindex"` - Fields []AuthenticationStore `json:"fields" datastore:"fields"` - Usage []AuthenticationUsage `json:"usage" datastore:"usage"` - WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"` - NodeCount int64 `json:"node_count" datastore:"node_count"` - OrgId string `json:"org_id" datastore:"org_id"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` - Defined bool `json:"defined" datastore:"defined"` -} - -type AuthenticationUsage struct { - WorkflowId string `json:"workflow_id" datastore:"workflow_id"` - Nodes []string `json:"nodes" datastore:"nodes"` -} - -// An app inside Shuffle -// Source string `json:"source" datastore:"soure" yaml:"source"` - downloadlocation -type WorkflowApp struct { - Name string `json:"name" yaml:"name" required:true datastore:"name"` - IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"` - ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"` - Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"` - AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"` - SharingConfig string `json:"sharing_config" yaml:"sharing_config" datastore:"sharing_config"` - Generated bool `json:"generated" yaml:"generated" required:false datastore:"generated"` - Downloaded bool `json:"downloaded" yaml:"downloaded" required:false datastore:"downloaded"` - Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` - Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` - Invalid bool `json:"invalid" yaml:"invalid" required:false datastore:"invalid"` - Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` - Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"` - Owner string `json:"owner" datastore:"owner" yaml:"owner"` - Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps - PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"` - Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"` - Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"` - SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` - ContactInfo struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Url string `json:"url" datastore:"url" yaml:"url"` - } `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false` - ReferenceInfo struct { - DocumentationUrl string `json:"documentation_url" datastore:"documentation_url"` - GithubUrl string `json:"github_url" datastore:"github_url"` - } - Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` - Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"` - Tags []string `json:"tags" yaml:"tags" required:false datastore:"activated"` - Categories []string `json:"categories" yaml:"categories" required:false datastore:"categories"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` - LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` -} - -type WorkflowAppActionParameter struct { - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - ID string `json:"id" datastore:"id" yaml:"id,omitempty"` - Name string `json:"name" datastore:"name" yaml:"name"` - Example string `json:"example" datastore:"example" yaml:"example"` - Value string `json:"value" datastore:"value,noindex" yaml:"value,omitempty"` - Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` - Options []string `json:"options" datastore:"options" yaml:"options"` - ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"` - Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"` - Required bool `json:"required" datastore:"required" yaml:"required"` - Configuration bool `json:"configuration" datastore:"configuration" yaml:"configuration"` - Tags []string `json:"tags" datastore:"tags" yaml:"tags"` - Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` - SkipMulticheck bool `json:"skip_multicheck" datastore:"skip_multicheck" yaml:"skip_multicheck"` - ValueReplace []Valuereplace `json:"value_replace" datastore:"value_replace,noindex" yaml:"value_replace,omitempty"` -} - -type Valuereplace struct { - Key string `json:"key" datastore:"key" yaml:"key"` - Value string `json:"value" datastore:"value" yaml:"value"` -} - -type SchemaDefinition struct { - Type string `json:"type" datastore:"type"` -} - -type WorkflowAppAction struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id" yaml:"id,omitempty"` - Name string `json:"name" datastore:"name"` - Label string `json:"label" datastore:"label"` - NodeType string `json:"node_type" datastore:"node_type"` - Environment string `json:"environment" datastore:"environment"` - Sharing bool `json:"sharing" datastore:"sharing"` - PrivateID string `json:"private_id" datastore:"private_id"` - AppID string `json:"app_id" datastore:"app_id"` - Tags []string `json:"tags" datastore:"tags" yaml:"tags"` - Authentication []AuthenticationStore `json:"authentication" datastore:"authentication,noindex" yaml:"authentication,omitempty"` - Tested bool `json:"tested" datastore:"tested" yaml:"tested"` - Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"` - ExecutionVariable struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"execution_variable" datastore:"execution_variables"` - Returns struct { - Description string `json:"description" datastore:"returns" yaml:"description,omitempty"` - Example string `json:"example" datastore:"example" yaml:"example"` - ID string `json:"id" datastore:"id" yaml:"id,omitempty"` - Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` - } `json:"returns" datastore:"returns"` - AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` - Example string `json:"example" datastore:"example" yaml:"example"` - AuthNotRequired bool `json:"auth_not_required" datastore:"auth_not_required" yaml:"auth_not_required"` -} +//type ExecutionRequest struct { +// ExecutionId string `json:"execution_id,omitempty"` +// ExecutionArgument string `json:"execution_argument,omitempty"` +// ExecutionSource string `json:"execution_source,omitempty"` +// WorkflowId string `json:"workflow_id,omitempty"` +// Environments []string `json:"environments,omitempty"` +// Authorization string `json:"authorization,omitempty"` +// Status string `json:"status,omitempty"` +// Start string `json:"start,omitempty"` +// Type string `json:"type,omitempty"` +//} +// +//type SyncFeatures struct { +// Webhook SyncData `json:"webhook" datastore:"webhook"` +// Schedules SyncData `json:"schedules" datastore:"schedules"` +// UserInput SyncData `json:"user_input" datastore:"user_input"` +// SendMail SyncData `json:"send_mail" datastore:"send_mail"` +// SendSms SyncData `json:"send_sms" datastore:"send_sms"` +// Updates SyncData `json:"updates" datastore:"updates"` +// Notifications SyncData `json:"notifications" datastore:"notifications"` +// EmailTrigger SyncData `json:"email_trigger" datastore:"email_trigger"` +// AppExecutions SyncData `json:"app_executions" datastore:"app_executions"` +// WorkflowExecutions SyncData `json:"workflow_executions" datastore:"workflow_executions"` +// Apps SyncData `json:"apps" datastore:"apps"` +// Workflows SyncData `json:"workflows" datastore:"workflows"` +// Autocomplete SyncData `json:"autocomplete" datastore:"autocomplete"` +// Authentication SyncData `json:"authentication" datastore:"authentication"` +// Schedule SyncData `json:"schedule" datastore:"schedule"` +//} +// +//type SyncData struct { +// Active bool `json:"active" datastore:"active"` +// Type string `json:"type,omitempty" datastore:"type"` +// Name string `json:"name,omitempty" datastore:"name"` +// Description string `json:"description,omitempty" datastore:"description"` +// Limit int64 `json:"limit,omitempty" datastore:"limit"` +// StartDate int64 `json:"start_date,omitempty" datastore:"start_date"` +// EndDate int64 `json:"end_date,omitempty" datastore:"end_date"` +// DataCollection int64 `json:"data_collection,omitempty" datastore:"data_collection"` +//} +// +//type SyncConfig struct { +// Interval int64 `json:"interval" datastore:"interval"` +// Apikey string `json:"api_key" datastore:"api_key"` +//} +// +//type PaymentSubscription struct { +// Active bool `json:"active" datastore:"active"` +// Startdate int64 `json:"startdate" datastore:"startdate"` +// CancellationDate int64 `json:"cancellationdate" datastore:"cancellationdate"` +// Enddate int64 `json:"enddate" datastore:"enddate"` +// Name string `json:"name" datastore:"name"` +// Recurrence string `json:"recurrence" datastore:"recurrence"` +// Reference string `json:"reference" datastore:"reference"` +// Level string `json:"level" datastore:"level"` +// Amount string `json:"amount" datastore:"amount"` +// Currency string `json:"currency" datastore:"currency"` +//} +// +//type Defaults struct { +// AppDownloadRepo string `json:"app_download_repo" datastore:"app_download_repo"` +// AppDownloadBranch string `json:"app_download_branch" datastore:"app_download_branch"` +// WorkflowDownloadRepo string `json:"workflow_download_repo" datastore:"workflow_download_repo"` +// WorkflowDownloadBranch string `json:"workflow_download_branch" datastore:"workflow_download_branch"` +//} +// +//type AppAuthenticationStorage struct { +// Active bool `json:"active" datastore:"active"` +// Label string `json:"label" datastore:"label"` +// Id string `json:"id" datastore:"id"` +// App WorkflowApp `json:"app" datastore:"app,noindex"` +// Fields []AuthenticationStore `json:"fields" datastore:"fields"` +// Usage []AuthenticationUsage `json:"usage" datastore:"usage"` +// WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"` +// NodeCount int64 `json:"node_count" datastore:"node_count"` +// OrgId string `json:"org_id" datastore:"org_id"` +// Created int64 `json:"created" datastore:"created"` +// Edited int64 `json:"edited" datastore:"edited"` +// Defined bool `json:"defined" datastore:"defined"` +//} +// +//type AuthenticationUsage struct { +// WorkflowId string `json:"workflow_id" datastore:"workflow_id"` +// Nodes []string `json:"nodes" datastore:"nodes"` +//} +// +//// An app inside Shuffle +//// Source string `json:"source" datastore:"soure" yaml:"source"` - downloadlocation +//type WorkflowApp struct { +// Name string `json:"name" yaml:"name" required:true datastore:"name"` +// IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"` +// ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"` +// Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"` +// AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"` +// SharingConfig string `json:"sharing_config" yaml:"sharing_config" datastore:"sharing_config"` +// Generated bool `json:"generated" yaml:"generated" required:false datastore:"generated"` +// Downloaded bool `json:"downloaded" yaml:"downloaded" required:false datastore:"downloaded"` +// Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` +// Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` +// Invalid bool `json:"invalid" yaml:"invalid" required:false datastore:"invalid"` +// Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` +// Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"` +// Owner string `json:"owner" datastore:"owner" yaml:"owner"` +// Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps +// PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"` +// Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"` +// Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"` +// SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` +// LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` +// ContactInfo struct { +// Name string `json:"name" datastore:"name" yaml:"name"` +// Url string `json:"url" datastore:"url" yaml:"url"` +// } `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false` +// ReferenceInfo struct { +// DocumentationUrl string `json:"documentation_url" datastore:"documentation_url"` +// GithubUrl string `json:"github_url" datastore:"github_url"` +// } +// FolderMount struct { +// FolderMount bool `json:"folder_mount" datastore:"folder_mount"` +// SourceFolder string `json:"source_folder" datastore:"source_folder"` +// DestinationFolder string `json:"destination_folder" datastore:"destination_folder"` +// } +// Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` +// Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"` +// Tags []string `json:"tags" yaml:"tags" required:false datastore:"activated"` +// Categories []string `json:"categories" yaml:"categories" required:false datastore:"categories"` +// Created int64 `json:"created" datastore:"created"` +// Edited int64 `json:"edited" datastore:"edited"` +// LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` +//} +// +//type WorkflowAppActionParameter struct { +// Description string `json:"description" datastore:"description,noindex" yaml:"description"` +// ID string `json:"id" datastore:"id" yaml:"id,omitempty"` +// Name string `json:"name" datastore:"name" yaml:"name"` +// Example string `json:"example" datastore:"example,noindex" yaml:"example"` +// Value string `json:"value" datastore:"value,noindex" yaml:"value,omitempty"` +// Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` +// Options []string `json:"options" datastore:"options" yaml:"options"` +// ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"` +// Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"` +// Required bool `json:"required" datastore:"required" yaml:"required"` +// Configuration bool `json:"configuration" datastore:"configuration" yaml:"configuration"` +// Tags []string `json:"tags" datastore:"tags" yaml:"tags"` +// Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` +// SkipMulticheck bool `json:"skip_multicheck" datastore:"skip_multicheck" yaml:"skip_multicheck"` +// ValueReplace []Valuereplace `json:"value_replace" datastore:"value_replace,noindex" yaml:"value_replace,omitempty"` +// UniqueToggled bool `json:"unique_toggled" datastore:"unique_toggled" yaml:"unique_toggled"` +//} +// +//type Valuereplace struct { +// Key string `json:"key" datastore:"key" yaml:"key"` +// Value string `json:"value" datastore:"value" yaml:"value"` +//} +// +//type SchemaDefinition struct { +// Type string `json:"type" datastore:"type"` +//} +// +//type WorkflowAppAction struct { +// Description string `json:"description" datastore:"description,noindex"` +// ID string `json:"id" datastore:"id" yaml:"id,omitempty"` +// Name string `json:"name" datastore:"name"` +// Label string `json:"label" datastore:"label"` +// NodeType string `json:"node_type" datastore:"node_type"` +// Environment string `json:"environment" datastore:"environment"` +// Sharing bool `json:"sharing" datastore:"sharing"` +// PrivateID string `json:"private_id" datastore:"private_id"` +// AppID string `json:"app_id" datastore:"app_id"` +// Tags []string `json:"tags" datastore:"tags" yaml:"tags"` +// Authentication []AuthenticationStore `json:"authentication" datastore:"authentication,noindex" yaml:"authentication,omitempty"` +// Tested bool `json:"tested" datastore:"tested" yaml:"tested"` +// Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"` +// ExecutionVariable struct { +// Description string `json:"description" datastore:"description,noindex"` +// ID string `json:"id" datastore:"id"` +// Name string `json:"name" datastore:"name"` +// Value string `json:"value" datastore:"value,noindex"` +// } `json:"execution_variable" datastore:"execution_variables"` +// Returns struct { +// Description string `json:"description" datastore:"returns" yaml:"description,omitempty"` +// Example string `json:"example" datastore:"example,noindex" yaml:"example"` +// ID string `json:"id" datastore:"id" yaml:"id,omitempty"` +// Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` +// } `json:"returns" datastore:"returns"` +// AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` +// Example string `json:"example,noindex" datastore:"example" yaml:"example"` +// AuthNotRequired bool `json:"auth_not_required" datastore:"auth_not_required" yaml:"auth_not_required"` +//} // FIXME: Generate a callback authentication ID? // FIXME: Add org check .. -type WorkflowExecution struct { - Type string `json:"type" datastore:"type"` - Status string `json:"status" datastore:"status"` - Start string `json:"start" datastore:"start"` - ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` - ExecutionId string `json:"execution_id" datastore:"execution_id"` - ExecutionSource string `json:"execution_source" datastore:"execution_source"` - ExecutionParent string `json:"execution_parent" datastore:"execution_parent"` - ExecutionOrg string `json:"execution_org" datastore:"execution_org"` - WorkflowId string `json:"workflow_id" datastore:"workflow_id"` - LastNode string `json:"last_node" datastore:"last_node"` - Authorization string `json:"authorization" datastore:"authorization"` - Result string `json:"result" datastore:"result,noindex"` - StartedAt int64 `json:"started_at" datastore:"started_at"` - CompletedAt int64 `json:"completed_at" datastore:"completed_at"` - ProjectId string `json:"project_id" datastore:"project_id"` - Locations []string `json:"locations" datastore:"locations"` - Workflow Workflow `json:"workflow" datastore:"workflow,noindex"` - Results []ActionResult `json:"results" datastore:"results,noindex"` - ExecutionVariables []struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"execution_variables,omitempty" datastore:"execution_variables,omitempty"` - OrgId string `json:"org_id" datastore:"org_id"` -} +//type WorkflowExecution struct { +// Type string `json:"type" datastore:"type"` +// Status string `json:"status" datastore:"status"` +// Start string `json:"start" datastore:"start"` +// ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` +// ExecutionId string `json:"execution_id" datastore:"execution_id"` +// ExecutionSource string `json:"execution_source" datastore:"execution_source"` +// ExecutionParent string `json:"execution_parent" datastore:"execution_parent"` +// ExecutionOrg string `json:"execution_org" datastore:"execution_org"` +// WorkflowId string `json:"workflow_id" datastore:"workflow_id"` +// LastNode string `json:"last_node" datastore:"last_node"` +// Authorization string `json:"authorization" datastore:"authorization"` +// Result string `json:"result" datastore:"result,noindex"` +// StartedAt int64 `json:"started_at" datastore:"started_at"` +// CompletedAt int64 `json:"completed_at" datastore:"completed_at"` +// ProjectId string `json:"project_id" datastore:"project_id"` +// Locations []string `json:"locations" datastore:"locations"` +// Workflow Workflow `json:"workflow" datastore:"workflow,noindex"` +// Results []ActionResult `json:"results" datastore:"results,noindex"` +// ExecutionVariables []struct { +// Description string `json:"description" datastore:"description,noindex"` +// ID string `json:"id" datastore:"id"` +// Name string `json:"name" datastore:"name"` +// Value string `json:"value" datastore:"value,noindex"` +// } `json:"execution_variables,omitempty" datastore:"execution_variables,omitempty"` +// OrgId string `json:"org_id" datastore:"org_id"` +//} // This is for the nodes in a workflow, NOT the app action itself. -type Action struct { - AppName string `json:"app_name" datastore:"app_name"` - AppVersion string `json:"app_version" datastore:"app_version"` - AppID string `json:"app_id" datastore:"app_id"` - Errors []string `json:"errors" datastore:"errors"` - ID string `json:"id" datastore:"id"` - IsValid bool `json:"is_valid" datastore:"is_valid"` - IsStartNode bool `json:"isStartNode,omitempty" datastore:"isStartNode"` - Sharing bool `json:"sharing,omitempty" datastore:"sharing"` - PrivateID string `json:"private_id,omitempty" datastore:"private_id"` - Label string `json:"label,omitempty" datastore:"label"` - SmallImage string `json:"small_image,omitempty" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image,omitempty" datastore:"large_image,noindex" yaml:"large_image" required:false` - Environment string `json:"environment,omitempty" datastore:"environment"` - Name string `json:"name" datastore:"name"` - Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` - ExecutionVariable struct { - Description string `json:"description,omitempty" datastore:"description,noindex"` - ID string `json:"id,omitempty" datastore:"id"` - Name string `json:"name,omitempty" datastore:"name"` - Value string `json:"value,omitempty" datastore:"value,noindex"` - } `json:"execution_variable,omitempty" datastore:"execution_variable,omitempty"` - Position struct { - X float64 `json:"x,omitempty" datastore:"x"` - Y float64 `json:"y,omitempty" datastore:"y"` - } `json:"position,omitempty"` - Priority int `json:"priority,omitempty" datastore:"priority"` - AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` - Example string `json:"example,omitempty" datastore:"example"` - AuthNotRequired bool `json:"auth_not_required,omitempty" datastore:"auth_not_required" yaml:"auth_not_required"` - Category string `json:"category" datastore:"category"` -} +//type Action struct { +// AppName string `json:"app_name" datastore:"app_name"` +// AppVersion string `json:"app_version" datastore:"app_version"` +// AppID string `json:"app_id" datastore:"app_id"` +// Errors []string `json:"errors" datastore:"errors"` +// ID string `json:"id" datastore:"id"` +// IsValid bool `json:"is_valid" datastore:"is_valid"` +// IsStartNode bool `json:"isStartNode,omitempty" datastore:"isStartNode"` +// Sharing bool `json:"sharing,omitempty" datastore:"sharing"` +// PrivateID string `json:"private_id,omitempty" datastore:"private_id"` +// Label string `json:"label,omitempty" datastore:"label"` +// SmallImage string `json:"small_image,omitempty" datastore:"small_image,noindex" required:false yaml:"small_image"` +// LargeImage string `json:"large_image,omitempty" datastore:"large_image,noindex" yaml:"large_image" required:false` +// Environment string `json:"environment,omitempty" datastore:"environment"` +// Name string `json:"name" datastore:"name"` +// Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` +// ExecutionVariable struct { +// Description string `json:"description,omitempty" datastore:"description,noindex"` +// ID string `json:"id,omitempty" datastore:"id"` +// Name string `json:"name,omitempty" datastore:"name"` +// Value string `json:"value,omitempty" datastore:"value,noindex"` +// } `json:"execution_variable,omitempty" datastore:"execution_variable,omitempty"` +// Position struct { +// X float64 `json:"x,omitempty" datastore:"x"` +// Y float64 `json:"y,omitempty" datastore:"y"` +// } `json:"position,omitempty"` +// Priority int `json:"priority,omitempty" datastore:"priority"` +// AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` +// Example string `json:"example,omitempty" datastore:"example,noindex"` +// AuthNotRequired bool `json:"auth_not_required,omitempty" datastore:"auth_not_required" yaml:"auth_not_required"` +// Category string `json:"category" datastore:"category"` +//} +// +//// Added environment for location to execute +//type Trigger struct { +// AppName string `json:"app_name" datastore:"app_name"` +// Description string `json:"description" datastore:"description,noindex"` +// LongDescription string `json:"long_description" datastore:"long_description"` +// Status string `json:"status" datastore:"status"` +// AppVersion string `json:"app_version" datastore:"app_version"` +// Errors []string `json:"errors" datastore:"errors"` +// ID string `json:"id" datastore:"id"` +// IsValid bool `json:"is_valid" datastore:"is_valid"` +// IsStartNode bool `json:"isStartNode" datastore:"isStartNode"` +// Label string `json:"label" datastore:"label"` +// SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` +// LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` +// Environment string `json:"environment" datastore:"environment"` +// TriggerType string `json:"trigger_type" datastore:"trigger_type"` +// Name string `json:"name" datastore:"name"` +// Tags []string `json:"tags" datastore:"tags" yaml:"tags"` +// Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` +// Position struct { +// X float64 `json:"x" datastore:"x"` +// Y float64 `json:"y" datastore:"y"` +// } `json:"position"` +// Priority int `json:"priority" datastore:"priority"` +//} +// +//type Branch struct { +// DestinationID string `json:"destination_id" datastore:"destination_id"` +// ID string `json:"id" datastore:"id"` +// SourceID string `json:"source_id" datastore:"source_id"` +// Label string `json:"label" datastore:"label"` +// HasError bool `json:"has_errors" datastore: "has_errors"` +// Conditions []Condition `json:"conditions" datastore: "conditions,noindex"` +//} +// +//// Same format for a lot of stuff +//type Condition struct { +// Condition WorkflowAppActionParameter `json:"condition" datastore:"condition"` +// Source WorkflowAppActionParameter `json:"source" datastore:"source"` +// Destination WorkflowAppActionParameter `json:"destination" datastore:"destination"` +//} +// +//type Schedule struct { +// Name string `json:"name" datastore:"name"` +// Frequency string `json:"frequency" datastore:"frequency"` +// ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` +// Id string `json:"id" datastore:"id"` +// OrgId string `json:"org_id" datastore:"org_id"` +// Environment string `json:"environment" datastore:"environment"` +//} -// Added environment for location to execute -type Trigger struct { - AppName string `json:"app_name" datastore:"app_name"` - Description string `json:"description" datastore:"description,noindex"` - LongDescription string `json:"long_description" datastore:"long_description"` - Status string `json:"status" datastore:"status"` - AppVersion string `json:"app_version" datastore:"app_version"` - Errors []string `json:"errors" datastore:"errors"` - ID string `json:"id" datastore:"id"` - IsValid bool `json:"is_valid" datastore:"is_valid"` - IsStartNode bool `json:"isStartNode" datastore:"isStartNode"` - Label string `json:"label" datastore:"label"` - SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` - Environment string `json:"environment" datastore:"environment"` - TriggerType string `json:"trigger_type" datastore:"trigger_type"` - Name string `json:"name" datastore:"name"` - Tags []string `json:"tags" datastore:"tags" yaml:"tags"` - Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` - Position struct { - X float64 `json:"x" datastore:"x"` - Y float64 `json:"y" datastore:"y"` - } `json:"position"` - Priority int `json:"priority" datastore:"priority"` -} +//type Workflow struct { +// Actions []Action `json:"actions" datastore:"actions,noindex"` +// Branches []Branch `json:"branches" datastore:"branches,noindex"` +// Triggers []Trigger `json:"triggers" datastore:"triggers,noindex"` +// Schedules []Schedule `json:"schedules" datastore:"schedules,noindex"` +// Configuration struct { +// ExitOnError bool `json:"exit_on_error" datastore:"exit_on_error"` +// StartFromTop bool `json:"start_from_top" datastore:"start_from_top"` +// } `json:"configuration,omitempty" datastore:"configuration"` +// Created int64 `json:"created" datastore:"created"` +// Edited int64 `json:"edited" datastore:"edited"` +// LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` +// Errors []string `json:"errors,omitempty" datastore:"errors"` +// Tags []string `json:"tags,omitempty" datastore:"tags"` +// ID string `json:"id" datastore:"id"` +// IsValid bool `json:"is_valid" datastore:"is_valid"` +// Name string `json:"name" datastore:"name"` +// Description string `json:"description" datastore:"description,noindex"` +// Start string `json:"start" datastore:"start"` +// Owner string `json:"owner" datastore:"owner"` +// Sharing string `json:"sharing" datastore:"sharing"` +// Org []Org `json:"org,omitempty" datastore:"org"` +// ExecutingOrg Org `json:"execution_org,omitempty" datastore:"execution_org"` +// OrgId string `json:"org_id,omitempty" datastore:"org_id"` +// WorkflowVariables []struct { +// Description string `json:"description" datastore:"description,noindex"` +// ID string `json:"id" datastore:"id"` +// Name string `json:"name" datastore:"name"` +// Value string `json:"value" datastore:"value,noindex"` +// } `json:"workflow_variables" datastore:"workflow_variables"` +// ExecutionVariables []struct { +// Description string `json:"description" datastore:"description,noindex"` +// ID string `json:"id" datastore:"id"` +// Name string `json:"name" datastore:"name"` +// Value string `json:"value" datastore:"value,noindex"` +// } `json:"execution_variables,omitempty" datastore:"execution_variables"` +// ExecutionEnvironment string `json:"execution_environment" datastore:"execution_environment"` +// PreviouslySaved bool `json:"previously_saved" datastore:"first_save"` +// Categories Categories `json:"categories" datastore:"categories"` +// ExampleArgument string `json:"example_argument" datastore:"example_argument,noindex"` +//} -type Branch struct { - DestinationID string `json:"destination_id" datastore:"destination_id"` - ID string `json:"id" datastore:"id"` - SourceID string `json:"source_id" datastore:"source_id"` - Label string `json:"label" datastore:"label"` - HasError bool `json:"has_errors" datastore: "has_errors"` - Conditions []Condition `json:"conditions" datastore: "conditions,noindex"` -} +//type Category struct { +// Name string `json:"name" datastore:"name"` +// Description string `json:"description" datastore:"description"` +// Count int64 `json:"count" datastore:"count"` +//} +// +//type Categories struct { +// SIEM Category `json:"siem" datastore:"siem"` +// Communication Category `json:"communication" datastore:"communication"` +// Assets Category `json:"assets" datastore:"assets"` +// Cases Category `json:"cases" datastore:"cases"` +// Network Category `json:"network" datastore:"network"` +// Intel Category `json:"intel" datastore:"intel"` +// EDR Category `json:"edr" datastore:"edr"` +// Other Category `json:"other" datastore:"other"` +//} -// Same format for a lot of stuff -type Condition struct { - Condition WorkflowAppActionParameter `json:"condition" datastore:"condition"` - Source WorkflowAppActionParameter `json:"source" datastore:"source"` - Destination WorkflowAppActionParameter `json:"destination" datastore:"destination"` -} - -type Schedule struct { - Name string `json:"name" datastore:"name"` - Frequency string `json:"frequency" datastore:"frequency"` - ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` - Id string `json:"id" datastore:"id"` - OrgId string `json:"org_id" datastore:"org_id"` - Environment string `json:"environment" datastore:"environment"` -} - -type Workflow struct { - Actions []Action `json:"actions" datastore:"actions,noindex"` - Branches []Branch `json:"branches" datastore:"branches,noindex"` - Triggers []Trigger `json:"triggers" datastore:"triggers,noindex"` - Schedules []Schedule `json:"schedules" datastore:"schedules,noindex"` - Configuration struct { - ExitOnError bool `json:"exit_on_error" datastore:"exit_on_error"` - StartFromTop bool `json:"start_from_top" datastore:"start_from_top"` - } `json:"configuration,omitempty" datastore:"configuration"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` - LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` - Errors []string `json:"errors,omitempty" datastore:"errors"` - Tags []string `json:"tags,omitempty" datastore:"tags"` - ID string `json:"id" datastore:"id"` - IsValid bool `json:"is_valid" datastore:"is_valid"` - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description,noindex"` - Start string `json:"start" datastore:"start"` - Owner string `json:"owner" datastore:"owner"` - Sharing string `json:"sharing" datastore:"sharing"` - Org []Org `json:"org,omitempty" datastore:"org"` - ExecutingOrg Org `json:"execution_org,omitempty" datastore:"execution_org"` - OrgId string `json:"org_id,omitempty" datastore:"org_id"` - WorkflowVariables []struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"workflow_variables" datastore:"workflow_variables"` - ExecutionVariables []struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"execution_variables,omitempty" datastore:"execution_variables"` - ExecutionEnvironment string `json:"execution_environment" datastore:"execution_environment"` - PreviouslySaved bool `json:"first_save" datastore:"first_save"` - Categories Categories `json:"categories" datastore:"categories"` - ExampleArgument string `json:"example_argument" datastore:"example_argument,noindex"` -} - -type Category struct { - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` - Count int64 `json:"count" datastore:"count"` -} - -type Categories struct { - SIEM Category `json:"siem" datastore:"siem"` - Communication Category `json:"communication" datastore:"communication"` - Assets Category `json:"assets" datastore:"assets"` - Cases Category `json:"cases" datastore:"cases"` - Network Category `json:"network" datastore:"network"` - Intel Category `json:"intel" datastore:"intel"` - EDR Category `json:"edr" datastore:"edr"` - Other Category `json:"other" datastore:"other"` -} - -type ActionResult struct { - Action Action `json:"action" datastore:"action,noindex"` - ExecutionId string `json:"execution_id" datastore:"execution_id"` - Authorization string `json:"authorization" datastore:"authorization"` - Result string `json:"result" datastore:"result,noindex"` - StartedAt int64 `json:"started_at" datastore:"started_at"` - CompletedAt int64 `json:"completed_at" datastore:"completed_at"` - Status string `json:"status" datastore:"status"` -} - -type Authentication struct { - Required bool `json:"required" datastore:"required" yaml:"required" ` - Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"` -} - -type AuthenticationParams struct { - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - ID string `json:"id" datastore:"id" yaml:"id"` - Name string `json:"name" datastore:"name" yaml:"name"` - Example string `json:"example" datastore:"example" yaml:"example"` - Value string `json:"value,omitempty" datastore:"value,noindex" yaml:"value"` - Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` - Required bool `json:"required" datastore:"required" yaml:"required"` - In string `json:"in" datastore:"in" yaml:"in"` - Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` - Scheme string `json:"scheme" datastore:"scheme" yaml:"scheme"` // Deprecated -} - -type AuthenticationStore struct { - Key string `json:"key" datastore:"key"` - Value string `json:"value" datastore:"value,noindex"` -} - -type ExecutionRequestWrapper struct { - Data []ExecutionRequest `json:"data"` -} - -type AppExecutionExample struct { - AppName string `json:"app_name" datastore:"app_name"` - AppVersion string `json:"app_version" datastore:"app_version"` - AppAction string `json:"app_action" datastore:"app_action"` - AppId string `json:"app_id" datastore:"app_id"` - ExampleId string `json:"example_id" datastore:"example_id"` - SuccessExamples []string `json:"success_examples" datastore:"success_examples,noindex"` - FailureExamples []string `json:"failure_examples" datastore:"failure_examples,noindex"` -} +//type ActionResult struct { +// Action Action `json:"action" datastore:"action,noindex"` +// ExecutionId string `json:"execution_id" datastore:"execution_id"` +// Authorization string `json:"authorization" datastore:"authorization"` +// Result string `json:"result" datastore:"result,noindex"` +// StartedAt int64 `json:"started_at" datastore:"started_at"` +// CompletedAt int64 `json:"completed_at" datastore:"completed_at"` +// Status string `json:"status" datastore:"status"` +//} +// +//type Authentication struct { +// Required bool `json:"required" datastore:"required" yaml:"required" ` +// Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"` +//} +// +//type AuthenticationParams struct { +// Description string `json:"description" datastore:"description,noindex" yaml:"description"` +// ID string `json:"id" datastore:"id" yaml:"id"` +// Name string `json:"name" datastore:"name" yaml:"name"` +// Example string `json:"example" datastore:"example,noindex" yaml:"example"` +// Value string `json:"value,omitempty" datastore:"value,noindex" yaml:"value"` +// Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` +// Required bool `json:"required" datastore:"required" yaml:"required"` +// In string `json:"in" datastore:"in" yaml:"in"` +// Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` +// Scheme string `json:"scheme" datastore:"scheme" yaml:"scheme"` // Deprecated +//} +// +//type AuthenticationStore struct { +// Key string `json:"key" datastore:"key"` +// Value string `json:"value" datastore:"value,noindex"` +//} +// +//type ExecutionRequestWrapper struct { +// Data []ExecutionRequest `json:"data"` +//} +// +//type AppExecutionExample struct { +// AppName string `json:"app_name" datastore:"app_name"` +// AppVersion string `json:"app_version" datastore:"app_version"` +// AppAction string `json:"app_action" datastore:"app_action"` +// AppId string `json:"app_id" datastore:"app_id"` +// ExampleId string `json:"example_id" datastore:"example_id"` +// SuccessExamples []string `json:"success_examples" datastore:"success_examples,noindex"` +// FailureExamples []string `json:"failure_examples" datastore:"failure_examples,noindex"` +//} // This might be... a bit off, but that's fine :) // This might also be stupid, as we want timelines and such @@ -540,7 +533,7 @@ func increaseStatisticsField(ctx context.Context, fieldname, id string, amount i return nil } -func setWorkflowQueue(ctx context.Context, executionRequest ExecutionRequest, env string) error { +func setWorkflowQueue(ctx context.Context, executionRequest shuffle.ExecutionRequest, env string) error { orgKey := fmt.Sprintf("workflowqueue-%s", env) key := datastore.NameKey(orgKey, executionRequest.ExecutionId, nil) @@ -566,16 +559,16 @@ func setWorkflowQueue(ctx context.Context, executionRequest ExecutionRequest, en // return nil //} -func getWorkflowQueue(ctx context.Context, id string) (ExecutionRequestWrapper, error) { +func getWorkflowQueue(ctx context.Context, id string) (shuffle.ExecutionRequestWrapper, error) { orgId := fmt.Sprintf("workflowqueue-%s", id) q := datastore.NewQuery(orgId).Limit(10) - executions := []ExecutionRequest{} + executions := []shuffle.ExecutionRequest{} _, err := dbclient.GetAll(ctx, q, &executions) if err != nil { - return ExecutionRequestWrapper{}, err + return shuffle.ExecutionRequestWrapper{}, err } - return ExecutionRequestWrapper{Data: executions}, nil + return shuffle.ExecutionRequestWrapper{Data: executions}, nil //key := datastore.NameKey("workflowqueue", id, nil) //executions := ExecutionRequestWrapper{} @@ -644,11 +637,12 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode log.Printf("WRAPPER BODY: \n%s", bodyWrapper) job := func() { request := &http.Request{ + URL: &url.URL{}, Method: "POST", Body: ioutil.NopCloser(strings.NewReader(bodyWrapper)), } - _, _, err := handleExecution(workflowId, Workflow{ExecutingOrg: Org{Id: orgId}}, request) + _, _, err := handleExecution(workflowId, shuffle.Workflow{ExecutingOrg: shuffle.Org{Id: orgId}}, request) if err != nil { log.Printf("Failed to execute %s: %s", workflowId, err) } @@ -735,7 +729,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque // Getting from the request //log.Println(string(body)) - var removeExecutionRequests ExecutionRequestWrapper + var removeExecutionRequests shuffle.ExecutionRequestWrapper err = json.Unmarshal(body, &removeExecutionRequests) if err != nil { log.Printf("Failed executionrequest in queue unmarshaling: %s", err) @@ -819,7 +813,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } if len(executionRequests.Data) == 0 { - executionRequests.Data = []ExecutionRequest{} + executionRequests.Data = []shuffle.ExecutionRequest{} } else { log.Printf("[INFO] Executionrequests (%s): %d", id, len(executionRequests.Data)) } @@ -849,7 +843,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { return } - var actionResult ActionResult + var actionResult shuffle.ActionResult err = json.Unmarshal(body, &actionResult) if err != nil { log.Printf("Failed ActionResult unmarshaling: %s", err) @@ -859,7 +853,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflowExecution, err := getWorkflowExecution(ctx, actionResult.ExecutionId) + workflowExecution, err := shuffle.GetWorkflowExecution(ctx, actionResult.ExecutionId) if err != nil { //log.Printf("Failed getting execution (streamresult) %s: %s", actionResult.ExecutionId, err) resp.WriteHeader(401) @@ -889,7 +883,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { // Finds the child nodes of a node in execution and returns them // Used if e.g. a node in a branch is exited, and all children have to be stopped -func findChildNodes(workflowExecution WorkflowExecution, nodeId string) []string { +func findChildNodes(workflowExecution shuffle.WorkflowExecution, nodeId string) []string { //log.Printf("\nNODE TO FIX: %s\n\n", nodeId) allChildren := []string{nodeId} @@ -943,14 +937,14 @@ func validateNewWorkerExecution(body []byte) error { //} ctx := context.Background() - var execution WorkflowExecution + var execution shuffle.WorkflowExecution err := json.Unmarshal(body, &execution) if err != nil { log.Printf("[WARNING] Failed execution unmarshaling: %s", err) return err } - baseExecution, err := getWorkflowExecution(ctx, execution.ExecutionId) + baseExecution, err := shuffle.GetWorkflowExecution(ctx, execution.ExecutionId) if err != nil { log.Printf("[ERROR] Failed getting execution (workflowqueue) %s: %s", execution.ExecutionId, err) return err @@ -969,6 +963,27 @@ func validateNewWorkerExecution(body []byte) error { return errors.New(fmt.Sprintf("Bad length of trigger: %d (probably normal app)", len(execution.Workflow.Triggers))) } + if baseExecution.Status != "WAITING" && baseExecution.Status != "EXECUTING" { + return errors.New(fmt.Sprintf("Workflow is already finished or failed. Can't update")) + } + + if execution.Status == "EXECUTING" { + //log.Printf("[INFO] Inside executing.") + extra := 0 + for _, trigger := range execution.Workflow.Triggers { + //log.Printf("Appname trigger (0): %s", trigger.AppName) + if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" { + extra += 1 + } + } + + if len(execution.Workflow.Actions)+extra == len(execution.Results) { + execution.Status = "FINISHED" + } + + log.Printf("[INFO] BASEEXECUTION LENGTH: %d", len(baseExecution.Workflow.Actions)+extra) + } + // FIXME: Add extra here //executionLength := len(baseExecution.Workflow.Actions) //if executionLength != len(execution.Results) { @@ -976,9 +991,9 @@ func validateNewWorkerExecution(body []byte) error { //} //log.Printf("\n\nSHOULD SET BACKEND DATA FOR EXEC \n\n") - err = setWorkflowExecution(ctx, execution, true) + err = shuffle.SetWorkflowExecution(ctx, execution, true) if err == nil { - log.Printf("[INFO] Set workflowexecution based on new worker (>0.8.53) for execution %s", baseExecution.ExecutionId) + log.Printf("[INFO] Set workflowexecution based on new worker (>0.8.53) for execution %s. Actions: %d, Triggers: %d, Results: %d", execution.ExecutionId, len(execution.Workflow.Actions), len(execution.Workflow.Triggers), len(execution.Results)) //log.Printf("[INFO] Successfully set the execution to wait.") } else { log.Printf("[WARNING] Failed to set the execution to wait.") @@ -1008,10 +1023,10 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Success"}`))) return } else { - //log.Printf("[WARNING] Failed to handle new execution variant: %s", err) + //log.Printf("[WARNING] Handling other execution variant: %s", err) } - var actionResult ActionResult + var actionResult shuffle.ActionResult err = json.Unmarshal(body, &actionResult) if err != nil { log.Printf("Failed ActionResult unmarshaling: %s", err) @@ -1027,7 +1042,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { // IF FAIL: Set executionstatus: abort or cancel ctx := context.Background() - workflowExecution, err := getWorkflowExecution(ctx, actionResult.ExecutionId) + workflowExecution, err := shuffle.GetWorkflowExecution(ctx, actionResult.ExecutionId) if err != nil { log.Printf("[ERROR] Failed getting execution (workflowqueue) %s: %s", actionResult.ExecutionId, err) resp.WriteHeader(401) @@ -1066,7 +1081,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { if actionResult.Status == "WAITING" && actionResult.Action.AppName == "User Input" { log.Printf("SHOULD WAIT A BIT AND RUN CLOUD STUFF WITH USER INPUT! WAITING!") - var trigger Trigger + var trigger shuffle.Trigger err = json.Unmarshal([]byte(actionResult.Result), &trigger) if err != nil { log.Printf("Failed unmarshaling actionresult for user input: %s", err) @@ -1086,7 +1101,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { actionResult.Result = fmt.Sprintf("Cloud error: %s", err) workflowExecution.Results = append(workflowExecution.Results, actionResult) workflowExecution.Status = "ABORTED" - err = setWorkflowExecution(ctx, *workflowExecution, true) + err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) if err != nil { log.Printf("Failed to set execution during wait") } else { @@ -1104,7 +1119,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { workflowExecution.Results = append(workflowExecution.Results, actionResult) workflowExecution.Status = actionResult.Status - err = setWorkflowExecution(ctx, *workflowExecution, true) + err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) if err != nil { log.Printf("Failed ") } else { @@ -1119,18 +1134,26 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { } // Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times -func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult ActionResult, resp http.ResponseWriter) { +func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult shuffle.ActionResult, resp http.ResponseWriter) { // Should start a tx for the execution here - workflowExecution, err := getWorkflowExecution(ctx, workflowExecutionId) + workflowExecution, err := shuffle.GetWorkflowExecution(ctx, workflowExecutionId) if err != nil { log.Printf("[ERROR] Failed getting execution cache: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution"}`))) return } - resultLength := len(workflowExecution.Results) + + //resultLength := len(workflowExecution.Results) dbSave := false setExecution := true + + if actionResult.Action.ID == "" { + //log.Printf("[ERROR] Failed handling EMPTY action %#v", actionResult) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't handle empty action"}`))) + return + } //tx, err := dbclient.NewTransaction(ctx) //if err != nil { // log.Printf("client.NewTransaction: %v", err) @@ -1152,7 +1175,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl if actionResult.Status == "ABORTED" || actionResult.Status == "FAILURE" { dbSave = true - newResults := []ActionResult{} + newResults := []shuffle.ActionResult{} childNodes := []string{} if workflowExecution.Workflow.Configuration.ExitOnError { log.Printf("[WARNING] Actionresult is %s for node %s in %s. Should set workflowExecution and exit all running functions", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) @@ -1161,6 +1184,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Find underlying nodes and add them } else { log.Printf("[WARNING] Actionresult is %s for node %s in %s. Continuing anyway because of workflow configuration.", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) + // Finds ALL childnodes to set them to SKIPPED childNodes = findChildNodes(*workflowExecution, actionResult.Action.ID) @@ -1173,7 +1197,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // 1. Find the action itself // 2. Create an actionresult - curAction := Action{ID: ""} + curAction := shuffle.Action{ID: ""} for _, action := range workflowExecution.Workflow.Actions { if action.ID == nodeId { curAction = action @@ -1217,7 +1241,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } if !skipNodeAdd { - newAction := Action{ + newAction := shuffle.Action{ AppName: curAction.AppName, AppVersion: curAction.AppVersion, Label: curAction.Label, @@ -1225,7 +1249,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl ID: curAction.ID, } - newResult := ActionResult{ + newResult := shuffle.ActionResult{ Action: newAction, ExecutionId: actionResult.ExecutionId, Authorization: actionResult.Authorization, @@ -1281,6 +1305,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // FIXME rebuild to be like this or something // workflowExecution/ExecutionId/Nodes/NodeId // Find the appropriate action + //log.Printf("[INFO] Setting value of %s in workflow %s to %s (1)", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) if len(workflowExecution.Results) > 0 { // FIXME skip := false @@ -1316,14 +1341,14 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } } - log.Printf("[INFO] Updating %s in workflow %s from %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, workflowExecution.Results[outerindex].Status, actionResult.Status) + log.Printf("[INFO] Updating %s in workflow %s from %s to %s (3)", actionResult.Action.ID, workflowExecution.ExecutionId, workflowExecution.Results[outerindex].Status, actionResult.Status) workflowExecution.Results[outerindex] = actionResult } else { - log.Printf("[INFO] Setting value of %s in workflow %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) + log.Printf("[INFO] Setting value of %s in workflow %s to %s (1)", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) workflowExecution.Results = append(workflowExecution.Results, actionResult) } } else { - log.Printf("[INFO] Setting value of %s in workflow %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) + log.Printf("[INFO] Setting value of %s in workflow %s to %s (2)", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) workflowExecution.Results = append(workflowExecution.Results, actionResult) } @@ -1432,7 +1457,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } // FIXME - why isn't this how it works otherwise, wtf? - //workflow, err := getWorkflow(workflowExecution.Workflow.ID) + //workflow, err := shuffle.GetWorkflow(workflowExecution.Workflow.ID) //newActions := []Action{} //for _, action := range workflowExecution.Workflow.Actions { // log.Printf("Name: %s, Env: %s", action.Name, action.Environment) @@ -1447,7 +1472,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Result string `json:"result" datastore:"result,noindex"` // Arbitrary reduction size maxSize := 500000 - newResults := []ActionResult{} + newResults := []shuffle.ActionResult{} for _, item := range workflowExecution.Results { if len(item.Result) > maxSize { item.Result = "[ERROR] Result too large to handle (https://github.com/frikky/shuffle/issues/171)" @@ -1463,10 +1488,12 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Validating that action results hasn't changed // Handled using cachhing, so actually pretty fast cacheKey := fmt.Sprintf("workflowexecution-%s", workflowExecution.ExecutionId) - if value, found := requestCache.Get(cacheKey); found { - parsedValue := value.(*WorkflowExecution) - if len(parsedValue.Results) > 0 && len(parsedValue.Results) != resultLength { - setExecution = false + cache, err := shuffle.GetCache(ctx, cacheKey) + if err == nil { + cacheData := []byte(cache.([]uint8)) + //log.Printf("CACHEDATA: %#v", cacheData) + err = json.Unmarshal(cacheData, &workflowExecution) + if err == nil { if attempts > 5 { //log.Printf("\n\nSkipping execution input - %d vs %d. Attempts: (%d)\n\n", len(parsedValue.Results), resultLength, attempts) } @@ -1479,8 +1506,24 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } } + //if value, found := requestCache.Get(cacheKey); found { + // parsedValue := value.(*shuffle.WorkflowExecution) + // if len(parsedValue.Results) > 0 && len(parsedValue.Results) != resultLength { + // setExecution = false + // if attempts > 5 { + // //log.Printf("\n\nSkipping execution input - %d vs %d. Attempts: (%d)\n\n", len(parsedValue.Results), resultLength, attempts) + // } + + // attempts += 1 + // if len(workflowExecution.Results) <= len(workflowExecution.Workflow.Actions) { + // runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp) + // return + // } + // } + //} + if setExecution || workflowExecution.Status == "FINISHED" || workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" { - err = setWorkflowExecution(ctx, *workflowExecution, dbSave) + err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, dbSave) if err != nil { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution actionresult: %s"}`, err))) @@ -1515,7 +1558,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // log.Printf("[ERROR] QUITTING: tx.Commit %d: %v", attempts, err) // workflowExecution.Status = "ABORTED" - // setWorkflowExecution(ctx, *workflowExecution, true) + // shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) // resp.WriteHeader(401) // resp.Write([]byte(`{"success": false}`)) @@ -1545,10 +1588,10 @@ func JSONCheck(str string) bool { return json.Unmarshal([]byte(str), &jsonStr) == nil } -func handleExecutionStatistics(execution WorkflowExecution) { +func handleExecutionStatistics(execution shuffle.WorkflowExecution) { // FIXME: CLEAN UP THE JSON THAT'S SAVED. // https://github.com/frikky/Shuffle/issues/172 - appResults := []AppExecutionExample{} + appResults := []shuffle.AppExecutionExample{} for _, result := range execution.Results { resultCheck := JSONCheck(result.Result) if !resultCheck { @@ -1583,7 +1626,7 @@ func handleExecutionStatistics(execution WorkflowExecution) { } else { // CREATE SuccessExamples or FailureExamples - executionExample := AppExecutionExample{ + executionExample := shuffle.AppExecutionExample{ AppName: result.Action.AppName, AppVersion: result.Action.AppVersion, AppAction: result.Action.Name, @@ -1630,7 +1673,7 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in getworkflows: %s", err) resp.WriteHeader(401) @@ -1661,7 +1704,7 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) { q = q.Order("-edited") - var workflows []Workflow + var workflows []shuffle.Workflow _, err = dbclient.GetAll(ctx, q, &workflows) if err != nil { if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { @@ -1716,240 +1759,13 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) { resp.Write(newjson) } -// FIXME - add to actual database etc -func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Error with body read: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - var workflow Workflow - err = json.Unmarshal(body, &workflow) - if err != nil { - log.Printf("Failed unmarshaling: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - workflow.ID = uuid.NewV4().String() - workflow.Owner = user.Id - workflow.Sharing = "private" - user.ActiveOrg.Users = []User{} - workflow.ExecutingOrg = user.ActiveOrg - workflow.OrgId = user.ActiveOrg.Id - //log.Printf("TRIGGERS: %d", len(workflow.Triggers)) - - ctx := context.Background() - //err = increaseStatisticsField(ctx, "total_workflows", workflow.ID, 1, workflow.OrgId) - //if err != nil { - // log.Printf("Failed to increase total workflows stats: %s", err) - //} - - if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} - } - if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} - } - if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} - } - if len(workflow.Errors) == 0 { - workflow.Errors = []string{} - } - - newActions := []Action{} - for _, action := range workflow.Actions { - if action.Environment == "" { - //action.Environment = baseEnvironment - action.IsValid = true - } - - action.LargeImage = "" - newActions = append(newActions, action) - } - - // Initialized without functions = adding a hello world node. - if len(newActions) == 0 { - //log.Printf("APPENDING NEW APP FOR NEW WORKFLOW") - - // Adds the Testing app if it's a new workflow - workflowapps, err := getAllWorkflowApps(ctx, 500) - if err == nil { - // FIXME: Add real env - envName := "Shuffle" - environments, err := getEnvironments(ctx, user.ActiveOrg.Id) - if err == nil { - for _, env := range environments { - if env.Default { - envName = env.Name - break - } - } - } - - for _, item := range workflowapps { - if item.Name == "Testing" && item.AppVersion == "1.0.0" { - nodeId := "40447f30-fa44-4a4f-a133-4ee710368737" - workflow.Start = nodeId - newActions = append(newActions, Action{ - Label: "Start node", - Name: "hello_world", - Environment: envName, - Parameters: []WorkflowAppActionParameter{}, - Position: struct { - X float64 "json:\"x,omitempty\" datastore:\"x\"" - Y float64 "json:\"y,omitempty\" datastore:\"y\"" - }{X: 449.5, Y: 446}, - Priority: 0, - Errors: []string{}, - ID: nodeId, - IsValid: true, - IsStartNode: true, - Sharing: true, - PrivateID: "", - SmallImage: "", - AppName: item.Name, - AppVersion: item.AppVersion, - AppID: item.ID, - LargeImage: item.LargeImage, - }) - break - } - } - } - } else { - log.Printf("[INFO] Has %d actions already", len(newActions)) - // FIXME: Check if they require authentication and if they exist locally - //log.Printf("\n\nSHOULD VALIDATE AUTHENTICATION") - //AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"` - //allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) - //if err == nil { - // log.Printf("AUTH: %#v", allAuths) - // for _, action := range newActions { - // log.Printf("ACTION: %#v", action) - // } - //} - } - - workflow.Actions = []Action{} - for _, item := range workflow.Actions { - oldId := item.ID - sourceIndexes := []int{} - destinationIndexes := []int{} - for branchIndex, branch := range workflow.Branches { - if branch.SourceID == oldId { - sourceIndexes = append(sourceIndexes, branchIndex) - } - - if branch.DestinationID == oldId { - destinationIndexes = append(destinationIndexes, branchIndex) - } - } - - item.ID = uuid.NewV4().String() - for _, index := range sourceIndexes { - workflow.Branches[index].SourceID = item.ID - } - - for _, index := range destinationIndexes { - workflow.Branches[index].DestinationID = item.ID - } - - newActions = append(newActions, item) - } - - newTriggers := []Trigger{} - for _, item := range workflow.Triggers { - oldId := item.ID - sourceIndexes := []int{} - destinationIndexes := []int{} - for branchIndex, branch := range workflow.Branches { - if branch.SourceID == oldId { - sourceIndexes = append(sourceIndexes, branchIndex) - } - - if branch.DestinationID == oldId { - destinationIndexes = append(destinationIndexes, branchIndex) - } - } - - item.ID = uuid.NewV4().String() - for _, index := range sourceIndexes { - workflow.Branches[index].SourceID = item.ID - } - - for _, index := range destinationIndexes { - workflow.Branches[index].DestinationID = item.ID - } - - item.Status = "uninitialized" - newTriggers = append(newTriggers, item) - } - - newSchedules := []Schedule{} - for _, item := range workflow.Schedules { - item.Id = uuid.NewV4().String() - newSchedules = append(newSchedules, item) - } - - timeNow := int64(time.Now().Unix()) - workflow.Actions = newActions - workflow.Triggers = newTriggers - workflow.Schedules = newSchedules - workflow.IsValid = true - workflow.Configuration.ExitOnError = false - workflow.Created = timeNow - - workflowjson, err := json.Marshal(workflow) - if err != nil { - log.Printf("Failed workflow json setting marshalling: %s", err) - resp.WriteHeader(http.StatusInternalServerError) - resp.Write([]byte(`{"success": false}`)) - return - } - - err = setWorkflow(ctx, workflow, workflow.ID) - if err != nil { - log.Printf("Failed setting workflow: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Printf("Saved new workflow %s with name %s", workflow.ID, workflow.Name) - //memcacheName := fmt.Sprintf("%s_workflows", user.Username) - //memcache.Delete(ctx, memcacheName) - - resp.WriteHeader(200) - //log.Println(string(workflowjson)) - resp.Write(workflowjson) -} - func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in deleting workflow: %s", err) resp.WriteHeader(401) @@ -1977,7 +1793,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Failed getting the workflow locally (delete workflow): %s", err) resp.WriteHeader(401) @@ -1986,7 +1802,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { } // FIXME - have a check for org etc too.. - if user.Id != workflow.Owner && user.Role != "admin" { + if user.Id != workflow.Owner { log.Printf("Wrong user (%s) for workflow %s", user.Username, workflow.ID) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) @@ -2006,7 +1822,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { // log.Printf("Failed to delete webhook: %s", err) //} } else if item.TriggerType == "EMAIL" { - err = handleOutlookSubRemoval(ctx, workflow.ID, item.ID) + err = handleOutlookSubRemoval(ctx, user, workflow.ID, item.ID) if err != nil { log.Printf("Failed to delete email sub: %s", err) } @@ -2019,7 +1835,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { } // FIXME - maybe delete workflow executions - log.Printf("Should delete workflow %s", fileId) + log.Printf("[INFO] Should have deleted workflow %s", fileId) err = DeleteKey(ctx, "workflow", fileId) if err != nil { log.Printf("Failed deleting key %s", fileId) @@ -2041,816 +1857,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": true}`)) } -// Adds app auth tracking -func updateAppAuth(auth AppAuthenticationStorage, workflowId, nodeId string, add bool) error { - workflowFound := false - workflowIndex := 0 - nodeFound := false - for index, workflow := range auth.Usage { - if workflow.WorkflowId == workflowId { - // Check if node exists - workflowFound = true - workflowIndex = index - for _, actionId := range workflow.Nodes { - if actionId == nodeId { - nodeFound = true - break - } - } - - break - } - } - - // FIXME: Add a way to use !add to remove - updateAuth := false - if !workflowFound && add { - log.Printf("[INFO] Adding workflow things to auth!") - usageItem := AuthenticationUsage{ - WorkflowId: workflowId, - Nodes: []string{nodeId}, - } - - auth.Usage = append(auth.Usage, usageItem) - auth.WorkflowCount += 1 - auth.NodeCount += 1 - updateAuth = true - } else if !nodeFound && add { - log.Printf("[INFO] Adding node things to auth!") - auth.Usage[workflowIndex].Nodes = append(auth.Usage[workflowIndex].Nodes, nodeId) - auth.NodeCount += 1 - updateAuth = true - } - - if updateAuth { - log.Printf("[INFO] Updating auth!") - ctx := context.Background() - err := setWorkflowAppAuthDatastore(ctx, auth, auth.Id) - if err != nil { - log.Printf("Failed setting up app auth %s: %s", auth.Id, err) - return err - } - } - - return nil -} - // Identifies what a category defined really is -func handleCategoryIncrease(categories Categories, action Action, workflowapps []WorkflowApp) Categories { - if action.Category == "" { - appName := action.AppName - for _, app := range workflowapps { - if appName != strings.ToLower(app.Name) { - continue - } - - if len(app.Categories) > 0 { - log.Printf("[INFO] Setting category for %s: %s", app.Name, app.Categories) - action.Category = app.Categories[0] - break - } - } - - //log.Printf("Should find app's categories as it's empty during save") - return categories - } - - //log.Printf("Action: %s, category: %s", action.AppName, action.Category) - // FIXME: Make this an "autodiscover" that's controlled by the category itself - // Should just be a list that's looped against :) - newCategory := strings.ToLower(action.Category) - if strings.Contains(newCategory, "case") || strings.Contains(newCategory, "ticket") || strings.Contains(newCategory, "alert") || strings.Contains(newCategory, "mssp") { - categories.Cases.Count += 1 - } else if strings.Contains(newCategory, "siem") || strings.Contains(newCategory, "event") || strings.Contains(newCategory, "log") || strings.Contains(newCategory, "search") { - categories.SIEM.Count += 1 - } else if strings.Contains(newCategory, "sms") || strings.Contains(newCategory, "comm") || strings.Contains(newCategory, "phone") || strings.Contains(newCategory, "call") || strings.Contains(newCategory, "chat") || strings.Contains(newCategory, "mail") || strings.Contains(newCategory, "phish") { - categories.Communication.Count += 1 - } else if strings.Contains(newCategory, "intel") || strings.Contains(newCategory, "crim") || strings.Contains(newCategory, "ti") { - categories.Intel.Count += 1 - } else if strings.Contains(newCategory, "sand") || strings.Contains(newCategory, "virus") || strings.Contains(newCategory, "malware") || strings.Contains(newCategory, "scan") || strings.Contains(newCategory, "edr") || strings.Contains(newCategory, "endpoint detection") { - // Sandbox lol - categories.EDR.Count += 1 - } else if strings.Contains(newCategory, "vuln") || strings.Contains(newCategory, "fim") || strings.Contains(newCategory, "fim") || strings.Contains(newCategory, "integrity") { - categories.Assets.Count += 1 - } else if strings.Contains(newCategory, "network") || strings.Contains(newCategory, "firewall") || strings.Contains(newCategory, "waf") || strings.Contains(newCategory, "switch") { - categories.Network.Count += 1 - } else { - categories.Other.Count += 1 - } - - return categories -} - -// Saves a workflow to an ID -func saveWorkflow(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - //log.Println("Start") - user, userErr := handleApiAuthentication(resp, request) - if userErr != nil { - log.Printf("Api authentication failed in edit workflow: %s", userErr) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - //log.Println("PostUser") - location := strings.Split(request.URL.String(), "/") - - var fileId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if len(fileId) != 36 { - log.Printf(`ID %s is not valid`, fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Workflow ID to save is not valid"}`)) - return - } - - // Here to check access rights - ctx := context.Background() - log.Println("GetWorkflow start") - - tmpworkflow, err := getWorkflow(ctx, fileId) - if err != nil { - log.Printf("Failed getting the workflow locally (save workflow): %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Println("GetWorkflow end") - - // FIXME - have a check for org etc too.. - if user.Id != tmpworkflow.Owner && user.Role != "admin" { - log.Printf("Wrong user (%s) for workflow %s (save)", user.Username, tmpworkflow.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Failed hook unmarshaling: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - var workflow Workflow - err = json.Unmarshal([]byte(body), &workflow) - //log.Printf(string(body)) - if err != nil { - log.Printf(string(body)) - log.Printf("[ERROR] Failed workflow unmarshaling: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - - // FIXME - auth and check if they should have access - if fileId != workflow.ID { - log.Printf("Path and request ID are not matching: %s:%s.", fileId, workflow.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // Fixing wrong owners when importing - if workflow.Owner == "" { - workflow.Owner = user.Id - } - - if len(workflow.ExecutingOrg.Id) == 0 { - log.Printf("Setting executing org for workflow") - user.ActiveOrg.Users = []User{} - workflow.ExecutingOrg = user.ActiveOrg - } - - // FIXME - this shouldn't be necessary with proper API checks - newActions := []Action{} - allNodes := []string{} - workflow.Categories = Categories{} - - workflowapps, apperr := getAllWorkflowApps(ctx, 500) - - //log.Printf("Action: %#v", action.Authentication) - for _, action := range workflow.Actions { - allNodes = append(allNodes, action.ID) - - if len(action.Errors) > 0 || !action.IsValid { - action.IsValid = true - action.Errors = []string{} - } - - if action.Environment == "" { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "An environment for %s is required"}`, action.Label))) - return - action.IsValid = true - } - - // FIXME: Have a good way of tracking errors. ID's or similar. - if !action.IsValid && len(action.Errors) > 0 { - log.Printf("Node %s is invalid and needs to be remade. Errors: %s", action.Label, strings.Join(action.Errors, "\n")) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Node %s is invalid and needs to be remade."}`, action.Label))) - return - action.IsValid = true - action.Errors = []string{} - } - - workflow.Categories = handleCategoryIncrease(workflow.Categories, action, workflowapps) - newActions = append(newActions, action) - } - - if !workflow.PreviouslySaved { - log.Printf("[WORKFLOW INIT] NOT PREVIOUSLY SAVED - SET ACTION AUTH!") - //AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"` - - allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) - if err == nil && len(workflowapps) > 0 && apperr == nil { - //log.Printf("Setting actions") - actionFixing := []Action{} - appsAdded := []string{} - for _, action := range newActions { - setAuthentication := false - if len(action.AuthenticationId) > 0 { - //found := false - authenticationFound := false - for _, auth := range allAuths { - if auth.Id == action.AuthenticationId { - authenticationFound = true - break - } - } - - if !authenticationFound { - setAuthentication = true - } - } else { - // FIXME: 1. Validate if the app needs auth - // 1. Validate if auth for the app exists - // var appAuth AppAuthenticationStorage - setAuthentication = true - - //App WorkflowApp `json:"app" datastore:"app,noindex"` - } - - if setAuthentication { - authSet := false - for _, auth := range allAuths { - if !auth.Active { - continue - } - - if !auth.Defined { - continue - } - - if auth.App.Name == action.AppName { - //log.Printf("FOUND AUTH FOR APP %s: %s", auth.App.Name, auth.Id) - action.AuthenticationId = auth.Id - authSet = true - break - } - } - - // FIXME: Only o this IF there isn't another one for the app already - if !authSet { - //log.Printf("Validate if the app NEEDS auth or not") - outerapp := WorkflowApp{} - for _, app := range workflowapps { - if app.Name == action.AppName { - outerapp = app - break - } - } - - if len(outerapp.ID) > 0 && outerapp.Authentication.Required { - found := false - for _, auth := range allAuths { - if auth.App.ID == outerapp.ID { - found = true - break - } - } - - for _, added := range appsAdded { - if outerapp.ID == added { - found = true - } - } - - // FIXME: Add app auth - if !found { - timeNow := int64(time.Now().Unix()) - authFields := []AuthenticationStore{} - for _, param := range outerapp.Authentication.Parameters { - authFields = append(authFields, AuthenticationStore{ - Key: param.Name, - Value: "", - }) - } - - appAuth := AppAuthenticationStorage{ - Active: true, - Label: fmt.Sprintf("default_%s", outerapp.Name), - Id: uuid.NewV4().String(), - App: outerapp, - Fields: authFields, - Usage: []AuthenticationUsage{}, - WorkflowCount: 0, - NodeCount: 0, - OrgId: user.ActiveOrg.Id, - Created: timeNow, - Edited: timeNow, - } - - err = setWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id) - if err != nil { - log.Printf("Failed setting appauth for with name %s", appAuth.Label) - } else { - appsAdded = append(appsAdded, outerapp.ID) - } - } - - action.Errors = append(action.Errors, "Requires authentication") - action.IsValid = false - workflow.IsValid = false - } - - //outerapp.Authentication.Required - // Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"` - //workflowapps, apperr := getAllWorkflowApps(ctx, 100) - } - } - - actionFixing = append(actionFixing, action) - } - - newActions = actionFixing - } else { - log.Printf("Err: %s - %s", err, apperr) - //workflowapps, apperr := getAllWorkflowApps(ctx, 100) - //allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) - } - - workflow.PreviouslySaved = true - } - - workflow.Actions = newActions - newTriggers := []Trigger{} - for _, trigger := range workflow.Triggers { - log.Printf("[INFO] Trigger %s: %s", trigger.TriggerType, trigger.Status) - - // Check if it's actually running - // FIXME: Do this for other triggers too - if trigger.TriggerType == "SCHEDULE" && trigger.Status != "uninitialized" { - schedule, err := getSchedule(ctx, trigger.ID) - if err != nil { - trigger.Status = "stopped" - } else if schedule.Id == "" { - trigger.Status = "stopped" - } - } else if trigger.TriggerType == "SUBFLOW" { - for index, param := range trigger.Parameters { - if len(param.Value) == 0 && param.Name != "argument" { - log.Printf("Param: %#v", param) - if param.Name == "user_apikey" { - apikey := "" - if len(user.ApiKey) > 0 { - apikey = user.ApiKey - } else { - user, err = generateApikey(ctx, user) - if err != nil { - workflow.IsValid = false - workflow.Errors = []string{"Trigger is missing a parameter: %s", param.Name} - - log.Printf("No type specified for user input node") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Trigger %s is missing the parameter %s"}`, trigger.Label, param.Name))) - return - } - - apikey = user.ApiKey - } - - log.Printf("[INFO] Set apikey in subflow trigger for user during save") - trigger.Parameters[index].Value = apikey - } else { - - workflow.IsValid = false - workflow.Errors = []string{"Trigger is missing a parameter: %s", param.Name} - - log.Printf("No type specified for user input node") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Trigger %s is missing the parameter %s"}`, trigger.Label, param.Name))) - return - } - } - } - } else if trigger.TriggerType == "WEBHOOK" && trigger.Status != "uninitialized" { - hook, err := getHook(ctx, trigger.ID) - if err != nil { - log.Printf("Failed getting webhook") - trigger.Status = "stopped" - } else if hook.Id == "" { - trigger.Status = "stopped" - } - } else if trigger.TriggerType == "USERINPUT" { - // E.g. check email - sms := "" - email := "" - triggerType := "" - triggerInformation := "" - for _, item := range trigger.Parameters { - if item.Name == "alertinfo" { - triggerInformation = item.Value - } else if item.Name == "type" { - triggerType = item.Value - } else if item.Name == "email" { - email = item.Value - } else if item.Name == "sms" { - sms = item.Value - } - } - - if len(triggerType) == 0 { - log.Printf("No type specified for user input node") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No contact option specified in user input"}`))) - return - } - - // FIXME: This is not the right time to send them, BUT it's well served for testing. Save -> send email / sms - _ = triggerInformation - if strings.Contains(triggerType, "email") { - if email == "test@test.com" { - log.Printf("Email isn't specified during save.") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Email field in user input can't be empty"}`))) - return - } - - log.Printf("Should send email to %s during execution.", email) - } - if strings.Contains(triggerType, "sms") { - if sms == "0000000" { - log.Printf("Email isn't specified during save.") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "SMS field in user input can't be empty"}`))) - return - } - - log.Printf("Should send SMS to %s during execution.", sms) - } - } - - //log.Println("TRIGGERS") - allNodes = append(allNodes, trigger.ID) - newTriggers = append(newTriggers, trigger) - } - - workflow.Triggers = newTriggers - - for _, variable := range workflow.WorkflowVariables { - if len(variable.Value) == 0 { - log.Printf("Can't have an empty variable: %s", variable.Name) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Variable %s can't be empty"}`, variable.Name))) - return - } - } - - if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} - } - if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} - } - if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} - } - if len(workflow.Errors) == 0 { - workflow.Errors = []string{} - } - - if len(workflow.ExecutionVariables) > 0 { - log.Printf("[INFO] Found %d execution variable(s)", len(workflow.ExecutionVariables)) - } - - if len(workflow.WorkflowVariables) > 0 { - log.Printf("[INFO] Found %d workflow variable(s)", len(workflow.WorkflowVariables)) - } - - // FIXME - do actual checks ROFL - // FIXME - minor issues with e.g. hello world and self.console_logger - // Nodechecks - foundNodes := []string{} - for _, node := range allNodes { - for _, branch := range workflow.Branches { - //log.Println("branch") - //log.Println(node) - //log.Println(branch.DestinationID) - if node == branch.DestinationID || node == branch.SourceID { - foundNodes = append(foundNodes, node) - break - } - } - } - - // FIXME - append all nodes (actions, triggers etc) to one single array here - if len(foundNodes) != len(allNodes) || len(workflow.Actions) <= 0 { - // This shit takes a few seconds lol - if !workflow.IsValid { - oldworkflow, err := getWorkflow(ctx, fileId) - if err != nil { - log.Printf("Workflow %s doesn't exist - oldworkflow.", fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Item already exists."}`)) - return - } - - oldworkflow.IsValid = false - err = setWorkflow(ctx, *oldworkflow, fileId) - if err != nil { - log.Printf("Failed saving workflow to database: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - } - - // FIXME - more checks here - force reload of data or something - //if len(allNodes) == 0 { - // resp.WriteHeader(401) - // resp.Write([]byte(`{"success": false, "reason": "Please insert a node"}`)) - // return - //} - - // Allowed with only a start node - //if len(allNodes) != 1 { - // resp.WriteHeader(401) - // resp.Write([]byte(`{"success": false, "reason": "There are nodes with no branches"}`)) - // return - //} - } - - // FIXME - might be a sploit to run someone elses app if getAllWorkflowApps - // doesn't check sharing=true - // Have to do it like this to add the user's apps - //log.Println("Apps set starting") - //log.Printf("EXIT ON ERROR: %#v", workflow.Configuration.ExitOnError) - workflowApps := []WorkflowApp{} - //memcacheName = "all_apps" - //if item, err := memcache.Get(ctx, memcacheName); err == memcache.ErrCacheMiss { - // // Not in cache - // log.Printf("Apps not in cache.") - workflowApps, err = getAllWorkflowApps(ctx, 100) - if err != nil { - log.Printf("Failed getting all workflow apps from database: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // Started getting the single apps, but if it's weird, this is faster - // 1. Check workflow.Start - // 2. Check if any node has "isStartnode" - if len(workflow.Actions) > 0 { - index := -1 - for indexFound, action := range workflow.Actions { - //log.Println("Apps set done") - if workflow.Start == action.ID { - index = indexFound - } - } - - if index >= 0 { - workflow.Actions[0].IsStartNode = true - } else { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "You need to set a startnode."}`))) - return - } - } - - allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) - if userErr != nil { - log.Printf("Api authentication failed in get all apps: %s", userErr) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // Check every app action and param to see whether they exist - newActions = []Action{} - for _, action := range workflow.Actions { - reservedApps := []string{ - "0ca8887e-b4af-4e3e-887c-87e9d3bc3d3e", - } - - //log.Printf("%s Action execution var: %s", action.Label, action.ExecutionVariable.Name) - - builtin := false - for _, id := range reservedApps { - if id == action.AppID { - builtin = true - break - } - } - - // Check auth - // 1. Find the auth in question - // 2. Update the node and workflow info in the auth - // 3. Get the values in the auth and add them to the action values - if len(action.AuthenticationId) > 0 { - authFound := false - for _, auth := range allAuths { - if auth.Id == action.AuthenticationId { - authFound = true - - // Updates the auth item itself IF necessary - go updateAppAuth(auth, workflow.ID, action.ID, true) - break - } - } - - if !authFound { - log.Printf("App auth %s doesn't exist. Setting error", action.AuthenticationId) - workflow.Errors = append(workflow.Errors, fmt.Sprintf("App authentication for %s doesn't exist!", action.AppName)) - workflow.IsValid = false - - action.Errors = append(action.Errors, "App authentication doesn't exist") - action.IsValid = false - action.AuthenticationId = "" - //resp.WriteHeader(401) - //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App auth %s doesn't exist"}`, action.AuthenticationId))) - //return - } - } - - if builtin { - newActions = append(newActions, action) - } else { - curapp := WorkflowApp{} - // FIXME - can this work with ONLY AppID? - for _, app := range workflowApps { - if app.ID == action.AppID { - curapp = app - break - } - - // Has to NOT be generated - if app.Name == action.AppName && app.AppVersion == action.AppVersion { - curapp = app - break - } - } - - // Check to see if the whole app is valid - if curapp.Name != action.AppName { - workflow.Errors = append(workflow.Errors, fmt.Sprintf("App %s doesn't exist", action.AppName)) - action.Errors = append(action.Errors, "This app doesn't exist.") - action.IsValid = false - workflow.IsValid = false - - // Append with errors - newActions = append(newActions, action) - log.Printf("App %s doesn't exist. Adding as error.", action.AppName) - //resp.WriteHeader(401) - //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App %s doesn't exist"}`, action.AppName))) - //return - } else { - // Check tosee if the appaction is valid - curappaction := WorkflowAppAction{} - for _, curAction := range curapp.Actions { - if action.Name == curAction.Name { - curappaction = curAction - break - } - } - - // Check to see if the action is valid - if curappaction.Name != action.Name { - log.Printf("[ERROR] Action %s in app %s doesn't exist.", action.Name, curapp.Name) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Action %s in app %s doesn't exist"}`, action.Name, curapp.Name))) - return - } - - // FIXME - check all parameters to see if they're valid - // Includes checking required fields - - newParams := []WorkflowAppActionParameter{} - for _, param := range curappaction.Parameters { - found := false - - // Handles check for parameter exists + value not empty in used fields - for _, actionParam := range action.Parameters { - if actionParam.Name == param.Name { - found = true - - if actionParam.Value == "" && actionParam.Variant == "STATIC_VALUE" && actionParam.Required == true { - log.Printf("Appaction %s with required param '%s' is empty.", action.Name, param.Name) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name))) - return - - } - - if actionParam.Variant == "" { - actionParam.Variant = "STATIC_VALUE" - } - - newParams = append(newParams, actionParam) - break - } - } - - // Handles check for required params - if !found && param.Required { - log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name) - action.Errors = append(action.Errors, "Parameter %s is required", param.Name) - //newActions = append(newActions, action) - //resp.WriteHeader(401) - //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name))) - //return - } - - } - - action.Parameters = newParams - newActions = append(newActions, action) - } - } - } - - workflow.Actions = newActions - workflow.IsValid = true - log.Printf("[INFO] Tags: %#v", workflow.Tags) - - // FIXME: Is this too drastic? May lead to issues in the future. - // Should maybe make a copy for the old org. - if workflow.OrgId != user.ActiveOrg.Id { - log.Printf("[WARNING] Editing workflow to be owned by %s", user.ActiveOrg.Id) - workflow.OrgId = user.ActiveOrg.Id - workflow.ExecutingOrg = user.ActiveOrg - workflow.Org = append(workflow.Org, user.ActiveOrg) - } - - err = setWorkflow(ctx, workflow, fileId) - if err != nil { - log.Printf("Failed saving workflow to database: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - totalOldActions := len(tmpworkflow.Actions) - totalNewActions := len(workflow.Actions) - err = increaseStatisticsField(ctx, "total_workflow_actions", workflow.ID, int64(totalNewActions-totalOldActions), workflow.OrgId) - if err != nil { - log.Printf("Failed to change total actions data: %s", err) - } - - type returnData struct { - Success bool `json:"success"` - Errors []string `json:"errors"` - } - - returndata := returnData{ - Success: true, - Errors: workflow.Errors, - } - - cacheKey := fmt.Sprintf("workflowapps-sorted-100") - requestCache.Delete(cacheKey) - cacheKey = fmt.Sprintf("workflowapps-sorted-500") - requestCache.Delete(cacheKey) - - log.Printf("[INFO] Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId) - resp.WriteHeader(200) - newBody, err := json.Marshal(returndata) - if err != nil { - resp.Write([]byte(`{"success": true}`)) - return - } - - resp.Write(newBody) -} func getWorkflowLocal(fileId string, request *http.Request) ([]byte, error) { fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s", localBase, fileId) @@ -2887,135 +1894,6 @@ func getWorkflowLocal(fileId string, request *http.Request) ([]byte, error) { return body, nil } -func abortExecution(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - location := strings.Split(request.URL.String(), "/") - var fileId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if len(fileId) != 36 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Workflow ID to abort is not valid"}`)) - return - } - - executionId := location[6] - if len(executionId) != 36 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "ExecutionID not valid"}`)) - return - } - - ctx := context.Background() - workflowExecution, err := getWorkflowExecution(ctx, executionId) - if err != nil { - log.Printf("[ERROR] Failed getting execution (abort) %s: %s", executionId, err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution ID %s because it doesn't exist (abort)."}`, executionId))) - return - } - - apikey := request.Header.Get("Authorization") - parsedKey := "" - if strings.HasPrefix(apikey, "Bearer ") { - apikeyCheck := strings.Split(apikey, " ") - if len(apikeyCheck) == 2 { - parsedKey = apikeyCheck[1] - } - } - - if workflowExecution.Authorization != parsedKey { - // FIXME: Check the execution if this fails. - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in abort workflow: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // FIXME - have a check for org etc too.. - if user.Id != workflowExecution.Workflow.Owner && user.Role != "admin" { - log.Printf("[INFO] Wrong user (%s) for workflowexecution workflow %s", user.Username, workflowExecution.Workflow.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - } else { - //log.Printf("[INFO] API key to abort/finish execution %s is correct.", executionId) - } - - if workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" || workflowExecution.Status == "FINISHED" { - log.Printf("[INFO] Stopped execution of %s with status %s", executionId, workflowExecution.Status) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Status for %s is %s, which can't be aborted."}`, executionId, workflowExecution.Status))) - return - } - - topic := "workflowexecution" - - workflowExecution.CompletedAt = int64(time.Now().Unix()) - workflowExecution.Status = "ABORTED" - - lastResult := "" - newResults := []ActionResult{} - // type ActionResult struct { - for _, result := range workflowExecution.Results { - if result.Status == "EXECUTING" { - result.Status = "ABORTED" - result.Result = "Aborted because of error in another node (1)" - } - - if len(result.Result) > 0 { - lastResult = result.Result - } - - newResults = append(newResults, result) - } - - workflowExecution.Results = newResults - if len(workflowExecution.Result) == 0 { - workflowExecution.Result = lastResult - } - - err = setWorkflowExecution(ctx, *workflowExecution, true) - if err != nil { - log.Printf("Error saving workflow execution for updates when aborting %s: %s", topic, err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution status to abort"}`))) - return - } - - err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) - if err != nil { - log.Printf("Failed to increase aborted execution stats: %s", err) - } - - // FIXME - allowed to edit it? idk - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) - - // Not sure what's up here - //if workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" { - // log.Printf("Workflowexecution is already aborted. No further action can be taken") - // resp.WriteHeader(401) - // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is aborted because of %s with result %s and status %s"}`, workflowExecution.LastNode, workflowExecution.Result, workflowExecution.Status))) - // return - //} -} - //// New execution with firestore func cleanupExecutions(resp http.ResponseWriter, request *http.Request) { @@ -3024,9 +1902,9 @@ func cleanupExecutions(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { - log.Printf("Api authentication failed in execute workflow: %s", err) + log.Printf("[INFO] Api authentication failed in cleanup executions: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "Not authenticated"}`)) return @@ -3044,7 +1922,7 @@ func cleanupExecutions(resp http.ResponseWriter, request *http.Request) { timestamp := int64(time.Now().AddDate(0, -2, 0).Unix()) log.Println(timestamp) q := datastore.NewQuery("workflowexecution").Filter("started_at <", timestamp) - var workflowExecutions []WorkflowExecution + var workflowExecutions []shuffle.WorkflowExecution _, err = dbclient.GetAll(ctx, q, &workflowExecutions) if err != nil { log.Printf("Error getting workflowexec (cleanup): %s", err) @@ -3057,13 +1935,13 @@ func cleanupExecutions(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": true}`)) } -func handleExecution(id string, workflow Workflow, request *http.Request) (WorkflowExecution, string, error) { +func handleExecution(id string, workflow shuffle.Workflow, request *http.Request) (shuffle.WorkflowExecution, string, error) { ctx := context.Background() if workflow.ID == "" || workflow.ID != id { - tmpworkflow, err := getWorkflow(ctx, id) + tmpworkflow, err := shuffle.GetWorkflow(ctx, id) if err != nil { log.Printf("Failed getting the workflow locally (execution cleanup): %s", err) - return WorkflowExecution{}, "Failed getting workflow", err + return shuffle.WorkflowExecution{}, "Failed getting workflow", err } workflow = *tmpworkflow @@ -3071,13 +1949,13 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if len(workflow.ExecutingOrg.Id) == 0 { log.Printf("[INFO] Stopped execution because there is no executing org for workflow %s", workflow.ID) - return WorkflowExecution{}, fmt.Sprintf("Workflow has no executing org defined"), errors.New("Workflow has no executing org defined") + return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow has no executing org defined"), errors.New("Workflow has no executing org defined") } if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} + workflow.Actions = []shuffle.Action{} } else { - newactions := []Action{} + newactions := []shuffle.Action{} for _, action := range workflow.Actions { action.LargeImage = "" action.SmallImage = "" @@ -3089,12 +1967,12 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} + workflow.Branches = []shuffle.Branch{} } if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} + workflow.Triggers = []shuffle.Trigger{} } else { - newtriggers := []Trigger{} + newtriggers := []shuffle.Trigger{} for _, trigger := range workflow.Triggers { trigger.LargeImage = "" trigger.SmallImage = "" @@ -3111,21 +1989,21 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if !workflow.IsValid { log.Printf("[ERROR] Stopped execution as workflow %s is not valid.", workflow.ID) - return WorkflowExecution{}, fmt.Sprintf(`workflow %s is invalid`, workflow.ID), errors.New("Failed getting workflow") + return shuffle.WorkflowExecution{}, fmt.Sprintf(`workflow %s is invalid`, workflow.ID), errors.New("Failed getting workflow") } workflowBytes, err := json.Marshal(workflow) if err != nil { log.Printf("Failed workflow unmarshal in execution: %s", err) - return WorkflowExecution{}, "", err + return shuffle.WorkflowExecution{}, "", err } //log.Println(workflow) - var workflowExecution WorkflowExecution + var workflowExecution shuffle.WorkflowExecution err = json.Unmarshal(workflowBytes, &workflowExecution.Workflow) if err != nil { log.Printf("Failed execution unmarshaling: %s", err) - return WorkflowExecution{}, "Failed unmarshal during execution", err + return shuffle.WorkflowExecution{}, "Failed unmarshal during execution", err } makeNew := true @@ -3134,11 +2012,11 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("[ERROR] Failed request POST read: %s", err) - return WorkflowExecution{}, "Failed getting body", err + return shuffle.WorkflowExecution{}, "Failed getting body", err } // This one doesn't really matter. - log.Printf("[INFO] Running POST execution with body of length %d", len(string(body))) + log.Printf("[INFO] Running POST execution with body of length %d for workflow %s", len(string(body)), workflowExecution.Workflow.ID) if len(body) >= 4 { if body[0] == 34 && body[len(body)-1] == 34 { @@ -3176,11 +2054,11 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf log.Printf("Body: %s", string(body)) } - var execution ExecutionRequest + var execution shuffle.ExecutionRequest err = json.Unmarshal(body, &execution) if err != nil { log.Printf("[WARNING] Failed execution POST unmarshaling - continuing anyway: %s", err) - //return WorkflowExecution{}, "", err + //return shuffle.WorkflowExecution{}, "", err } if execution.Start == "" && len(body) > 0 { @@ -3211,12 +2089,12 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if !found { log.Printf("[ERROR] ACTION %s WAS NOT FOUND!", workflow.Start) - return WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", workflow.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", workflow.Start)) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", workflow.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", workflow.Start)) } } else if len(execution.Start) > 0 { log.Printf("[ERROR] START ACTION %s IS WRONG ID LENGTH %d!", execution.Start, len(execution.Start)) - return WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", execution.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", execution.Start)) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", execution.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", execution.Start)) } if len(execution.ExecutionId) == 36 { @@ -3238,18 +2116,18 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf log.Printf("Should update reference and return, no need for further execution!") // Get the reference execution - oldExecution, err := getWorkflowExecution(ctx, referenceId[0]) + oldExecution, err := shuffle.GetWorkflowExecution(ctx, referenceId[0]) if err != nil { log.Printf("Failed getting execution (execution) %s: %s", referenceId[0], err) - return WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err + return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err } if oldExecution.Workflow.ID != id { log.Println("Wrong workflowid!") - return WorkflowExecution{}, fmt.Sprintf("Bad ID %s", referenceId), errors.New("Bad ID") + return shuffle.WorkflowExecution{}, fmt.Sprintf("Bad ID %s", referenceId), errors.New("Bad ID") } - newResults := []ActionResult{} + newResults := []shuffle.ActionResult{} //log.Printf("%#v", oldExecution.Results) for _, result := range oldExecution.Results { log.Printf("%s - %s", result.Action.ID, start[0]) @@ -3273,23 +2151,23 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } oldExecution.Results = newResults - err = setWorkflowExecution(ctx, *oldExecution, true) + err = shuffle.SetWorkflowExecution(ctx, *oldExecution, true) if err != nil { log.Printf("Error saving workflow execution actionresult setting: %s", err) - return WorkflowExecution{}, fmt.Sprintf("Failed setting workflowexecution actionresult in execution: %s", err), err + return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed setting workflowexecution actionresult in execution: %s", err), err } - return WorkflowExecution{}, "", nil + return shuffle.WorkflowExecution{}, "", nil } } if referenceok { log.Printf("Handling an old execution continuation!") // Will use the old name, but still continue with NEW ID - oldExecution, err := getWorkflowExecution(ctx, referenceId[0]) + oldExecution, err := shuffle.GetWorkflowExecution(ctx, referenceId[0]) if err != nil { log.Printf("Failed getting execution (execution) %s: %s", referenceId[0], err) - return WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err + return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err } workflowExecution = *oldExecution @@ -3315,7 +2193,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf // FIXME - regex uuid, and check if already exists? if len(workflowExecution.ExecutionId) != 36 { log.Printf("Invalid uuid: %s", workflowExecution.ExecutionId) - return WorkflowExecution{}, "Invalid uuid", err + return shuffle.WorkflowExecution{}, "Invalid uuid", err } // FIXME - find owner of workflow @@ -3349,7 +2227,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf log.Printf("[INFO] No execution source (trigger) specified. Setting to default") workflowExecution.ExecutionSource = "default" } else { - log.Printf("[INFO] Execution source is %s for execution ID %s", workflowExecution.ExecutionSource, workflowExecution.ExecutionId) + log.Printf("[INFO] Execution source is %s for execution ID %s in workflow %s", workflowExecution.ExecutionSource, workflowExecution.ExecutionId, workflowExecution.Workflow.ID) } workflowExecution.ExecutionVariables = workflow.ExecutionVariables @@ -3376,10 +2254,10 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf topic := "workflows" startFound := false // FIXME - remove this? - newActions := []Action{} - defaultResults := []ActionResult{} + newActions := []shuffle.Action{} + defaultResults := []shuffle.ActionResult{} - allAuths := []AppAuthenticationStorage{} + allAuths := []shuffle.AppAuthenticationStorage{} for _, action := range workflowExecution.Workflow.Actions { //action.LargeImage = "" if action.ID == workflowExecution.Start { @@ -3388,20 +2266,20 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf //log.Println(action.Environment) if action.Environment == "" { - return WorkflowExecution{}, fmt.Sprintf("Environment is not defined for %s", action.Name), errors.New("Environment not defined!") + return shuffle.WorkflowExecution{}, fmt.Sprintf("Environment is not defined for %s", action.Name), errors.New("Environment not defined!") } // FIXME: Authentication parameters if len(action.AuthenticationId) > 0 { if len(allAuths) == 0 { - allAuths, err = getAllWorkflowAppAuth(ctx, workflow.ExecutingOrg.Id) + allAuths, err = shuffle.GetAllWorkflowAppAuth(ctx, workflow.ExecutingOrg.Id) if err != nil { log.Printf("Api authentication failed in get all app auth: %s", err) - return WorkflowExecution{}, fmt.Sprintf("Api authentication failed in get all app auth: %s", err), err + return shuffle.WorkflowExecution{}, fmt.Sprintf("Api authentication failed in get all app auth: %s", err), err } } - curAuth := AppAuthenticationStorage{Id: ""} + curAuth := shuffle.AppAuthenticationStorage{Id: ""} for _, auth := range allAuths { if auth.Id == action.AuthenticationId { curAuth = auth @@ -3410,11 +2288,11 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } if len(curAuth.Id) == 0 { - return WorkflowExecution{}, fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId), errors.New(fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId)) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId), errors.New(fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId)) } // Rebuild params with the right data. This is to prevent issues on the frontend - newParams := []WorkflowAppActionParameter{} + newParams := []shuffle.WorkflowAppActionParameter{} for _, param := range action.Parameters { for _, authparam := range curAuth.Fields { @@ -3443,7 +2321,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf // as it's not a childnode of the startnode // This is a configuration item for the workflow itself. if len(workflowExecution.Results) > 0 { - defaultResults = []ActionResult{} + defaultResults = []shuffle.ActionResult{} for _, result := range workflowExecution.Results { if result.Status == "WAITING" { result.Status = "FINISHED" @@ -3467,7 +2345,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } //log.Printf("[WARNING] Set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID) - curaction := Action{ + curaction := shuffle.Action{ AppName: action.AppName, AppVersion: action.AppVersion, Label: action.Label, @@ -3476,7 +2354,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } //action //curaction.Parameters = [] - defaultResults = append(defaultResults, ActionResult{ + defaultResults = append(defaultResults, shuffle.ActionResult{ Action: curaction, ExecutionId: workflowExecution.ExecutionId, Authorization: workflowExecution.Authorization, @@ -3510,15 +2388,15 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if !found { //log.Printf("SHOULD SET TRIGGER %s TO BE SKIPPED", trigger.ID) - curaction := Action{ - AppName: trigger.AppName, + curaction := shuffle.Action{ + AppName: "shuffle-subflow", AppVersion: trigger.AppVersion, Label: trigger.Label, Name: trigger.Name, ID: trigger.ID, } - defaultResults = append(defaultResults, ActionResult{ + defaultResults = append(defaultResults, shuffle.ActionResult{ Action: curaction, ExecutionId: workflowExecution.ExecutionId, Authorization: workflowExecution.Authorization, @@ -3528,15 +2406,15 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf Status: "SKIPPED", }) } else { - log.Printf("SHOULD KEEP TRIGGER %s", trigger.ID) + //log.Printf("SHOULD KEEP TRIGGER %s", trigger.ID) } } } //childNodes := findChildNodes(workflowExecution, workflowExecution.Start) if !startFound { - log.Printf("Startnode %s doesn't exist!", workflowExecution.Start) - return WorkflowExecution{}, fmt.Sprintf("Workflow action %s doesn't exist in workflow", workflowExecution.Start), errors.New(fmt.Sprintf(`Workflow start node "%s" doesn't exist. Exiting!`, workflowExecution.Start)) + log.Printf("[ERROR] Startnode %s doesn't exist!!", workflowExecution.Start) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow action %s doesn't exist in workflow", workflowExecution.Start), errors.New(fmt.Sprintf(`Workflow start node "%s" doesn't exist. Exiting!`, workflowExecution.Start)) } // Verification for execution environments @@ -3549,14 +2427,14 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf workflowExecution.ExecutionOrg = workflow.ExecutingOrg.Id } - var allEnvs []Environment + var allEnvs []shuffle.Environment if len(workflowExecution.ExecutionOrg) > 0 { - log.Printf("[INFO] Executing ORG: %s", workflowExecution.ExecutionOrg) + //log.Printf("[INFO] Executing ORG: %s", workflowExecution.ExecutionOrg) - allEnvironments, err := getEnvironments(ctx, workflowExecution.ExecutionOrg) + allEnvironments, err := shuffle.GetEnvironments(ctx, workflowExecution.ExecutionOrg) if err != nil { log.Printf("Failed finding environments: %s", err) - return WorkflowExecution{}, fmt.Sprintf("Workflow environments not found for this org"), errors.New(fmt.Sprintf("Workflow environments not found for this org")) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow environments not found for this org"), errors.New(fmt.Sprintf("Workflow environments not found for this org")) } for _, curenv := range allEnvironments { @@ -3568,12 +2446,12 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } } else { log.Printf("[ERROR] No org identified for execution of %s. Returning", workflowExecution.Workflow.ID) - return WorkflowExecution{}, "No org identified for execution", errors.New("No org identified for execution") + return shuffle.WorkflowExecution{}, "No org identified for execution", errors.New("No org identified for execution") } if len(allEnvs) == 0 { log.Printf("[ERROR] No active environments found for org: %s", workflowExecution.ExecutionOrg) - return WorkflowExecution{}, "No active environments found", errors.New(fmt.Sprintf("No active env found for org %s", workflowExecution.ExecutionOrg)) + return shuffle.WorkflowExecution{}, "No active environments found", errors.New(fmt.Sprintf("No active env found for org %s", workflowExecution.ExecutionOrg)) } // Check if the actions are children of the startnode? @@ -3592,7 +2470,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf onpremExecution = true } else { log.Printf("[ERROR] No handler for environment type %s", env.Type) - return WorkflowExecution{}, "No active environments found", errors.New(fmt.Sprintf("No handler for environment type %s", env.Type)) + return shuffle.WorkflowExecution{}, "No active environments found", errors.New(fmt.Sprintf("No handler for environment type %s", env.Type)) } break } @@ -3600,7 +2478,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if !found { log.Printf("[ERROR] Couldn't find environment %s. Maybe it's inactive?", action.Environment) - return WorkflowExecution{}, "Couldn't find the environment", errors.New(fmt.Sprintf("Couldn't find env %s in org %s", action.Environment, workflowExecution.ExecutionOrg)) + return shuffle.WorkflowExecution{}, "Couldn't find the environment", errors.New(fmt.Sprintf("Couldn't find env %s in org %s", action.Environment, workflowExecution.ExecutionOrg)) } found = false @@ -3625,7 +2503,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf err = imageCheckBuilder(imageNames) if err != nil { log.Printf("[ERROR] Failed building the required images from %#v: %s", imageNames, err) - return WorkflowExecution{}, "Failed building missing Docker images", err + return shuffle.WorkflowExecution{}, "Failed building missing Docker images", err } //b, err := json.Marshal(workflowExecution) @@ -3635,17 +2513,18 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf // //workflowExecution.ExecutionOrg.SyncFeatures = Org{} //} - workflowExecution.Workflow.ExecutingOrg = Org{ + workflowExecution.Workflow.ExecutingOrg = shuffle.Org{ Id: workflowExecution.Workflow.ExecutingOrg.Id, } - workflowExecution.Workflow.Org = []Org{ + workflowExecution.Workflow.Org = []shuffle.Org{ workflowExecution.Workflow.ExecutingOrg, } + //Org []Org `json:"org,omitempty" datastore:"org"` - err = setWorkflowExecution(ctx, workflowExecution, true) + err = shuffle.SetWorkflowExecution(ctx, workflowExecution, true) if err != nil { log.Printf("Error saving workflow execution for updates %s: %s", topic, err) - return WorkflowExecution{}, "Failed getting workflowexecution", err + return shuffle.WorkflowExecution{}, "Failed getting workflowexecution", err } // Adds queue for onprem execution @@ -3654,9 +2533,9 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf // FIXME - tmp name based on future companyname-companyId // This leads to issues with overlaps. Should set limits and such instead for _, environment := range environments { - log.Printf("[INFO] Execution: %s should execute onprem with execution environment \"%s\"", workflowExecution.ExecutionId, environment) + log.Printf("[INFO] Execution: %s should execute onprem with execution environment \"%s\". Workflow: %s", workflowExecution.ExecutionId, environment, workflowExecution.Workflow.ID) - executionRequest := ExecutionRequest{ + executionRequest := shuffle.ExecutionRequest{ ExecutionId: workflowExecution.ExecutionId, WorkflowId: workflowExecution.Workflow.ID, Authorization: workflowExecution.Authorization, @@ -3686,7 +2565,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if !featuresList.Workflows.Active || err != nil { log.Printf("Error: %s", err) log.Printf("[ERROR] Cloud not implemented yet. May need to work on app checking and such") - return WorkflowExecution{}, "Cloud not implemented yet", errors.New("Cloud not implemented yet") + return shuffle.WorkflowExecution{}, "Cloud not implemented yet", errors.New("Cloud not implemented yet") } // What it needs to know: @@ -3696,11 +2575,11 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf //cloudExecuteAction(workflowExecution.ExecutionId, workflowExecution.Workflow.Actions[0], workflowExecution.ExecutionOrg, workflowExecution.Workflow.ID) cloudExecuteAction(workflowExecution) - return WorkflowExecution{}, "Cloud not implemented yet (1)", errors.New("Cloud not implemented yet") + return shuffle.WorkflowExecution{}, "Cloud not implemented yet (1)", errors.New("Cloud not implemented yet") } else { // If it's here, it should be controlled by Worker. // If worker, should this backend be a proxy? I think so. - return WorkflowExecution{}, "Cloud not implemented yet (2)", errors.New("Cloud not implemented yet") + return shuffle.WorkflowExecution{}, "Cloud not implemented yet (2)", errors.New("Cloud not implemented yet") } } @@ -3713,21 +2592,21 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } // This updates stuff locally from remote executions -func cloudExecuteAction(execution WorkflowExecution) error { +func cloudExecuteAction(execution shuffle.WorkflowExecution) error { ctx := context.Background() - org, err := getOrg(ctx, execution.ExecutionOrg) + org, err := shuffle.GetOrg(ctx, execution.ExecutionOrg) if err != nil { return err } type ExecutionStruct struct { - ExecutionId string `json:"execution_id" datastore:"execution_id"` - Action Action `json:"action" datastore:"action"` - Authorization string `json:"authorization" datastore:"authorization"` - Results []ActionResult `json:"results" datastore:"results,noindex"` - ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` - WorkflowId string `json:"workflow_id" datastore:"workflow_id"` - ExecutionSource string `json:"execution_source" datastore:"execution_source"` + ExecutionId string `json:"execution_id" datastore:"execution_id"` + Action shuffle.Action `json:"action" datastore:"action"` + Authorization string `json:"authorization" datastore:"authorization"` + Results []shuffle.ActionResult `json:"results" datastore:"results,noindex"` + ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` + WorkflowId string `json:"workflow_id" datastore:"workflow_id"` + ExecutionSource string `json:"execution_source" datastore:"execution_source"` } data := ExecutionStruct{ @@ -3789,9 +2668,9 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { - log.Printf("Api authentication failed in execute workflow: %s", err) + log.Printf("[INFO] Api authentication failed in execute workflow: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -3818,7 +2697,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { //memcacheName := fmt.Sprintf("%s_%s", user.Username, fileId) ctx := context.Background() - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Failed getting the workflow locally (execute workflow): %s", err) resp.WriteHeader(401) @@ -3828,7 +2707,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { // FIXME - have a check for org etc too.. // FIXME - admin check like this? idk - if user.Id != workflow.Owner && user.Role != "admin" && user.Role != "scheduler" && user.Role != fmt.Sprintf("workflow_%s", fileId) { + if user.Id != workflow.Owner && user.Role != "scheduler" && user.Role != fmt.Sprintf("workflow_%s", fileId) { log.Printf("Wrong user (%s) for workflow %s (execute)", user.Username, workflow.ID) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) @@ -3837,7 +2716,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { log.Printf("[INFO] Starting execution of %s!", fileId) - user.ActiveOrg.Users = []User{} + user.ActiveOrg.Users = []shuffle.User{} workflow.ExecutingOrg = user.ActiveOrg workflowExecution, executionResp, err := handleExecution(fileId, *workflow, request) @@ -3857,7 +2736,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in schedule workflow: %s", err) resp.WriteHeader(401) @@ -3893,9 +2772,9 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { - log.Printf("Failed getting the workflow locally (stop schedule): %s", err) + log.Printf("[WARNING] Failed getting the workflow locally (stop schedule): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -3903,27 +2782,26 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { // FIXME - have a check for org etc too.. // FIXME - admin check like this? idk - if user.Id != workflow.Owner && user.Role != "admin" && user.Role != "scheduler" { - log.Printf("Wrong user (%s) for workflow %s (stop schedule)", user.Username, workflow.ID) + if user.Id != workflow.Owner && user.Role != "scheduler" { + log.Printf("[WARNING] Wrong user (%s) for workflow %s (stop schedule)", user.Username, workflow.ID) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } - schedule, err := getSchedule(ctx, scheduleId) + schedule, err := shuffle.GetSchedule(ctx, scheduleId) if err != nil { - log.Printf("Failed finding schedule %s", scheduleId) + log.Printf("[WARNING] Failed finding schedule %s", scheduleId) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } - log.Printf("Schedule: %#v", schedule) + //log.Printf("Schedule: %#v", schedule) if schedule.Environment == "cloud" { log.Printf("[INFO] Should STOP a cloud schedule for workflow %s with schedule ID %s", fileId, scheduleId) - // https://shuffler.io/v1/hooks/webhook_80184973-3e82-4852-842e-0290f7f34d7c - org, err := getOrg(ctx, user.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) if err != nil { log.Printf("Failed finding org %s: %s", org.Id, err) return @@ -3931,7 +2809,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { // 1. Send request to cloud // 2. Remove schedule if success - action := CloudSyncJob{ + action := shuffle.CloudSyncJob{ Type: "schedule", Action: "stop", OrgId: org.Id, @@ -3942,15 +2820,15 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { err = executeCloudAction(action, org.SyncConfig.Apikey) if err != nil { - log.Printf("Failed cloud action STOP schedule: %s", err) + log.Printf("[WARNING] Failed cloud action STOP schedule: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } else { - log.Printf("Successfully ran cloud action STOP schedule") + log.Printf("[INFO] Successfully ran cloud action STOP schedule") err = DeleteKey(ctx, "schedules", scheduleId) if err != nil { - log.Printf("Failed deleting cloud schedule onprem..") + log.Printf("[WARNING] Failed deleting cloud schedule onprem..") resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting cloud schedule"}`))) return @@ -3964,7 +2842,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { err = deleteSchedule(ctx, scheduleId) if err != nil { - log.Printf("Failed deleting schedule: %s", err) + log.Printf("[WARNING] Failed deleting schedule: %s", err) if strings.Contains(err.Error(), "Job not found") { resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) @@ -3986,7 +2864,7 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in schedule workflow: %s", err) resp.WriteHeader(401) @@ -4022,7 +2900,7 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Failed getting the workflow locally (stop schedule GCP): %s", err) resp.WriteHeader(401) @@ -4032,7 +2910,7 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { // FIXME - have a check for org etc too.. // FIXME - admin check like this? idk - if user.Id != workflow.Owner && user.Role != "admin" && user.Role != "scheduler" { + if user.Id != workflow.Owner && user.Role != "scheduler" { log.Printf("Wrong user (%s) for workflow %s (stop schedule)", user.Username, workflow.ID) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) @@ -4040,13 +2918,13 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { } if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} + workflow.Actions = []shuffle.Action{} } if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} + workflow.Branches = []shuffle.Branch{} } if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} + workflow.Triggers = []shuffle.Trigger{} } if len(workflow.Errors) == 0 { workflow.Errors = []string{} @@ -4115,7 +2993,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in schedule workflow: %s", err) resp.WriteHeader(401) @@ -4143,7 +3021,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Failed getting the workflow locally (schedule workflow): %s", err) resp.WriteHeader(401) @@ -4153,7 +3031,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { // FIXME - have a check for org etc too.. // FIXME - admin check like this? idk - if user.Id != workflow.Owner && user.Role != "admin" && user.Role != "scheduler" { + if user.Id != workflow.Owner && user.Role != "scheduler" { log.Printf("Wrong user (%s) for workflow %s", user.Username, workflow.ID) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) @@ -4161,13 +3039,13 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { } if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} + workflow.Actions = []shuffle.Action{} } if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} + workflow.Branches = []shuffle.Branch{} } if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} + workflow.Triggers = []shuffle.Trigger{} } if len(workflow.Errors) == 0 { workflow.Errors = []string{} @@ -4181,7 +3059,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { return } - var schedule Schedule + var schedule shuffle.Schedule err = json.Unmarshal(body, &schedule) if err != nil { log.Printf("Failed schedule POST unmarshaling: %s", err) @@ -4244,8 +3122,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { if schedule.Environment == "cloud" { log.Printf("[INFO] Should START a cloud schedule for workflow %s with schedule ID %s", workflow.ID, schedule.Id) - // https://shuffler.io/v1/hooks/webhook_80184973-3e82-4852-842e-0290f7f34d7c - org, err := getOrg(ctx, user.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) if err != nil { log.Printf("Failed finding org %s: %s", org.Id, err) return @@ -4255,7 +3132,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { // 2 = schedule (cron, frequency) // 3 = workflowId // 4 = execution argument - action := CloudSyncJob{ + action := shuffle.CloudSyncJob{ Type: "schedule", Action: "start", OrgId: org.Id, @@ -4326,7 +3203,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { } workflow.Schedules = append(workflow.Schedules, schedule) - err = setWorkflow(ctx, *workflow, workflow.ID) + err = shuffle.SetWorkflow(ctx, *workflow, workflow.ID) if err != nil { log.Printf("Failed setting workflow for schedule: %s", err) resp.WriteHeader(401) @@ -4339,230 +3216,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { return } -// FIXME - add to actual database etc -func getSpecificWorkflow(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in getting specific workflow: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - location := strings.Split(request.URL.String(), "/") - - var fileId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if strings.Contains(fileId, "?") { - fileId = strings.Split(fileId, "?")[0] - } - - if len(fileId) != 36 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Workflow ID when getting workflow is not valid"}`)) - return - } - - ctx := context.Background() - //memcacheName := fmt.Sprintf("%s_%s", user.Username, fileId) - //if item, err := memcache.Get(ctx, memcacheName); err == memcache.ErrCacheMiss { - // // Not in cache - // log.Printf("User %s not in cache.", memcacheName) - //} else if err != nil { - // log.Printf("Error getting item: %v", err) - //} else { - // log.Printf("Got workflow %s from cache", fileId) - // // FIXME - verify if value is ok? Can unmarshal etc. - // resp.WriteHeader(200) - // resp.Write(item.Value) - // return - //} - - workflow, err := getWorkflow(ctx, fileId) - if err != nil { - log.Printf("Workflow %s doesn't exist.", fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Item already exists."}`)) - return - } - - // CHECK orgs of user, or if user is owner - // FIXME - add org check too, and not just owner - // Check workflow.Sharing == private / public / org too - if user.Id != workflow.Owner && user.Role != "admin" { - log.Printf("Wrong user (%s) for workflow %s (get workflow)", user.Username, workflow.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} - } - if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} - } - if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} - } - if len(workflow.Errors) == 0 { - workflow.Errors = []string{} - } - - // Only required for individuals I think - //newactions := []Action{} - //for _, item := range workflow.Actions { - // item.LargeImage = "" - // item.SmallImage = "" - // newactions = append(newactions, item) - //} - //workflow.Actions = newactions - - //newtriggers := []Trigger{} - //for _, item := range workflow.Triggers { - // item.LargeImage = "" - // newtriggers = append(newtriggers, item) - //} - //workflow.Triggers = newtriggers - - body, err := json.Marshal(workflow) - if err != nil { - log.Printf("Failed workflow GET marshalling: %s", err) - resp.WriteHeader(http.StatusInternalServerError) - resp.Write([]byte(`{"success": false}`)) - return - } - - //item := &memcache.Item{ - // Key: memcacheName, - // Value: body, - // Expiration: time.Minute * 60, - //} - //if err := memcache.Add(ctx, item); err == memcache.ErrNotStored { - // if err := memcache.Set(ctx, item); err != nil { - // log.Printf("Error setting item: %v", err) - // } - //} else if err != nil { - // log.Printf("error adding item: %v", err) - //} else { - // //log.Printf("Set cache for %s", item.Key) - //} - - resp.WriteHeader(200) - resp.Write(body) -} - -func setWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecution, dbSave bool) error { - //log.Printf("\n\n\nRESULT: %s\n\n\n", workflowExecution.Status) - if len(workflowExecution.ExecutionId) == 0 { - log.Printf("Workflowexeciton executionId can't be empty.") - return errors.New("ExecutionId can't be empty.") - } - - cacheKey := fmt.Sprintf("workflowexecution-%s", workflowExecution.ExecutionId) - requestCache.Set(cacheKey, &workflowExecution, cache.DefaultExpiration) - if !dbSave && workflowExecution.Status == "EXECUTING" && len(workflowExecution.Results) > 1 { - //log.Printf("[WARNING] SHOULD skip DB saving for execution") - return nil - } - - // New struct, to not add body, author etc - key := datastore.NameKey("workflowexecution", workflowExecution.ExecutionId, nil) - if _, err := dbclient.Put(ctx, key, &workflowExecution); err != nil { - log.Printf("Error adding workflow_execution: %s", err) - return err - } - - return nil -} - -func getWorkflowExecution(ctx context.Context, id string) (*WorkflowExecution, error) { - workflowExecution := &WorkflowExecution{} - cacheKey := fmt.Sprintf("workflowexecution-%s", id) - if value, found := requestCache.Get(cacheKey); found { - parsedValue := value.(*WorkflowExecution) - //log.Printf("Found execution for id %s with %d results", parsedValue.ExecutionId, len(parsedValue.Results)) - return parsedValue, nil - - //log.Printf("[INFO] FOUND key %s with value length %d", cacheKey, len(parsedValue)) - //err := json.Unmarshal([]byte(parsedValue), &workflowExecution) - //if err == nil { - // log.Printf("SHOULD RETURN CACHED EXECUTION of length %d", len(parsedValue)) - //} else { - // log.Printf("Failed unmarshalling cached value: %s", err) - //} - } else { - //log.Printf("[ERROR] Couldn't find key %s", cacheKey) - } - - key := datastore.NameKey("workflowexecution", strings.ToLower(id), nil) - if err := dbclient.Get(ctx, key, workflowExecution); err != nil { - return &WorkflowExecution{}, err - } - - return workflowExecution, nil -} - -func getApp(ctx context.Context, id string) (*WorkflowApp, error) { - key := datastore.NameKey("workflowapp", strings.ToLower(id), nil) - workflowApp := &WorkflowApp{} - if err := dbclient.Get(ctx, key, workflowApp); err != nil { - return &WorkflowApp{}, err - - } - - return workflowApp, nil -} - -func getWorkflow(ctx context.Context, id string) (*Workflow, error) { - key := datastore.NameKey("workflow", strings.ToLower(id), nil) - workflow := &Workflow{} - if err := dbclient.Get(ctx, key, workflow); err != nil { - return &Workflow{}, err - } - - return workflow, nil -} - -func getEnvironments(ctx context.Context, orgId string) ([]Environment, error) { - var environments []Environment - q := datastore.NewQuery("Environments").Filter("org_id =", orgId) - - _, err := dbclient.GetAll(ctx, q, &environments) - if err != nil { - return []Environment{}, err - } - - return environments, nil -} - -func getAllWorkflows(ctx context.Context, orgId string) ([]Workflow, error) { - var allworkflows []Workflow - q := datastore.NewQuery("workflow").Filter("org_id = ", orgId) - - _, err := dbclient.GetAll(ctx, q, &allworkflows) - if err != nil { - return []Workflow{}, err - } - - return allworkflows, nil -} - -func setExampleresult(ctx context.Context, result AppExecutionExample) error { +func setExampleresult(ctx context.Context, result shuffle.AppExecutionExample) error { // FIXME: Reintroduce this for stats //key := datastore.NameKey("example_result", result.ExampleId, nil) @@ -4575,74 +3229,6 @@ func setExampleresult(ctx context.Context, result AppExecutionExample) error { return nil } -// Hmm, so I guess this should use uuid :( -// Consistency PLX -func setWorkflow(ctx context.Context, workflow Workflow, id string) error { - workflow.Edited = int64(time.Now().Unix()) - key := datastore.NameKey("workflow", id, nil) - - // New struct, to not add body, author etc - if _, err := dbclient.Put(ctx, key, &workflow); err != nil { - log.Printf("Error adding workflow: %s", err) - return err - } - - return nil -} - -func deleteAppAuthentication(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, userErr := handleApiAuthentication(resp, request) - if userErr != nil { - log.Printf("Api authentication failed in edit workflow: %s", userErr) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if user.Role != "admin" { - log.Printf("Need to be admin to delete appauth") - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - location := strings.Split(request.URL.String(), "/") - log.Printf("%#v", location) - var fileId string - if location[1] == "api" { - if len(location) <= 5 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[5] - } - - // FIXME: Set affected workflows to have errors - // 1. Get the auth - // 2. Loop the workflows (.Usage) and set them to have errors - // 3. Loop the nodes in workflows and do the same - - log.Printf("ID: %s", fileId) - ctx := context.Background() - err := DeleteKey(ctx, "workflowappauth", fileId) - if err != nil { - log.Printf("Failed deleting workflowapp") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting workflow app"}`))) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) -} - // FIXME: Not suitable for cloud right now :O func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) @@ -4650,7 +3236,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { return } - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Api authentication failed in edit workflow: %s", userErr) resp.WriteHeader(401) @@ -4673,7 +3259,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() log.Printf("ID: %s", fileId) - app, err := getApp(ctx, fileId) + app, err := shuffle.GetApp(ctx, fileId, user) if err != nil { log.Printf("Error getting app (delete) %s: %s", fileId, err) resp.WriteHeader(401) @@ -4685,23 +3271,23 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { // FIXME - actually delete other than private apps too.. private := false if app.Downloaded && user.Role == "admin" { - log.Printf("Deleting downloaded app (authenticated users can do this)") - } else if user.Id != app.Owner && user.Role != "admin" { - log.Printf("Wrong user (%s) for app %s (delete)", user.Username, app.Name) + log.Printf("[INFO] Deleting downloaded app (authenticated users can do this)") + } else if user.Id != app.Owner { + log.Printf("[WARNING] Wrong user (%s) for app %s (delete)", user.Username, app.Name) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } else { - log.Printf("App to be deleted is private") + log.Printf("[WARNING] App to be deleted is private") private = true } // FIXME: Make workflows track themself INSIDE apps, or with a reference q := datastore.NewQuery("workflow").Filter("org_id = ", user.ActiveOrg.Id).Limit(30) - var workflows []Workflow + var workflows []shuffle.Workflow _, err = dbclient.GetAll(ctx, q, &workflows) if err != nil { - log.Printf("Failed getting related workflows for the app: %s", err) + log.Printf("[WARNING] Failed getting related workflows for the app: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return @@ -4712,7 +3298,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { for _, workflow := range workflows { found := false - newActions := []Action{} + newActions := []shuffle.Action{} for _, action := range workflow.Actions { if action.AppName == app.Name && action.AppVersion == app.AppVersion { found = true @@ -4744,7 +3330,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { //} } - err = setWorkflow(ctx, workflow, workflow.ID) + err = shuffle.SetWorkflow(ctx, workflow, workflow.ID) if err != nil { log.Printf("Failed setting workflow when deleting app: %s", err) continue @@ -4763,7 +3349,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { // Not really deleting it, just removing from user cache if private { log.Printf("[INFO] Deleting private app") - var privateApps []WorkflowApp + var privateApps []shuffle.WorkflowApp for _, item := range user.PrivateApps { if item.ID == fileId { continue @@ -4773,7 +3359,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { } user.PrivateApps = privateApps - err = setUser(ctx, &user) + err = shuffle.SetUser(ctx, &user) if err != nil { log.Printf("[ERROR] Failed removing %s app for user %s: %s", app.Name, user.Username, err) resp.WriteHeader(401) @@ -4796,9 +3382,9 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { log.Printf("Failed to increase total apps loaded stats: %s", err) } cacheKey := fmt.Sprintf("workflowapps-sorted-100") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) //err = memcache.Delete(request.Context(), sessionToken) resp.WriteHeader(200) @@ -4825,7 +3411,7 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { fileId = location[4] } - app, err := getApp(ctx, fileId) + app, err := shuffle.GetApp(ctx, fileId, shuffle.User{}) if err != nil { log.Printf("[WARNING] Error getting app %s (app config): %s", fileId, err) resp.WriteHeader(401) @@ -4870,7 +3456,7 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { return } - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("[WARNING] Api authentication failed in get app: %s", userErr) resp.WriteHeader(401) @@ -4878,7 +3464,7 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { return } - if user.Id != app.Owner && user.Role != "admin" { + if user.Id != app.Owner { log.Printf("[WARNING] Wrong user (%s) for app %s", user.Username, app.Name) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) @@ -4912,506 +3498,6 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { resp.Write(data) } -func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, userErr := handleApiAuthentication(resp, request) - if userErr != nil { - log.Printf("Api authentication failed in get all apps: %s", userErr) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if user.Role != "admin" { - log.Printf("[WARNING] User isn't admin during auth edit config") - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Must be admin to perform this action"}`))) - return - } - - var fileId string - location := strings.Split(request.URL.String(), "/") - if location[1] == "api" { - if len(location) <= 5 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[5] - } - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Error with body read: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - type configAuth struct { - Id string `json:"id"` - Action string `json:"action"` - } - - var config configAuth - err = json.Unmarshal(body, &config) - if err != nil { - log.Printf("Failed unmarshaling (appauth): %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if config.Id != fileId { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Bad ID match"}`)) - return - } - - ctx := context.Background() - auth, err := getWorkflowAppAuthDatastore(ctx, fileId) - if err != nil { - log.Printf("Authget error: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": ":("}`)) - return - } - - if auth.OrgId != user.ActiveOrg.Id { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "User can't edit this org"}`)) - return - } - - if config.Action == "assign_everywhere" { - log.Printf("Should set authentication config") - q := datastore.NewQuery("workflow").Filter("org_id =", user.ActiveOrg.Id) - q = q.Order("-edited").Limit(35) - - var workflows []Workflow - _, err = dbclient.GetAll(ctx, q, &workflows) - if err != nil { - log.Printf("Getall error in auth update: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed getting workflows to update"}`)) - return - } - - // FIXME: Add function to remove auth from other auth's - actionCnt := 0 - workflowCnt := 0 - authenticationUsage := []AuthenticationUsage{} - for _, workflow := range workflows { - newActions := []Action{} - edited := false - usage := AuthenticationUsage{ - WorkflowId: workflow.ID, - Nodes: []string{}, - } - - for _, action := range workflow.Actions { - if action.AppName == auth.App.Name { - action.AuthenticationId = auth.Id - - edited = true - actionCnt += 1 - usage.Nodes = append(usage.Nodes, action.ID) - } - - newActions = append(newActions, action) - } - - workflow.Actions = newActions - if edited { - //auth.Usage = usage - authenticationUsage = append(authenticationUsage, usage) - err = setWorkflow(ctx, workflow, workflow.ID) - if err != nil { - log.Printf("Failed setting (authupdate) workflow: %s", err) - continue - } - - workflowCnt += 1 - } - } - - //Usage []AuthenticationUsage `json:"usage" datastore:"usage"` - log.Printf("[INFO] Found %d workflows, %d actions", workflowCnt, actionCnt) - if actionCnt > 0 && workflowCnt > 0 { - auth.WorkflowCount = int64(workflowCnt) - auth.NodeCount = int64(actionCnt) - auth.Usage = authenticationUsage - auth.Defined = true - - err = setWorkflowAppAuthDatastore(ctx, *auth, auth.Id) - if err != nil { - log.Printf("Failed setting appauth: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed setting app auth for all workflows"}`)) - return - } else { - // FIXME: Remove ALL workflows from other auths using the same - } - } - } - - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) - //var config configAuth - - //log.Printf("Should set %s -} - -func addAppAuthentication(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, userErr := handleApiAuthentication(resp, request) - if userErr != nil { - log.Printf("Api authentication failed in get all apps: %s", userErr) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Error with body read: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - var appAuth AppAuthenticationStorage - err = json.Unmarshal(body, &appAuth) - if err != nil { - log.Printf("Failed unmarshaling (appauth): %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - ctx := context.Background() - if len(appAuth.Id) == 0 { - appAuth.Id = uuid.NewV4().String() - } else { - auth, err := getWorkflowAppAuthDatastore(ctx, appAuth.Id) - if err == nil { - // OrgId string `json:"org_id" datastore:"org_id"` - if auth.OrgId != user.ActiveOrg.Id { - log.Printf("[WARNING] User isn't a part of the right org during auth edit") - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": ":("}`))) - return - } - - if user.Role != "admin" { - log.Printf("[WARNING] User isn't admin during auth edit") - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": ":("}`))) - return - } - - if !auth.Active { - log.Printf("[WARNING] Auth isn't active for edit") - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't update an inactive auth"}`))) - return - } - - if auth.App.Name != appAuth.App.Name { - log.Printf("[WARNING] User tried to modify auth") - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad app configuration: need to specify correct name"}`))) - return - } - } - } - - if len(appAuth.Label) == 0 { - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Label can't be empty"}`))) - return - } - - // Super basic check - if len(appAuth.App.ID) != 36 && len(appAuth.App.ID) != 32 { - log.Printf("Bad ID for app: %s", appAuth.App.ID) - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App has to be defined"}`))) - return - } - - // FIXME: Doens't validate Org - app, err := getApp(ctx, appAuth.App.ID) - if err != nil { - log.Printf("[WARNING] Failed finding app %s while setting auth. Finding it by looping apps.", appAuth.App.ID) - workflowapps, err := getAllWorkflowApps(ctx, 500) - if err != nil { - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - - foundIndex := -1 - for i, workflowapp := range workflowapps { - if workflowapp.Name == appAuth.App.Name { - foundIndex = i - break - } - } - - if foundIndex >= 0 { - log.Printf("[INFO] Found app %s by looping auth", appAuth.App.ID) - } else { - log.Printf("[ERROR] Failed finding app %s which has auth after looping", appAuth.App.ID) - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - } - - // Check if the items are correct - for _, field := range appAuth.Fields { - found := false - for _, param := range app.Authentication.Parameters { - if field.Key == param.Name { - found = true - } - } - - if !found { - log.Printf("Failed finding field %s in appauth fields", field.Key) - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "All auth fields required"}`))) - return - } - } - - //appAuth.LargeImage = "" - appAuth.OrgId = user.ActiveOrg.Id - appAuth.Defined = true - err = setWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id) - if err != nil { - log.Printf("Failed setting up app auth %s: %s", appAuth.Id, err) - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) -} - -func getAppAuthentication(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, userErr := handleApiAuthentication(resp, request) - if userErr != nil { - log.Printf("Api authentication failed in get all apps: %s", userErr) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // FIXME: Auth to get the right ones only - //if user.Role != "admin" { - // log.Printf("User isn't admin") - // resp.WriteHeader(401) - // resp.Write([]byte(`{"success": false}`)) - // return - //} - ctx := context.Background() - allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) - if err != nil { - log.Printf("Api authentication failed in get all app auth: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if len(allAuths) == 0 { - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true, "data": []}`)) - return - } - - // Cleanup for frontend - newAuth := []AppAuthenticationStorage{} - for _, auth := range allAuths { - newAuthField := auth - for index, _ := range auth.Fields { - newAuthField.Fields[index].Value = "auth placeholder (replaced during execution)" - } - - newAuth = append(newAuth, newAuthField) - } - - newbody, err := json.Marshal(allAuths) - if err != nil { - log.Printf("Failed unmarshalling all app auths: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflow app auth"}`))) - return - } - - data := fmt.Sprintf(`{"success": true, "data": %s}`, string(newbody)) - - resp.WriteHeader(200) - resp.Write([]byte(data)) - - /* - data := `{ - "success": true, - "data": [ - { - "app": { - "name": "thehive", - "description": "what", - "app_version": "1.0.0", - "id": "4f97da9d-1caf-41cc-aa13-67104d8d825c", - "large_image": "asd" - }, - "fields": { - "apikey": "hello", - "url": "url" - }, - "usage": [{ - "workflow_id": "asd", - "nodes": [{ - "node_id": "" - }] - }], - "label": "Original", - "id": "4f97da9d-1caf-41cc-aa13-67104d8d825d", - "active": true - }, - { - "app": { - "name": "thehive", - "description": "what", - "app_version": "1.0.0", - "id": "4f97da9d-1caf-41cc-aa13-67104d8d825c", - "large_image": "asd" - }, - "fields": { - "apikey": "hello", - "url": "url" - }, - "usage": [{ - "workflow_id": "asd", - "nodes": [{ - "node_id": "" - }] - }], - "label": "Number 2", - "id": "4f97da9d-1caf-41cc-aa13-67104d8d825d", - "active": true - } - ] - }` - */ -} -func updateWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, userErr := handleApiAuthentication(resp, request) - if userErr != nil { - log.Printf("Api authentication failed in get all apps: %s", userErr) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - location := strings.Split(request.URL.String(), "/") - var fileId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - ctx := context.Background() - app, err := getApp(ctx, fileId) - if err != nil { - log.Printf("Error getting app (update app): %s", fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if user.Id != app.Owner && user.Role != "admin" { - log.Printf("Wrong user (%s) for app %s in update app", user.Username, app.Name) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Error with body read in update app: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - type updatefields struct { - Sharing bool `json:"sharing"` - SharingConfig string `json:"sharing_config"` - } - - var tmpfields updatefields - err = json.Unmarshal(body, &tmpfields) - if err != nil { - log.Printf("Error with unmarshal body in update app: %s\n%s", err, string(body)) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if tmpfields.Sharing != app.Sharing { - app.Sharing = tmpfields.Sharing - } - - if tmpfields.SharingConfig != app.SharingConfig { - app.SharingConfig = tmpfields.SharingConfig - } - - err = setWorkflowAppDatastore(ctx, *app, app.ID) - if err != nil { - log.Printf("Failed patching workflowapp: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - cacheKey := fmt.Sprintf("workflowapps-sorted-100") - requestCache.Delete(cacheKey) - cacheKey = fmt.Sprintf("workflowapps-sorted-500") - requestCache.Delete(cacheKey) - - log.Printf("Changed workflow app %s", app.ID) - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) -} - func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -5425,7 +3511,7 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() // Just need to be logged in // FIXME - need to be logged in? - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Continuing with apps even without auth") //log.Printf("Api authentication failed in get all apps: %s", userErr) @@ -5462,7 +3548,7 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { // return //} - workflowapps, err := getAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting apps (getworkflowapps): %s", err) resp.WriteHeader(401) @@ -5575,7 +3661,7 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { // Bad check for workflowapps :) // FIXME - use tags and struct reflection -func checkWorkflowApp(workflowApp WorkflowApp) error { +func checkWorkflowApp(workflowApp shuffle.WorkflowApp) error { // Validate fields if workflowApp.Name == "" { return errors.New("App field name doesn't exist") @@ -5623,7 +3709,7 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) { // Just need to be logged in // FIXME - should have some permissions? - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new app: %s", err) resp.WriteHeader(401) @@ -5655,7 +3741,7 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) { // FIXME - continue the search here with github repos etc. // Caching might be smart :D ctx := context.Background() - workflowapps, err := getAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Error: Failed getting workflowapps: %s", err) resp.WriteHeader(401) @@ -5663,7 +3749,7 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) { return } - returnValues := []WorkflowApp{} + returnValues := []shuffle.WorkflowApp{} search := strings.ToLower(tmpBody.Search) for _, app := range workflowapps { if !app.Activated && app.Generated { @@ -5698,7 +3784,7 @@ func validateAppInput(resp http.ResponseWriter, request *http.Request) { // Just need to be logged in // FIXME - should have some permissions? - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new app: %s", err) resp.WriteHeader(401) @@ -5957,7 +4043,7 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { // Just need to be logged in // FIXME - should have some permissions? - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in load apps: %s", err) resp.WriteHeader(401) @@ -6017,11 +4103,15 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("[INFO] Starting app hotloading") + ctx := context.Background() + cacheKey := fmt.Sprintf("workflowapps-sorted-100") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") + shuffle.DeleteCache(ctx, cacheKey) // Just need to be logged in // FIXME - should have some permissions? - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in app hotload: %s", err) resp.WriteHeader(401) @@ -6042,15 +4132,20 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("[INFO] Hotloading from %s", location) - err = handleAppHotload(location, true) + log.Printf("[INFO] Starting hotloading from %s", location) + err = handleAppHotload(ctx, location, true) if err != nil { - log.Printf("Failed app hotload: %s", err) + log.Printf("[WARNING] Failed app hotload: %s", err) resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed loading apps: %s"}`, err))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } + cacheKey = fmt.Sprintf("workflowapps-sorted-100") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") + shuffle.DeleteCache(ctx, cacheKey) + resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } @@ -6063,7 +4158,7 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { // Just need to be logged in // FIXME - should have some permissions? - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in load specific apps: %s", err) resp.WriteHeader(401) @@ -6139,6 +4234,20 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { } else { log.Printf("Updating apps with updates") } + + if tmpBody.ForceUpdate { + ctx := context.Background() + dockercli, err := client.NewEnvClient() + if err == nil { + _, err := dockercli.ImagePull(ctx, "frikky/shuffle:app_sdk", types.ImagePullOptions{}) + if err != nil { + log.Printf("[WARNING] Failed to download apps with the new App SDK: %s", err) + } + } else { + log.Printf("[WARNING] Failed to download apps with the new App SDK because of docker cli: %s", err) + } + } + iterateAppGithubFolders(fs, dir, "", "", tmpBody.ForceUpdate) } else if strings.Contains(tmpBody.URL, "s3") { @@ -6163,10 +4272,11 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { return } + ctx := context.Background() cacheKey := fmt.Sprintf("workflowapps-sorted-100") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) @@ -6175,7 +4285,7 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error { ctx := context.Background() - workflowapps, err := getAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) appCounter := 0 if err != nil { log.Printf("Failed to get existing generated apps") @@ -6218,7 +4328,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, } if contOuter { - log.Printf("Skipping %s", filename) + //log.Printf("Skipping %s", filename) continue } @@ -6262,7 +4372,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, } //log.Printf("Should generate yaml") - swagger, api, _, err := generateYaml(swagger, parsedOpenApi.ID) + swagger, api, _, err := shuffle.GenerateYaml(swagger, parsedOpenApi.ID) if err != nil { log.Printf("Failed building and generating yaml in loop (%s): %s", filename, err) continue @@ -6287,7 +4397,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, } if !found { - err = setWorkflowAppDatastore(ctx, api, api.ID) + err = shuffle.SetWorkflowAppDatastore(ctx, api, api.ID) if err != nil { log.Printf("Failed setting workflowapp in loop: %s", err) continue @@ -6303,9 +4413,9 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, } cacheKey := fmt.Sprintf("workflowapps-sorted-100") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) } } else { //log.Printf("Skipped upload of %s (%s)", api.Name, api.ID) @@ -6326,9 +4436,25 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, // Onlyname is used to func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname, userId, orgId string) error { var err error + secondsOffset := 0 + // sort file names + filenames := []string{} for _, file := range dir { - if len(onlyname) > 0 && file.Name() != onlyname { + filename := file.Name() + filenames = append(filenames, filename) + } + sort.Strings(filenames) + + // iterate through sorted filenames + for _, filename := range filenames { + secondsOffset -= 10 + if len(onlyname) > 0 && filename != onlyname { + continue + } + + file, err := fs.Stat(filename) + if err != nil { continue } @@ -6349,7 +4475,6 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra } case mode.IsRegular(): // Check the file - filename := file.Name() if strings.HasSuffix(filename, ".json") { path := fmt.Sprintf("%s%s", extra, file.Name()) fileReader, err := fs.Open(path) @@ -6364,7 +4489,7 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra continue } - var workflow Workflow + var workflow shuffle.Workflow err = json.Unmarshal(readFile, &workflow) if err != nil { continue @@ -6377,11 +4502,11 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra workflow.ID = uuid.NewV4().String() workflow.OrgId = orgId - workflow.ExecutingOrg = Org{ + workflow.ExecutingOrg = shuffle.Org{ Id: orgId, } - workflow.Org = append(workflow.Org, Org{ + workflow.Org = append(workflow.Org, shuffle.Org{ Id: orgId, }) workflow.IsValid = false @@ -6402,8 +4527,9 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra } */ + log.Printf("Import workflow from file: %s", filename) ctx := context.Background() - err = setWorkflow(ctx, workflow, workflow.ID) + err = shuffle.SetWorkflow(ctx, workflow, workflow.ID, secondsOffset) if err != nil { log.Printf("Failed setting (download) workflow: %s", err) continue @@ -6427,16 +4553,18 @@ type buildLaterStruct struct { func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) ([]buildLaterStruct, []buildLaterStruct, error) { var err error - allapps := []WorkflowApp{} + allapps := []shuffle.WorkflowApp{} + + // These are slow apps to build with some funky mechanisms reservedNames := []string{ "OWA", "NLP", + "YARA", } + // It's here to prevent getting them in every iteration buildLaterFirst := []buildLaterStruct{} buildLaterList := []buildLaterStruct{} - - // It's here to prevent getting them in every iteration ctx := context.Background() for _, file := range dir { if len(onlyname) > 0 && file.Name() != onlyname { @@ -6455,10 +4583,6 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin // Go routine? Hmm, this can be super quick I guess buildFirst, buildLast, err := iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate) - if err != nil { - log.Printf("Error reading folder: %s", err) - continue - } for _, item := range buildFirst { buildLaterFirst = append(buildLaterFirst, item) @@ -6468,6 +4592,15 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin buildLaterList = append(buildLaterList, item) } + if err != nil { + log.Printf("[WARNING] Error reading folder: %s", err) + //buildFirst, buildLast, err := iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate) + + if !forceUpdate { + return buildLaterFirst, buildLaterList, err + } + } + case mode.IsRegular(): // Check the file filename := file.Name() @@ -6534,11 +4667,12 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin combined = append(combined, dockerfileData...) md5 := md5sum(combined) - var workflowapp WorkflowApp + var workflowapp shuffle.WorkflowApp err = gyaml.Unmarshal(appfileData, &workflowapp) if err != nil { - log.Printf("Failed unmarshaling workflowapp %s: %s", fullPath, err) - continue + log.Printf("[WARNING] Failed building workflowapp %s: %s", extra, err) + return buildLaterFirst, buildLaterList, errors.New(fmt.Sprintf("Failed building %s: %s", extra, err)) + //continue } newName := workflowapp.Name @@ -6549,7 +4683,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } if len(allapps) == 0 { - allapps, err = getAllWorkflowApps(ctx, 500) + allapps, err = shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting apps to verify: %s", err) continue @@ -6580,7 +4714,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin // Fixes (appends) authentication parameters if they're required if workflowapp.Authentication.Required { - log.Printf("Checking authentication fields and appending for %s!", workflowapp.Name) + log.Printf("[INFO] Checking authentication fields and appending for %s!", workflowapp.Name) // FIXME: // Might require reflection into the python code to append the fields as well for index, action := range workflowapp.Actions { @@ -6593,7 +4727,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin // 2. Check if they're present in the action // 3. Add them IF they DONT exist // 4. Fix python code with reflection (FIXME) - appendParams := []WorkflowAppActionParameter{} + appendParams := []shuffle.WorkflowAppActionParameter{} for _, fieldname := range workflowapp.Authentication.Parameters { found := false for index, param := range action.Parameters { @@ -6607,7 +4741,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } if !found { - appendParams = append(appendParams, WorkflowAppActionParameter{ + appendParams = append(appendParams, shuffle.WorkflowAppActionParameter{ Name: fieldname.Name, Description: fieldname.Description, Example: fieldname.Example, @@ -6634,7 +4768,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin if len(removeApps) > 0 { for _, item := range removeApps { - log.Printf("[WARNING] Removing duplicate: %s", item) + log.Printf("[WARNING] Removing duplicate app: %s", item) err = DeleteKey(ctx, "workflowapp", item) if err != nil { log.Printf("[ERROR] Failed deleting duplicate %s: %s", item, err) @@ -6648,22 +4782,25 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin workflowapp.Sharing = true workflowapp.Downloaded = true workflowapp.Hash = md5 + workflowapp.Public = true - err = setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) + err = shuffle.SetWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err != nil { log.Printf("Failed setting workflowapp: %s", err) continue } - err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1, "") - if err != nil { - log.Printf("Failed to increase total apps created stats: %s", err) - } + /* + err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1, "") + if err != nil { + log.Printf("Failed to increase total apps created stats: %s", err) + } - err = increaseStatisticsField(ctx, "total_apps_loaded", workflowapp.ID, 1, "") - if err != nil { - log.Printf("Failed to increase total apps loaded stats: %s", err) - } + err = increaseStatisticsField(ctx, "total_apps_loaded", workflowapp.ID, 1, "") + if err != nil { + log.Printf("Failed to increase total apps loaded stats: %s", err) + } + */ //log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion) @@ -6698,6 +4835,12 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin return buildLaterFirst, buildLaterList, err } + // This is getting silly + cacheKey := fmt.Sprintf("workflowapps-sorted-100") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") + shuffle.DeleteCache(ctx, cacheKey) + //log.Printf("BUILDLATERFIRST: %d, BUILDLATERLIST: %d", len(buildLaterFirst), len(buildLaterList)) if len(extra) == 0 { log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst)) @@ -6708,6 +4851,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } else { if len(item.Tags) > 0 { log.Printf("[INFO] Successfully built image %s", item.Tags[0]) + } else { log.Printf("[INFO] Successfully built Docker image") } @@ -6740,7 +4884,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { // Just need to be logged in // FIXME - should have some permissions? - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new app: %s", err) resp.WriteHeader(401) @@ -6756,7 +4900,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { return } - var workflowapp WorkflowApp + var workflowapp shuffle.WorkflowApp err = json.Unmarshal(body, &workflowapp) if err != nil { log.Printf("Failed unmarshaling: %s", err) @@ -6766,7 +4910,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - allapps, err := getAllWorkflowApps(ctx, 500) + allapps, err := shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting apps to verify: %s", err) resp.WriteHeader(401) @@ -6807,7 +4951,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { workflowapp.Generated = false workflowapp.Activated = true - err = setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) + err = shuffle.SetWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err != nil { log.Printf("Failed setting workflowapp: %s", err) resp.WriteHeader(401) @@ -6817,11 +4961,10 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion) } - //memcache.Delete(ctx, "all_apps") cacheKey := fmt.Sprintf("workflowapps-sorted-100") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) @@ -6833,7 +4976,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in getting specific workflow: %s", err) resp.WriteHeader(401) @@ -6861,7 +5004,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Failed getting the workflow %s locally (get executions): %s", fileId, err) resp.WriteHeader(401) @@ -6870,7 +5013,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { } // FIXME - have a check for org etc too.. - if user.Id != workflow.Owner && user.Role != "admin" { + if user.Id != workflow.Owner { log.Printf("Wrong user (%s) for workflow %s (get execution)", user.Username, workflow.ID) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) @@ -6880,7 +5023,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { // Query for the specifci workflowId maxAmount := 30 q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(maxAmount) - var workflowExecutions []WorkflowExecution + var workflowExecutions []shuffle.WorkflowExecution _, err = dbclient.GetAll(ctx, q, &workflowExecutions) if err != nil { @@ -6901,7 +5044,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { it := dbclient.Run(ctx, q) //_, err = it.Next(&app) for { - var workflowExecution WorkflowExecution + var workflowExecution shuffle.WorkflowExecution _, err := it.Next(&workflowExecution) if err != nil { break @@ -6934,8 +5077,12 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { } if err != nil { - log.Printf("Cursorerror: %s", err) - break + if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { + log.Printf("[WARNING] Cursorerror in app grab WARNING: %s", err) + } else { + log.Printf("[ERROR] Cursorerror in app grab: %s", err) + break + } } else { //log.Printf("NEXTCURSOR: %s", nextCursor) nextStr := fmt.Sprintf("%s", nextCursor) @@ -6991,169 +5138,169 @@ func getAllSchedules(ctx context.Context, orgId string) ([]ScheduleOld, error) { } //FIXME: Add cursor -func getAllWorkflowApps(ctx context.Context, maxLen int) ([]WorkflowApp, error) { - var apps []WorkflowApp - query := datastore.NewQuery("workflowapp").Order("-edited").Limit(20) - //query := datastore.NewQuery("workflowapp").Order("-edited").Limit(40) +//func shuffle.GetAllWorkflowApps(ctx context.Context, maxLen int) ([]shuffle.WorkflowApp, error) { +// var apps []WorkflowApp +// query := datastore.NewQuery("workflowapp").Order("-edited").Limit(10) +// //query := datastore.NewQuery("workflowapp").Order("-edited").Limit(40) +// +// cacheKey := fmt.Sprintf("workflowapps-sorted-%d", maxLen) +// if value, found := requestCache.Get(cacheKey); found { +// parsedValue := value.(*[]WorkflowApp) +// log.Printf("[INFO] Returning %d apps from cache", len(*parsedValue)) +// return *parsedValue, nil +// } +// +// cursorStr := "" +// +// // NOT BEING UPDATED +// // FIXME: Update the app with the correct actions. HOW DOES THIS WORK?? +// // Seems like only actions are wrong. Could get the app individually. +// // Guessing it's a memory issue. +// //Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` +// //errors.New(nil) +// var err error +// for { +// it := dbclient.Run(ctx, query) +// //_, err = it.Next(&app) +// for { +// var app WorkflowApp +// _, err := it.Next(&app) +// if err != nil { +// break +// } +// +// if app.Name == "Shuffle Subflow" { +// continue +// } +// +// found := false +// //log.Printf("ACTIONS: %d - %s", len(app.Actions), app.Name) +// for _, innerapp := range apps { +// if innerapp.Name == app.Name { +// found = true +// break +// } +// } +// +// if !found { +// apps = append(apps, app) +// } +// } +// +// if err != iterator.Done { +// //log.Printf("[INFO] Failed fetching results: %v", err) +// //break +// } +// +// // Get the cursor for the next page of results. +// nextCursor, err := it.Cursor() +// if err != nil { +// log.Printf("Cursorerror: %s", err) +// break +// } else { +// //log.Printf("NEXTCURSOR: %s", nextCursor) +// nextStr := fmt.Sprintf("%s", nextCursor) +// if cursorStr == nextStr { +// break +// } +// +// cursorStr = nextStr +// query = query.Start(nextCursor) +// //cursorStr = nextCursor +// //break +// } +// +// if len(apps) > maxLen { +// break +// } +// } +// +// if len(apps) > 20 { +// log.Printf("[INFO] Setting %d apps in cache", len(apps)) +// requestCache.Set(cacheKey, &apps, cache.DefaultExpiration) +// } +// +// //var allworkflowapps []WorkflowApp +// //_, err := dbclient.GetAll(ctx, query, &allworkflowapps) +// //if err != nil { +// // if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { +// // //datastore.NewQuery("workflowapp").Limit(30).Order("-edited") +// // query = datastore.NewQuery("workflowapp").Order("-edited").Limit(25) +// // //q := q.Limit(25) +// // _, err := dbclient.GetAll(ctx, query, &allworkflowapps) +// // if err != nil { +// // return []WorkflowApp{}, err +// // } +// // } else { +// // return []WorkflowApp{}, err +// // } +// //} +// +// return apps, nil +//} - cacheKey := fmt.Sprintf("workflowapps-sorted-%d", maxLen) - if value, found := requestCache.Get(cacheKey); found { - parsedValue := value.(*[]WorkflowApp) - log.Printf("[INFO] Returning %d apps from cache", len(*parsedValue)) - return *parsedValue, nil - } - - cursorStr := "" - - // NOT BEING UPDATED - // FIXME: Update the app with the correct actions. HOW DOES THIS WORK?? - // Seems like only actions are wrong. Could get the app individually. - // Guessing it's a memory issue. - //Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` - //errors.New(nil) - var err error - for { - it := dbclient.Run(ctx, query) - //_, err = it.Next(&app) - for { - var app WorkflowApp - _, err := it.Next(&app) - if err != nil { - break - } - - if app.Name == "Shuffle Subflow" { - continue - } - - found := false - //log.Printf("ACTIONS: %d - %s", len(app.Actions), app.Name) - for _, innerapp := range apps { - if innerapp.Name == app.Name { - found = true - break - } - } - - if !found { - apps = append(apps, app) - } - } - - if err != iterator.Done { - //log.Printf("[INFO] Failed fetching results: %v", err) - //break - } - - // Get the cursor for the next page of results. - nextCursor, err := it.Cursor() - if err != nil { - log.Printf("Cursorerror: %s", err) - break - } else { - //log.Printf("NEXTCURSOR: %s", nextCursor) - nextStr := fmt.Sprintf("%s", nextCursor) - if cursorStr == nextStr { - break - } - - cursorStr = nextStr - query = query.Start(nextCursor) - //cursorStr = nextCursor - //break - } - - if len(apps) > maxLen { - break - } - } - - if len(apps) > 20 { - log.Printf("[INFO] Setting %d apps in cache", len(apps)) - requestCache.Set(cacheKey, &apps, cache.DefaultExpiration) - } - - //var allworkflowapps []WorkflowApp - //_, err := dbclient.GetAll(ctx, query, &allworkflowapps) - //if err != nil { - // if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { - // //datastore.NewQuery("workflowapp").Limit(30).Order("-edited") - // query = datastore.NewQuery("workflowapp").Order("-edited").Limit(25) - // //q := q.Limit(25) - // _, err := dbclient.GetAll(ctx, query, &allworkflowapps) - // if err != nil { - // return []WorkflowApp{}, err - // } - // } else { - // return []WorkflowApp{}, err - // } - //} - - return apps, nil -} - -func getAllWorkflowAppAuth(ctx context.Context, OrgId string) ([]AppAuthenticationStorage, error) { - var allworkflowapps []AppAuthenticationStorage - q := datastore.NewQuery("workflowappauth").Filter("org_id = ", OrgId) - - _, err := dbclient.GetAll(ctx, q, &allworkflowapps) - if err != nil { - return []AppAuthenticationStorage{}, err - } - - return allworkflowapps, nil -} - -func getWorkflowAppAuthDatastore(ctx context.Context, id string) (*AppAuthenticationStorage, error) { - - key := datastore.NameKey("workflowappauth", id, nil) - appAuth := &AppAuthenticationStorage{} - // New struct, to not add body, author etc - if err := dbclient.Get(ctx, key, appAuth); err != nil { - return &AppAuthenticationStorage{}, err - } - - return appAuth, nil -} - -func setWorkflowAppAuthDatastore(ctx context.Context, workflowappauth AppAuthenticationStorage, id string) error { - timeNow := int64(time.Now().Unix()) - if workflowappauth.Created == 0 { - workflowappauth.Created = timeNow - } - - workflowappauth.Edited = timeNow - - key := datastore.NameKey("workflowappauth", id, nil) - - // New struct, to not add body, author etc - if _, err := dbclient.Put(ctx, key, &workflowappauth); err != nil { - log.Printf("Error adding workflow app auth: %s", err) - return err - } - - return nil -} - -// Hmm, so I guess this should use uuid :( -// Consistency PLX -func setWorkflowAppDatastore(ctx context.Context, workflowapp WorkflowApp, id string) error { - timeNow := int64(time.Now().Unix()) - if workflowapp.Created == 0 { - workflowapp.Created = timeNow - } - - workflowapp.Edited = timeNow - key := datastore.NameKey("workflowapp", id, nil) - - // New struct, to not add body, author etc - if _, err := dbclient.Put(ctx, key, &workflowapp); err != nil { - log.Printf("Error adding workflow app: %s", err) - return err - } - - return nil -} +//func shuffle.GetAllWorkflowAppAuth(ctx context.Context, OrgId string) ([]shuffle.AppAuthenticationStorage, error) { +// var allworkflowapps []AppAuthenticationStorage +// q := datastore.NewQuery("workflowappauth").Filter("org_id = ", OrgId) +// +// _, err := dbclient.GetAll(ctx, q, &allworkflowapps) +// if err != nil { +// return []AppAuthenticationStorage{}, err +// } +// +// return allworkflowapps, nil +//} +// +//func getWorkflowAppAuthDatastore(ctx context.Context, id string) (*AppAuthenticationStorage, error) { +// +// key := datastore.NameKey("workflowappauth", id, nil) +// appAuth := &AppAuthenticationStorage{} +// // New struct, to not add body, author etc +// if err := dbclient.Get(ctx, key, appAuth); err != nil { +// return &AppAuthenticationStorage{}, err +// } +// +// return appAuth, nil +//} +// +//func shuffle.SetWorkflowAppAuthDatastore(ctx context.Context, workflowappauth AppAuthenticationStorage, id string) error { +// timeNow := int64(time.Now().Unix()) +// if workflowappauth.Created == 0 { +// workflowappauth.Created = timeNow +// } +// +// workflowappauth.Edited = timeNow +// +// key := datastore.NameKey("workflowappauth", id, nil) +// +// // New struct, to not add body, author etc +// if _, err := dbclient.Put(ctx, key, &workflowappauth); err != nil { +// log.Printf("Error adding workflow app auth: %s", err) +// return err +// } +// +// return nil +//} +// +//// Hmm, so I guess this should use uuid :( +//// Consistency PLX +//func SetWorkflowAppDatastore(ctx context.Context, workflowapp WorkflowApp, id string) error { +// timeNow := int64(time.Now().Unix()) +// if workflowapp.Created == 0 { +// workflowapp.Created = timeNow +// } +// +// workflowapp.Edited = timeNow +// key := datastore.NameKey("workflowapp", id, nil) +// +// // New struct, to not add body, author etc +// if _, err := dbclient.Put(ctx, key, &workflowapp); err != nil { +// log.Printf("Error adding workflow app: %s", err) +// return err +// } +// +// return nil +//} // Starts a new webhook func handleStopHook(resp http.ResponseWriter, request *http.Request) { @@ -7162,7 +5309,7 @@ func handleStopHook(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -7198,7 +5345,7 @@ func handleStopHook(resp http.ResponseWriter, request *http.Request) { return } - if user.Id != hook.Owner && user.Role != "admin" { + if user.Id != hook.Owner { log.Printf("Wrong user (%s) for workflow %s", user.Username, hook.Id) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) @@ -7239,123 +5386,6 @@ func handleStopHook(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": true, "reason": "Stopped webhook"}`)) } -func handleDeleteHook(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - location := strings.Split(request.URL.String(), "/") - - var fileId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if len(fileId) != 36 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Workflow ID when deleting hook is not valid"}`)) - return - } - - ctx := context.Background() - hook, err := getHook(ctx, fileId) - if err != nil { - log.Printf("Failed getting hook %s (delete): %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if user.Id != hook.Owner && user.Role != "admin" && user.ActiveOrg.Id != hook.OrgId { - log.Printf("Wrong user (%s) for workflow %s", user.Username, hook.Id) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if len(hook.Workflows) > 0 { - //err = increaseStatisticsField(ctx, "total_workflow_triggers", hook.Workflows[0], -1, user.ActiveOrg.Id) - //if err != nil { - // log.Printf("Failed to increase total workflows: %s", err) - //} - } - - hook.Status = "stopped" - err = setHook(ctx, *hook) - if err != nil { - log.Printf("Failed setting hook: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Printf("Hook: %#v", hook) - if hook.Environment == "cloud" { - log.Printf("[INFO] Should STOP cloud webhook https://shuffler.io/api/v1/hooks/webhook_%s", hook.Id) - org, err := getOrg(ctx, user.ActiveOrg.Id) - if err != nil { - log.Printf("Failed finding org %s: %s", org.Id, err) - return - } - - action := CloudSyncJob{ - Type: "webhook", - Action: "stop", - OrgId: org.Id, - PrimaryItemId: hook.Id, - } - - if len(hook.Workflows) > 0 { - action.SecondaryItem = hook.Workflows[0] - } - - err = executeCloudAction(action, org.SyncConfig.Apikey) - if err != nil { - log.Printf("Failed cloud action STOP execution: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - // https://shuffler.io/v1/hooks/webhook_80184973-3e82-4852-842e-0290f7f34d7c - } - - // This is here to force stop and remove the old webhook - //image := "webhook" - //err = removeWebhookFunction(ctx, fileId) - //if err != nil { - // log.Printf("Function removal issue for %s-%s: %s", image, fileId, err) - // if strings.Contains(err.Error(), "does not exist") { - // resp.WriteHeader(200) - // resp.Write([]byte(`{"success": true, "reason": "Stopped webhook"}`)) - - // } else { - // resp.WriteHeader(401) - // resp.Write([]byte(`{"success": false, "reason": "Couldn't stop webhook, please try again later"}`)) - // } - - // return - //} - - log.Printf("Successfully deleted webhook %s", fileId) - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true, "reason": "Stopped webhook"}`)) -} - func removeWebhookFunction(ctx context.Context, hookid string) error { service, err := cloudfunctions.NewService(ctx) if err != nil { @@ -7386,7 +5416,7 @@ func handleStartHook(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -7422,7 +5452,7 @@ func handleStartHook(resp http.ResponseWriter, request *http.Request) { return } - if user.Id != hook.Owner && user.Role != "admin" { + if user.Id != hook.Owner { log.Printf("Wrong user (%s) for workflow %s", user.Username, hook.Id) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) @@ -7441,7 +5471,7 @@ func handleStartHook(resp http.ResponseWriter, request *http.Request) { environmentVariables := map[string]string{ "FUNCTION_APIKEY": user.ApiKey, - "CALLBACKURL": "https://shuffler.io", + "CALLBACKURL": syncUrl, "HOOKID": fileId, } @@ -7494,7 +5524,7 @@ func removeOutlookTriggerFunction(ctx context.Context, triggerId string) error { return nil } -func handleUserInput(trigger Trigger, organizationId string, workflowId string, referenceExecution string) error { +func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId string, referenceExecution string) error { // E.g. check email sms := "" email := "" @@ -7521,7 +5551,7 @@ func handleUserInput(trigger Trigger, organizationId string, workflowId string, ctx := context.Background() startNode := trigger.ID if strings.Contains(triggerType, "email") { - action := CloudSyncJob{ + action := shuffle.CloudSyncJob{ Type: "user_input", Action: "send_email", OrgId: organizationId, @@ -7532,7 +5562,7 @@ func handleUserInput(trigger Trigger, organizationId string, workflowId string, FifthItem: referenceExecution, } - org, err := getOrg(ctx, organizationId) + org, err := shuffle.GetOrg(ctx, organizationId) if err != nil { log.Printf("Failed email send to cloud (1): %s", err) return err @@ -7547,7 +5577,7 @@ func handleUserInput(trigger Trigger, organizationId string, workflowId string, log.Printf("Should send email to %s during execution.", email) } if strings.Contains(triggerType, "sms") { - action := CloudSyncJob{ + action := shuffle.CloudSyncJob{ Type: "user_input", Action: "send_sms", OrgId: organizationId, @@ -7558,7 +5588,7 @@ func handleUserInput(trigger Trigger, organizationId string, workflowId string, FifthItem: referenceExecution, } - org, err := getOrg(ctx, organizationId) + org, err := shuffle.GetOrg(ctx, organizationId) if err != nil { log.Printf("Failed sms send to cloud (3): %s", err) return err diff --git a/backend/tests/dockerpull.sh b/backend/tests/dockerpull.sh new file mode 100644 index 00000000..3df0d1e5 --- /dev/null +++ b/backend/tests/dockerpull.sh @@ -0,0 +1,2 @@ +#!/bin/sh +curl -XPOST http://localhost:5001/api/v1/get_docker_image -d '{"name":"frikky/shuffle:Testing_1.0.0"}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" diff --git a/backend/tests/files.sh b/backend/tests/files.sh new file mode 100755 index 00000000..d27a4755 --- /dev/null +++ b/backend/tests/files.sh @@ -0,0 +1,16 @@ +curl http://192.168.3.6:5001/api/v1/files/create -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -d '{"filename": "file.txt", "org_id": "b199646b-16d2-456d-9fd6-b9972e929466", "workflow_id": "global"}' + +curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -d '{"filename": "file.txt", "org_id": "b199646b-16d2-456d-9fd6-b9972e929466", "workflow_id": "global"}' + +echo +curl http://localhost:5001/api/v1/files/e19cffe4-e2da-47e9-809e-904f5cb03687/upload -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -F 'shuffle_file=@files.sh' + +curl http://localhost:5001/api/v1/files/1915981b-b897-4db1-8a2e-44bc34cead3b/content -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl http://localhost:5001/api/v1/files/e19cffe4-e2da-47e9-809e-904f5cb03687 -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XDELETE http://localhost:5001/api/v1/files/e19cffe4-e2da-47e9-809e-904f5cb03687 -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" + + #r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS") + #r.HandleFunc("/api/v1/files/create", handleCreateFile).Methods("POST", "OPTIONS") + #r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS") + #r.HandleFunc("/api/v1/files/{fileId}", handleGetFileMeta).Methods("GET", "OPTIONS") + #r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS") diff --git a/backend/tests/hotload.sh b/backend/tests/hotload.sh new file mode 100644 index 00000000..d55544f6 --- /dev/null +++ b/backend/tests/hotload.sh @@ -0,0 +1 @@ +curl http://localhost:5001/api/v1/apps/run_hotload -H "Authorization: Bearer e08c6f22-9a55-4557-b008-04388cc51fb0" diff --git a/backend/tests/list_files.sh b/backend/tests/list_files.sh new file mode 100644 index 00000000..4c63e246 --- /dev/null +++ b/backend/tests/list_files.sh @@ -0,0 +1 @@ +curl -XGET http://192.168.3.6:5001/api/v1/files -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" diff --git a/backend/tests/schedules.sh b/backend/tests/schedules.sh index 339f982d..dc0c15d8 100644 --- a/backend/tests/schedules.sh +++ b/backend/tests/schedules.sh @@ -1,2 +1,2 @@ # Fails cus of unmarshal -curl -XPOST http://localhost:5001/api/v1/workflows/1d9d8ce2-566e-4c3f-8a37-5d6c7d2000b5/schedule -d '{"name": "hey", "frequency": "*/1 * * * *", "execution_argument": "{\"test\": \"hey\"}"}' -H "Authorization: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjYwZjQwNjBlNThkNzVmZDNmNzBiZWZmODhjNzk0YTc3NTMyN2FhMzEiLCJ0eXAiOiJKV1QifQ.eyJhdWQiOiJodHRwczovL3NodWZmbGVyLmlvL2FwaS92MS93b3JrZmxvd3MvMWQ5ZDhjZTItNTY2ZS00YzNmLThhMzctNWQ2YzdkMjAwMGI1L2V4ZWN1dGUiLCJhenAiOiIxMDMwNzY3ODIwNjE0MjQ2MTg0MjIiLCJlbWFpbCI6InNjaGVkdWxlckBzaHVmZmxlLTI0MTUxNy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJleHAiOjE1NjU1Mjc1NTEsImlhdCI6MTU2NTUyMzk1MSwiaXNzIjoiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tIiwic3ViIjoiMTAzMDc2NzgyMDYxNDI0NjE4NDIyIn0.r0EDq9fjhf_5CPTiltyfk_L3uYJp577Uy0yYPcCAl2nv50_z_oUtbWGBpQLL8gcj-NGd3g4E52Qur8k6hCMIQweLS6WAb1279vGffEoCNDfkWb3Oy-yJGP1kzwLvqFJqnHLkSWYXNWvSyWnEimW8Rryx_m1BXS5wcA8l4NIr83kS7fPZrTwjnwFSeGSThwk91DVARzapQb8r0GEgOUyHZ1aBXnV98mikzSUt-5xFKe9eMdD22YJAj0Ru-DxAxs5nOqghX4PMRysWjshjOMrlR1piPWxqAmewp8YKZDCQ5gXskpeAFBDoULT971Wsx_NCohnJsFqx1JfPS9ZYMTW2oQ" +curl -XPOST http://localhost:5001/api/v1/workflows/1d9d8ce2-566e-4c3f-8a37-5d6c7d2000b5/schedule -d '{"name": "hey", "frequency": "*/1 * * * *", "execution_argument": "{\"test\": \"hey\"}"}' -H "Authorization: Bearer WUT" diff --git a/backend/tests/users.sh b/backend/tests/users.sh new file mode 100644 index 00000000..fa78cd6b --- /dev/null +++ b/backend/tests/users.sh @@ -0,0 +1,13 @@ +curl http://localhost:5001/api/v1/users/register -H "Authorization: Bearer 36a3eb38-0070-41c6-b20a-2a3e0941d10e" -d '{"username": "username1", "password": ""}' + +echo +curl http://localhost:5001/api/v1/users -H "Authorization: Bearer 36a3eb38-0070-41c6-b20a-2a3e0941d10e" + +echo UPDATE +curl -XPUT http://localhost:5001/api/v1/users/updateuser -H "Authorization: Bearer 36a3eb38-0070-41c6-b20a-2a3e0941d10e" -d '{"user_id": "id", "role": "admin"}' + +echo +curl -XDELETE http://localhost:5001/api/v1/users/userid -H "Authorization: Bearer 36a3eb38-0070-41c6-b20a-2a3e0941d10e" + +echo +curl -XPOST http://localhost:5001/api/v1/users/generateapikey -H "Authorization: Bearer 36a3eb38-0070-41c6-b20a-2a3e0941d10e" -d '{"user_id": "390efa79-73a3-454b-8b1d-38f56eec14ad"}' diff --git a/docker-compose.yml b/docker-compose.yml index 67fe27a6..56fd9a3c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.60 + image: ghcr.io/frikky/shuffle-frontend:0.8.71 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.60 + image: ghcr.io/frikky/shuffle-backend:0.8.71 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -35,9 +35,11 @@ services: - SHUFFLE_FILE_LOCATION=/shuffle-files - ORG_ID=${ORG_ID} - SHUFFLE_APP_DOWNLOAD_LOCATION=${SHUFFLE_APP_DOWNLOAD_LOCATION} + - SHUFFLE_DOWNLOAD_AUTH_BRANCH=${SHUFFLE_DOWNLOAD_AUTH_BRANCH} - SHUFFLE_DEFAULT_USERNAME=${SHUFFLE_DEFAULT_USERNAME} - SHUFFLE_DEFAULT_PASSWORD=${SHUFFLE_DEFAULT_PASSWORD} - SHUFFLE_DEFAULT_APIKEY=${SHUFFLE_DEFAULT_APIKEY} + - SHUFFLE_APP_FORCE_UPDATE=${SHUFFLE_APP_FORCE_UPDATE} - HTTP_PROXY=${SHUFFLE_HTTP_PROXY} - HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY} restart: unless-stopped @@ -45,7 +47,7 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: ghcr.io/frikky/shuffle-orborus:0.8.60 + image: ghcr.io/frikky/shuffle-orborus:0.8.71 container_name: shuffle-orborus hostname: shuffle-orborus networks: @@ -54,7 +56,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock environment: - SHUFFLE_APP_SDK_VERSION=0.8.60 - - SHUFFLE_WORKER_VERSION=0.8.60 + - SHUFFLE_WORKER_VERSION=0.8.71 - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} diff --git a/frontend/Dockerfile b/frontend/Dockerfile index f48b048d..4204651c 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,5 +1,5 @@ # Build environment -FROM node as builder +FROM node:14 as builder RUN mkdir /usr/src/app @@ -18,14 +18,10 @@ COPY ./src /usr/src/app/src/ COPY ./*.sh /usr/src/app/ COPY ./*.json /usr/src/app/ -# There were issues with the webpack installer from package.json -RUN rm -rf /usr/src/app/node_modules/webpack -#RUN yarn add webpack@4.42.0 - RUN yarn build # Production environment -FROM nginx:latest +FROM nginx:1.19 RUN mkdir -p /usr/share/nginx/html/build RUN mkdir -p /usr/share/nginx/html/css diff --git a/frontend/package.json b/frontend/package.json index 0b99f34c..d2b2ef26 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -5,6 +5,7 @@ "private": true, "dependencies": { "@material-ui/core": "^4.5.2", + "@material-ui/data-grid": "^4.0.0-alpha.22", "@material-ui/icons": "^4.5.1", "@material-ui/styles": "^4.5.2", "@use-it/interval": "^1.0.0", @@ -35,6 +36,7 @@ "react": "^16.14.0", "react-alert": "^5.5.0", "react-alert-template-basic": "^1.0.0", + "react-avatar-editor": "^11.1.0", "react-beforeunload": "^2.2.1", "react-chartjs-2": "^2.11.1", "react-cookie": "^4.0.1", @@ -57,7 +59,7 @@ "shellwords": "^0.1.1", "simplebar": "^4.2.3", "styled-components": "^4.4.0", - "webpack": "^4.42.0", + "webpack": "4.44.2", "websocket": "^1.0.30", "yaml": "^1.7.2", "yamljs": "^0.3.0", diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 18456036..99465936 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -13,10 +13,9 @@ import EditWebhook from "./views/EditWebhook"; import AngularWorkflow from "./views/AngularWorkflow"; import Header from './components/Header'; +import theme from './theme' import Apps from './views/Apps'; import AppCreator from './views/AppCreator'; -import Contact from './views/Contact'; -import Oauth2 from './views/Oauth2'; import Dashboard from "./views/Dashboard"; import AdminSetup from "./views/AdminSetup"; @@ -28,11 +27,14 @@ import LandingPageNew from "./views/LandingpageNew"; import LoginPage from "./views/LoginPage"; import SettingsPage from "./views/SettingsPage"; +import MyView from "./views/MyView"; + import { createMuiTheme, MuiThemeProvider } from '@material-ui/core/styles'; import ScrollToTop from "./components/ScrollToTop"; import AlertTemplate from "./components/AlertTemplate"; import { positions, Provider } from "react-alert"; +import {isMobile} from "react-device-detect"; // Production - backend proxy forwarding in nginx var globalUrl = window.location.origin @@ -43,34 +45,8 @@ if (window.location.protocol == "http:" && window.location.port === "3000") { //globalUrl = "http://localhost:5002" } -const theme = createMuiTheme({ - palette: { - primary: { - main: "#f85a3e" - }, - secondary: { - main: '#e8eaf6', - }, - surfaceColor: "#27292d", - inputColor: "#383B40" - }, - typography: { - useNextVariants: true - }, - overrides: { - MuiMenu: { - list: { - backgroundColor: "#383B40", - }, - }, - }, -}); - - -// FIXME - set client side cookies const App = (message, props) => { const [userdata, setUserData] = useState({}); - //const [homePage, ] = useState(true); const [cookies, setCookie, removeCookie] = useCookies([]); const [isLoggedIn, setIsLoggedIn] = useState(false); const [dataset, setDataset] = useState(false); @@ -130,8 +106,6 @@ const App = (message, props) => {
- } /> - } /> } /> } /> } /> @@ -147,10 +121,11 @@ const App = (message, props) => { } /> } /> } /> - } /> + } /> { window.location.pathname = "/docs/about" }} /> } /> } /> + } /> } />
diff --git a/frontend/src/assets/img/bag.svg b/frontend/src/assets/img/bag.svg new file mode 100644 index 00000000..c63d8f9c --- /dev/null +++ b/frontend/src/assets/img/bag.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/frontend/src/assets/img/book.svg b/frontend/src/assets/img/book.svg new file mode 100644 index 00000000..c64ae378 --- /dev/null +++ b/frontend/src/assets/img/book.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/frontend/src/assets/img/mobile.svg b/frontend/src/assets/img/mobile.svg new file mode 100644 index 00000000..0608e2c8 --- /dev/null +++ b/frontend/src/assets/img/mobile.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/frontend/src/components/AlertTemplate.js b/frontend/src/components/AlertTemplate.js index a476afc7..58ff46cc 100644 --- a/frontend/src/components/AlertTemplate.js +++ b/frontend/src/components/AlertTemplate.js @@ -18,6 +18,7 @@ const alertStyle = { width: 400, boxSizing: 'border-box', zIndex: 100001, + overflow: "hidden", } const buttonStyle = { diff --git a/frontend/src/components/ConfigureWorkflow.jsx b/frontend/src/components/ConfigureWorkflow.jsx new file mode 100644 index 00000000..0e83f81b --- /dev/null +++ b/frontend/src/components/ConfigureWorkflow.jsx @@ -0,0 +1,97 @@ +import React, {useState} from 'react'; + +import {Typography, } from '@material-ui/core'; + +const Workflow = (props) => { + const { workflow, appAuthentication, apps } = props + const [requiredActions, setRequiredActions] = React.useState([]) + const [firstLoad, setFirstLoad] = React.useState("") + + // Rofl + if (workflow === undefined || workflow === null) { + return null + } + + if (apps === undefined || apps === null) { + return null + } + + if (appAuthentication === undefined || appAuthentication === null) { + return null + } + + if (firstLoad.length === 0 || firstLoad !== workflow.id) { + setFirstLoad(workflow.id) + const newactions = [] + for (var key in workflow.actions) { + var newaction = { + "large_image": "", + "app_name": "", + "app_version": "", + "must_activate": false, + "must_authenticate": false, + "action_ids": [], + } + + const action = workflow.actions[key] + console.log(action) + const app = apps.find(app => app.name === action.app_name && app.app_version === action.app_version) + if (app === undefined || app === null) { + console.log("COULDNT FIND APP - SEARCH BACKEND") + + newaction.app_name = action.app_name + newaction.app_version = action.app_version + } else { + newaction.app_name = app.name + newaction.app_version = app.app_version + + console.log("APP: ", app) + if (action.authentication_id === "" && app.authentication.required === true) { + console.log("Requires auth!") + newaction.must_authenticate = true + newaction.action_ids.push(action.id) + } + + //newaction.app_name = action.app_name + //newaction.app_name = action.app_version + } + + if (action.errors !== undefined && action.errors !== null && action.errors.length > 0) { + console.log("Has errors!") + } + + console.log("NEWACTION: ", newaction) + if (newaction.must_authenticate || newaction.must_activate) { + newactions.push(newaction) + } + } + + console.log("ACTIONS: ", newactions) + setRequiredActions(newactions) + } + + const AppSection = (props) => { + const {action} = props + + return ( +
+ Name: {action.app_name}:{action.app_version}. +
+ ) + } + + console.log(requiredActions) + + return ( +
+ Workflow: {workflow.id} + {requiredActions.map((data, index) => { + return ( + + ) + })} +
+ ) +} + +export default Workflow diff --git a/frontend/src/components/Header.js b/frontend/src/components/Header.js index 1143a696..eb6d6e0c 100644 --- a/frontend/src/components/Header.js +++ b/frontend/src/components/Header.js @@ -118,9 +118,43 @@ const Header = props => { // Should be based on some path - const logoCheck = !homePage ? null : null + const avatarMenu = + + { + setAnchorEl(event.currentTarget); + }}> + + + { + handleClose() + }} + > + { + event.preventDefault() + handleClose() + }}> + + Settings + + + { + event.preventDefault() + handleClose() + handleClickLogout() + }}> + Logout + + + + // Handle top bar or something + const logoCheck = !homePage ? null : null const loginTextBrowser = !isLoggedIn ?
@@ -197,36 +231,7 @@ const Header = props => {
- { - setAnchorEl(event.currentTarget); - }}> - - - { - handleClose() - }} - > - { - event.preventDefault() - handleClose() - }}> - - Settings - - - { - event.preventDefault() - handleClose() - handleClickLogout() - }}> - Logout - - + {avatarMenu} {userdata === undefined || userdata.admin === undefined || userdata.admin === null || !userdata.admin ? null :
- - -
- Logout -
- - {logoCheck} - - - - - - - + {avatarMenu}
- + // const loadedCheck = diff --git a/frontend/src/components/RenderCytoscape.js b/frontend/src/components/RenderCytoscape.js new file mode 100644 index 00000000..3a88dcf8 --- /dev/null +++ b/frontend/src/components/RenderCytoscape.js @@ -0,0 +1,146 @@ +import React, {useState, useEffect, useLayoutEffect} from 'react'; +import * as cytoscape from 'cytoscape'; +import CytoscapeComponent from 'react-cytoscapejs'; +import cystyle from '../defaultCytoscapeStyle'; + +const surfaceColor = "#27292D" +const CytoscapeWrapper = (props) => { + const { globalUrl, inworkflow } = props; + + const [elements, setElements] = useState([]) + const [workflow, setWorkflow] = useState(inworkflow) + const [cy, setCy] = React.useState() + const bodyWidth = 200 + const bodyHeight = 150 + + const setupGraph = () => { + const actions = workflow.actions.map(action => { + const node = {} + node.position = action.position + node.data = action + + node.data._id = action["id"] + node.data.type = "ACTION" + node.isStartNode = action["id"] === workflow.start + + + var example = "" + if (action.example !== undefined && action.example !== null && action.example.length > 0) { + example = action.example + } + + node.data.example = example + return node; + }) + + const triggers = workflow.triggers.map(trigger => { + const node = {} + node.position = trigger.position + node.data = trigger + + node.data._id = trigger["id"] + node.data.type = "TRIGGER" + + return node; + }) + + // FIXME - tmp branch update + var insertedNodes = [].concat(actions, triggers) + const edges = workflow.branches.map((branch, index) => { + //workflow.branches[index].conditions = [{ + + const edge = { }; + var conditions = workflow.branches[index].conditions + if (conditions === undefined || conditions === null) { + conditions = [] + } + + var label = "" + if (conditions.length === 1) { + label = conditions.length+" condition" + } else if (conditions.length > 1) { + label = conditions.length+" conditions" + } + + edge.data = { + id: branch.id, + _id: branch.id, + source: branch.source_id, + target: branch.destination_id, + label: label, + conditions: conditions, + hasErrors: branch.has_errors + }; + + // This is an attempt at prettier edges. The numbers are weird to work with. + /* + //http://manual.graphspace.org/projects/graphspace-python/en/latest/demos/edge-types.html + const sourcenode = actions.find(node => node.data._id === branch.source_id) + const destinationnode = actions.find(node => node.data._id === branch.destination_id) + if (sourcenode !== undefined && destinationnode !== undefined && branch.source_id !== branch.destination_id) { + //node.data._id = action["id"] + console.log("SOURCE: ", sourcenode.position) + console.log("DESTINATIONNODE: ", destinationnode.position) + + var opposite = true + if (sourcenode.position.x > destinationnode.position.x) { + opposite = false + } else { + opposite = true + } + + edge.style = { + 'control-point-distance': opposite ? ["25%", "-75%"] : ["-10%", "90%"], + 'control-point-weight': ['0.3', '0.7'], + } + } + */ + + return edge; + }) + + setWorkflow(workflow) + + // Verifies if a branch is valid and skips others + var newedges = [] + for (var key in edges) { + var item = edges[key] + + const sourcecheck = insertedNodes.find(data => data.data.id === item.data.source) + const destcheck = insertedNodes.find(data => data.data.id === item.data.target) + if (sourcecheck === undefined || destcheck === undefined) { + continue + } + + newedges.push(item) + } + + insertedNodes = insertedNodes.concat(newedges) + setElements(insertedNodes) + } + + if (elements.length === 0) { + setupGraph() + } + + return ( + { + // FIXME: There's something specific loading when + // you do the first hover of a node. Why is this different? + //console.log("CY: ", incy) + setCy(incy) + }} + /> + ) +} + +export default CytoscapeWrapper diff --git a/frontend/src/defaultCytoscapeStyle.js b/frontend/src/defaultCytoscapeStyle.js index e0732bac..8fd5d326 100644 --- a/frontend/src/defaultCytoscapeStyle.js +++ b/frontend/src/defaultCytoscapeStyle.js @@ -6,7 +6,7 @@ const data = [{ 'font-family': 'Segoe UI, Tahoma, Geneva, Verdana, sans-serif, sans-serif', 'font-weight': 'lighter', 'margin-right': '10px', - 'font-size': '15px', + 'font-size': '18px', 'width': '80px', 'height': '80px', 'color': 'white', @@ -20,14 +20,15 @@ const data = [{ selector: 'edge', css: { 'target-arrow-shape': 'triangle', - 'target-arrow-color': 'yellow', + 'target-arrow-color': 'grey', 'curve-style': 'unbundled-bezier', 'label': 'data(label)', 'text-margin-y': '-15px', + 'width': '2px', "color": "white", "line-fill": "linear-gradient", - "line-gradient-stop-colors": ["cyan", "yellow"], "line-gradient-stop-positions": ["0.0", "100"], + "line-gradient-stop-colors": ["grey", "grey"], }, }, { @@ -38,6 +39,21 @@ const data = [{ 'border-color': '#81c784', 'background-width': '100%', 'background-height': '100%', + 'border-radius': '5px', + }, + }, + { + selector: `node[app_name="Shuffle Tools"]`, + css: { + 'width': '30px', + 'height': '30px', + }, + }, + { + selector: `node[app_name="Testing"]`, + css: { + 'width': '30px', + 'height': '30px', }, }, { @@ -101,6 +117,8 @@ const data = [{ css: { 'shape': 'ellipse', 'border-color': '#80deea', + 'width': '80px', + 'height': '80px', }, }, { @@ -111,7 +129,7 @@ const data = [{ }, }, { - selector: 'node:selected', + selector: ':selected', css: { 'background-color': '#77b0d0', 'border-color': '#77b0d0', @@ -163,7 +181,7 @@ const data = [{ css: { 'background-color': '#ffef47', 'border-color': '#ffef47', - 'border-width': '5px', + 'border-width': '8px', 'transition-property': 'border-width', 'transition-duration': '0.25s', }, @@ -183,9 +201,11 @@ const data = [{ css: { 'background-color': "#f85a3e", 'border-color': '#f85a3e', - 'border-width': '5px', + 'border-width': '12px', 'transition-property': 'border-width', 'transition-duration': '0.25s', + 'font-size': '30px', + 'label': 'data(label)', }, }, { @@ -211,10 +231,13 @@ const data = [{ selector: 'edge.success-highlight', css: { 'width': '5px', - 'target-arrow-color': '#399645', - 'line-color': '#399645', + 'target-arrow-color': '#41dcab', + 'line-color': '#41dcab', 'transition-property': 'line-color, width', 'transition-duration': '0.5s', + "line-fill": "linear-gradient", + "line-gradient-stop-positions": ["0.0", "100"], + "line-gradient-stop-colors": ["#41dcab", "#41dcab"], }, }, { @@ -222,7 +245,10 @@ const data = [{ css: { 'target-arrow-color': '#991818', 'line-color': '#991818', - 'line-style': 'dashed' + 'line-style': 'dashed', + "line-fill": "linear-gradient", + "line-gradient-stop-positions": ["0.0", "100"], + "line-gradient-stop-colors": ["#991818", "#991818"], }, }, { diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 62f926a4..dfefcd5f 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -1,60 +1,18 @@ import React, { useState, useEffect } from 'react'; import { makeStyles } from '@material-ui/styles'; +import { useTheme } from '@material-ui/core/styles'; import {Link} from 'react-router-dom'; -import Paper from '@material-ui/core/Paper'; -import Card from '@material-ui/core/Card'; -import Tooltip from '@material-ui/core/Tooltip'; -import FormControlLabel from '@material-ui/core/FormControlLabel'; -import Typography from '@material-ui/core/Typography'; -import Switch from '@material-ui/core/Switch'; -import Select from '@material-ui/core/Select'; -import MenuItem from '@material-ui/core/MenuItem'; -import Divider from '@material-ui/core/Divider'; -import TextField from '@material-ui/core/TextField'; -import Button from '@material-ui/core/Button'; -import Tabs from '@material-ui/core/Tabs'; -import Tab from '@material-ui/core/Tab'; -import Grid from '@material-ui/core/Grid'; -import List from '@material-ui/core/List'; -import ListItem from '@material-ui/core/ListItem'; -import ListItemText from '@material-ui/core/ListItemText'; -import ListItemAvatar from '@material-ui/core/ListItemAvatar'; -import ListItemSecondaryAction from '@material-ui/core/ListItemSecondaryAction'; -import IconButton from '@material-ui/core/IconButton'; -import Avatar from '@material-ui/core/Avatar'; -import Zoom from '@material-ui/core/Zoom'; + +import {Paper, Card, Tooltip, FormControlLabel, Typography, Switch, Select, MenuItem, Divider, TextField, Button, Tabs, Tab, Grid, List, ListItem, ListItemText, ListItemAvatar, ListItemSecondaryAction, IconButton, Avatar, Zoom, Dialog, DialogTitle, DialogActions, DialogContent, CircularProgress } from '@material-ui/core'; + +import {Edit as EditIcon, FileCopy as FileCopyIcon, Publish as PublishIcon, SelectAll as SelectAllIcon, OpenInNew as OpenInNewIcon, CloudDownload as CloudDownloadIcon, Description as DescriptionIcon, Polymer as PolymerIcon, CheckCircle as CheckCircleIcon, Close as CloseIcon, Apps as AppsIcon, Image as ImageIcon, Delete as DeleteIcon, Cached as CachedIcon, AccessibilityNew as AccessibilityNewIcon, Lock as LockIcon, Eco as EcoIcon, Schedule as ScheduleIcon, Cloud as CloudIcon, Business as BusinessIcon} from '@material-ui/icons'; + import { useAlert } from "react-alert"; import Dropzone from '../components/Dropzone'; - -import { Dialog, DialogTitle, DialogActions, DialogContent } from '@material-ui/core'; -import { useTheme } from '@material-ui/core/styles'; import HandlePayment from './HandlePayment' import OrgHeader from '../components/OrgHeader' -import CircularProgress from '@material-ui/core/CircularProgress'; -import EditIcon from '@material-ui/icons/Edit'; -import FileCopyIcon from '@material-ui/icons/FileCopy'; -import PublishIcon from '@material-ui/icons/Publish'; -import SelectAllIcon from '@material-ui/icons/SelectAll'; -import OpenInNewIcon from '@material-ui/icons/OpenInNew'; -import CloudDownloadIcon from '@material-ui/icons/CloudDownload'; -import DescriptionIcon from '@material-ui/icons/Description'; -import PolymerIcon from '@material-ui/icons/Polymer'; -import CheckCircleIcon from '@material-ui/icons/CheckCircle'; -import CloseIcon from '@material-ui/icons/Close'; -import AppsIcon from '@material-ui/icons/Apps'; -import ImageIcon from '@material-ui/icons/Image'; -import DeleteIcon from '@material-ui/icons/Delete'; -import CachedIcon from '@material-ui/icons/Cached'; -import AccessibilityNewIcon from '@material-ui/icons/AccessibilityNew'; -import LockIcon from '@material-ui/icons/Lock'; -import EcoIcon from '@material-ui/icons/Eco'; -import ScheduleIcon from '@material-ui/icons/Schedule'; -import CloudIcon from '@material-ui/icons/Cloud'; -import BusinessIcon from '@material-ui/icons/Business'; - - const useStyles = makeStyles({ notchedOutline: { borderColor: "#f85a3e !important" @@ -233,6 +191,45 @@ const Admin = (props) => { }); } + const handleStopOrgSync = (org_id) => { + if (org_id === undefined || org_id === null) { + alert.error("Couldn't get org "+org_id) + return + } + + const data = {} + + const url = globalUrl + '/api/v1/orgs/' + org_id + "/stop_sync"; + fetch(url, { + mode: 'cors', + method: 'POST', + body: JSON.stringify(data), + credentials: 'include', + crossDomain: true, + withCredentials: true, + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + }) + .then(response => { + if (response.status === 200) { + console.log("Cloud sync success?") + alert.success("Successfully stopped cloud sync") + } else { + console.log("Cloud sync fail?") + alert.error("Failed stopping sync. Try again, and contact support if this persists.") + } + + return response.json() + }) + .then((responseJson) => { + handleGetOrg(org_id) + }) + .catch(error => { + alert.error("Err: " + error.toString()) + }) + } + const enableCloudSync = (apikey, organization, disableSync) => { setOrgSyncResponse("") @@ -390,7 +387,9 @@ const Admin = (props) => { const deleteUser = (data) => { // Just use this one? - const url = globalUrl + '/api/v1/users/' + data.id + const userId = isCloud ? data.username : data.id + + const url = globalUrl + '/api/v1/users/' + userId fetch(url, { method: 'DELETE', credentials: "include", @@ -419,6 +418,11 @@ const Admin = (props) => { } const handleGetOrg = (orgId) => { + if (orgId.length === 0) { + alert.error("Organization ID not defined. Please contact us on https://shuffler.io if this persists logout.") + return + } + // Just use this one? var baseurl = globalUrl const url = baseurl + '/api/v1/orgs/'+orgId @@ -472,11 +476,13 @@ const Admin = (props) => { const submitUser = (data) => { console.log("INPUT: ", data) + setLoginInfo("") // Just use this one? var data = { "username": data.Username, "password": data.Password } var baseurl = globalUrl const url = baseurl + '/api/v1/users/register'; + fetch(url, { method: 'POST', credentials: "include", @@ -488,7 +494,7 @@ const Admin = (props) => { .then(response => response.json().then(responseJson => { if (responseJson["success"] === false) { - setLoginInfo("Error in input: " + responseJson.reason) + setLoginInfo("Error: " + responseJson.reason) } else { setLoginInfo("") setModalOpen(false) @@ -910,7 +916,8 @@ const Admin = (props) => { }) .then((response) => { if (response.status !== 200) { - window.location.pathname = "/workflows" + // Ahh, this happens because they're not admin + // window.location.pathname = "/workflows" return } @@ -954,11 +961,11 @@ const Admin = (props) => { } else if (newValue === 2) { getAppAuthentication() } else if (newValue === 3) { - getEnvironments() + getFiles() } else if (newValue === 4) { getSchedules() } else if (newValue === 5) { - getFiles() + getEnvironments() } else if (newValue === 6) { getOrgs() } @@ -1072,7 +1079,8 @@ const Admin = (props) => { - const generateApikey = (userId) => { + const generateApikey = (user) => { + const userId = isCloud ? user.username : user.id const data = { "user_id": userId } fetch(globalUrl + "/api/v1/generateapikey", { @@ -1199,11 +1207,11 @@ const Admin = (props) => { }, }} > - + Edit user
{ style={{}} variant="outlined" color="primary" - onClick={() => generateApikey(selectedUser.id)} + onClick={() => generateApikey(selectedUser)} > Get new API key @@ -1399,14 +1407,14 @@ const Admin = (props) => { const cancelSubscriptions = (subscription_id) => { console.log(selectedOrganization) + const orgId = selectedOrganization.id const data = { "subscription_id": subscription_id, "action": "cancel", "org_id": selectedOrganization.id, } - - const url = globalUrl + `/api/v1/orgs/${selectedOrganization.id}`; + const url = globalUrl + `/api/v1/orgs/${orgId}`; fetch(url, { mode: 'cors', method: 'POST', @@ -1441,7 +1449,7 @@ const Admin = (props) => { } const organizationView = curTab === 0 && selectedOrganization.id !== undefined ? -
+

Organization overview

@@ -1457,6 +1465,25 @@ const Admin = (props) => {
:
+ + { + const elementName = "copy_element_shuffle" + const org_id = selectedOrganization.id + var copyText = document.getElementById(elementName); + if (copyText !== null && copyText !== undefined) { + navigator.clipboard.writeText(org_id) + copyText.select(); + copyText.setSelectionRange(0, 99999); /* For mobile devices */ + + /* Copy the text inside the text field */ + document.execCommand("copy"); + + alert.info(org_id + " copied to clipboard") + } + }}> + + + {selectedOrganization.name.length > 0 ? : @@ -1487,27 +1514,41 @@ const Admin = (props) => { Your Apikey - -
+
+ + {selectedOrganization.cloud_sync_active ? + + : null} +
+
:
@@ -1626,7 +1667,7 @@ const Admin = (props) => { }
- +
: null @@ -1743,7 +1784,6 @@ const Admin = (props) => {
+ { + style={{ maxWidth: 100, minWidth: 100, }} + primary={data.apikey === undefined || data.apikey.length === 0 ? "" : + + { + const elementName = "copy_element_shuffle" + var copyText = document.getElementById(elementName); + if (copyText !== null && copyText !== undefined) { + navigator.clipboard.writeText(data.apikey) + copyText.select(); + copyText.setSelectionRange(0, 99999); /* For mobile devices */ + + /* Copy the text inside the text field */ + document.execCommand("copy"); + + alert.info("Apikey copied to clipboard") + } + }}> + + + + }/> { + console.log("VALUE: ", e.target.value) + + if (isCloud) { + setUser(data.username, "role", e.target.value) + } else { + setUser(data.id, "role", e.target.value) } }} - value={data.role} - fullWidth - onChange={(e) => { - console.log("VALUE: ", e.target.value) - setUser(data.id, "role", e.target.value) - }} style={{ backgroundColor: theme.palette.surfaceColor, color: "white", height: "50px" }} - > - - Admin - - - User - - } - style = {{ minWidth: 150, maxWidth: 150}} + > + + Admin + + + User + + + } + style ={{ minWidth: 135, maxWidth: 135, marginRight: 15,}} /> - + + - + ) })} @@ -1890,7 +1960,7 @@ const Admin = (props) => { uploadFiles(files) } - const filesView = curTab === 5 ? + const filesView = curTab === 3 ? 1366 ? 1366 : 1200, margin: "auto", padding: 20 }} onDrop={uploadFile}>
@@ -1939,7 +2009,7 @@ const Admin = (props) => { /> { primary={new Date(file.created_at*1000).toISOString()} /> { /> { /* Copy the text inside the text field */ document.execCommand("copy"); - alert.info(file.id + "copied to clipboard") + alert.info(file.id + " copied to clipboard") } }}> @@ -2228,7 +2298,7 @@ const Admin = (props) => { /> { /> {
: null - const environmentView = curTab === 3 ? + const environmentView = curTab === 5 ?

Environments

@@ -2578,10 +2648,10 @@ const Admin = (props) => { > Organization/> Users /> - {isCloud ? null : App Authentication/>} + App Authentication/> + Files /> + Schedules /> {isCloud ? null : Environments/>} - {isCloud ? null : Schedules />} - {isCloud ? null : Files />} {window.location.protocol == "http:" && window.location.port === "3000" ? Hybrid/> : null} {window.location.protocol == "http:" && window.location.port === "3000" ? Organizations/> : null} {window.location.protocol === "http:" && window.location.port === "3000" ? Categories/> : null} diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 79979409..bb6e1d40 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1,75 +1,16 @@ import React, {useState, useEffect, useLayoutEffect} from 'react'; import { useInterval } from 'react-powerhooks'; +import { useTheme } from '@material-ui/core/styles'; import uuid from "uuid"; - import {Link} from 'react-router-dom'; import { Prompt } from 'react-router' -import TextField from '@material-ui/core/TextField'; -import Drawer from '@material-ui/core/Drawer'; -import Button from '@material-ui/core/Button'; -import Paper from '@material-ui/core/Paper'; -import Grid from '@material-ui/core/Grid'; -import Tabs from '@material-ui/core/Tabs'; -import InputAdornment from '@material-ui/core/InputAdornment'; -import Tab from '@material-ui/core/Tab'; -import ButtonBase from '@material-ui/core/ButtonBase'; -import Tooltip from '@material-ui/core/Tooltip'; -import Select from '@material-ui/core/Select'; -import MenuItem from '@material-ui/core/MenuItem'; -import Divider from '@material-ui/core/Divider'; -import Dialog from '@material-ui/core/Dialog'; -import Modal from '@material-ui/core/Modal'; -import DialogActions from '@material-ui/core/DialogActions'; -import DialogTitle from '@material-ui/core/DialogTitle'; -import InputLabel from '@material-ui/core/InputLabel'; -import DialogContent from '@material-ui/core/DialogContent'; -import FormControl from '@material-ui/core/FormControl'; -import IconButton from '@material-ui/core/IconButton'; -import Menu from '@material-ui/core/Menu'; -import Input from '@material-ui/core/Input'; -import FormGroup from '@material-ui/core/FormGroup'; -import FormControlLabel from '@material-ui/core/FormControlLabel'; -import Typography from '@material-ui/core/Typography'; -import Checkbox from '@material-ui/core/Checkbox'; -import Breadcrumbs from '@material-ui/core/Breadcrumbs'; -import CircularProgress from '@material-ui/core/CircularProgress'; -import Switch from '@material-ui/core/Switch'; import ReactJson from 'react-json-view' import { useBeforeunload } from 'react-beforeunload'; import NestedMenuItem from "material-ui-nested-menu-item"; -import Fade from '@material-ui/core/Fade'; -import ArrowUpwardIcon from '@material-ui/icons/ArrowUpward'; -import VisibilityIcon from '@material-ui/icons/Visibility'; -import DoneIcon from '@material-ui/icons/Done'; -import CloseIcon from '@material-ui/icons/Close'; -import ErrorIcon from '@material-ui/icons/Error'; -import FindReplaceIcon from '@material-ui/icons/FindReplace'; -import ArrowLeftIcon from '@material-ui/icons/ArrowLeft'; -import CachedIcon from '@material-ui/icons/Cached'; -import AddIcon from '@material-ui/icons/Add'; -import DirectionsRunIcon from '@material-ui/icons/DirectionsRun'; -import PolymerIcon from '@material-ui/icons/Polymer'; -import FormatListNumberedIcon from '@material-ui/icons/FormatListNumbered'; -import CreateIcon from '@material-ui/icons/Create'; -import PlayArrowIcon from '@material-ui/icons/PlayArrow'; -import AspectRatioIcon from '@material-ui/icons/AspectRatio'; -import MoreVertIcon from '@material-ui/icons/MoreVert'; -import AppsIcon from '@material-ui/icons/Apps'; -import ScheduleIcon from '@material-ui/icons/Schedule'; -import FavoriteBorderIcon from '@material-ui/icons/FavoriteBorder'; -import PauseIcon from '@material-ui/icons/Pause'; -import DeleteIcon from '@material-ui/icons/Delete'; -import AddCircleOutlineIcon from '@material-ui/icons/AddCircleOutline'; -import SaveIcon from '@material-ui/icons/Save'; -import KeyboardArrowLeftIcon from '@material-ui/icons/KeyboardArrowLeft'; -import KeyboardArrowRightIcon from '@material-ui/icons/KeyboardArrowRight'; -import ArrowBackIcon from '@material-ui/icons/ArrowBack'; -import SettingsIcon from '@material-ui/icons/Settings'; -import LockOpenIcon from '@material-ui/icons/LockOpen'; -import ExpandMoreIcon from '@material-ui/icons/ExpandMore'; -import VpnKeyIcon from '@material-ui/icons/VpnKey'; +import {TextField, Drawer, Button, Paper, Grid, Tabs, InputAdornment, Tab, ButtonBase, Tooltip, Select, MenuItem, Divider, Dialog, Modal, DialogActions, DialogTitle, InputLabel, DialogContent, FormControl, IconButton, Menu, Input, FormGroup, FormControlLabel, Typography, Checkbox, Breadcrumbs, CircularProgress, Switch, Fade} from '@material-ui/core'; +import {GetApp as GetAppIcon, Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons'; import * as cytoscape from 'cytoscape'; import * as edgehandles from 'cytoscape-edgehandles'; @@ -85,6 +26,7 @@ import { w3cwebsocket as W3CWebSocket } from "websocket"; import { useAlert } from "react-alert"; import { validateJson } from "./Workflows.jsx"; import { GetParsedPaths } from "./Apps.jsx"; +import ConfigureWorkflow from '../components/ConfigureWorkflow.jsx'; const surfaceColor = "#27292D" const inputColor = "#383B40" @@ -109,6 +51,34 @@ function useWindowSize() { return size; } +function removeParam(key, sourceURL) { + if (sourceURL === undefined) { + return + } + + var rtn = sourceURL.split("?")[0], + param, + params_arr = [], + queryString = (sourceURL.indexOf("?") !== -1) ? sourceURL.split("?")[1] : ""; + + if (queryString !== "") { + params_arr = queryString.split("&"); + for (var i = params_arr.length - 1; i >= 0; i -= 1) { + param = params_arr[i].split("=")[0]; + if (param === key) { + params_arr.splice(i, 1); + } + } + rtn = rtn + "?" + params_arr.join("&"); + } + + if (rtn === "?") { + return "" + } + + return rtn; +} + const splitter = "|~|" //const referenceUrl = "https://shuffler.io/functions/webhooks/" //const referenceUrl = window.location.origin+"/api/v1/hooks/" @@ -117,17 +87,17 @@ const AngularWorkflow = (props) => { const { globalUrl, isLoggedIn, isLoaded, userdata } = props; const referenceUrl = globalUrl+"/api/v1/hooks/" const alert = useAlert() - const borderRadius = 3 + const borderRadius = 5 + const theme = useTheme(); + const green = "#86c142" + const yellow = "#FECC00" const [bodyWidth, bodyHeight] = useWindowSize(); - const appBarSize = 74 - const headerSize = 60 var to_be_copied = "" const [cystyle, ] = useState(cytoscapestyle) const [cy, setCy] = React.useState() - const [appSearch, setAppSearch] = React.useState("") const [currentView, setCurrentView] = React.useState(0) const [triggerAuthentication, setTriggerAuthentication] = React.useState({}) const [triggerFolders, setTriggerFolders] = React.useState([]) @@ -152,11 +122,14 @@ const AngularWorkflow = (props) => { const [newVariableDescription, setNewVariableDescription] = React.useState(""); const [newVariableValue, setNewVariableValue] = React.useState(""); const [workflowDone, setWorkflowDone] = React.useState(false) + const [authLoaded, setAuthLoaded] = React.useState(false) const [localFirstrequest, setLocalFirstrequest] = React.useState(true) const [requiresAuthentication, setRequiresAuthentication] = React.useState(false) const [rightSideBarOpen, setRightSideBarOpen] = React.useState(false) const [showSkippedActions, setShowSkippedActions] = React.useState(false) const [lastExecution, setLastExecution] = React.useState("") + const [configureWorkflowModalOpen, setConfigureWorkflowModalOpen] = React.useState(false) + const [curpath, setCurpath] = useState(typeof window === 'undefined' || window.location === undefined ? "" : window.location.pathname) // 0 = normal, 1 = just done, 2 = normal const [savingState, setSavingState] = React.useState(0) @@ -199,7 +172,6 @@ const AngularWorkflow = (props) => { const [selectedApp, setSelectedApp] = React.useState({}); const [selectedAction, setSelectedAction] = React.useState({}); - const [selectedActionName, setSelectedActionName] = React.useState({}); const [selectedActionEnvironment, setSelectedActionEnvironment] = React.useState({}); const [executionRequest, setExecutionRequest] = React.useState({}) @@ -217,21 +189,27 @@ const AngularWorkflow = (props) => { const [workflowExecutions, setWorkflowExecutions] = React.useState([]); const [defaultEnvironmentIndex, setDefaultEnvironmentIndex] = React.useState(0) + // This should all be set once, not on every iteration + // Use states and don't update lol const cloudSyncEnabled = props.userdata !== undefined && props.userdata.active_org !== null && props.userdata.active_org !== undefined ? props.userdata.active_org.cloud_sync === true : false //const triggerEnvironments = cloudSyncEnabled ? ["cloud", "onprem"] : environments const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" + const appBarSize = isCloud ? 75 : 60 const triggerEnvironments = isCloud ? ["cloud"] : ["onprem", "cloud"] - const unloadText = 'Are you sure you want to leave without saving (CTRL+S)?' + useBeforeunload(() => { if (!lastSaved) { return unloadText } }) + const [elements, setElements] = useState([]) + // No point going as fast, as the nodes aren't realtime anymore, but bulk updated. + // Set it from 2500 to 6000 to reduce overall load const { start, stop } = useInterval({ - duration: 2500, + duration: 6000, startImmediate: false, callback: () => { fetchUpdates() @@ -306,14 +284,14 @@ const AngularWorkflow = (props) => { }) .then((response) => { if (response.status !== 200) { - console.log("Status not 200 for WORKFLOW EXECUTION :O!") + console.log("Status not 200 for APIKEY gen :O!") } return response.json() }) .then((responseJson) => { setUserSettings(responseJson) - }) + }) .catch(error => { console.log(error) }); @@ -330,7 +308,7 @@ const AngularWorkflow = (props) => { }) .then((response) => { if (response.status !== 200) { - console.log("Status not 200 for WORKFLOW EXECUTION :O!") + console.log("Status not 200 for get settings :O!") } return response.json() @@ -372,7 +350,7 @@ const AngularWorkflow = (props) => { setAuthenticationModalOpen(false) // Needs a refresh with the new authentication.. - alert.success("Successfully saved new app auth") + //alert.success("Successfully saved new app auth") } }) .catch(error => { @@ -380,7 +358,7 @@ const AngularWorkflow = (props) => { }) } - const getWorkflowExecution = (id) => { + const getWorkflowExecution = (id, execution_id) => { fetch(globalUrl+"/api/v1/workflows/"+id+"/executions", { method: 'GET', headers: { @@ -399,16 +377,33 @@ const AngularWorkflow = (props) => { .then((responseJson) => { if (responseJson.length > 0) { // FIXME: Sort this by time - setWorkflowExecutions(responseJson) + + // - means it's opposite + const newkeys = sortByKey(responseJson, "-started_at") + setWorkflowExecutions(newkeys) + //console.log("NEWKEYS: ", newkeys) + //setWorkflowExecutions(responseJson) const cursearch = typeof window === 'undefined' || window.location === undefined ? "" : window.location.search - const tmpView = new URLSearchParams(cursearch).get("execution_id") + var tmpView = new URLSearchParams(cursearch).get("execution_id") + if (execution_id !== undefined && execution_id !== null && execution_id.length > 0 && (tmpView === undefined || tmpView === null || tmpView.length === 0)) { + tmpView = execution_id + } + if (tmpView !== undefined && tmpView !== null && tmpView.length > 0) { - //console.log("SHOW EXECUTION ", tmpView) const execution = responseJson.find(data => data.execution_id === tmpView) if (execution !== null && execution !== undefined) { setExecutionData(execution) setExecutionModalView(1) + start() + + setExecutionRequest({ + "execution_id": execution.execution_id, + "authorization": execution.authorization, + }) + + const newitem = removeParam("execution_id", cursearch) + props.history.push(curpath+newitem) } } } @@ -459,7 +454,7 @@ const AngularWorkflow = (props) => { return response.json() }) .then((responseJson) => { - handleUpdateResults(responseJson) + handleUpdateResults(responseJson, executionRequest) }) .catch(error => { console.log("Error: ", error) @@ -482,7 +477,7 @@ const AngularWorkflow = (props) => { }) .then((response) => { if (response.status !== 200) { - console.log("Status not 200 for WORKFLOW EXECUTION :O!") + console.log("Status not 200 for ABORT EXECUTION :O!") } else { alert.success("Execution aborted") } @@ -496,7 +491,7 @@ const AngularWorkflow = (props) => { // Controls the colors and direction of execution results. // Style is in defaultCytoscapeStyle.js - const handleUpdateResults = (responseJson) => { + const handleUpdateResults = (responseJson, executionRequest) => { //console.log(responseJson) // Loop nodes and find results // Update on every interval? idk @@ -508,11 +503,16 @@ const AngularWorkflow = (props) => { setExecutionData(responseJson) } } + + //console.log("PRE LOOPING RESULTS: !", responseJson.execution_id, executionRequest.execution_id) + if (responseJson.execution_id !== executionRequest.execution_id) { cy.elements().removeClass('success-highlight failure-highlight executing-highlight') return } + //console.log("LOOPING RESULTS!") + if (responseJson.results !== null && responseJson.results !== []) { for (var key in responseJson.results) { var item = responseJson.results[key] @@ -574,6 +574,8 @@ const AngularWorkflow = (props) => { currentnode.removeClass('shuffle-hover-highlight') currentnode.removeClass('awaiting-data-highlight') currentnode.addClass('success-highlight') + incomingEdges.addClass('success-highlight') + outgoingEdges.addClass('success-highlight') if (visited !== undefined && visited !== null && !visited.includes(item.action.label)) { if (executionRunning) { @@ -615,7 +617,7 @@ const AngularWorkflow = (props) => { currentnode.addClass('failure-highlight') if (!visited.includes(item.action.label)) { - if (!item.action.result.includes("failed condition")) { + if (item.action.result !== undefined && item.action.result !== null && !item.action.result.includes("failed condition")) { alert.error("Error for "+item.action.label+" with result "+item.result) } visited.push(item.action.label) @@ -655,12 +657,12 @@ const AngularWorkflow = (props) => { } } - getWorkflowExecution(props.match.params.key) + getWorkflowExecution(props.match.params.key, "") } else if (responseJson.status === "FINISHED") { - console.log("STOPPING BECAUSE ITS OVAH!") + //console.log("STOPPING BECAUSE ITS OVAH!") setExecutionRunning(false) stop() - getWorkflowExecution(props.match.params.key) + getWorkflowExecution(props.match.params.key, "") setUpdate(Math.random()) } } @@ -786,16 +788,41 @@ const AngularWorkflow = (props) => { console.log(responseJson) alert.error("Failed to save: "+responseJson.reason) } else { + if (responseJson.new_id !== undefined && responseJson.new_id !== null) { + window.location.pathname = "/workflows/"+responseJson.new_id + } + success = true if (responseJson.errors !== undefined) { //console.log(responseJson) workflow.errors = responseJson.errors if (responseJson.errors.length === 0) { workflow.isValid = true + workflow.is_valid = true + + //console.log("ELEMENTS: ", cy.elements()) + //const setupGraph = () => { + const cyelements = cy.elements() + for (var i = 0; i < cyelements.length; i++) { + cyelements[i].removeStyle() + cyelements[i].data().is_valid = true + cyelements[i].data().errors = [] + } + + for (var key in workflow.actions) { + workflow.actions[key].is_valid = true + workflow.actions[key].errors = [] + } + } + + for (var key in workflow.errors) { + //console.log("Error: ", workflow.errors[key]) + alert.info(workflow.errors[key]) } setWorkflow(workflow) } + //alert.success("Successfully saved workflow") setSavingState(1) setTimeout(() => { @@ -829,6 +856,11 @@ const AngularWorkflow = (props) => { console.log("FIXME: Might have forgotten to save before executing.") } + if (workflow.public) { + alert.info("Save it to get a new version") + } + + var returncheck = monitorUpdates() if (!returncheck) { alert.error("No startnode set.") @@ -845,7 +877,7 @@ const AngularWorkflow = (props) => { curelements[i].addClass("not-executing-highlight") } - if (executionArgument.length > 0) { + if (executionArgument !== undefined && executionArgument !== null && executionArgument.length > 0) { //alert.success("Starting execution WITH an execution argument") } else { //alert.success("Starting execution") @@ -884,6 +916,19 @@ const AngularWorkflow = (props) => { setExecutionRequestStarted(false) } + if (responseJson.execution_id === "" || responseJson.execution_id === undefined || responseJson.authorization === "" || responseJson.authorization === undefined ) { + alert.error("Something went wrong during execution startup") + console.log("BAD RESPONSE FOR EXECUTION: ", responseJson) + setExecutionRunning(false) + setExecutionRequestStarted(false) + stop() + + for (var i = 0; i < curelements.length; i++) { + curelements[i].removeClass("not-executing-highlight") + } + return + } + setExecutionRequest({ "execution_id": responseJson.execution_id, "authorization": responseJson.authorization, @@ -990,7 +1035,6 @@ const AngularWorkflow = (props) => { .then((response) => { if (response.status !== 200) { console.log("Status not 200 for apps :O!") - return } return response.json() @@ -1006,23 +1050,32 @@ const AngularWorkflow = (props) => { newauth.push(responseJson.data[key]) } - if (reset === true) { - console.log("APP RESET = reset cy") + if (cy !== undefined) { + console.log("NEW AUTH = reset cy's onnodeselect") + + // Remove the old listener for select, run with new one + cy.removeListener('select') + cy.on('select', 'node', (e) => onNodeSelect(e, newauth)) + cy.on('select', 'edge', (e) => onEdgeSelect(e)) } + setAppAuthentication(newauth) + setAuthLoaded(true) } else { - alert.error("Failed getting authentications") + setAuthLoaded(true) + //alert.error("Failed getting authentications") } }) .catch(error => { - alert.error(error.toString()) - }); + setAuthLoaded(true) + alert.error("Auth loading error: "+error.toString()) + }) } const getApps = () => { - fetch(globalUrl+"/api/v1/workflows/apps", { + fetch(globalUrl+"/api/v1/apps", { method: 'GET', headers: { 'Content-Type': 'application/json', @@ -1043,11 +1096,16 @@ const AngularWorkflow = (props) => { //var tmpapps = [] //tmpapps = tmpapps.concat(getExtraApps()) //tmpapps = tmpapps.concat(responseJson) + //console.log("APPS: ", responseJson) setApps(responseJson) - //getAppAuthentication() - setFilteredApps(responseJson.filter(app => !internalIds.includes(app.name))) - setPrioritizedApps(responseJson.filter(app => internalIds.includes(app.name))) + if (isCloud) { + setFilteredApps(responseJson.filter(app => !internalIds.includes(app.name))) + setPrioritizedApps(responseJson.filter(app => internalIds.includes(app.name))) + } else { + setFilteredApps(responseJson.filter(app => !internalIds.includes(app.name) && !(!app.activated && app.generated))) + setPrioritizedApps(responseJson.filter(app => internalIds.includes(app.name))) + } }) .catch(error => { alert.error(error.toString()) @@ -1076,10 +1134,16 @@ const AngularWorkflow = (props) => { if (responseJson.isValid === undefined) { responseJson.isValid = true } + if (responseJson.errors === undefined) { responseJson.errors = [] } + if (responseJson.public) { + alert.info("This workflow is public. You will have to save it to make it your own!") + setLastSaved(false) + } + setWorkflow(responseJson) setWorkflowDone(true) }) @@ -1170,28 +1234,81 @@ const AngularWorkflow = (props) => { setSelectedTrigger({}) } + // Comparing locations between nodes and setting views + const onNodeDrag = (event) => { + //console.log("DRAGGING: ", event.target) + //console.log("LEN2: ", event.target.edges.length) + + const nodedata = event.target.data() + if (nodedata.app_name == "Shuffle Tools" || nodedata.app_name == "Testing") { + //console.log("NODE: ", + //selector: `node[app_name="Shuffle Tools"]`, + console.log(event.target) + + // 1. Find location of node + // 2. Check if it's within view of another node (inside) + // 3. If it is, then hide text + } + + + /* + event.target.animate({ + style: { + "border-width": "12px", + "border-opacity": ".7", + } + }, { + duration: animationDuration, + }) + event.target.animate({ + style: { + "border-width": "12px", + "border-opacity": ".7", + } + }, { + duration: animationDuration, + }) + */ + } + + // Nodeselectbatching: + // https://stackoverflow.com/questions/16677856/cy-onselect-callback-only-once const onNodeSelect = (event, newAppAuth) => { + const data = event.target.data() setLastSaved(false) - const branch = workflow.branches.filter(branch => branch.source_id === data.id || branch.destination_id === data.id) + + //const node = cy.getElementById(data.id) + //if (node.length > 0) { + // node.addClass('shuffle-hover-highlight') + //} + + //const branch = workflow.branches.filter(branch => branch.source_id === data.id || branch.destination_id === data.id) console.log("NODE: ", data) - console.log("BRANCHES: ", branch) + //console.log("APPAUTH: ", newAppAuth) + //console.log("BRANCHES: ", branch) if (data.type === "ACTION") { + + // FIXME - unselect //console.log(cy.elements('[_id!="${data._id}"]`)) // Does it choose the wrong action? var curaction = workflow.actions.find(a => a.id === data.id) if (!curaction || curaction === undefined) { + //event.target.unselect() //alert.error("Action not found. Please remake it.") return } + const curapp = apps.find(a => a.name === curaction.app_name && a.app_version === curaction.app_version) if (!curapp || curapp === undefined) { - alert.error("App "+curaction.app_name+" not found. Did someone delete it?") + alert.error("App "+curaction.app_name+" not found. Is it activated?") //return } else { + + console.log("AUTHENTICATION: ", curapp.authentication) setRequiresAuthentication(curapp.authentication.required && curapp.authentication.parameters !== undefined && curapp.authentication.parameters !== null) if (curapp.authentication.required) { // Setup auth here :) @@ -1202,7 +1319,9 @@ const AngularWorkflow = (props) => { } var tmpAuth = JSON.parse(JSON.stringify(newAppAuth)) - console.log("Checking authentication: ", tmpAuth) + console.log("FOUND AUTH: ", tmpAuth) + + //console.log("Checking authentication: ", tmpAuth) for (var key in tmpAuth) { var item = tmpAuth[key] @@ -1220,8 +1339,9 @@ const AngularWorkflow = (props) => { } } + console.log("OPTIONS: ", authenticationOptions) curaction.authentication = authenticationOptions - console.log("Authentication: ", authenticationOptions) + //console.log("Authentication: ", authenticationOptions) if (curaction.selectedAuthentication === null || curaction.selectedAuthentication === undefined || curaction.selectedAuthentication.length === "") { curaction.selectedAuthentication = {} } @@ -1232,6 +1352,7 @@ const AngularWorkflow = (props) => { } setSelectedApp(curapp) + setSelectedAction(curaction) } if (environments !== undefined && environments !== null) { @@ -1243,37 +1364,12 @@ const AngularWorkflow = (props) => { setSelectedActionEnvironment(env) } - setSelectedActionName(curaction.name) - setSelectedAction(curaction) - - /* - var params = [] - const fixedName = "$"+curaction.label.toLowerCase().replace(" ", "_") - for (var actionkey in workflow.actions) { - if (workflow.actions[actionkey].id === curaction.id) { - continue - } - - for (var paramkey in workflow.actions[actionkey].parameters) { - const param = workflow.actions[actionkey].parameters[paramkey] - if (param.value === null || param.value === undefined || !param.value.includes("$")) { - continue - } - - const innername = param.value.toLowerCase().replace(" ", "_") - if (innername.includes(fixedName)) { - console.log("FOUND!: ", innername) - } - } - } - */ } else if (data.type === "TRIGGER") { //console.log("Should handle trigger "+data.triggertype) //console.log(data) const trigger_index = workflow.triggers.findIndex(a => a.id === data.id) setSelectedTriggerIndex(trigger_index) setSelectedTrigger(data) - setSelectedActionName(data.name) setSelectedActionEnvironment(data.env) if (data.app_name === "Shuffle Workflow") { @@ -1421,7 +1517,7 @@ const AngularWorkflow = (props) => { // might just be confusing cy.nodes().some(function( ele ) { if (ele.id() !== workflow.start && ele.data()["label"] !== undefined) { - alert.success("Changed startnode to "+ele.data()["label"]) + //alert.success("Changed startnode to "+ele.data()["label"]) ele.data("isStartNode", true) workflow.start = ele.id() //throw BreakException @@ -1580,6 +1676,17 @@ const AngularWorkflow = (props) => { }); } + + if (!firstrequest && graphSetup && established && props.match.params.key !== workflow.id && workflow.id !== undefined && workflow.id !== null && workflow.id.length > 0) { + //console.log(props.match.params.key, workflow.id) + //getWorkflow() + //setCy() + //getWorkflowExecution(props.match.params.key, "") + //setEstablished(false) + //setGraphSetup(false) + window.location.pathname = "/workflows/"+props.match.params.key + } + useEffect(() => { if (firstrequest) { setFirstrequest(false) @@ -1587,7 +1694,7 @@ const AngularWorkflow = (props) => { getApps() getAppAuthentication() getEnvironments() - getWorkflowExecution(props.match.params.key) + getWorkflowExecution(props.match.params.key, "") getAvailableWorkflows(-1) getSettings() @@ -1595,15 +1702,29 @@ const AngularWorkflow = (props) => { const tmpView = new URLSearchParams(cursearch).get("view") if (tmpView !== undefined && tmpView !== null && tmpView === "executions") { setExecutionModalOpen(true) + + const newitem = removeParam("view", cursearch) + props.history.push(curpath+newitem) } return } + // App length necessary cus of cy initialization - if (elements.length === 0 && !graphSetup && Object.getOwnPropertyNames(workflow).length > 0) { + //console.log("PRE ELEMENTS: !", workflow.actions, graphSetup, apps, authLoaded, cy) + if (elements.length === 0 && workflow.actions !== undefined && !graphSetup && Object.getOwnPropertyNames(workflow).length > 0) { setGraphSetup(true) setupGraph() - } else if (!established && cy !== undefined && apps.length > 0 && Object.getOwnPropertyNames(workflow).length > 0){ + + //console.log("IN ELEMENT CHECK!") + } else if (!established && cy !== undefined && apps !== null && apps !== undefined && apps.length > 0 && Object.getOwnPropertyNames(workflow).length > 0 && authLoaded){ + //This part has to load LAST, as it's kind of not async. + //This means we need everything else to happen first. + // + //console.log("IN THIS PART AGAIN") + + //console.log("IN ESTABLISHED!") + setEstablished(true) cy.edgehandles({ handleNodes: (el) => el.isNode(), @@ -1630,6 +1751,9 @@ const AngularWorkflow = (props) => { cy.on('mouseover', 'node', (e) => onNodeHover(e)) cy.on('mouseout', 'node', (e) => onNodeHoverOut(e)) + // Handles dragging + cy.on('drag', 'node', (e) => onNodeDrag(e)) + //cy.on('mouseover', 'node', () => $(targetElement).addClass('mouseover')); //cy.on('cxttapstart', 'node', (e) => edgeHandler.start(e.target)) @@ -1664,7 +1788,7 @@ const AngularWorkflow = (props) => { const onNodeHover = (event) => { event.target.animate({ style: { - "border-width": "5px", + "border-width": "7px", "border-opacity": ".7", } }, { @@ -1680,17 +1804,23 @@ const AngularWorkflow = (props) => { // This is here to have a proper transition for lines const onEdgeHover = (event) => { + if (event === null || event === undefined) { + return + } + const sourcecolor = cy.getElementById(event.target.data("source")).style("border-color") const targetcolor = cy.getElementById(event.target.data("target")).style("border-color") - event.target.animate({ - style: { - "line-fill": "linear-gradient", - 'target-arrow-color': targetcolor, - "line-gradient-stop-colors": [sourcecolor, targetcolor], - "line-gradient-stop-positions": [0, 1], - }, - duration: 0, - }) + if (sourcecolor !== null && sourcecolor !== undefined && targetcolor !== null && targetcolor !== undefined) { + event.target.animate({ + style: { + "line-fill": "linear-gradient", + 'target-arrow-color': targetcolor, + "line-gradient-stop-colors": [sourcecolor, targetcolor], + "line-gradient-stop-positions": [0, 1], + }, + duration: 0, + }) + } } @@ -1704,6 +1834,7 @@ const AngularWorkflow = (props) => { node.data.type = "ACTION" node.isStartNode = action["id"] === workflow.start + var example = "" if (action.example !== undefined && action.example !== null && action.example.length > 0) { example = action.example @@ -1711,6 +1842,8 @@ const AngularWorkflow = (props) => { node.data.example = example + //node.data.is_valid = false + return node; }) @@ -1752,6 +1885,31 @@ const AngularWorkflow = (props) => { conditions: conditions, hasErrors: branch.has_errors }; + + // This is an attempt at prettier edges. The numbers are weird to work with. + /* + //http://manual.graphspace.org/projects/graphspace-python/en/latest/demos/edge-types.html + const sourcenode = actions.find(node => node.data._id === branch.source_id) + const destinationnode = actions.find(node => node.data._id === branch.destination_id) + if (sourcenode !== undefined && destinationnode !== undefined && branch.source_id !== branch.destination_id) { + //node.data._id = action["id"] + console.log("SOURCE: ", sourcenode.position) + console.log("DESTINATIONNODE: ", destinationnode.position) + + var opposite = true + if (sourcenode.position.x > destinationnode.position.x) { + opposite = false + } else { + opposite = true + } + + edge.style = { + 'control-point-distance': opposite ? ["25%", "-75%"] : ["-10%", "90%"], + 'control-point-weight': ['0.3', '0.7'], + } + } + */ + return edge; }) @@ -1778,7 +1936,6 @@ const AngularWorkflow = (props) => { const removeNode = () => { setSelectedApp({}) setSelectedAction({}) - setSelectedActionName("") const selectedNode = cy.$(':selected') if (selectedNode.data() === undefined) { @@ -1973,7 +2130,7 @@ const AngularWorkflow = (props) => {
{ }}> -
+
{ setNewVariableName(variable.name) @@ -2040,11 +2197,12 @@ const AngularWorkflow = (props) => {
{ }}> -
+
{ - setNewVariableName(variable.name) - setExecutionVariablesModalOpen(true)}}> + setNewVariableName(variable.name) + setExecutionVariablesModalOpen(true) + }}> Name: {variable.name}
@@ -2107,7 +2265,7 @@ const AngularWorkflow = (props) => { const HandleLeftView = () => { // Defaults to apps. - var thisview = + var thisview = if (currentView === 1) { thisview = } else if (currentView === 2) { @@ -2232,7 +2390,7 @@ const AngularWorkflow = (props) => { "description": "Add your email provider", "trigger_type": "EMAIL", "errors": null, - "is_valid": true ? false : cloudSyncEnabled, + "is_valid": (cloudSyncEnabled && !isCloud) ? true : false, "label": "Email", "environment": "cloud", "large_image": 'data:image/png;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4QAYRXhpZgAASUkqAAgAAAAAAAAAAAAAAP/hAytodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENTNiAoV2luZG93cykiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6Nzg4QTJBMjVEMDI1MTFFN0EwQUVDODc5QjYyQkFCMUQiIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6Nzg4QTJBMjZEMDI1MTFFN0EwQUVDODc5QjYyQkFCMUQiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo3ODhBMkEyM0QwMjUxMUU3QTBBRUM4NzlCNjJCQUIxRCIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo3ODhBMkEyNEQwMjUxMUU3QTBBRUM4NzlCNjJCQUIxRCIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/Pv/bAEMAAwICAgICAwICAgMDAwMEBgQEBAQECAYGBQYJCAoKCQgJCQoMDwwKCw4LCQkNEQ0ODxAQERAKDBITEhATDxAQEP/AAAsIAGQAZAEBEQD/xAAeAAABAwUBAQAAAAAAAAAAAAAAAQgJAgQFBwoGA//EAEoQAAECBAMEBwMDEQgDAAAAAAECAwAEBREGBxIIITFRCRMiMkFSYRRicSNCQxUWGBk2U1dYc3WBlJWzwdLTJDNjZXKDkeEmgqH/2gAIAQEAAD8Ak8JKipSlBwuDSpSeDw8qeRguQQrUAQNAV4JH3s+sA7OnT8n1fcv9Bfzc7wWAAToIAOsJ8Uq++H0gI1XSUlYWdSkji6fMnkBAdS9SidesWUpI3OjyjkYXtXCgbEDSFW3JT5D6+sIOzp0/J9X3NX0F/NzvBYABOggA6wnxSr74fSAjVdJSVhZ1KSOLp8yeQEBJUSVKCysaVKHB0eVPIwXIIVqAIGgK8Ej72fWFS640kNtzjcukcGli6k/GENwVagAQO0EcEjmj1g33AATe1wD3SnmffgG/Tp337mv535T+EaB2k9tzInZilVyuMq+up4iUjWxh+mFLs8s23dbv0stnmsgnwBiNPOHpddozHExMSmWsrSsA0tZIaMs0JudCfV50FKf/AFQPjDYsT7S+0LjJ8zGJc68aTqiSrSqtPoQD6ISoJH6BGFkM5c3qW8Jim5qYvlXArUFNVuZSb89y43Tlv0ju15ltMtKZzUmsRSbZGuSxC0mebcHIrV8r/wALEP02c+l2ywzAmZXDGeND+saqvEIRVWXFP0x1Z3AOE9thPx1JHioQ/un1Kn1eQZqtLn2ZuSmkJdamZVwOIWlQuktKTuUg8xFybgnUACB2gjgkc0e9BvuAAm9rgHulPM+/CpDik3bal1p8FPd8/GEtp7Ojq+r36OPUe96wWv2dF79vR5/8T/qI/ukM6RMZMGcyWyUqDMzjZ5vRWKwmy26UlQ3IQOBmLHx3IFibncIeqvWKtiCqTVbrtSmahUJ11T0zNTLqnHXnFG5UpSiSSeZi0ggggh2GxRt8Y92X69K4cr83N1zLuadCZumLXrcp4Ue0/KXPZPiW+6r0O+JxsF4zwvmFhSl42wXV5eo0Sqy6ZySmWFakJbUO/wDHiCk7wQQd4jN2v2dF79vR5v8AE/6g6rrflPYfar/S69Or9EIAAE6QoAHshfFJ5r9Ibpt3bTrOzBkbPYhpb7f11V5aqZh9le8iZUm65i3i20ntcirQPGIA6nU6jWqlNVirzr05PTzy5iZmHllTjrqyVKWoneSSSSYtoIIIIIIkI6J/axmsA4+Rs84yqZ+tvFb5VQ1vL7EjVCNyN/Bt4C1vOEn5xiYndYghVr3IHeKuY9yKVJbJu63MrV4qY7h+EVA6rEKKwvclSuLp8quQiDnpWM5ZnMrafn8HS04pykZfy6KMw2D2BNEByZUPXWQj/aEM0h3uwvk5PYmoeLM4HcnqHmth/CU1KytdwrNy5M+uUdQtZmZBYIu83oN2z3wbcbWk7ym2Zej6zuwXJ49y5yXwbUqXNgpUPZlpelnh32XmyrU24k7ik7/0WMey+wI2OPxesJfqyv5oPsCNjj8XrCX6sr+aD7AjY4/F6wl+rq/mhs2b2SGzXjPGc3s9bKuzngiq4zZGjEWJ3pRTlKwkyrcVOKCrOzVr6GRex73AiIe65TvqRWqhSet632Kadl9enTq0LKb28L24R86ZUp6jVKUrFMmVy85IvtzMu8g2U24hQUlQPMEAx0h7O+abGdWR+DM0mnAF16lMuzRTxamgNDzQHIOJWI2IpxDZ0Lm3ZdQ4tti6U/CEdeDaHJhxYWAklahwdAF9KeRjmXzRxJMYxzLxXiyacUt2sVqdnlFRuflHlq/jHmIlv6D37is1T/mlM/cvQ5zNnZ7xtl/jWc2htlFUtIYrfs7iXCLy+rpeLGk7zcDczN2vpdFrnvcSTsrIPaHwNtBYcmKlhsv02t0h4ydfw7UE9XUKPOJJC2Xmzv3KBAWNyrbvEDaDjjbTa3XVpQhAKlKUbBIHEk+ENLxdnDj/AGr8T1LJ7Zgra6NgymPmSxhmU0LhB+kkaV4OPkGynu6gG4PAlwGUuT2AMjsDy2BMuqGin06XBcdWTrfm3j3333D2nHFHeVH/AOCwjmnxr92Ve/Oc1+9VGGibDogMVP1vZWmKI+4b4dxJOybS1G4S06ht7QPipxf/ADD4kurbGhE21LpHBtwXUn4xbVNpb9OnGAAFrl3EkI4JukgFHrHMFWpdyUrE/KvAhxmZdbUDxBCyDFnEuHQej/wjNU/5rTP3L0PQ2hs1cR4f+pOUWU/VTGZeOtbFK1jW3SZNO6YqkwPBtlJ7IPfcKUi++NdYh2H5LB1CoWLtnXFD2Fs1sLS6rV+ZUXG8SqWouPtVVP0yXnCo6+8gqFtwAGAbkdpzbFcTgXNLB1Qyay9pBEri1iXm9VQxPNo/vJeVdT/dyJ3XcG9YNgTvt6TFODKZsY4nlc2MsaEmRypn25em45oMi2eqpiUANsVllA8gsiYtvUiyzcpJhz8rOylSkGqjITTUzKzTKXmHmlhSHG1C6VJI3EEEEGOXnGv3ZV785zX71UYaJjehfk3mdn/GM4oHRM4sWEBfcsiUZ1Eeu+JBUhxSbttS60+Cnu+fjCAaDbR1fV9rRx6n3vW8c5+2Bl1MZV7TGYmDXmlIaZrkxNyhIsFy0wrr2lD00OJjT0SfdE5mphvJnIrOXHmJutdalatSmZSSlxqmKhNuNOpYlWU8VOOLISAOdzuBiQLZ5yrxJQTVs382Q0/mVjrQ9VAk6m6RJp3y9Llz4NtA9ojvuFSjfdG54N8fGekZOpyUxTajKtTMrNNLYfYdSFIdbULKSpJ3EEEgiG4ZXz07s0Zis7OuJpp1zAmJFvP5cVSYWSJVYut2hurPzkC62Ce83dHFFogGxr92Ve/Oc1+9VGGiezo0MupnLzY/wezPy5bm8RqmMROsqFiUvr+SWf8AaQ2besOl6rrflPYfar/S69Or9EIAAE6QoAHshfFJ5r9Ii76Y7Z4mJgUHaSw7IqWhlCKHiLQN6RcmWmP9Nypsn8mIizj3GWGdOY2T9Xka1gSuJlH6bO/VKWbflm5hlubDam0v9U4lSC4lClBKiLp1G1iY3v8AbSdtr8LTP7Ekf6UL9tJ22vwss/sOR/pQfbSdtr8LLP7Dkf6UH20nba/C0z+w5H+lHmMxOkB2qc1MOKwrjjMNmfkPaGZxrTSZRl1iYZWFtPNOobC21pULhSSDx8DDe5uamJ6aenZt1Tr8w4p11auKlqNyT8SY2ZszZH1raIzqw1ldSG1hqozSXKlMAHTKyLZCn3VHwsi4HvKSPGOjSj0im0CjyVBpMqJen06XalZZhG7Q22kJQE+4AAIulJbJu63MrV4qY7h+EVA6rEKKwvclSuLp8quQjB45wVhrMbB9YwNjGnIn6JW5VyQnWVjihYtoTysbEKHAgGOf7a72U8abKeZszhStMOzVAnVreoNXCfk5uXvuSojcHUAgLTz3jcRGi4IIIIIuqVSqnXKnK0ajSD89PzzyJeWlpdsrcecUbJQlI3kkkAAROd0eGxqjZiy7XiLF8sy5j/FjaFVEiyhJMDeiSB9D2lkbiqw4JEO58CrUQAdJV4pPkHu+sIpxDZ0Lm3ZdQ4tti6U/CFJKiVKUFle5Sk8HR5U8jBcghWoAgaArwSPIfe9Y8PnJkvl1nzgScy7zMoDVQpMyLtlXZekHfmvNucULHgR8DcEiIZtqro1s58gZucxFg6QmsbYJQVOonpFkqnJNrw9pYTcgAfSJuk8Tp4Qz9SSklKgQQbEHwgggjYeTOz9m7n/iFGHMq8Fz1YdCgJiZSjRKSiT8955XYQB6m58AYmN2LejtwJsyoYxri52XxVmC43unA3/ZpAEb0ygVvv4F09ojgEgm7wSSq5Kgsr3KUODo8qeRguQQrUAQNAV4JHkPvesKl1bY0Im2pdI4NuC6k/GENwVagAQO0EcEjmj1g33AATe1wD3SnmffgG/Tp337mv535T+EG4i4KiCbAnvE8j7kaHzf2Hdl/O1+YqONcraexU3jd6p0i8jNFfPU1ZLnxWlUNjxL0LOTs6+tzC+bWLKSkHUWpmXl5xKR4BJAbJjD0/oTcEoeH1Uz5rbzfe0sUZlolH+pTigFelo3Nlr0UuyZgSYZn6vQ6zjKaQQpr6uz3yBI462WQhNvRVxDscMYVwvgujMYfwfh+n0Wly/ZZlJCVQw2k8tCABp9YypsL6iQAe0U8Unkj3YDcE6gAQO0EcEjmj3oN9wAE3tcA90p5n34VIcUm7bUutPgp7vn4wOpS05MNtiyZdAW0PKo+MASkuIbIulbPXKHNfOEa+V9m6zf7Vq633rcIpSoqbQ6T2lvdQo80coVxRbRMLRuVLuBts+VJ4iKnEhtb6ECwl0BxseVR8YAlJcQ2RdK2euUOa+cI18r7N1m/wBq1db71uEUpUVNodJ7S3uoUeaOUK4otomFo3Kl3A22fKk8RFTiQ2t9CBYS6A42PKo+MASkuIbIulbPXKHNfOPtKSkvNy6JiYaC3F71KJO+P//Z', @@ -2258,7 +2416,7 @@ const AngularWorkflow = (props) => { : - const color = trigger.is_valid ? "green" : "orange" + const color = trigger.is_valid ? green : yellow return( { } if (data.is_valid === false) { - alert.error(data.name+" is only available on https://shuffler.io so far") + alert.error(data.name+" trigger isn't available yet") return } @@ -2335,7 +2493,7 @@ const AngularWorkflow = (props) => { const newAppData = { app_name: data.name, app_version: "1.0.0", - environment: data.environment, + environment: isCloud ? "cloud" : data.environment, description: data.description, long_description: data.long_description, errors: [], @@ -2471,7 +2629,8 @@ const AngularWorkflow = (props) => { authentication: [], execution_variable: undefined, example: example, - category: app.categories !== null && app.categories !== undefined && app.categories.length > 0 ? app.categories[0] : "" + category: app.categories !== null && app.categories !== undefined && app.categories.length > 0 ? app.categories[0] : "", + authentication_id: "", } // FIXME: overwrite category if the ACTION chosen has a different category @@ -2532,6 +2691,56 @@ const AngularWorkflow = (props) => { console.log("SHOULD STITCH WITH STARTNODE") cy.add(edgeToBeAdded) } + + // AUTHENTICATION + if (app.authentication.required) { + // Setup auth here :) + const authenticationOptions = [] + var findAuthId = "" + if (newAppData.authentication_id !== null && newAppData.authentication_id !== undefined && newAppData.authentication_id.length > 0) { + findAuthId = newAppData.authentication_id + } + + var tmpAuth = JSON.parse(JSON.stringify(appAuthentication)) + for (var key in tmpAuth) { + var item = tmpAuth[key] + + const newfields = {} + for (var filterkey in item.fields) { + newfields[item.fields[filterkey].key] = item.fields[filterkey].value + } + + item.fields = newfields + if (item.app.name === app.name) { + authenticationOptions.push(item) + if (item.id === findAuthId) { + newAppData.selectedAuthentication = item + } + } + } + + if (authenticationOptions !== undefined && authenticationOptions !== null && authenticationOptions.length > 0) { + for (var key in authenticationOptions) { + const option = authenticationOptions[key] + if (option.active) { + newAppData.selectedAuthentication = option + newAppData.authentication_id = option.id + break + } + } + } + + //newAppData.authentication = authenticationOptions + //if (newAppData.selectedAuthentication === null || newAppData.selectedAuthentication === undefined || newAppData.selectedAuthentication.length === "") { + // newAppData.selectedAuthentication = {} + //} else { + // console.log("CAN WE SELECT AUTH?: ", authenticationOptions) + //} + } else { + newAppData.authentication = [] + newAppData.authentication_id = "" + newAppData.selectedAuthentication = {} + } workflow.actions.push(newAppData) setWorkflow(workflow) @@ -2552,78 +2761,85 @@ const AngularWorkflow = (props) => { const appScrollStyle = { overflow: "scroll", - maxHeight: bodyHeight-appBarSize-55, - minHeight: bodyHeight-appBarSize-55, + maxHeight: bodyHeight-appBarSize-55-50, + minHeight: bodyHeight-appBarSize-55-50, + marginTop: 1, overflowY: "auto", overflowX: "hidden", } - const runAppSearch = (event) => { - setAppSearch(event.target.value) - setFilteredApps(apps.filter(app => app.name.toLowerCase().includes(event.target.value.trim().toLowerCase()))) - } + const AppView = (props) => { + const { allApps, prioritizedApps, filteredApps } = props; + const [visibleApps, setVisibleApps] = React.useState(prioritizedApps.concat(filteredApps.filter(innerapp => !internalIds.includes(innerapp.id)))) - const ParsedAppPaper = (props) => { - const app = props.app - const [hover, setHover] = React.useState(false) + const ParsedAppPaper = (props) => { + const app = props.app + const [hover, setHover] = React.useState(false) - // FIXME - add label to apps, as this might be slow with A LOT of apps - var newAppname = app.name - newAppname = newAppname.replace("_", " ").charAt(0).toUpperCase()+newAppname.substring(1) - const maxlen = 24 - if (newAppname.length > maxlen) { - newAppname = newAppname.slice(0, maxlen)+".." - } + const maxlen = 24 + var newAppname = app.name + newAppname = newAppname.charAt(0).toUpperCase()+newAppname.substring(1) + if (newAppname.length > maxlen) { + newAppname = newAppname.slice(0, maxlen)+".." + } + app.name.replaceAll("_", " ", -1) - //const image = "url("+app.large_image+")" - const image = app.large_image - const newAppStyle = JSON.parse(JSON.stringify(paperAppStyle)) - const pixelSize = !hover ? "2px" : "4px" - newAppStyle.borderLeft = app.is_valid ? `${pixelSize} solid green` : `${pixelSize} solid orange` - + //const image = "url("+app.large_image+")" + const image = app.large_image + const newAppStyle = JSON.parse(JSON.stringify(paperAppStyle)) + const pixelSize = !hover ? "2px" : "4px" + newAppStyle.borderLeft = app.is_valid ? `${pixelSize} solid ${green}` : `${pixelSize} solid ${yellow}` - return ( - {handleAppDrag(e, app)}} - onStop={(e) => {handleDragStop(e, app)}} - key={app.id} - dragging={false} - position={{ - x: 0, - y: 0, - }} - > - {setHover(true)}} onMouseOut={() => {setHover(false)}}> - - - {newAppname} - - - -

{newAppname}

+ return ( + {handleAppDrag(e, app)}} + onStop={(e) => {handleDragStop(e, app)}} + key={app.id} + dragging={false} + position={{ + x: 0, + y: 0, + }} + > + {setHover(true)}} onMouseOut={() => {setHover(false)}}> + + + {newAppname} - - Version: {app.app_version} - - - {app.description} + + + {newAppname} + + + + Version: {app.app_version} + + + + + {app.description} + + - - ) - } + ) + } + + const runSearch = (event) => { + if (event.target.value.length > 0) { + setVisibleApps(allApps.filter(app => app.name.toLowerCase().includes(event.target.value.trim().toLowerCase()))) + } else { + setVisibleApps(prioritizedApps.concat(filteredApps.filter(innerapp => !internalIds.includes(innerapp.id)))) + } + } - const AppView = () => { return(
-
- {/* { maxWidth: "95%", fontSize: "1em", }, + endAdornment: ( + + + + + + ) }} fullWidth color="primary" - placeholder={"Search apps"} - onChange={(event) => { - runAppSearch(event) + placeholder={"Search Active Apps"} + id="appsearch" + onBlur={(event) => { + runSearch(event) }} /> - */} - {prioritizedApps.map((app, index) => { - return( - - ) - })} - {filteredApps.filter(innerapp => !internalIds.includes(innerapp.id)).map((app, index) => { - if (app.invalid) { - return null - } + {visibleApps.length > 0 ? +
+ {visibleApps.map((app, index) => { + if (app.invalid) { + return null + } - return( - - ) - })} -
+ return( + + ) + })} +
+ : +
+ + + Loading apps + +
+ }
) @@ -2701,7 +2929,7 @@ const AngularWorkflow = (props) => { //setSelectedActionEnvironment(env) setSelectedAction(selectedAction) - setSelectedActionName(e.target.value) + setUpdate(Math.random()) } // APPSELECT at top @@ -2710,17 +2938,19 @@ const AngularWorkflow = (props) => { // ACTION select // const selectedNameChange = (event) => { - console.log("OLDNAME: ", selectedActionName) - event.target.value = event.target.value.replace("(", "") - event.target.value = event.target.value.replace(")", "") - event.target.value = event.target.value.replace("$", "") - event.target.value = event.target.value.replace("#", "") - event.target.value = event.target.value.replace(".", "") - event.target.value = event.target.value.replace(",", "") - event.target.value = event.target.value.replace(" ", "_") + //console.log("OLDNAME: ", selectedAction.name) + event.target.value = event.target.value.replaceAll("(", "") + event.target.value = event.target.value.replaceAll(")", "") + event.target.value = event.target.value.replaceAll("$", "") + event.target.value = event.target.value.replaceAll("#", "") + event.target.value = event.target.value.replaceAll(".", "") + event.target.value = event.target.value.replaceAll(",", "") + event.target.value = event.target.value.replaceAll(" ", "_") selectedAction.label = event.target.value setSelectedAction(selectedAction) + //console.log("SHOULD CHANGE NAME EVERYWHERE ITS USED TOO BASED ON OLD NAME!") + /* if (nodeaction.label !== curaction.label) { console.log("BEACH!") @@ -2806,10 +3036,10 @@ const AngularWorkflow = (props) => { const AppActionArguments = (props) => { const [selectedActionParameters, setSelectedActionParameters] = React.useState([]) const [selectedVariableParameter, setSelectedVariableParameter] = React.useState("") - const [showDropdown, setShowDropdown] = React.useState(false) - const [showDropdownNumber, setShowDropdownNumber] = React.useState(0) const [actionlist, setActionlist] = React.useState([]) const [jsonList, setJsonList] = React.useState([]) + const [showDropdown, setShowDropdown] = React.useState(false) + const [showDropdownNumber, setShowDropdownNumber] = React.useState(0) const [showAutocomplete, setShowAutocomplete] = React.useState(false) const [menuPosition, setMenuPosition] = useState(null) @@ -2854,7 +3084,8 @@ const AngularWorkflow = (props) => { var exampledata = item.example === undefined ? "" : item.example // Find previous execution and their variables - if (exampledata === "" && workflowExecutions.length > 0) { + //exampledata === "" && + if (workflowExecutions.length > 0) { // Look for the ID const found = false for (var key in workflowExecutions) { @@ -2938,6 +3169,7 @@ const AngularWorkflow = (props) => { selectedActionParameters[count]["value_replace"] = paramcheck selectedAction.parameters[count]["value_replace"] = paramcheck setSelectedAction(selectedAction) + //setUpdate(Math.random()) return } } @@ -3030,6 +3262,7 @@ const AngularWorkflow = (props) => { selectedActionParameters[count].value = event.target.value selectedAction.parameters[count].value = event.target.value setSelectedAction(selectedAction) + //setUpdate(Math.random()) //setUpdate(event.target.value) } @@ -3044,7 +3277,6 @@ const AngularWorkflow = (props) => { selectedActionParameters[count].action_field = fieldvalue selectedAction.parameters = selectedActionParameters - setSelectedActionName(selectedActionName) setSelectedApp(selectedApp) setSelectedAction(selectedAction) setUpdate(fieldvalue) @@ -3065,11 +3297,10 @@ const AngularWorkflow = (props) => { // FIXME - check if startnode // Set value - setSelectedActionName(selectedActionName) setSelectedApp(selectedApp) setSelectedAction(selectedAction) - setUpdate(fieldvalue) + setUpdate(Math.random()) } const changeActionParameterVariant = (data, count) => { @@ -3092,7 +3323,6 @@ const AngularWorkflow = (props) => { selectedAction.parameters = selectedActionParameters // This is a stupid workaround to make it refresh rofl - setSelectedActionName({}) setSelectedAction({}) setSelectedTrigger({}) setSelectedApp({}) @@ -3100,9 +3330,9 @@ const AngularWorkflow = (props) => { // FIXME - check if startnode // Set value - setSelectedActionName(selectedActionName) setSelectedApp(selectedApp) setSelectedAction(selectedAction) + setUpdate(Math.random()) } // FIXME: Issue #40 - selectedActionParameters not reset @@ -3123,6 +3353,7 @@ const AngularWorkflow = (props) => { selectedActionParameters[count].value = selectedAction.selectedAuthentication.fields[data.name] selectedAction.parameters[count].value = selectedAction.selectedAuthentication.fields[data.name] setSelectedAction(selectedAction) + //setUpdate(Math.random()) return null } @@ -3231,8 +3462,8 @@ const AngularWorkflow = (props) => { { setMenuPosition({ - top: event.pageY, - left: event.pageX, + top: event.pageY+10, + left: event.pageX+10, }) setShowDropdownNumber(count) setShowDropdown(true) @@ -3294,8 +3525,8 @@ const AngularWorkflow = (props) => { { setMenuPosition({ - top: event.pageY, - left: event.pageX, + top: event.pageY+10, + left: event.pageX+10, }) setShowDropdownNumber(count) setShowDropdown(true) @@ -3504,10 +3735,11 @@ const AngularWorkflow = (props) => { // Handles the fields under OpenAPI body to be parsed. if (data.name.startsWith("${") && data.name.endsWith("}")) { + console.log("INSIDE VALUE REPLACE: ", data.name, toComplete) // PARAM FIX - Gonna use the ID field, even though it's a hack const paramcheck = selectedAction.parameters.find(param => param.name === "body") if (paramcheck !== undefined) { - if (paramcheck["value_replace"] === undefined) { + if (paramcheck["value_replace"] === undefined || paramcheck["value_replace"] === null) { paramcheck["value_replace"] = [{ "key": data.name, "value": toComplete, @@ -3645,7 +3877,7 @@ const AngularWorkflow = (props) => { // FIXME: Should be recursive in here const icon = pathdata.type === "value" ? : pathdata.type === "list" ? : return ( - {}} + {}} onClick={() => { handleItemClick([innerdata, pathdata]) }} @@ -3690,11 +3922,12 @@ const AngularWorkflow = (props) => { } tmpitem = tmpitem.charAt(0).toUpperCase()+tmpitem.substring(1) + tmpitem = tmpitem.replaceAll("_", " ") const description = data.description === undefined ? "" : data.description return (
-
+
{data.configuration === true ? @@ -3704,15 +3937,15 @@ const AngularWorkflow = (props) => { }}/> : -
+
} -
+
{tmpitem}
- {selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 ? null : + {/*selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 ? null :
{ @@ -3741,6 +3974,29 @@ const AngularWorkflow = (props) => {
+ */} + {selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 && selectedActionParameters[count].required === true && selectedActionParameters[count].unique_toggled !== undefined ? null : +
+ +
{}}> + { + //console.log("CHECKED!: ", selectedActionParameters[count]) + selectedActionParameters[count].unique_toggled = !selectedActionParameters[count].unique_toggled + selectedAction.parameters[count].unique_toggled = selectedActionParameters[count].unique_toggled + setSelectedActionParameters(selectedActionParameters) + setSelectedAction(selectedAction) + setUpdate(Math.random()) + }} + name="requires_unique" + /> +
+ +
}
{datafield} @@ -3875,6 +4131,14 @@ const AngularWorkflow = (props) => { return [] } + if (key.startsWith("-") && key.length > 2) { + key = key.slice(1, key.length) + return array.sort(function(a, b) { + var x = a[key]; var y = b[key] + return ((x < y) ? -1 : ((x > y) ? 1 : 0)) + }).reverse() + } + return array.sort(function(a, b) { var x = a[key]; var y = b[key] return ((x < y) ? -1 : ((x > y) ? 1 : 0)) @@ -3884,7 +4148,7 @@ const AngularWorkflow = (props) => { const rightsidebarStyle = { position: "fixed", right: 0, - top: headerSize+1, + top: appBarSize+1, height: "100%", bottom: 0, minWidth: 350, @@ -3915,7 +4179,7 @@ const AngularWorkflow = (props) => {
-

{selectedAction.app_name}

+

{selectedAction.app_name.replaceAll("_", " ")}

{ console.log("FIND EXAMPLE RESULTS FOR ", selectedAction) @@ -3938,7 +4202,7 @@ const AngularWorkflow = (props) => { } } }}> - + @@ -3953,10 +4217,16 @@ const AngularWorkflow = (props) => {
-
- +
+ {selectedAction.id === workflow.start ? null : + + + + }
@@ -3975,14 +4245,13 @@ const AngularWorkflow = (props) => { /> {selectedApp.name !== undefined && selectedAction.authentication !== undefined && selectedAction.authentication.length === 0 && requiresAuthentication ?
- Authenticate {selectedApp.name}: - +
@@ -4111,7 +4380,7 @@ const AngularWorkflow = (props) => { Actions
{ - //setTriggerFolderWrapper(e) - setTriggerFolderWrapperMulti(e) - }} - fullWidth - input={} - key={selectedTrigger} - > - {triggerFolders.map(folder => { - var folderItem = - if (folder.childFolderCount > 0) { - // Here to handle subfolders sometime later - folderItem = - - } + {triggerFolders === undefined || triggerFolders === null ? + null : + +
+
+
+ Select a folder +
+
+ } + key={selectedTrigger} + > + {triggerFolders.map(folder => { + //console.log("FOLDER: ", folder) + //var folderItem = - return folderItem - })} - + if (folder.childFolderCount > 0) { + // Here to handle subfolders sometime later + folderItem = + + } + + return folderItem + })} + + + }
} else if (triggerAuthentication.type === "gmail") { triggerInfo = "SPECIAL GMAIL" @@ -5031,15 +5343,17 @@ const AngularWorkflow = (props) => {
- Login to either: + Login to either
{outlookButton} + {/* + */}
return( @@ -5115,6 +5429,11 @@ const AngularWorkflow = (props) => { } const SubflowSidebar = () => { + const [menuPosition, setMenuPosition] = useState(null) + const [showDropdown, setShowDropdown] = React.useState(false) + const [showDropdownNumber, setShowDropdownNumber] = React.useState(0) + const [showAutocomplete, setShowAutocomplete] = React.useState(false) + if (Object.getOwnPropertyNames(selectedTrigger).length > 0) { if (workflow.triggers[selectedTriggerIndex] === undefined) { return null @@ -5192,7 +5511,17 @@ const AngularWorkflow = (props) => { if (e.target.value.id !== workflow.id) { const startnode = e.target.value.actions.find(action => action.id === e.target.value.start) if (startnode !== undefined && startnode !== null) { + //ddsetSubworkflowStartnode(innernode) setSubworkflowStartnode(startnode) + + try { + workflow.triggers[selectedTriggerIndex].parameters[3].value = e.target.value.id + } catch { + workflow.triggers[selectedTriggerIndex].parameters[3] = { + "name": "startnode", + "value": e.target.value.id, + } + } } console.log("STARTNODE: ", startnode) } @@ -5214,50 +5543,55 @@ const AngularWorkflow = (props) => { })} } - {workflow.triggers[selectedTriggerIndex].parameters[0].value.length === 0 ? null : Explore selected workflow} + {workflow.triggers[selectedTriggerIndex].parameters[0].value.length === 0 ? null : + workflow.triggers[selectedTriggerIndex].parameters[0].value === props.match.params.key ? null : + Explore selected workflow + } -
-
-
- Select the Startnode -
-
{subworkflow === undefined || subworkflow === null || subworkflow.id === undefined || subworkflow.actions === null || subworkflow.actions === undefined || subworkflow.actions.length === 0 ? null : - { - setSubworkflowStartnode(e.target.value) - - try { - workflow.triggers[selectedTriggerIndex].parameters[3].value = e.target.value.id - } catch { - workflow.triggers[selectedTriggerIndex].parameters[3] = { - "name": "startnode", - "value": e.target.value.id, } - } + }} + fullWidth + onChange={(e) => { + setSubworkflowStartnode(e.target.value) - setWorkflow(workflow) - //setUpdate(Math.random()) - }} - style={{backgroundColor: inputColor, color: "white", height: "50px"}} - > - {subworkflow.actions.map((action, index) => { - //console.log(action) - return ( - parent.id === action.id)} key={index} style={{backgroundColor: inputColor, color: "white"}} value={action}> - {action.label} - - ) - })} - + try { + workflow.triggers[selectedTriggerIndex].parameters[3].value = e.target.value.id + } catch { + workflow.triggers[selectedTriggerIndex].parameters[3] = { + "name": "startnode", + "value": e.target.value.id, + } + } + + setWorkflow(workflow) + //setUpdate(Math.random()) + }} + style={{backgroundColor: inputColor, color: "white", height: "50px"}} + > + {subworkflow.actions.map((action, index) => { + //console.log(action) + return ( + parent.id === action.id)} key={index} style={{backgroundColor: inputColor, color: "white"}} value={action}> + {action.label} + + ) + })} + + }
@@ -5274,6 +5608,21 @@ const AngularWorkflow = (props) => { maxWidth: "95%", fontSize: "1em", }, + endAdornment: ( + + + { + setMenuPosition({ + top: event.pageY+10, + left: event.pageX+10, + }) + //setShowDropdownNumber(count) + setShowDropdown(true) + setShowAutocomplete(true) + }}/> + + + ) }} rows="6" multiline @@ -5499,10 +5848,10 @@ const AngularWorkflow = (props) => { if (trigger.id === undefined) { return } - alert.info("Stopping trigger") + alert.info("Deleting mail trigger") fetch(globalUrl+"/api/v1/workflows/"+props.match.params.key+"/outlook/"+trigger.id, { - method: 'DELETE', + method: 'DELETE', headers: { 'Content-Type': 'application/json', 'Accept': 'application/json', @@ -5537,6 +5886,10 @@ const AngularWorkflow = (props) => { const startMailSub = (trigger, triggerindex) => { var folders = [] + if (triggerFolders === null || triggerFolders === undefined) { + return null + } + const splitItem = workflow.triggers[selectedTriggerIndex].parameters[0].value.split(splitter) for (var key in splitItem) { const item = splitItem[key] @@ -5884,7 +6237,7 @@ const AngularWorkflow = (props) => { if (Object.getOwnPropertyNames(selectedTrigger).length > 0 && workflow.triggers[selectedTriggerIndex] !== undefined) { if (workflow.triggers[selectedTriggerIndex].parameters === undefined || workflow.triggers[selectedTriggerIndex].parameters === null || workflow.triggers[selectedTriggerIndex].parameters.length === 0) { workflow.triggers[selectedTriggerIndex].parameters = [] - workflow.triggers[selectedTriggerIndex].parameters[0] = {"name": "cron", "value": "120"} + workflow.triggers[selectedTriggerIndex].parameters[0] = {"name": "cron", "value": isCloud ? "*/15 * * * *" : "120"} workflow.triggers[selectedTriggerIndex].parameters[1] = {"name": "execution_argument", "value": '{"example": {"json": "is cool"}}'} setWorkflow(workflow) } @@ -5936,7 +6289,7 @@ const AngularWorkflow = (props) => { setSelectedTrigger(selectedTrigger) if (e.target.value === "cloud") { console.log("Set cloud config") - workflow.triggers[selectedTriggerIndex].parameters[0].value = "*/2 * * * *" + workflow.triggers[selectedTriggerIndex].parameters[0].value = "*/15 * * * *" //var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/") //const urlpath = tmpvalue.slice(3, tmpvalue.length) @@ -6048,7 +6401,7 @@ const AngularWorkflow = (props) => { return null } - const cytoscapeViewWidths = 750 + const cytoscapeViewWidths = 800 const bottomBarStyle = { position: "fixed", right: 20, @@ -6064,28 +6417,36 @@ const AngularWorkflow = (props) => { const topBarStyle= { position: "fixed", right: 0, - left: leftBarSize, - top: appBarSize, + left: leftBarSize+20, + top: appBarSize+20, + /* minWidth: cytoscapeViewWidths, maxWidth: cytoscapeViewWidths, - marginLeft: 20, - marginBottom: 20, + */ } const TopCytoscapeBar = () => { return (
-
+
-

+

Workflows

-

+

{workflow.name}

+ + {workflow.public ? +

+ Public Workflow PREVIEW +

+ : + null + }
@@ -6210,7 +6571,7 @@ const AngularWorkflow = (props) => { - @@ -6280,7 +6642,7 @@ const AngularWorkflow = (props) => { - + + {workflow.public ? + + + + + + : null} {/* */} - + {workflow.configuration !== null && workflow.configuration !== undefined && workflow.configuration.exit_on_error !== undefined ? : null}
) @@ -6376,9 +6760,11 @@ const AngularWorkflow = (props) => { //}}>Execute websocket // - const leftView = leftViewOpen ?
- -
: + const leftView = leftViewOpen ? +
+ +
+ :
{ setLeftViewOpen(true) @@ -6455,13 +6841,11 @@ const AngularWorkflow = (props) => { if (execution.execution_source === "webhook") { return {"webhook"} trigger.trigger_type === "WEBHOOK").large_image} style={{width: size, height: size}} /> - } - - if (execution.execution_source === "schedule") { + } else if (execution.execution_source === "outlook") { + return {"email"} trigger.trigger_type === "EMAIL").large_image} style={{width: size, height: size}} /> + } else if (execution.execution_source === "schedule") { return {"schedule"} trigger.trigger_type === "SCHEDULE").large_image} style={{width: size, height: size}} /> - } - - if (execution.execution_source === "EMAIL") { + } else if (execution.execution_source === "EMAIL") { return {"email"} trigger.trigger_type === "EMAIL").large_image} style={{width: size, height: size}} /> } @@ -6507,7 +6891,7 @@ const AngularWorkflow = (props) => { // console.log("HANDLE INPUT FIELD FOR COPY!") //} - to_be_copied.replace(" ", "_") + to_be_copied.replaceAll(" ", "_") const elementName = "copy_element_shuffle" var copyText = document.getElementById(elementName); if (copyText !== null && copyText !== undefined) { @@ -6539,7 +6923,7 @@ const AngularWorkflow = (props) => { style={{borderRadius: "0px"}} variant="outlined" onClick={() => { - getWorkflowExecution(props.match.params.key) + getWorkflowExecution(props.match.params.key, "") }} color="primary"> Refresh executions @@ -6548,7 +6932,7 @@ const AngularWorkflow = (props) => { {workflowExecutions.length > 0 ?
{workflowExecutions.map((data, index) => { - const statusColor = data.status === "FINISHED" ? "green" : data.status === "ABORTED" || data.status === "FAILED" ? "red" : "orange" + const statusColor = data.status === "FINISHED" ? green : data.status === "ABORTED" || data.status === "FAILED" ? "red" : yellow const timeElapsed = data.completed_at-data.started_at const resultsLength = data.results !== undefined && data.results !== null ? data.results.length : 0 @@ -6566,17 +6950,21 @@ const AngularWorkflow = (props) => { {}} onMouseOut={() => {}} onClick={() => { - if (data.result === undefined || data.result === null || data.result.length === 0) { - setExecutionRequest({ - "execution_id": data.execution_id, - "authorization": data.authorization, - }) + if ((data.result === undefined || data.result === null || data.result.length === 0) && data.status !== "FINISHED" && data.status !== "ABORTED") { + start() setExecutionRunning(true) setExecutionRequestStarted(false) } + + const cur_execution = { + "execution_id": data.execution_id, + "authorization": data.authorization, + } + setExecutionRequest(cur_execution) setExecutionModalView(1) setExecutionData(data) + handleUpdateResults(data, cur_execution) }}>
@@ -6619,7 +7007,7 @@ const AngularWorkflow = (props) => { { setExecutionRunning(false) stop() - getWorkflowExecution(props.match.params.key) + getWorkflowExecution(props.match.params.key, "") setExecutionModalView(0) setLastExecution(executionData.execution_id) }}> @@ -6649,30 +7037,59 @@ const AngularWorkflow = (props) => {
{executionData.status !== undefined && executionData.status.length > 0 ? -
- Status:   {executionData.status} +
+ + Status   + + + {executionData.status} +
: null } {executionData.execution_source !== undefined && executionData.execution_source !== null && executionData.execution_source.length > 0 && executionData.execution_source !== "default" ? -
- Source:   {executionData.execution_parent !== null && executionData.execution_parent !== undefined && executionData.execution_parent.length > 0 ? - Parent Workflow - : - executionData.execution_source - } +
+ + Source    + + + {executionData.execution_parent !== null && executionData.execution_parent !== undefined && executionData.execution_parent.length > 0 ? + executionData.execution_source === props.match.params.key ? + + { + getWorkflowExecution(props.match.params.key, executionData.execution_parent) + }}> + Parent Execution + + + : + Parent Workflow + : + executionData.execution_source + } +
: null } {executionData.started_at !== undefined ? -
- Started:  {new Date(executionData.started_at*1000).toISOString()} +
+ + Started    + + + {new Date(executionData.started_at*1000).toISOString()} +
: null } {executionData.completed_at !== undefined && executionData.completed_at !== null && executionData.completed_at > 0 ? -
- Finished: {new Date(executionData.completed_at*1000).toISOString()} +
+ + Finished   + + + {new Date(executionData.completed_at*1000).toISOString()} +
: null } @@ -6696,7 +7113,7 @@ const AngularWorkflow = (props) => { Actions
- {executionData.status !== undefined && executionData.status !== "ABORTED" && executionData.status !== "FINISHED" && executionData.status !== "FAILURE" && executionData.status !== "WAITING" ? : null} + {executionData.status !== undefined && executionData.status !== "ABORTED" && executionData.status !== "FINISHED" && executionData.status !== "FAILURE" && executionData.status !== "WAITING" && !(executionData.results === undefined || executionData.results === null || executionData.results.length === 0 && executionData.status === "EXECUTING")? : null}
{executionData.results === undefined || executionData.results === null || executionData.results.length === 0 && executionData.status === "EXECUTING" ? @@ -6713,28 +7130,34 @@ const AngularWorkflow = (props) => { const curapp = apps.find(a => a.name === data.action.app_name && a.app_version === data.action.app_version) const imgsize = 50 - const statusColor = data.status === "FINISHED" || data.status === "SUCCESS" ? "green" : data.status === "ABORTED" || data.status === "FAILURE" ? "red" : "orange" + const statusColor = data.status === "FINISHED" || data.status === "SUCCESS" ? green : data.status === "ABORTED" || data.status === "FAILURE" ? "red" : yellow var imgSrc = curapp === undefined ? "" : curapp.large_image - if (imgSrc.length === 0) { + if (imgSrc.length === 0 && workflow.actions !== undefined && workflow.actions !== null) { // Look for the node in the workflow const action = workflow.actions.find(action => action.id === data.action.id) if (action !== undefined && action !== null) { imgSrc = action.large_image } + + /* + if (imgSrc.length === 0) { + console.log("CHECK IF ITS A + } + */ } var actionimg = curapp === null ? null : - {data.action.app_name} + {data.action.app_name} if (triggers.length > 2) { if (data.action.app_name === "shuffle-subflow") { - actionimg = {"Shuffle + actionimg = {"Shuffle } if (data.action.app_name === "User Input") { - actionimg = {"Shuffle + actionimg = {"Shuffle } } @@ -6743,7 +7166,17 @@ const AngularWorkflow = (props) => { } return ( -
+
{ + var currentnode = cy.getElementById(data.action.id) + if (currentnode.length !== 0) { + currentnode.addClass('shuffle-hover-highlight') + } + }} onMouseOut={() => { + var currentnode = cy.getElementById(data.action.id) + if (currentnode.length !== 0) { + currentnode.removeClass('shuffle-hover-highlight') + } + }}>
{ setSelectedResult(data) @@ -6756,10 +7189,21 @@ const AngularWorkflow = (props) => { {actionimg}
{data.action.label}
-
{data.action.name}
+
+ + {data.action.name} + +
-
Status {data.status}
+
+ + Status   + + + {data.status} + +
{validate.valid ? { }} name={"Results for "+data.action.label} /> - {data.action.app_name === "shuffle-subflow" ? + {data.action.app_name === "shuffle-subflow" && validate.result.success !== undefined && validate.result.success === true ? {validate.valid && data.action.parameters !== undefined && data.action.parameters !== null ? - See subflow execution + data.action.parameters[0].value === props.match.params.key ? + { + getWorkflowExecution(props.match.params.key, validate.result.execution_id) + }}> + See sub-execution + + : + { + }}>See subflow execution : "TBD: Load subexecution result for" } @@ -6783,9 +7235,13 @@ const AngularWorkflow = (props) => { } : -
- Result  - {data.result} +
+ + Result  + + + {data.result} +
}
@@ -6799,7 +7255,7 @@ const AngularWorkflow = (props) => { // This sucks :) const curapp = !codeModalOpen ? {} : selectedResult.action.app_name === "shuffle-subflow" ? triggers[1] : selectedResult.action.app_name === "User Input" ? triggers[2] : apps.find(a => a.name === selectedResult.action.app_name && a.app_version === selectedResult.action.app_version) const imgsize = 50 - const statusColor = !codeModalOpen ? "red" : selectedResult.status === "FINISHED" || selectedResult.status === "SUCCESS" ? "green" : selectedResult.status === "ABORTED" || selectedResult.status === "FAILURE" ? "red" : "orange" + const statusColor = !codeModalOpen ? "red" : selectedResult.status === "FINISHED" || selectedResult.status === "SUCCESS" ? green : selectedResult.status === "ABORTED" || selectedResult.status === "FAILURE" ? "red" : yellow const validate = !codeModalOpen ? "" : validateJson(selectedResult.result.trim()) if (validate.valid && typeof(validate.result) === "string") { validate.result = JSON.parse(validate.result) @@ -6923,9 +7379,7 @@ const AngularWorkflow = (props) => { -
{ - //event.preventDefault() - }}> +
{curapp === null ? null : {selectedResult.app_name}} @@ -6972,7 +7426,7 @@ const AngularWorkflow = (props) => { - const newView = isLoggedIn ? + const newView = //isLoggedIn ?
{leftView} @@ -6980,10 +7434,11 @@ const AngularWorkflow = (props) => { elements={elements} minZoom={0.35} maxZoom={2.00} - style={{width: bodyWidth-leftBarSize-15, height: bodyHeight-appBarSize-1, backgroundColor: surfaceColor}} + style={{width: bodyWidth-leftBarSize-15, height: bodyHeight-appBarSize-5, backgroundColor: surfaceColor}} stylesheet={cystyle} boxSelectionEnabled={true} autounselectify={false} + showGrid={true} cy={(incy) => { // FIXME: There's something specific loading when // you do the first hover of a node. Why is this different? @@ -6997,10 +7452,13 @@ const AngularWorkflow = (props) => {
+ + /* :
TMP FOR NOT LOGGED IN
+ */ const executionVariableModal = executionVariablesModalOpen ? { Execution Variable - Execution Variables are TEMPORARY variables that you can ony be set and used during execution. Learn more here + Execution Variables are TEMPORARY variables that you can ony be set and used during execution. Learn more here setNewVariableName(event.target.value)} color="primary" @@ -7400,6 +7858,24 @@ const AngularWorkflow = (props) => { ) } + const configureWorkflowModal = configureWorkflowModalOpen && apps.length !== 0 ? + { + setConfigureWorkflowModalOpen(false) + }} + PaperProps={{ + style: { + backgroundColor: surfaceColor, + color: "white", + minWidth: 600, + padding: 15, + }, + }} + > + + + : null // This whole part is redundant. Made it part of Arguments instead. @@ -7422,7 +7898,8 @@ const AngularWorkflow = (props) => { : null - const loadedCheck = isLoaded && isLoggedIn && workflowDone ? + //const loadedCheck = isLoaded && isLoggedIn && workflowDone ? + const loadedCheck = isLoaded && workflowDone ?
{newView} {variablesModal} @@ -7430,6 +7907,7 @@ const AngularWorkflow = (props) => { {conditionsModal} {authenticationModal} {codePopoutModal} + {configureWorkflowModal} { const checkQuery = () => { var urlParams = new URLSearchParams(window.location.search) if (!urlParams.has("id")) { + setActionAmount(0) + setIsAppLoaded(true) return } @@ -340,12 +342,10 @@ const AppCreator = (props) => { return response.json() }) .then((responseJson) => { - console.log("THE BODY IS HERE") setIsAppLoaded(true) if (!responseJson.success) { alert.error("Failed to verify") } else{ - console.log("HMM 2") var jsonvalid = false var tmpvalue = "" try { @@ -569,7 +569,7 @@ const AppCreator = (props) => { } } - console.log(methodvalue["requestBody"]["content"]) + //console.log(methodvalue["requestBody"]["content"]) if (methodvalue["requestBody"]["content"]["multipart/form-data"] !== undefined) { if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"] !== null) { if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["type"] === "object") { @@ -1260,6 +1260,7 @@ const AppCreator = (props) => { }) setActions(actions) + setActionAmount(actionAmount-1) setUpdate(Math.random()) } @@ -1303,13 +1304,13 @@ const AppCreator = (props) => { id: 'outlined-age-simple', }} > - {apikeySelection.map(data => { + {apikeySelection.map((data, index) => { if (data === undefined) { return null } return ( - + {data} )} @@ -1327,7 +1328,7 @@ const AppCreator = (props) => { const requiredColor = data.required === true ? "green" : "red" //const required = data.required === true ?
{data.required.toString()}
:
{flipRequired(index)}} style={{display: "inline", color: "red", cursor: "pointer"}}>{data.required.toString()}
return ( - +
{flipRequired(index)}}> Required:
{data.required.toString()}
@@ -1339,7 +1340,9 @@ const AppCreator = (props) => { placeholder={'Query name'} helperText={Click required switch} onBlur={(e) => { - urlPathQueries[index].name = e.target.value + console.log("IN BLUR: ", e.target.value) + urlPathQueries[index].name = e.target.value.replaceAll("=", "") + setUrlPathQueries(urlPathQueries) }} InputProps={{ @@ -1367,7 +1370,7 @@ const AppCreator = (props) => { {actions.slice(0,actionAmount).map((data, index) => { var error = data.errors.length > 0 ? - + : @@ -1391,7 +1394,7 @@ const AppCreator = (props) => { const url = data.url const hasFile = data["file_field"] !== undefined && data["file_field"] !== null && data["file_field"].length > 0 return ( - + {error}
{ @@ -1564,9 +1567,9 @@ const AppCreator = (props) => { } // Url verification - if (currentAction.url.length === 0) { - errormessage.push("URL path can't be empty.") - } else if (!currentAction.url.startsWith("/") && baseUrl.length > 0) { + //if (currentAction.url.length === 0) { + // errormessage.push("URL path can't be empty.") + if (!currentAction.url.startsWith("/") && baseUrl.length > 0 && currentAction.url.length > 0) { errormessage.push("URL must start with /") } @@ -1795,8 +1798,8 @@ const AppCreator = (props) => { ) })} - {actionBodyRequest.map(data => ( - + {actionBodyRequest.map((data, index) => ( + {data} ))} @@ -1832,8 +1835,8 @@ const AppCreator = (props) => { console.log("URL: ", parsedurl) if (parsedurl.includes("<") && parsedurl.includes(">")) { console.log("REPLACE") - parsedurl = parsedurl.replace("<", "{") - parsedurl = parsedurl.replace(">", "}") + parsedurl = parsedurl.replaceAll("<", "{") + parsedurl = parsedurl.replaceAll(">", "}") } if (parsedurl.startsWith("PUT ") || parsedurl.startsWith("GET ") ||parsedurl.startsWith("POST ") || parsedurl.startsWith("DELETE ") ||parsedurl.startsWith("PATCH ") || parsedurl.startsWith("CONNECT ")) { @@ -1936,7 +1939,7 @@ const AppCreator = (props) => { {fileUploadEnabled ? { const categories = [ "Communication", "Cases", - "EDR", - "Intel", "SIEM", - "Network", "Assets", + "Intel", + "IAM", + "Network", + "Eradication", "Other", ] + const tagView =
{/* @@ -2057,8 +2062,8 @@ const AppCreator = (props) => { value={newWorkflowCategories.length === 0 ? "Select a category" : newWorkflowCategories[0]} style={{backgroundColor: inputColor, color: "white", height: "50px"}} > - {categories.map(data => ( - + {categories.map((data, index) => ( + {data} ))} @@ -2089,8 +2094,24 @@ const AppCreator = (props) => {
const actionView = -
-

Actions ({actions.length})

+
+
+ {actionAmount > 0 && actionAmount < actions.length ? + + : null} +
+

Actions {actionAmount > 0 ? ({actionAmount} / {actions.length}) : null}

Actions are the tasks performed by an app. Read more about actions and apps here.
@@ -2113,18 +2134,7 @@ const AppCreator = (props) => { setActionsModalOpen(true) }}>New action {/* - {actionAmount} {actions.length} - {actionAmount > 0 && actionAmount < actions.length ? null : - - } + {actionAmount} {actions.length} */}
@@ -2184,8 +2194,120 @@ const AppCreator = (props) => { //
: // - const imageData = file.length > 0 ? file : fileBase64 - const imageInfo = + const [imageUploadError, setImageUploadError] = useState(""); + const [openImageModal, setOpenImageModal] = useState(""); + const [scale, setScale] = useState(1); + const [rotate, setRotatation] = useState(0); + const [disableImageUpload, setDisableImageUpload] = useState(true); + + let imageData = fileBase64; + let croppedData = file.length > 0 ? file : fileBase64 + + const imageInfo = + + const alternateImg = { + upload.click() + }}/> + + const zoomIn = () => { + console.log("ZOOOMING IN") + setScale(scale+0.1); + } + + const zoomOut = () => { + setScale(scale-0.1); + } + const rotatation = () => { + setRotatation(rotate+10); + } + + const onPositionChange = () => { + setDisableImageUpload(false); + } + + const onCancelSaveAppIcon = () => { + setFile(""); + setOpenImageModal(false) + setImageUploadError("") + } + + let editor; + const setEditorRef = (imgEditor) => { editor = imgEditor; } + + const onSaveAppIcon = () => { + if(editor){ + setFile(""); + const canvas = editor.getImageScaledToCanvas(); + setFileBase64(canvas.toDataURL()); + setOpenImageModal(false) + setDisableImageUpload(true); + } + } + + const errorText = imageUploadError.length > 0 ?
Error: {imageUploadError}
: null + const imageUploadModalView = openImageModal ? + + +
Upload App Icon
 + {errorText} + + setRotatation(0)} + /> + + + + + + + + + + + + + + + + + + + + + + : null; // Random names for type & autoComplete. Didn't research :^) const landingpageDataBrowser = @@ -2201,14 +2323,25 @@ const AppCreator = (props) => { {name} {actions === null || actions === undefined || actions.length === 0 ? null : ({actions.length})} + {imageUploadModalView} + upload = ref} onChange={editHeaderImage} />

General information

Click here to learn more about app creation
-
{upload.click()}}> - upload = ref} onChange={editHeaderImage} /> +
{ + /* + if (fileBase64.length === 0) { + upload.click() + } + */ + + setOpenImageModal(true) + }}> + {!imageData && (alternateImg)} {imageInfo} + upload = ref} onChange={editHeaderImage} />
@@ -2332,8 +2465,8 @@ const AppCreator = (props) => { value={authenticationOption} style={{backgroundColor: inputColor, color: "white", height: "50px"}} > - {authenticationOptions.map(data => ( - + {authenticationOptions.map((data, index) => ( + {data} ))} @@ -2371,7 +2504,9 @@ const AppCreator = (props) => { }}> {appBuilding ? : "Save"} - {errorCode.length > 0 ? `Error: ${errorCode}` : null} + + {errorCode.length > 0 ? `Error: ${errorCode}` : null} +
diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 2ee8e716..40d2cb4e 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -2,47 +2,24 @@ import React, { useEffect } from 'react'; import { useInterval } from 'react-powerhooks'; -import AppsIcon from '@material-ui/icons/Apps'; -import Grid from '@material-ui/core/Grid'; -import Select from '@material-ui/core/Select'; -import Paper from '@material-ui/core/Paper'; -import Divider from '@material-ui/core/Divider'; -import ButtonBase from '@material-ui/core/ButtonBase'; -import Button from '@material-ui/core/Button'; -import TextField from '@material-ui/core/TextField'; -import FormControl from '@material-ui/core/FormControl'; -import MenuItem from '@material-ui/core/MenuItem'; -import Tooltip from '@material-ui/core/Tooltip'; -import FormControlLabel from '@material-ui/core/FormControlLabel'; -import Switch from '@material-ui/core/Switch'; -import Input from '@material-ui/core/Input'; -import YAML from 'yaml' -import {Link} from 'react-router-dom'; -import Breadcrumbs from '@material-ui/core/Breadcrumbs'; -import ReactJson from 'react-json-view' -import Chip from '@material-ui/core/Chip'; +import {IconButton, Typography, Grid, Select, Paper, Divider, ButtonBase, Button, TextField, FormControl, MenuItem, Tooltip, FormControlLabel, Switch, Input, Breadcrumbs, Chip, Dialog, DialogTitle, DialogActions, DialogContent, CircularProgress} from '@material-ui/core'; +import {OpenInNew as OpenInNewIcon,Apps as AppsIcon, Cached as CachedIcon, Publish as PublishIcon, CloudDownload as CloudDownloadIcon, Edit as EditIcon, Delete as DeleteIcon} from '@material-ui/icons'; + import { useTheme } from '@material-ui/core/styles'; -import CachedIcon from '@material-ui/icons/Cached'; -import CloudDownloadIcon from '@material-ui/icons/CloudDownload'; -import PublishIcon from '@material-ui/icons/Publish'; -import CloudDownload from '@material-ui/icons/CloudDownload'; -import EditIcon from '@material-ui/icons/Edit'; -import DeleteIcon from '@material-ui/icons/Delete'; - +import YAML from 'yaml' +import {Link} from 'react-router-dom'; +import ReactJson from 'react-json-view' import { useAlert } from "react-alert"; - -import Dialog from '@material-ui/core/Dialog'; -import DialogTitle from '@material-ui/core/DialogTitle'; -import DialogActions from '@material-ui/core/DialogActions'; -import DialogContent from '@material-ui/core/DialogContent'; -import CircularProgress from '@material-ui/core/CircularProgress'; - import Dropzone from '../components/Dropzone'; const surfaceColor = "#27292D" const inputColor = "#383B40" +const chipStyle = { + backgroundColor: "#3d3f43", height: 30, marginRight: 5, paddingLeft: 5, paddingRight: 5, height: 28, cursor: "pointer", borderColor: "#3d3f43", color: "white", +} + // Parses JSON data into keys that can be used everywhere :) export const GetParsedPaths = (inputdata, basekey) => { const splitkey = " > " @@ -200,7 +177,7 @@ const Apps = (props) => { minHeight: 130, maxHeight: 130, minWidth: "100%", - maxWidth: "100%", + maxWidth: 612.5, marginBottom: 5, borderRadius: 5, color: "white", @@ -222,26 +199,57 @@ const Apps = (props) => { setIsLoading(false) if (response.status !== 200) { console.log("Status not 200 for apps :O!") + + if (isCloud) { + window.location.pathname = "/search" + } } return response.json() }) .then((responseJson) => { //console.log("Apps: ", responseJson) - responseJson = sortByKey(responseJson, "large_image") + //responseJson = sortByKey(responseJson, "large_image") + //responseJson = sortByKey(responseJson, "is_valid") + //setFilteredApps(responseJson.filter(app => !internalIds.includes(app.name) && !(!app.activated && app.generated))) + + var privateapps = [] + var valid = [] + var invalid = [] + for (var key in responseJson) { + const app = responseJson[key] + if (app.is_valid && !(!app.activated && app.generated)) { + privateapps.push(app) + } else if (app.private_id !== undefined && app.private_id.length > 0) { + valid.push(app) + } else { + invalid.push(app) + } + } - setApps(responseJson) - setFilteredApps(responseJson) - if (responseJson.length > 0) { - setSelectedApp(responseJson[0]) - if (responseJson[0].actions !== null && responseJson[0].actions.length > 0) { - setSelectedAction(responseJson[0].actions[0]) + //console.log(privateapps) + //console.log(valid) + //console.log(invalid) + //console.log(privateapps) + //privateapps.reverse() + privateapps.push(...valid) + privateapps.push(...invalid) + + setApps(privateapps) + setFilteredApps(privateapps) + if (privateapps.length > 0) { + if (selectedApp.id === undefined || selectedApp.id === null) { + setSelectedApp(privateapps[0]) + } + + if (privateapps[0].actions !== null && privateapps[0].actions.length > 0) { + setSelectedAction(privateapps[0].actions[0]) } else { setSelectedAction({}) } } - runAppSearch("") + //runAppSearch("") }) .catch(error => { alert.error(error.toString()) @@ -337,11 +345,9 @@ const Apps = (props) => { //console.log("IMG LOADED!: ", event.target) }} /> - // FIXME - add label to apps, as this might be slow with A LOT of apps var newAppname = data.name - newAppname = newAppname.replace("_", " ") newAppname = newAppname.charAt(0).toUpperCase()+newAppname.substring(1) - + newAppname = newAppname.replaceAll("_", " ") var sharing = "public" if (!data.sharing) { sharing = "private" @@ -357,7 +363,7 @@ const Apps = (props) => { } var description = data.description - const maxDescLen = 51 + const maxDescLen = 60 if (description.length > maxDescLen) { description = data.description.slice(0, maxDescLen)+"..." } @@ -366,6 +372,7 @@ const Apps = (props) => { return ( { if (selectedApp.id !== data.id) { + data.name = newAppname setSelectedApp(data) console.log(data) @@ -376,7 +383,7 @@ const Apps = (props) => { } if (data.sharing) { - setSharingConfiguration("everyone") + setSharingConfiguration(isCloud ? "public" : "everyone") } } }}> @@ -384,19 +391,22 @@ const Apps = (props) => { {imageline} -
-
- - +
+ + -

{newAppname}

+ + {newAppname} + -
- - {description} +
+ + + {description} +
- + {data.tags === null || data.tags === undefined ? null : data.tags.map((tag, index) => { if (index >= 3) { return null @@ -405,7 +415,7 @@ const Apps = (props) => { return ( { {data.activated && data.private_id !== undefined && data.private_id.length > 0 && data.generated ? {downloadApp(data)}}> - + : null} @@ -447,7 +457,7 @@ const Apps = (props) => { // FIXME - add label to apps, as this might be slow with A LOT of apps var newAppname = selectedApp.name if (newAppname !== undefined && newAppname.length > 0) { - newAppname = newAppname.replace("_", " ") + newAppname = newAppname.replaceAll("_", " ") newAppname = newAppname.charAt(0).toUpperCase()+newAppname.substring(1) } else { newAppname = "" @@ -467,7 +477,7 @@ const Apps = (props) => { color="primary" style={{marginTop: 10, marginRight: 8}} > - + : null @@ -599,22 +609,36 @@ const Apps = (props) => { const userRoles = [ "you", - "everyone", + isCloud ? "public" : "everyone", ] //fetch(globalUrl+"/api/v1/get_openapi/"+urlParams.get("id"), var baseInfo = newAppname.length > 0 ? -
+
{imageline}
-

{newAppname}

-

Version {selectedApp.app_version}

-

{description}

+ + {newAppname} + + + Version {selectedApp.app_version} + + + {description} +
+ {isCloud ? + + + + + + : null} + {activateButton} {(props.userdata !== undefined && (props.userdata.role === "admin" || props.userdata.id === selectedApp.owner) || !selectedApp.generated) ?
@@ -633,7 +657,7 @@ const Apps = (props) => { return ( { {props.userdata !== undefined && props.userdata.id === selectedApp.owner ?
{/*

ID: {selectedApp.id}

*/} - Sharing: + Sharing upload = ref} onChange={importFiles} /> + {workflows.length > 0 ? + + + + : null} + + + + + + const useStyles = makeStyles((theme) => ({ + root: { + border: 0, + '& .MuiDataGrid-columnsContainer': { + backgroundColor: theme.palette.type === 'light' ? '#fafafa' : '#1d1d1d', + }, + '& .MuiDataGrid-iconSeparator': { + display: 'none', + }, + '& .MuiDataGrid-colCell, .MuiDataGrid-cell': { + borderRight: `1px solid ${ + theme.palette.type === 'light' ? 'white' : '#303030' + }`, + }, + '& .MuiDataGrid-columnsContainer, .MuiDataGrid-cell': { + borderBottom: `1px solid ${ + theme.palette.type === 'light' ? '#f0f0f0' : '#303030' + }`, + }, + '& .MuiDataGrid-cell': { + color: + theme.palette.type === 'light' + ? 'white' + : 'rgba(255,255,255,0.65)', + }, + '& .MuiPaginationItem-root, .MuiTablePagination-actions, .MuiTablePagination-caption': { + borderRadius: 0, + color: "white", + }, + }, + })); + const classes = useStyles(); + + const WorkflowGridView = () => { + let workflowData = ""; + if (workflows.length > 0) { + const columns = [ + { field: 'title', headerName: 'Title', width: 330, }, + { field: 'actions', headerName: 'Actions', width: 200, sortable: false, + disableClickEventBubbling: true, + renderCell: (params) => { + const data = params.row.record; + let [triggers, schedules, webhooks, subflows] = getWorkflowMeta(data); + + return + + + + + + + + executeWorkflow(data.id)} /> + + + + + {webhooks > 0 ? + + + + : null} + {schedules > 0 ? + + + + : null} + + } + }, + { field: 'tags', headerName: 'Tags', width: 390, sortable: false, + disableClickEventBubbling: true, + renderCell: (params) => { + const data = params.row.record; + return + {data.tags !== undefined ? + data.tags.map((tag, index) => { + if (index >= 3) { + return null + } + + return ( + + ) + }) + : null} + + } + }, + ]; + let rows = []; + rows = workflows.map((data, index) => { + let obj = {"id":index+1, "title":data.name, "record":data,}; + return obj; + }); + workflowData = + } + return ( +
+ {workflowData} +
+ ); + } + + const WorkflowView = () => { + if (workflows.length === 0) { + return ( +
+ +
+

Welcome to Shuffle

+
+
+

+ Shuffle is a flexible, easy to use, automation platform allowing users to integrate their services and devices freely. It's made to significantly reduce the amount of manual labor, and is focused on security applications. Click here to learn more. +

+
+
+ If you want to jump straight into it, click here to create your first workflow: +
+
+ + + + ..OR + + {workflowButtons} + +
+ +
+ ) + } + + return ( +
+
+
+
+

Workflows

+
+
+ +
+
+
+
+
+
{workflows.length}
+
ACTIVE WORKFLOWS
+
+
+
+
+
+
+
+
{workflows.length}
+
AVAILABE WORKFLOWS
+
+
+
+
+
+
+
+
{workflows.length}
+
NOTIFICATIONS
+
+
+
+
+ +
+
+ This is your workflow view. Learn more about Workflows +
+
+ {workflowButtons} +
+
+
+ {view === "grid" && ( + + {workflows.map((data, index) => { + return ( + + ) + })} + + )} + + {view === "list" && ( + + )} + +
+
+
+ ) + } + + const importWorkflowsFromUrl = (url) => { + console.log("IMPORT WORKFLOWS FROM ", downloadUrl) + + const parsedData = { + "url": url, + "field_3": downloadBranch || 'master' + } + + if (field1.length > 0) { + parsedData["field_1"] = field1 + } + + if (field2.length > 0) { + parsedData["field_2"] = field2 + } + + alert.success("Getting specific workflows from your URL.") + var cors = "cors" + fetch(globalUrl+"/api/v1/workflows/download_remote", { + method: "POST", + mode: "cors", + headers: { + 'Accept': 'application/json', + }, + body: JSON.stringify(parsedData), + credentials: "include", + }) + .then((response) => { + if (response.status === 200) { + alert.success("Successfully loaded workflows from "+downloadUrl) + getAvailableWorkflows() + } + + return response.json() + }) + .then((responseJson) => { + console.log("DATA: ", responseJson) + if (!responseJson.success) { + if (responseJson.reason !== undefined) { + alert.error("Failed loading: "+responseJson.reason) + } else { + alert.error("Failed loading") + } + } + }) + .catch(error => { + alert.error(error.toString()) + }) + } + + const handleGithubValidation = () => { + importWorkflowsFromUrl(downloadUrl) + setLoadWorkflowsModalOpen(false) + } + + const workflowDownloadModalOpen = loadWorkflowsModalOpen ? + { + }} + PaperProps={{ + style: { + backgroundColor: surfaceColor, + color: "white", + minWidth: "800px", + minHeight: "320px", + }, + }} + > + +
+ Load workflows from github repo +
+ + + +
+
+ + + Repository (supported: github, gitlab, bitbucket) + 0 ? userdata.active_org.defaults.workflow_download_repo : downloadUrl} + InputProps={{ + style:{ + color: "white", + height: "50px", + fontSize: "1em", + }, + }} + onChange={e => setDownloadUrl(e.target.value)} + placeholder="https://github.com/frikky/shuffle-apps" + fullWidth + /> + + Branch (default value is "master"): +
+ 0 ? userdata.active_org.defaults.workflow_download_branch : downloadBranch} + InputProps={{ + style:{ + color: "white", + height: "50px", + fontSize: "1em", + }, + }} + onChange={e => setDownloadBranch(e.target.value)} + placeholder="master" + fullWidth + /> +
+ + Authentication (optional - private repos etc): +
+ setField1(e.target.value)} + type="username" + placeholder="Username / APIkey (optional)" + fullWidth + /> + setField2(e.target.value)} + type="password" + placeholder="Password (optional)" + fullWidth + /> +
+ + + + + + + : null + + const loadedCheck = isLoaded && isLoggedIn && workflowDone ? +
+ + {modalView} + {deleteModal} + {workflowDownloadModalOpen} +
+ : +
+ + + Loading Workflows + +
+ + + // Maybe use gridview or something, idk + return ( +
+ {loadedCheck} +
+ ) +} + +export default MyView diff --git a/frontend/src/views/SettingsPage.jsx b/frontend/src/views/SettingsPage.jsx index 31dfde8d..b1f53c1c 100644 --- a/frontend/src/views/SettingsPage.jsx +++ b/frontend/src/views/SettingsPage.jsx @@ -1,11 +1,7 @@ import React, {useState, useEffect} from 'react'; -import Paper from '@material-ui/core/Paper'; -import Button from '@material-ui/core/Button'; -import Divider from '@material-ui/core/Divider'; +import {Paper, Button, Divider, TextField} from '@material-ui/core'; import {Link} from 'react-router-dom'; - -import TextField from '@material-ui/core/TextField'; import { useAlert } from "react-alert"; import { useTheme } from '@material-ui/core/styles'; diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 05bc5196..edb39cce 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -1,31 +1,13 @@ import React, { useEffect} from 'react'; import { useInterval } from 'react-powerhooks'; +import { makeStyles } from '@material-ui/core/styles'; -import Grid from '@material-ui/core/Grid'; -import Paper from '@material-ui/core/Paper'; -import Tooltip from '@material-ui/core/Tooltip'; -import Divider from '@material-ui/core/Divider'; -import Button from '@material-ui/core/Button'; -import TextField from '@material-ui/core/TextField'; -import FormControl from '@material-ui/core/FormControl'; -import IconButton from '@material-ui/core/IconButton'; -import Menu from '@material-ui/core/Menu'; -import MenuItem from '@material-ui/core/MenuItem'; -import FormControlLabel from '@material-ui/core/FormControlLabel'; -import Chip from '@material-ui/core/Chip'; -import Switch from '@material-ui/core/Switch'; -import Typography from '@material-ui/core/Typography'; -import Zoom from '@material-ui/core/Zoom'; +import {Avatar, Grid, Paper, Tooltip, Divider, Button, TextField, FormControl, IconButton, Menu, MenuItem, FormControlLabel, Chip, Switch, Typography, Zoom, CircularProgress, Dialog, DialogTitle, DialogActions, DialogContent} from '@material-ui/core'; +import {FileCopy as FileCopyIcon, Delete as DeleteIcon, BubbleChart as BubbleChartIcon, Restore as RestoreIcon, Cached as CachedIcon, GetApp as GetAppIcon, Apps as AppsIcon, Edit as EditIcon, MoreVert as MoreVertIcon, PlayArrow as PlayArrowIcon, Add as AddIcon, Publish as PublishIcon, CloudUpload as CloudUploadIcon, CloudDownload as CloudDownloadIcon} from '@material-ui/icons'; +//import {Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons'; + +import {DataGrid, GridToolbarContainer, GridDensitySelector, GridToolbar} from '@material-ui/data-grid'; -import CircularProgress from '@material-ui/core/CircularProgress'; -import CachedIcon from '@material-ui/icons/Cached'; -import GetAppIcon from '@material-ui/icons/GetApp'; -import AppsIcon from '@material-ui/icons/Apps'; -import EditIcon from '@material-ui/icons/Edit'; -import MoreVertIcon from '@material-ui/icons/MoreVert'; -import PlayArrowIcon from '@material-ui/icons/PlayArrow'; -import AddIcon from '@material-ui/icons/Add'; -import PublishIcon from '@material-ui/icons/Publish'; //import JSONPretty from 'react-json-pretty'; //import JSONPrettyMon from 'react-json-pretty/dist/monikai' import ReactJson from 'react-json-view' @@ -34,16 +16,91 @@ import Dropzone from '../components/Dropzone'; import {Link} from 'react-router-dom'; import { useAlert } from "react-alert"; import ChipInput from 'material-ui-chip-input' +import uuid from "uuid" +import CytoscapeWrapper from '../components/RenderCytoscape' -import Dialog from '@material-ui/core/Dialog'; -import DialogTitle from '@material-ui/core/DialogTitle'; -import DialogActions from '@material-ui/core/DialogActions'; -import DialogContent from '@material-ui/core/DialogContent'; -import CloudDownloadIcon from '@material-ui/icons/CloudDownload'; +//import mobileImage from '../assets/img/mobile.svg'; +//import bagImage from '../assets/img/bag.svg'; +//import bookImage from '../assets/img/book.svg'; const inputColor = "#383B40" const surfaceColor = "#27292D" +const flexContainerStyle = { + display: "flex", + flexDirection: "row", + justifyContent: "left", + alignContent: "space-between", +} + +const flexBoxStyle = { + height: 125, + borderRadius: 4, + boxSizing: "border-box", + letterSpacing: "0.4px", + color: "#D6791E", + margin: 10, + flex: 1, +} + +const useStyles = makeStyles((theme) => ({ + root: { + border: 0, + '& .MuiDataGrid-columnsContainer': { + backgroundColor: theme.palette.type === 'light' ? '#fafafa' : '#1d1d1d', + }, + '& .MuiDataGrid-iconSeparator': { + display: 'none', + }, + '& .MuiDataGrid-colCell, .MuiDataGrid-cell': { + borderRight: `1px solid ${ + theme.palette.type === 'light' ? 'white' : '#303030' + }`, + }, + '& .MuiDataGrid-columnsContainer, .MuiDataGrid-cell': { + borderBottom: `1px solid ${ + theme.palette.type === 'light' ? '#f0f0f0' : '#303030' + }`, + }, + '& .MuiDataGrid-cell': { + color: + theme.palette.type === 'light' + ? 'white' + : 'rgba(255,255,255,0.65)', + }, + '& .MuiPaginationItem-root, .MuiTablePagination-actions, .MuiTablePagination-caption': { + borderRadius: 0, + color: "white", + }, + }, +})); + +//const activeWorkflowStyle = {backgroundColor: "#FFF5EE"} +//const notificationStyle = {backgroundColor: "#E5F9FF"} +//const activeWorkflowStyle = {backgroundColor: "#3d3f43"} +const availableWorkflowStyle = {backgroundColor: "#3d3f43"} +const notificationStyle = {backgroundColor: "#3d3f43"} +const activeWorkflowStyle = {backgroundColor: "#3d3f43"} + +const fontSize_16 = {fontSize: "16px",} +const counterStyle = {fontSize: "36px",fontWeight:"bold"} +const blockRightStyle = {textAlign: "right",padding: "20px 20px 0px 0px",width:"100%"} + +const chipStyle = { + backgroundColor: "#3d3f43", height: 30, marginRight: 5, paddingLeft: 5, paddingRight: 5, height: 28, cursor: "pointer", borderColor: "#3d3f43", color: "white", +} + +const flexContentStyle = { + display: "flex", + flexDirection: "row" +} + +const iconStyle = { + width: "75px", + height: "75px", + padding: "20px" +} + export const validateJson = (showResult) => { //showResult = showResult.split(" None").join(" \"None\"") showResult = showResult.split(" False").join(" false") @@ -81,11 +138,13 @@ const Workflows = (props) => { document.title = "Shuffle - Workflows" const alert = useAlert() + const classes = useStyles(); var upload = "" const [file, setFile] = React.useState(""); const [workflows, setWorkflows] = React.useState([]); + const [filteredWorkflows, setFilteredWorkflows] = React.useState([]); const [selectedWorkflow, setSelectedWorkflow] = React.useState({}); const [selectedExecution, setSelectedExecution] = React.useState({}); const [workflowExecutions, setWorkflowExecutions] = React.useState([]); @@ -109,40 +168,83 @@ const Workflows = (props) => { const [deleteModalOpen, setDeleteModalOpen] = React.useState(false); const [editingWorkflow, setEditingWorkflow] = React.useState({}) const [executionLoading, setExecutionLoading] = React.useState(false) + const [importLoading, setImportLoading] = React.useState(false) const [isDropzone, setIsDropzone] = React.useState(false); + const [view, setView] = React.useState("grid") + const [filters, setFilters] = React.useState([]) + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" - const { start, stop } = useInterval({ - duration: 5000, - startImmediate: false, - callback: () => { - //getWorkflowExecution(selectedWorkflow.id) - } - }) + const findWorkflow = (filters) => { + if (filters.length === 0) { + setFilteredWorkflows(workflows) + return + } - // DEBUG HERE - const handleClickLogout = () => { - //console.log("Cookies: ", cookies) - //console.log("SHOULD LOG OUT") - //console.log(isLoggedIn) + var newWorkflows = [] + for (var workflowKey in workflows) { + const curWorkflow = workflows[workflowKey] - // Don't really care about the logout - //fetch(globalUrl+"/api/v1/logout", { - // credentials: "include", - // method: 'POST', - // headers: { - // 'Content-Type': 'application/json', - // }, - //}) - //.then(() => { - // // Log out anyway - // removeCookie("session_token", {path: "/"}) - // //window.location = "/login" - //}) - //.catch(error => { - // console.log(error) - // removeCookie("session_token", {path: "/"}) - //}); - } + var found = [false] + if (curWorkflow.tags === undefined || curWorkflow.tags === null) { + found = filters.map(filter => curWorkflow.name.toLowerCase().includes(filter)) + } else { + found = filters.map(filter => curWorkflow.name.toLowerCase().includes(filter.toLowerCase()) || curWorkflow.tags.includes(filter)) + } + //console.log("FOUND: ", found) + //if (found) { + if (found.every(v => v === true)) { + newWorkflows.push(curWorkflow) + continue + } + } + + if (newWorkflows.length !== workflows.length) { + setFilteredWorkflows(newWorkflows) + } + } + + const addFilter = (data) => { + if (data === null || data === undefined) { + return + } + + if (data.includes("<") && data.includes(">")) { + return + } + + if (filters.includes(data)) { + return + } + + filters.push(data.toLowerCase()) + setFilters(filters) + + findWorkflow(filters) + } + + const removeFilter = (index) => { + var newfilters = filters + + if (index < 0) { + console.log("Can't handle index: ", index) + return + } + + + //console.log("Removing filter index", index) + newfilters.splice(index, 1) + //console.log("FILTER LENGTH: ", filters.length) + + if (newfilters.length === 0) { + newfilters = [] + setFilters(newfilters) + } else { + setFilters(newfilters) + } + //console.log("FILTERS: ", newfilters) + + findWorkflow(newfilters) + } const deleteModal = deleteModalOpen ? { .then((response) => { if (response.status !== 200) { console.log("Status not 200 for workflows :O!: ", response.status) + + if (isCloud) { + window.location.pathname = "/login" + } + alert.info("Failed getting workflows.") setWorkflowDone(true) @@ -257,22 +364,22 @@ const Workflows = (props) => { .then((responseJson) => { setSelectedExecution({}) setWorkflowExecutions([]) + //console.log(responseJson) if (responseJson !== undefined) { setWorkflows(responseJson) + setFilteredWorkflows(responseJson) setWorkflowDone(true) } else { if (isLoggedIn) { alert.error("An error occurred while loading workflows") - } else { - handleClickLogout() } return } if (responseJson.length > 0){ - setSelectedWorkflow(responseJson[0]) + //setSelectedWorkflow(responseJson[0]) //getWorkflowExecution(responseJson[0].id) } }) @@ -295,7 +402,7 @@ const Workflows = (props) => { minWidth: 1024, maxWidth: 1024, margin: "auto", - maxHeight: "90vh", + /*maxHeight: "90vh",*/ } const emptyWorkflowStyle = { @@ -323,78 +430,36 @@ const Workflows = (props) => { overflowY: "auto", } + const paperAppContainer = { + display: "flex", + flexWrap: 'wrap', + alignContent: "space-between", + } + const paperAppStyle = { - minHeight: "100px", - minWidth: "100%", - maxWidth: "100%", - marginTop: "5px", + minHeight: 130, + width: "100%", color: "white", backgroundColor: surfaceColor, + padding: "12px 12px 0px 15px", borderRadius: 5, - padding: 10, - cursor: "pointer", display: "flex", + boxSizing: "border-box", + position: "relative", } - const getWorkflowExecution = (id) => { - setExecutionLoading(true) - fetch(globalUrl+"/api/v1/workflows/"+id+"/executions", { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - }, - credentials: "include", - }) - .then((response) => { - setExecutionLoading(false) - if (response.status !== 200) { - console.log("Status not 200 for WORKFLOW EXECUTION :O!") - } - - return response.json() - }) - .then((responseJson) => { - if (responseJson.success === false) { - alert.error("Failed getting executions") - } else { - if (responseJson.length > 0) { - setSelectedExecution(responseJson[0]) - setWorkflowExecutions(responseJson) - } else { - //alert.info("Couldn't find executions for the workflow") - setSelectedExecution({}) - setWorkflowExecutions([]) - } - } - }) - .catch(error => { - setExecutionLoading(false) - alert.error(error.toString()) - }); + const gridContainer = { + height: "auto", + color: "white", + margin: "10px", + backgroundColor: surfaceColor, } - const abortExecution = (workflowid, executionid) => { - alert.success("Aborting execution") - fetch(globalUrl+"/api/v1/workflows/"+workflowid+"/executions/"+executionid+"/abort", { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - }, - credentials: "include", - }) - .then((response) => { - if (response.status !== 200) { - console.log("Status not 200 for WORKFLOW EXECUTION :O!") - } - //getWorkflowExecution(workflowid) - - return response.json() - }) - .catch(error => { - alert.error(error.toString()) - }); + const workflowActionStyle = { + display: "flex", + width: 160, + height: 44, + justifyContent: "space-between", } const executeWorkflow = (id) => { @@ -423,13 +488,6 @@ const Workflows = (props) => { .catch(error => { alert.error(error.toString()) }); - - if (id === selectedWorkflow.id) { - sleep(2000).then(() => { - stop() - start() - }) - } } function sleep (time) { @@ -442,14 +500,9 @@ const Workflows = (props) => { } } - const exportWorkflow = (data) => { - console.log("export") - let dataStr = JSON.stringify(data) - - let dataUri = 'data:application/json;charset=utf-8,'+ encodeURIComponent(dataStr); - let exportFileDefaultName = data.name+'.json'; - + const sanitizeWorkflow = (data) => { data["owner"] = "" + console.log("Sanitize start: ", data) if (data.triggers !== null && data.triggers !== undefined) { for (var key in data.triggers) { const trigger = data.triggers[key] @@ -462,6 +515,23 @@ const Workflows = (props) => { if (trigger.status == "running") { trigger.status = "stopped" } + + const newId = uuid.v4() + for (var branchkey in data.branches) { + const branch = data.branches[branchkey] + if (branch.source_id === trigger.id) { + //console.log("CHANGING SOURCE ID") + branch.source_id = newId + } + + if (branch.destination_id === trigger.id) { + //console.log("CHANGING DESTINATION ID") + branch.destination_id = newId + } + } + + trigger.environment = isCloud ? "cloud" : "Shuffle" + trigger.id = newId } } @@ -472,9 +542,33 @@ const Workflows = (props) => { for (var subkey in data.actions[key].parameters) { const param = data.actions[key].parameters[subkey] if (param.name.includes("key") || param.name.includes("user") || param.name.includes("pass") || param.name.includes("api") || param.name.includes("auth") || param.name.includes("secret")) { - param.value = "" + // FIXME: This may be a vuln if api-keys are generated that start with $ + if (param.value.startsWith("$")) { + console.log("Skipping field, as it's referencing a variable") + } else { + param.value = "" + param.is_valid = false + } } } + + const newId = uuid.v4() + for (var branchkey in data.branches) { + const branch = data.branches[branchkey] + if (branch.source_id === data.actions[key].id) { + //console.log("CHANGING SOURCE ID IN ACTION") + branch.source_id = newId + } + + if (branch.destination_id === data.actions[key].id) { + //console.log("CHANGING DESTINATION ID IN ACTION") + branch.destination_id = newId + } + } + + //data.actions[key].environment = isCloud ? "cloud" : "Shuffle" + data.actions[key].environment = "" + data.actions[key].id = newId } } @@ -483,6 +577,7 @@ const Workflows = (props) => { const param = data.workflow_variables[key] if (param.name.includes("key") || param.name.includes("user") || param.name.includes("pass") || param.name.includes("api") || param.name.includes("auth") || param.name.includes("secret")) { param.value = "" + param.is_valid = false } } } @@ -492,21 +587,77 @@ const Workflows = (props) => { data["org"] = [] data["org_id"] = "" - data.execution_org = {"id": ""} - console.log(data) + data["execution_org"] = {} + // These are backwards.. True = saved before. Very confuse. + data["previously_saved"] = false + data["first_save"] = false + console.log("Sanitize end: ", data) + + return data + } + + const exportWorkflow = (data) => { + let exportFileDefaultName = data.name+'.json'; + data = sanitizeWorkflow(data) + + //console.log("EXPORT: ", data) + //return + + let dataStr = JSON.stringify(data) + let dataUri = 'data:application/json;charset=utf-8,'+ encodeURIComponent(dataStr); let linkElement = document.createElement('a'); linkElement.setAttribute('href', dataUri); linkElement.setAttribute('download', exportFileDefaultName); linkElement.click(); } + const publishWorkflow = (data) => { + data = JSON.parse(JSON.stringify(data)) + data = sanitizeWorkflow(data) + alert.info("Sanitizing and publishing "+data.name) + + // This ALWAYS talks to Shuffle cloud + fetch(globalUrl+"/api/v1/workflows/"+data.id+"/publish", { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + body: JSON.stringify(data), + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for workflow publish :O!") + } else { + if (isCloud) { + alert.success("Successfully published workflow") + } else { + alert.success("Successfully published workflow to https://shuffler.io") + } + } + + return response.json() + }) + .then((responseJson) => { + if (responseJson.reason !== undefined) { + alert.error("Failed publishing: ", responseJson.reason) + } + + getAvailableWorkflows() + }) + .catch(error => { + alert.error(error.toString()) + }) + } + const copyWorkflow = (data) => { data = JSON.parse(JSON.stringify(data)) alert.success("Copying workflow "+data.name) - console.log("data: ", data) data.id = "" data.name = data.name+"_copy" + console.log("COPIED DATA: ", data) //return fetch(globalUrl+"/api/v1/workflows", { @@ -530,12 +681,11 @@ const Workflows = (props) => { }) .catch(error => { alert.error(error.toString()) - }); + }) } const deleteWorkflow = (id) => { - alert.success("Deleted workflow "+id) fetch(globalUrl+"/api/v1/workflows/"+id, { method: 'DELETE', headers: { @@ -547,6 +697,9 @@ const Workflows = (props) => { .then((response) => { if (response.status !== 200) { console.log("Status not 200 for setting workflows :O!") + alert.error("Failed deleting workflow") + } else { + alert.success("Deleted workflow "+id) } return response.json() @@ -559,7 +712,11 @@ const Workflows = (props) => { }); } - // dropdown with copy etc I guess + + const handleChipClick = (e) => { + addFilter(e.target.innerHTML) + } + const WorkflowPaper = (props) => { const { data } = props; const [open, setOpen] = React.useState(false); @@ -570,9 +727,13 @@ const Workflows = (props) => { boxWidth = "4px" } - var boxColor = "orange" + var boxColor = "#FECC00" if (data.is_valid) { - boxColor = "green" + boxColor = "#86c142" + } + + if (!data.previously_saved) { + boxColor = "#f85a3e" } const menuClick = (event) => { @@ -580,62 +741,140 @@ const Workflows = (props) => { setAnchorEl(event.currentTarget); } - const actions = data.actions !== null ? data.actions.length : 0 - var schedules = 0 - var webhooks = 0 - var webhookImg = "" - var scheduleImg = "" - if (data.triggers !== undefined && data.triggers !== null && data.triggers.length > 0) { - for (var key in data.triggers) { - if (data.triggers[key].app_name === "Webhook") { - webhooks += 1 - webhookImg = data.triggers[key].large_image - } else if (data.triggers[key].app_name === "Schedule") { - schedules += 1 - scheduleImg = data.triggers[key].large_image - } - } + var parsedName = data.name + if (parsedName !== undefined && parsedName !== null && parsedName.length > 25) { + parsedName = parsedName.slice(0,26)+".." } - const imgSize = 25 - return ( - { - }}> -
- - - -
{ - if (selectedWorkflow.id !== data.id) { - setSelectedWorkflow(data) - //getWorkflowExecution(data.id) - } - }}> - - {data.name} - -
-
- - - - { - setOpen(false) - setAnchorEl(null) - }} - > + const actions = data.actions !== null ? data.actions.length : 0 + const [triggers, schedules, webhooks, subflows] = getWorkflowMeta(data) + + return ( + + +
+ + + + + + {parsedName} + + + + + + + + + + {actions} + + + + + + + + {triggers} + + + + + { + if (subflows === 0) { + alert.info("No subflows for "+data.name) + return + } + + var newWorkflows = [data] + for (var key in data.triggers) { + const trigger = data.triggers[key] + if (trigger.app_name !== "Shuffle Workflow") { + continue + } + + if (trigger.parameters !== undefined && trigger.parameters !== null && trigger.parameters.length > 0 && trigger.parameters[0].name === "workflow") { + const newWorkflow = workflows.find(item => item.id === trigger.parameters[0].value) + if (newWorkflow !== null && newWorkflow !== undefined) { + newWorkflows.push(newWorkflow) + continue + } + } + } + + setFilters(["Subflows of "+data.name]) + setFilteredWorkflows(newWorkflows) + }}> + + + + + {subflows} + + + + {/* + + + + + + + : null} + {schedules > 0 ? + + + + : null} + */} + + + {data.tags !== undefined ? + data.tags.map((tag, index) => { + if (index >= 3) { + return null + } + + + return ( + + ) + }) + : null} + + + {data.actions !== undefined && data.actions !== null ? + + + + + + { + setOpen(false) + setAnchorEl(null) + }} + > { setModalOpen(true) setEditingWorkflow(data) @@ -644,142 +883,59 @@ const Workflows = (props) => { if (data.tags !== undefined && data.tags !== null) { setNewWorkflowTags(JSON.parse(JSON.stringify(data.tags))) } - }} key={"change"}>{"Change details"} + }} key={"change"}> + + {"Change details"} + + { + publishWorkflow(data) + }} key={"publish"}> + + {"Publish Workflow"} + { copyWorkflow(data) setOpen(false) - }} key={"copy"}>{"Copy"} + }} key={"duplicate"}> + + {"Duplicate Workflow"} + { exportWorkflow(data) setOpen(false) - }} key={"export"}>{"Export"} - { + }} key={"export"}> + + {"Export Workflow"} + + { setDeleteModalOpen(true) setSelectedWorkflowId(data.id) setOpen(false) - }} key={"delete"}>{"Delete"} + }} key={"delete"}> + + {"Delete Workflow"} + - -
+ -
{ - if (selectedWorkflow.id !== data.id) { - setSelectedWorkflow(data) - //getWorkflowExecution(data.id) - } - }}> - - - - - - - - - - + {/* + + + + - {data.tags !== undefined ? - data.tags.map((tag, index) => { - if (index >= 3) { - return null - } - - return ( - - ) - }) - : null} - -
- - - {data.actions !== undefined && data.actions !== null ? - - - - - - {webhooks > 0 ? - - {data.title} - - : null} - {schedules > 0 ? - - {data.title} - - : null} - - : null} - - ) - } - - const executionPaper = (data) => { - var boxWidth = "2px" - if (selectedExecution.execution_id === data.execution_id) { - boxWidth = "4px" - } - - var boxColor = "orange" - if (data.status === "ABORTED" || data.status === "UNFINISHED" || data.status === "FAILURE"){ - boxColor = "red" - } else if (data.status === "FINISHED") { - boxColor = "green" - } - - var t = new Date(data.started_at*1000) - if (data.workflow.actions === null || data.workflow.actions === undefined ) { - return null - } - - if (data.workflow.actions === null || data.workflow.actions === undefined) { - return null - } - - var actions = data.workflow.actions.length - if (data.results !== null) { - var results = data.results.length - } - - return ( - { - setSelectedExecution(data) - }}> -
- - - -
-

Status: {data.status}

- Actions: {results}/{actions} -
-
- -
+ -
- - Started: {t.toISOString()} - -
+ */} - - + : null} + + ) } + + const dividerColor = "rgb(225, 228, 232)" const resultPaperAppStyle = { @@ -974,32 +1130,6 @@ const Workflows = (props) => { ) } - const ExecutionsView = () => { - if (workflowExecutions.length > 0) { - const sortedWorkflows = workflowExecutions.sort((a, b) => a.started_at - b.started_at).reverse() - - return ( -
- {sortedWorkflows.map(data => { - return ( - executionPaper(data) - ) - })} -
- ) - } - return ( - executionLoading ? -
- -
- : -

- Executions have been moved to the Workflow itself.
Click here to see them -

- ) - } - // Can create and set workflows const setNewWorkflow = (name, description, tags, editingWorkflow, redirect) => { @@ -1010,7 +1140,7 @@ const Workflows = (props) => { if (editingWorkflow.id !== undefined) { console.log("Building original workflow") method = "PUT" - extraData = "/"+editingWorkflow.id + extraData = "/"+editingWorkflow.id+"?skip_save=true" workflowdata = editingWorkflow console.log("REMOVING OWNER") @@ -1048,6 +1178,7 @@ const Workflows = (props) => { } else if (!redirect) { // Update :) getAvailableWorkflows() + setImportLoading(false) } else { alert.info("Successfully changed basic info for workflow") } @@ -1056,12 +1187,15 @@ const Workflows = (props) => { }) .catch(error => { alert.error(error.toString()) + setImportLoading(false) }); } const importFiles = (event) => { console.log("Importing!") + + setImportLoading(true) const file = event.target.value if (event.target.files.length > 0) { for (var key in event.target.files) { @@ -1082,6 +1216,7 @@ const Workflows = (props) => { data = JSON.parse(reader.result) } catch (e) { alert.error("Invalid JSON: "+e) + setImportLoading(false) return } @@ -1091,6 +1226,9 @@ const Workflows = (props) => { if (response !== undefined) { // SET THE FULL THING data.id = response.id + data.first_save = false + data.previously_saved = false + data.is_valid = false // Actually create it const ret = setNewWorkflow(data.name, data.description, data.tags, data, false) @@ -1114,6 +1252,111 @@ const Workflows = (props) => { setLoadWorkflowsModalOpen(false) } + const getWorkflowMeta = (data) => { + let triggers = 0 + let schedules = 0 + let webhooks = 0 + let subflows = 0 + if (data.triggers !== undefined && data.triggers !== null && data.triggers.length > 0) { + triggers = data.triggers.length + for (let key in data.triggers) { + + if (data.triggers[key].app_name === "Webhook") { + webhooks += 1 + //webhookImg = data.triggers[key].large_image + } else if (data.triggers[key].app_name === "Schedule") { + schedules += 1 + //scheduleImg = data.triggers[key].large_image + } else if (data.triggers[key].app_name === "Shuffle Workflow") { + subflows += 1 + } + } + } + + return [triggers, schedules, webhooks, subflows] + } + + const WorkflowGridView = () => { + let workflowData = ""; + if (workflows.length > 0) { + const columns = [ + { field: 'title', headerName: 'Title', width: 330, }, + { field: 'actions', headerName: 'Actions', width: 200, sortable: false, + disableClickEventBubbling: true, + renderCell: (params) => { + const data = params.row.record; + let [triggers, schedules, webhooks, subflows] = getWorkflowMeta(data); + + return + + + + + + + + executeWorkflow(data.id)} /> + + + + + {webhooks > 0 ? + + + + : null} + {schedules > 0 ? + + + + : null} + + } + }, + { field: 'tags', headerName: 'Tags', width: 390, sortable: false, + disableClickEventBubbling: true, + renderCell: (params) => { + const data = params.row.record; + return + {data.tags !== undefined ? + data.tags.map((tag, index) => { + if (index >= 3) { + return null + } + + return ( + + ) + }) + : null} + + } + }, + ]; + let rows = []; + rows = workflows.map((data, index) => { + let obj = {"id":index+1, "title":data.name, "record":data,}; + return obj; + }); + workflowData = + } + return ( +
+ {workflowData} +
+ ); + } + const modalView = modalOpen ? { fullWidth /> { : null} - + {importLoading ? + + : + + } upload = ref} onChange={importFiles} /> {workflows.length > 0 ? @@ -1258,11 +1507,13 @@ const Workflows = (props) => { : null} + {isCloud ? null : + } const WorkflowView = () => { @@ -1299,64 +1550,106 @@ const Workflows = (props) => {
-
-

Workflows ({workflows.length})

+
+

Workflows

-
+
+ + {/* +
+
+
+
+
+
{workflows.length}
+
ACTIVE WORKFLOWS
+
+
+
+
+
+
+
+
{workflows.length}
+
AVAILABE WORKFLOWS
+
+
+
+
+
+
+
+
{workflows.length}
+
NOTIFICATIONS
+
+
+
+
+ */} + + {/* + chipRenderer={({ value, isFocused, isDisabled, handleClick, handleRequestDelete }, key) => { + console.log("VALUE: ", value) + + return ( + + {value} + + ) + }} + */} +
+ +
+ { + addFilter(chip) + }} + onDelete={(chip, index) => { + removeFilter(index) + }} + /> +
+
{workflowButtons}
- - -
- {workflows.map((data, index) => { - return ( +
+ {view === "grid" && ( + + {filteredWorkflows.map((data, index) => { + return ( - ) - })} -
+ ) + })} + + )} + + {view === "list" && ( + + )} + +
-
-
-
-

Executions: {selectedWorkflow.name}

-
- {/* -
- -
- */} -
- -
- -
-
- {/* -
-
-
-

Execution Timeline

-
-
- Collapse results
} - control={ {setCollapseJson(!collapseJson)}} />} - /> -
-
- -
- -
-
- */}
) } @@ -1459,7 +1752,7 @@ const Workflows = (props) => { onChange={e => setDownloadUrl(e.target.value)} placeholder="https://github.com/frikky/shuffle-apps" fullWidth - /> + /> Branch (default value is "master"):
diff --git a/functions/onprem/orborus/Dockerfile b/functions/onprem/orborus/Dockerfile index 3db4c027..6479c09e 100644 --- a/functions/onprem/orborus/Dockerfile +++ b/functions/onprem/orborus/Dockerfile @@ -1,11 +1,17 @@ -from golang as builder +FROM golang:1.16.0-buster as builder RUN mkdir /app WORKDIR /app -RUN go get github.com/docker/docker/api/types github.com/docker/docker/api/types/container github.com/docker/docker/client COPY orborus.go /app/orborus.go RUN go mod init orborus +RUN go get github.com/docker/docker/api/types && \ + go get github.com/docker/docker/api/types/container && \ + go get github.com/docker/docker/client && \ + go get github.com/mackerelio/go-osstat/cpu && \ + go get github.com/mackerelio/go-osstat/memory && \ + go get github.com/satori/go.uuid && \ + go get github.com/frikky/shuffle-shared RUN go build RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o orborus . diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index a0fcec2c..c6df2439 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.8.60 +VERSION=0.8.71 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 47154cda..2991cf7f 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -3,17 +3,18 @@ module orborus go 1.13 require ( + github.com/Microsoft/go-winio v0.4.16 // indirect github.com/containerd/containerd v1.4.3 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v20.10.1+incompatible github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect + github.com/frikky/shuffle-shared v0.0.23 // indirect github.com/gogo/protobuf v1.3.1 // indirect - github.com/mackerelio/go-osstat v0.1.0 // indirect + github.com/mackerelio/go-osstat v0.1.0 github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/satori/go.uuid v1.2.0 github.com/sirupsen/logrus v1.7.0 // indirect - google.golang.org/grpc v1.34.0 // indirect ) diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 84c5d0ec..d3ea7e29 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -1,7 +1,54 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.75.0 h1:XgtDnVJRCPEUG21gjFiRPz4zI1Mjg16R+NYQjfmU4XY= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastore v1.4.0 h1:CFDJm15RpYXeEblQ0TMDUrYtqmBmbAWTy536nA8JIc8= +cloud.google.com/go/datastore v1.4.0/go.mod h1:d18825/a9bICdAIJy2EkHs9joU4RlIZ1t6l8WDdbdY0= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lUX4= +cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/containerd/containerd v1.4.3 h1:ijQT13JedHSHrQGWFcGEwzcNKrAGIiZ+jSD5QQG07SY= github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= @@ -17,14 +64,42 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/frikky/kin-openapi v0.38.0 h1:V7ttwIJS8Vks4KL+mZVj1ZSqhIcQtgaG8akeqXEQgsE= +github.com/frikky/kin-openapi v0.38.0/go.mod h1:Fr28TtCHL4K0kIqtqui8HWxN1LG5uAh3z/tDfFyiA1s= +github.com/frikky/shuffle-shared v0.0.12 h1:+0EIfThmK47Po+LogPYZR4XjbS4Ds19WNMFu2YUSjhw= +github.com/frikky/shuffle-shared v0.0.12/go.mod h1:SEY432/xs4oBkOUnGwxiYKCZWZLRaheGInhAoW7N8ww= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -33,16 +108,56 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mackerelio/go-osstat v0.1.0 h1:e57QHeHob8kKJ5FhcXGdzx5O6Ktuc5RHMDIkeqhgkFA= github.com/mackerelio/go-osstat v0.1.0/go.mod h1:1K3NeYLhMHPvzUu+ePYXtoB58wkaRpxZsGClZBJyIFw= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= @@ -51,51 +166,310 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423 h1:/hEknzWkMPCjTo7StMHRrBRa8YBbXuBWfck8680k3RE= +golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190410235845-0ad05ae3009d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3 h1:kzM6+9dur93BcC2kVlYl34cHU+TYZLanmpSJHVMmL64= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963 h1:K+NlvTLy0oONtRtkl1jRD9xIhnItbG2PiE7YOdjPb+k= +golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQWYWo= +google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0 h1:l2Nfbl2GPXdWorv+dT2XfinX2jOOw4zv1VhLstx+6rE= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595 h1:x7nk+/4+SvuTDI4wnzQUlhvi+DTpyfncXBo3QWTFs7U= +google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0 h1:raiipEjMOIC/TO2AvyTxP25XFdLxNIBwzDh3FM3XztI= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.34.1 h1:ugq+9++ZQPFzM2pKUMCIK8gj9M0pFyuUWO9Q8kwEDQw= +google.golang.org/grpc v1.34.1/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -104,9 +478,25 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index bf7f2919..a3fd36dd 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -5,6 +5,8 @@ package main */ import ( + "github.com/frikky/shuffle-shared" + "bytes" "context" "encoding/json" @@ -55,19 +57,6 @@ var runningMode = strings.ToLower(os.Getenv("RUNNING_MODE")) var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP")) var executionIds = []string{} -type ExecutionRequestWrapper struct { - Data []ExecutionRequest `json:"data"` -} - -type ExecutionRequest struct { - ExecutionId string `json:"execution_id"` - ExecutionArgument string `json:"execution_argument"` - WorkflowId string `json:"workflow_id"` - Authorization string `json:"authorization"` - Status string `json:"status"` - Type string `json:"type"` -} - var dockercli *dockerclient.Client var containerId string @@ -106,7 +95,7 @@ func getThisContainerId() { } if fCol != "" { - cmd := fmt.Sprintf("cat /proc/self/cgroup | grep memory | tail -1 | cut -d/ -f%s", fCol) + cmd := fmt.Sprintf("cat /proc/self/cgroup | grep memory | tail -1 | cut -d/ -f%s | grep -o -E '[0-9A-z]{64}'", fCol) out, err := exec.Command("bash", "-c", cmd).Output() if err == nil { containerId = strings.TrimSpace(string(out)) @@ -119,12 +108,14 @@ func getThisContainerId() { //docker-76c537e9a4b7c7233011f5d70e6b7f2d600b6413ac58a96519b8dca7a3f7117a.scope } } else { - containerId = "shuffle-orborus" - log.Printf("[WARNING] Failed getting container ID: %s", err) + if fCol == "0" { + containerId = "shuffle-orborus" + log.Printf("[WARNING] Failed getting container ID: %s", err) + } } } - log.Printf("Started with containerId %s", containerId) + log.Printf(`[INFO] Started with containerId "%s"`, containerId) } // Deploys the internal worker whenever something happens @@ -257,7 +248,7 @@ func initializeImages() { log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion) } if workerVersion == "" { - workerVersion = "0.8.60" + workerVersion = "0.8.70" log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion) } @@ -475,7 +466,7 @@ func main() { continue } - var executionRequests ExecutionRequestWrapper + var executionRequests shuffle.ExecutionRequestWrapper err = json.Unmarshal(body, &executionRequests) if err != nil { log.Printf("[WARNING] Failed executionrequest in queue unmarshaling: %s", err) @@ -521,7 +512,7 @@ func main() { } // New, abortable version. Should check executionid and remove everything else - var toBeRemoved ExecutionRequestWrapper + var toBeRemoved shuffle.ExecutionRequestWrapper for _, execution := range executionRequests.Data { if len(execution.ExecutionArgument) > 0 { log.Printf("[INFO] Argument: %#v", execution.ExecutionArgument) @@ -546,8 +537,8 @@ func main() { // Doesn't work because of USER INPUT if found { - //log.Printf("[INFO] Skipping duplicate %s", execution.ExecutionId) - //continue + log.Printf("[INFO] Skipping duplicate %s", execution.ExecutionId) + continue } else { //log.Printf("[INFO] Adding to be ran %s", execution.ExecutionId) executionIds = append(executionIds, execution.ExecutionId) diff --git a/functions/onprem/worker/Dockerfile b/functions/onprem/worker/Dockerfile index 483ab136..826ef4d3 100644 --- a/functions/onprem/worker/Dockerfile +++ b/functions/onprem/worker/Dockerfile @@ -1,21 +1,28 @@ -from golang as builder +FROM golang:1.16.0-buster as builder WORKDIR /app +RUN go get github.com/docker/docker/api/types github.com/docker/docker/api/types/container github.com/docker/docker/client -RUN go get -u github.com/docker/docker/api/types -RUN go get -u github.com/docker/docker/api/types/container -RUN go get -u github.com/docker/docker/client -RUN go get -u github.com/gorilla/mux -RUN go get -u github.com/patrickmn/go-cache - +#RUN go env -w GO111MODULE=auto COPY worker.go /app/worker.go +RUN go mod init worker +RUN go get github.com/docker/docker/api/types && \ + go get github.com/docker/docker/api/types/container && \ + go get github.com/docker/docker/client && \ + go get github.com/gorilla/mux && \ + go get github.com/patrickmn/go-cache && \ + go get github.com/frikky/shuffle-shared && \ + go get github.com/satori/go.uuid + +RUN go build RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker . +## ALPINE IMAGE FROM alpine:3.12 ENV SHUFFLE_BASE_IMAGE_REGISTRY=docker.io ENV SHUFFLE_BASE_IMAGE_NAME=frikky/shuffle -ENV SHUFFLE_BASE_IMAGE_TAG_SUFFIX=0.8.5 +ENV SHUFFLE_BASE_IMAGE_TAG_SUFFIX=0.8.70 RUN apk add --no-cache bash COPY --from=builder /app/ / diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index ba1b2e38..10f2c70c 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.8.60 +VERSION=0.8.71 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . @@ -10,5 +10,5 @@ docker build . -t frikky/shuffle:$NAME -t frikky/shuffle:$NAME_$VERSION -t docke #docker push frikky/shuffle:$NAME_$VERSION #docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION #docker tag frikky/shuffle:0.8.51 ghcr.io/frikky/shuffle-worker:0.8.5 -docker tag frikky/shuffle:$NAME ghcr.io/frikky/shuffle-worker:0.8.52 +#docker tag frikky/shuffle:$NAME ghcr.io/frikky/shuffle-worker:0.8.52 docker push ghcr.io/frikky/$NAME:$VERSION diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod new file mode 100644 index 00000000..44d51077 --- /dev/null +++ b/functions/onprem/worker/go.mod @@ -0,0 +1,18 @@ +module worker + +go 1.15 + +require ( + github.com/containerd/containerd v1.4.4 // indirect + github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/docker/docker v20.10.5+incompatible // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.4.0 // indirect + github.com/frikky/shuffle-shared v0.0.20 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/gorilla/mux v1.8.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect +) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index dcda8b3e..cc981636 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -1,6 +1,8 @@ package main import ( + "github.com/frikky/shuffle-shared" + "bytes" "context" "encoding/json" @@ -11,6 +13,7 @@ import ( "log" "net" "net/http" + "net/url" "os" "os/exec" "strings" @@ -18,7 +21,10 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" + //"github.com/docker/docker/api/types/filters" + "github.com/docker/docker/api/types/mount" dockerclient "github.com/docker/docker/client" + "github.com/satori/go.uuid" "github.com/gorilla/mux" "github.com/patrickmn/go-cache" @@ -52,8 +58,12 @@ var containerId string // form container id of current running container func getThisContainerId() string { + if len(containerId) > 0 { + return containerId + } + id := "" - cmd := fmt.Sprintf("cat /proc/self/cgroup | grep memory | tail -1 | cut -d/ -f3") + cmd := fmt.Sprintf("cat /proc/self/cgroup | grep memory | tail -1 | cut -d/ -f3 | grep -o -E '[0-9A-z]{64}'") out, err := exec.Command("bash", "-c", cmd).Output() if err == nil { id = strings.TrimSpace(string(out)) @@ -76,701 +86,23 @@ func init() { } } -type Userapi struct { - Username string `datastore:"username"` - ApiKey string `datastore:"apikey"` -} - -type ExecutionInfo struct { - TotalApiUsage int64 `json:"total_api_usage" datastore:"total_api_usage"` - TotalWorkflowExecutions int64 `json:"total_workflow_executions" datastore:"total_workflow_executions"` - TotalAppExecutions int64 `json:"total_app_executions" datastore:"total_app_executions"` - TotalCloudExecutions int64 `json:"total_cloud_executions" datastore:"total_cloud_executions"` - TotalOnpremExecutions int64 `json:"total_onprem_executions" datastore:"total_onprem_executions"` - DailyApiUsage int64 `json:"daily_api_usage" datastore:"daily_api_usage"` - DailyWorkflowExecutions int64 `json:"daily_workflow_executions" datastore:"daily_workflow_executions"` - DailyAppExecutions int64 `json:"daily_app_executions" datastore:"daily_app_executions"` - DailyCloudExecutions int64 `json:"daily_cloud_executions" datastore:"daily_cloud_executions"` - DailyOnpremExecutions int64 `json:"daily_onprem_executions" datastore:"daily_onprem_executions"` -} - -type StatisticsData struct { - Timestamp int64 `json:"timestamp" datastore:"timestamp"` - Id string `json:"id" datastore:"id"` - Amount int64 `json:"amount" datastore:"amount"` -} - -type StatisticsItem struct { - Total int64 `json:"total" datastore:"total"` - Fieldname string `json:"field_name" datastore:"field_name"` - Data []StatisticsData `json:"data" datastore:"data"` -} - -// "Execution by status" -// Execution history -//type GlobalStatistics struct { -// BackendExecutions int64 `json:"backend_executions" datastore:"backend_executions"` -// WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"` -// ExecutionCount int64 `json:"execution_count" datastore:"execution_count"` -// ExecutionSuccessCount int64 `json:"execution_success_count" datastore:"execution_success_count"` -// ExecutionAbortCount int64 `json:"execution_abort_count" datastore:"execution_abort_count"` -// ExecutionFailureCount int64 `json:"execution_failure_count" datastore:"execution_failure_count"` -// ExecutionPendingCount int64 `json:"execution_pending_count" datastore:"execution_pending_count"` -// AppUsageCount int64 `json:"app_usage_count" datastore:"app_usage_count"` -// TotalAppsCount int64 `json:"total_apps_count" datastore:"total_apps_count"` -// SelfMadeAppCount int64 `json:"self_made_app_count" datastore:"self_made_app_count"` -// WebhookUsageCount int64 `json:"webhook_usage_count" datastore:"webhook_usage_count"` -// Baseline map[string]int64 `json:"baseline" datastore:"baseline"` -//} - -type ParsedOpenApi struct { - Body string `datastore:"body,noindex" json:"body"` - ID string `datastore:"id" json:"id"` - Success bool `datastore:"success,omitempty" json:"success,omitempty"` -} - -// Limits set for a user so that they can't do a shitload -type UserLimits struct { - DailyApiUsage int64 `json:"daily_api_usage" datastore:"daily_api_usage"` - DailyWorkflowExecutions int64 `json:"daily_workflow_executions" datastore:"daily_workflow_executions"` - DailyCloudExecutions int64 `json:"daily_cloud_executions" datastore:"daily_cloud_executions"` - DailyTriggers int64 `json:"daily_triggers" datastore:"daily_triggers"` - DailyMailUsage int64 `json:"daily_mail_usage" datastore:"daily_mail_usage"` - MaxTriggers int64 `json:"max_triggers" datastore:"max_triggers"` - MaxWorkflows int64 `json:"max_workflows" datastore:"max_workflows"` -} - -type retStruct struct { - Success bool `json:"success"` - SyncFeatures SyncFeatures `json:"sync_features"` - SessionKey string `json:"session_key"` - IntervalSeconds int64 `json:"interval_seconds"` - Reason string `json:"reason"` -} - -// Saves some data, not sure what to have here lol -type UserAuth struct { - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - Name string `json:"name" datastore:"name" yaml:"name"` - Workflows []string `json:"workflows" datastore:"workflows"` - Username string `json:"username" datastore:"username"` - Fields []UserAuthField `json:"fields" datastore:"fields"` -} - -type UserAuthField struct { - Key string `json:"key" datastore:"key"` - Value string `json:"value" datastore:"value,noindex"` -} - -// Not environment, but execution environment -type Environment struct { - Name string `datastore:"name"` - Type string `datastore:"type"` - Registered bool `datastore:"registered"` - Default bool `datastore:"default" json:"default"` - Archived bool `datastore:"archived" json:"archived"` - Id string `datastore:"id" json:"id"` - OrgId string `datastore:"org_id" json:"org_id"` -} - -type User struct { - Username string `datastore:"Username" json:"username"` - Password string `datastore:"password,noindex" password:"password,omitempty"` - Session string `datastore:"session,noindex" json:"session"` - Verified bool `datastore:"verified,noindex" json:"verified"` - PrivateApps []WorkflowApp `datastore:"privateapps" json:"privateapps":` - Role string `datastore:"role" json:"role"` - Roles []string `datastore:"roles" json:"roles"` - VerificationToken string `datastore:"verification_token" json:"verification_token"` - ApiKey string `datastore:"apikey" json:"apikey"` - ResetReference string `datastore:"reset_reference" json:"reset_reference"` - Executions ExecutionInfo `datastore:"executions" json:"executions"` - Limits UserLimits `datastore:"limits" json:"limits"` - Authentication []UserAuth `datastore:"authentication,noindex" json:"authentication"` - ResetTimeout int64 `datastore:"reset_timeout,noindex" json:"reset_timeout"` - Id string `datastore:"id" json:"id"` - Orgs []string `datastore:"orgs" json:"orgs"` - CreationTime int64 `datastore:"creation_time" json:"creation_time"` - ActiveOrg Org `json:"active_org" datastore:"active_org"` - Active bool `datastore:"active" json:"active"` -} - -// timeout maybe? idk -type session struct { - Username string `datastore:"Username,noindex"` - Id string `datastore:"Id,noindex"` - Session string `datastore:"session,noindex"` -} - -type loginStruct struct { - Username string `json:"username"` - Password string `json:"password"` -} - -type Contact struct { - Firstname string `json:"firstname"` - Lastname string `json:"lastname"` - Title string `json:"title"` - Companyname string `json:"companyname"` - Phone string `json:"phone"` - Email string `json:"email"` - Message string `json:"message"` -} - -type Translator struct { - Src struct { - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - Description string `json:"description" datastore:"description,noindex"` - Required string `json:"required" datastore:"required"` - Type string `json:"type" datastore:"type"` - Schema struct { - Type string `json:"type" datastore:"type"` - } `json:"schema" datastore:"schema"` - } `json:"src" datastore:"src"` - Dst struct { - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - Type string `json:"type" datastore:"type"` - Description string `json:"description" datastore:"description,noindex"` - Required string `json:"required" datastore:"required"` - Schema struct { - Type string `json:"type" datastore:"type"` - } `json:"schema" datastore:"schema"` - } `json:"dst" datastore:"dst"` -} - -type Appconfig struct { - Key string `json:"key" datastore:"key"` - Value string `json:"value" datastore:"value,noindex"` -} - -type ScheduleApp struct { - Foldername string `json:"foldername" datastore:"foldername,noindex"` - Name string `json:"name" datastore:"name,noindex"` - Id string `json:"id" datastore:"id,noindex"` - Description string `json:"description" datastore:"description,noindex"` - Action string `json:"action" datastore:"action,noindex"` - Config []Appconfig `json:"config,omitempty" datastore:"config,noindex"` -} - -type AppInfo struct { - SourceApp ScheduleApp `json:"sourceapp,omitempty" datastore:"sourceapp,noindex"` - DestinationApp ScheduleApp `json:"destinationapp,omitempty" datastore:"destinationapp,noindex"` -} - -// May 2020: Reused for onprem schedules - Id, Seconds, WorkflowId and argument -type ScheduleOld struct { - Id string `json:"id" datastore:"id"` - StartNode string `json:"start_node" datastore:"start_node"` - Seconds int `json:"seconds" datastore:"seconds"` - WorkflowId string `json:"workflow_id" datastore:"workflow_id", ` - Argument string `json:"argument" datastore:"argument"` - WrappedArgument string `json:"wrapped_argument" datastore:"wrapped_argument"` - AppInfo AppInfo `json:"appinfo" datastore:"appinfo,noindex"` - Finished bool `json:"finished" finished:"id"` - BaseAppLocation string `json:"base_app_location" datastore:"baseapplocation,noindex"` - Translator []Translator `json:"translator,omitempty" datastore:"translator"` - Org string `json:"org" datastore:"org"` - CreatedBy string `json:"createdby" datastore:"createdby"` - Availability string `json:"availability" datastore:"availability"` - CreationTime int64 `json:"creationtime" datastore:"creationtime,noindex"` - LastModificationtime int64 `json:"lastmodificationtime" datastore:"lastmodificationtime,noindex"` - LastRuntime int64 `json:"lastruntime" datastore:"lastruntime,noindex"` - Frequency string `json:"frequency" datastore:"frequency,noindex"` - Environment string `json:"environment" datastore:"environment"` -} - -// Returned from /GET /schedules -type Schedules struct { - Schedules []ScheduleOld `json:"schedules"` - Success bool `json:"success"` -} - -type ScheduleApps struct { - Apps []ApiYaml `json:"apps"` - Success bool `json:"success"` -} - -// The yaml that is uploaded -type ApiYaml struct { - Name string `json:"name" yaml:"name" required:"true datastore:"name"` - Foldername string `json:"foldername" yaml:"foldername" required:"true datastore:"foldername"` - Id string `json:"id" yaml:"id",required:"true, datastore:"id"` - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - AppVersion string `json:"app_version" yaml:"app_version",datastore:"app_version"` - ContactInfo struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Url string `json:"url" datastore:"url" yaml:"url"` - } `json:"contact_info" datastore:"contact_info" yaml:"contact_info"` - Types []string `json:"types" datastore:"types" yaml:"types"` - Input []struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - InputParameters []struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - Required string `json:"required" datastore:"required" yaml:"required"` - Schema struct { - Type string `json:"type" datastore:"type" yaml:"type"` - } `json:"schema" datastore:"schema" yaml:"schema"` - } `json:"inputparameters" datastore:"inputparameters" yaml:"inputparameters"` - OutputParameters []struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - Required string `json:"required" datastore:"required" yaml:"required"` - Schema struct { - Type string `json:"type" datastore:"type" yaml:"type"` - } `json:"schema" datastore:"schema" yaml:"schema"` - } `json:"outputparameters" datastore:"outputparameters" yaml:"outputparameters"` - Config []struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - Required string `json:"required" datastore:"required" yaml:"required"` - Schema struct { - Type string `json:"type" datastore:"type" yaml:"type"` - } `json:"schema" datastore:"schema" yaml:"schema"` - } `json:"config" datastore:"config" yaml:"config"` - } `json:"input" datastore:"input" yaml:"input"` - Output []struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - Config []struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - Required string `json:"required" datastore:"required" yaml:"required"` - Schema struct { - Type string `json:"type" datastore:"type" yaml:"type"` - } `json:"schema" datastore:"schema" yaml:"schema"` - } `json:"config" datastore:"config" yaml:"config"` - InputParameters []struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - Required string `json:"required" datastore:"required" yaml:"required"` - Schema struct { - Type string `json:"type" datastore:"type" yaml:"type"` - } `json:"schema" datastore:"schema" yaml:"schema"` - } `json:"inputparameters" datastore:"inputparameters" yaml:"inputparameters"` - OutputParameters []struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - Required string `json:"required" datastore:"required" yaml:"required"` - Schema struct { - Type string `json:"type" datastore:"type" yaml:"type"` - } `json:"schema" datastore:"schema" yaml:"schema"` - } `json:"outputparameters" datastore:"outputparameters" yaml:"outputparameters"` - } `json:"output" datastore:"output" yaml:"output"` -} - -type Hooks struct { - Hooks []Hook `json:"hooks"` - Success bool `json:"-"` -} - -type Info struct { - Url string `json:"url" datastore:"url"` - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description,noindex"` -} - -// Actions to be done by webhooks etc -// Field is the actual field to use from json -type HookAction struct { - Type string `json:"type" datastore:"type"` - Name string `json:"name" datastore:"name"` - Id string `json:"id" datastore:"id"` - Field string `json:"field" datastore:"field"` -} - -type Hook struct { - Id string `json:"id" datastore:"id"` - Start string `json:"start" datastore:"start"` - Info Info `json:"info" datastore:"info"` - Actions []HookAction `json:"actions" datastore:"actions,noindex"` - Type string `json:"type" datastore:"type"` - Owner string `json:"owner" datastore:"owner"` - Status string `json:"status" datastore:"status"` - Workflows []string `json:"workflows" datastore:"workflows"` - Running bool `json:"running" datastore:"running"` - OrgId string `json:"org_id" datastore:"org_id"` - Environment string `json:"environment" datastore:"environment"` -} - -type ExecutionRequest struct { - ExecutionId string `json:"execution_id,omitempty"` - ExecutionArgument string `json:"execution_argument,omitempty"` - ExecutionSource string `json:"execution_source,omitempty"` - WorkflowId string `json:"workflow_id,omitempty"` - Environments []string `json:"environments,omitempty"` - Authorization string `json:"authorization,omitempty"` - Status string `json:"status,omitempty"` - Start string `json:"start,omitempty"` - Type string `json:"type,omitempty"` -} - -type SyncFeatures struct { - Webhook SyncData `json:"webhook" datastore:"webhook"` - Schedules SyncData `json:"schedules" datastore:"schedules"` - UserInput SyncData `json:"user_input" datastore:"user_input"` - SendMail SyncData `json:"send_mail" datastore:"send_mail"` - SendSms SyncData `json:"send_sms" datastore:"send_sms"` - Updates SyncData `json:"updates" datastore:"updates"` - Notifications SyncData `json:"notifications" datastore:"notifications"` - EmailTrigger SyncData `json:"email_trigger" datastore:"email_trigger"` - AppExecutions SyncData `json:"app_executions" datastore:"app_executions"` - WorkflowExecutions SyncData `json:"workflow_executions" datastore:"workflow_executions"` - Apps SyncData `json:"apps" datastore:"apps"` - Workflows SyncData `json:"workflows" datastore:"workflows"` - Autocomplete SyncData `json:"autocomplete" datastore:"autocomplete"` - Authentication SyncData `json:"authentication" datastore:"authentication"` - Schedule SyncData `json:"schedule" datastore:"schedule"` -} - -type SyncData struct { - Active bool `json:"active" datastore:"active"` - Type string `json:"type,omitempty" datastore:"type"` - Name string `json:"name,omitempty" datastore:"name"` - Description string `json:"description,omitempty" datastore:"description"` - Limit int64 `json:"limit,omitempty" datastore:"limit"` - StartDate int64 `json:"start_date,omitempty" datastore:"start_date"` - EndDate int64 `json:"end_date,omitempty" datastore:"end_date"` - DataCollection int64 `json:"data_collection,omitempty" datastore:"data_collection"` -} - -type SyncConfig struct { - Interval int64 `json:"interval" datastore:"interval"` - Apikey string `json:"api_key" datastore:"api_key"` -} - -// Role is just used for feedback for a user -type Org struct { - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` - Image string `json:"image" datastore:"image,noindex"` - Id string `json:"id" datastore:"id"` - Org string `json:"org" datastore:"org"` - Users []User `json:"users" datastore:"users"` - Role string `json:"role" datastore:"role"` - Roles []string `json:"roles" datastore:"roles"` - CloudSync bool `json:"cloud_sync" datastore:"CloudSync"` - SyncConfig SyncConfig `json:"sync_config" datastore:"sync_config"` - SyncFeatures SyncFeatures `json:"sync_features" datastore:"sync_features"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` -} - -type AppAuthenticationStorage struct { - Active bool `json:"active" datastore:"active"` - Label string `json:"label" datastore:"label"` - Id string `json:"id" datastore:"id"` - App WorkflowApp `json:"app" datastore:"app,noindex"` - Fields []AuthenticationStore `json:"fields" datastore:"fields"` - Usage []AuthenticationUsage `json:"usage" datastore:"usage"` - WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"` - NodeCount int64 `json:"node_count" datastore:"node_count"` - OrgId string `json:"org_id" datastore:"org_id"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` -} - -type AuthenticationUsage struct { - WorkflowId string `json:"workflow_id" datastore:"workflow_id"` - Nodes []string `json:"nodes" datastore:"nodes"` -} - -// An app inside Shuffle -// Source string `json:"source" datastore:"soure" yaml:"source"` - downloadlocation -type WorkflowApp struct { - Name string `json:"name" yaml:"name" required:true datastore:"name"` - IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"` - ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"` - Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"` - AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"` - SharingConfig string `json:"sharing_config" yaml:"sharing_config" datastore:"sharing_config"` - Generated bool `json:"generated" yaml:"generated" required:false datastore:"generated"` - Downloaded bool `json:"downloaded" yaml:"downloaded" required:false datastore:"downloaded"` - Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` - Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` - Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` - Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"` - Owner string `json:"owner" datastore:"owner" yaml:"owner"` - Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps - PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"` - Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"` - Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"` - SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` - ContactInfo struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Url string `json:"url" datastore:"url" yaml:"url"` - } `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false` - Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` - Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"` - Tags []string `json:"tags" yaml:"tags" required:false datastore:"activated"` - Categories []string `json:"categories" yaml:"categories" required:false datastore:"categories"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` - LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` -} - -type WorkflowAppActionParameter struct { - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - ID string `json:"id" datastore:"id" yaml:"id,omitempty"` - Name string `json:"name" datastore:"name" yaml:"name"` - Example string `json:"example" datastore:"example" yaml:"example"` - Value string `json:"value" datastore:"value,noindex" yaml:"value,omitempty"` - Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` - Options []string `json:"options" datastore:"options" yaml:"options"` - ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"` - Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"` - Required bool `json:"required" datastore:"required" yaml:"required"` - Configuration bool `json:"configuration" datastore:"configuration" yaml:"configuration"` - Tags []string `json:"tags" datastore:"tags" yaml:"tags"` - Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` - SkipMulticheck bool `json:"skip_multicheck" datastore:"skip_multicheck" yaml:"skip_multicheck"` - ValueReplace []Valuereplace `json:"value_replace" datastore:"value_replace,noindex" yaml:"value_replace,omitempty"` -} - -type Valuereplace struct { - Key string `json:"key" datastore:"key" yaml:"key"` - Value string `json:"value" datastore:"value" yaml:"value"` -} - -type SchemaDefinition struct { - Type string `json:"type" datastore:"type"` -} - -type WorkflowAppAction struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id" yaml:"id,omitempty"` - Name string `json:"name" datastore:"name"` - Label string `json:"label" datastore:"label"` - NodeType string `json:"node_type" datastore:"node_type"` - Environment string `json:"environment" datastore:"environment"` - Sharing bool `json:"sharing" datastore:"sharing"` - PrivateID string `json:"private_id" datastore:"private_id"` - AppID string `json:"app_id" datastore:"app_id"` - Tags []string `json:"tags" datastore:"tags" yaml:"tags"` - Authentication []AuthenticationStore `json:"authentication" datastore:"authentication,noindex" yaml:"authentication,omitempty"` - Tested bool `json:"tested" datastore:"tested" yaml:"tested"` - Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"` - ExecutionVariable struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"execution_variable" datastore:"execution_variables"` - Returns struct { - Description string `json:"description" datastore:"returns" yaml:"description,omitempty"` - Example string `json:"example" datastore:"example" yaml:"example"` - ID string `json:"id" datastore:"id" yaml:"id,omitempty"` - Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` - } `json:"returns" datastore:"returns"` - AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` - Example string `json:"example" datastore:"example" yaml:"example"` - AuthNotRequired bool `json:"auth_not_required" datastore:"auth_not_required" yaml:"auth_not_required"` -} - -type WorkflowExecution struct { - Type string `json:"type" datastore:"type"` - Status string `json:"status" datastore:"status"` - Start string `json:"start" datastore:"start"` - ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` - ExecutionId string `json:"execution_id" datastore:"execution_id"` - ExecutionSource string `json:"execution_source" datastore:"execution_source"` - ExecutionParent string `json:"execution_parent" datastore:"execution_parent"` - ExecutionOrg string `json:"execution_org" datastore:"execution_org"` - WorkflowId string `json:"workflow_id" datastore:"workflow_id"` - LastNode string `json:"last_node" datastore:"last_node"` - Authorization string `json:"authorization" datastore:"authorization"` - Result string `json:"result" datastore:"result,noindex"` - StartedAt int64 `json:"started_at" datastore:"started_at"` - CompletedAt int64 `json:"completed_at" datastore:"completed_at"` - ProjectId string `json:"project_id" datastore:"project_id"` - Locations []string `json:"locations" datastore:"locations"` - Workflow Workflow `json:"workflow" datastore:"workflow,noindex"` - Results []ActionResult `json:"results" datastore:"results,noindex"` - ExecutionVariables []struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"execution_variables,omitempty" datastore:"execution_variables,omitempty"` - OrgId string `json:"org_id" datastore:"org_id"` -} - -type Action struct { - AppName string `json:"app_name,omitempty" datastore:"app_name"` - AppVersion string `json:"app_version,omitempty" datastore:"app_version"` - AppID string `json:"app_id,omitempty" datastore:"app_id"` - Errors []string `json:"errors,omitempty" datastore:"errors"` - ID string `json:"id,omitempty" datastore:"id"` - IsValid bool `json:"is_valid,omitempty" datastore:"is_valid"` - IsStartNode bool `json:"isStartNode,omitempty" datastore:"isStartNode"` - Sharing bool `json:"sharing,omitempty" datastore:"sharing"` - PrivateID string `json:"private_id,omitempty" datastore:"private_id"` - Label string `json:"label,omitempty" datastore:"label"` - SmallImage string `json:"small_image,omitempty" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image,omitempty" datastore:"large_image,noindex" yaml:"large_image" required:false` - Environment string `json:"environment,omitempty" datastore:"environment"` - Name string `json:"name,omitempty" datastore:"name"` - Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` - ExecutionVariable struct { - Description string `json:"description,omitempty" datastore:"description,noindex"` - ID string `json:"id,omitempty" datastore:"id"` - Name string `json:"name,omitempty" datastore:"name"` - Value string `json:"value,omitempty" datastore:"value,noindex"` - } `json:"execution_variable,omitempty" datastore:"execution_variable,omitempty"` - Position struct { - X float64 `json:"x,omitempty" datastore:"x"` - Y float64 `json:"y,omitempty" datastore:"y"` - } `json:"position,omitempty"` - Priority int `json:"priority,omitempty" datastore:"priority"` - AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"` - Example string `json:"example,omitempty" datastore:"example"` - AuthNotRequired bool `json:"auth_not_required,omitempty" datastore:"auth_not_required" yaml:"auth_not_required"` -} - -// Added environment for location to execute -type Trigger struct { - AppName string `json:"app_name" datastore:"app_name"` - Description string `json:"description" datastore:"description,noindex"` - LongDescription string `json:"long_description" datastore:"long_description"` - Status string `json:"status" datastore:"status"` - AppVersion string `json:"app_version" datastore:"app_version"` - Errors []string `json:"errors" datastore:"errors"` - ID string `json:"id" datastore:"id"` - IsValid bool `json:"is_valid" datastore:"is_valid"` - IsStartNode bool `json:"isStartNode" datastore:"isStartNode"` - Label string `json:"label" datastore:"label"` - SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` - Environment string `json:"environment" datastore:"environment"` - TriggerType string `json:"trigger_type" datastore:"trigger_type"` - Name string `json:"name" datastore:"name"` - Tags []string `json:"tags" datastore:"tags" yaml:"tags"` - Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` - Position struct { - X float64 `json:"x" datastore:"x"` - Y float64 `json:"y" datastore:"y"` - } `json:"position"` - Priority int `json:"priority" datastore:"priority"` -} - -type Branch struct { - DestinationID string `json:"destination_id" datastore:"destination_id"` - ID string `json:"id" datastore:"id"` - SourceID string `json:"source_id" datastore:"source_id"` - Label string `json:"label" datastore:"label"` - HasError bool `json:"has_errors" datastore: "has_errors"` - Conditions []Condition `json:"conditions" datastore: "conditions,noindex"` -} - -// Same format for a lot of stuff -type Condition struct { - Condition WorkflowAppActionParameter `json:"condition" datastore:"condition"` - Source WorkflowAppActionParameter `json:"source" datastore:"source"` - Destination WorkflowAppActionParameter `json:"destination" datastore:"destination"` -} - -type Schedule struct { - Name string `json:"name" datastore:"name"` - Frequency string `json:"frequency" datastore:"frequency"` - ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` - Id string `json:"id" datastore:"id"` - OrgId string `json:"org_id" datastore:"org_id"` - Environment string `json:"environment" datastore:"environment"` -} - -type Workflow struct { - Actions []Action `json:"actions" datastore:"actions,noindex"` - Branches []Branch `json:"branches" datastore:"branches,noindex"` - Triggers []Trigger `json:"triggers" datastore:"triggers,noindex"` - Schedules []Schedule `json:"schedules" datastore:"schedules,noindex"` - Configuration struct { - ExitOnError bool `json:"exit_on_error" datastore:"exit_on_error"` - StartFromTop bool `json:"start_from_top" datastore:"start_from_top"` - } `json:"configuration,omitempty" datastore:"configuration"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` - LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` - Errors []string `json:"errors,omitempty" datastore:"errors"` - Tags []string `json:"tags,omitempty" datastore:"tags"` - ID string `json:"id" datastore:"id"` - IsValid bool `json:"is_valid" datastore:"is_valid"` - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description,noindex"` - Start string `json:"start" datastore:"start"` - Owner string `json:"owner" datastore:"owner"` - Sharing string `json:"sharing" datastore:"sharing"` - Org []Org `json:"org,omitempty" datastore:"org"` - ExecutingOrg Org `json:"execution_org,omitempty" datastore:"execution_org"` - OrgId string `json:"org_id,omitempty" datastore:"org_id"` - WorkflowVariables []struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"workflow_variables" datastore:"workflow_variables"` - ExecutionVariables []struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"execution_variables,omitempty" datastore:"execution_variables"` - ExecutionEnvironment string `json:"execution_environment" datastore:"execution_environment"` -} - -type ActionResult struct { - Action Action `json:"action" datastore:"action,noindex"` - ExecutionId string `json:"execution_id" datastore:"execution_id"` - Authorization string `json:"authorization" datastore:"authorization"` - Result string `json:"result" datastore:"result,noindex"` - StartedAt int64 `json:"started_at" datastore:"started_at"` - CompletedAt int64 `json:"completed_at" datastore:"completed_at"` - Status string `json:"status" datastore:"status"` -} - -type Authentication struct { - Required bool `json:"required" datastore:"required" yaml:"required" ` - Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"` -} - -type AuthenticationParams struct { - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - ID string `json:"id" datastore:"id" yaml:"id"` - Name string `json:"name" datastore:"name" yaml:"name"` - Example string `json:"example" datastore:"example" yaml:"example"` - Value string `json:"value,omitempty" datastore:"value,noindex" yaml:"value"` - Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` - Required bool `json:"required" datastore:"required" yaml:"required"` - In string `json:"in" datastore:"in" yaml:"in"` - Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` - Scheme string `json:"scheme" datastore:"scheme" yaml:"scheme"` // Deprecated -} - -type AuthenticationStore struct { - Key string `json:"key" datastore:"key"` - Value string `json:"value" datastore:"value,noindex"` -} - -type ExecutionRequestWrapper struct { - Data []ExecutionRequest `json:"data"` -} - -type AppExecutionExample struct { - AppName string `json:"app_name" datastore:"app_name"` - AppVersion string `json:"app_version" datastore:"app_version"` - AppAction string `json:"app_action" datastore:"app_action"` - AppId string `json:"app_id" datastore:"app_id"` - ExampleId string `json:"example_id" datastore:"example_id"` - SuccessExamples []string `json:"success_examples" datastore:"success_examples,noindex"` - FailureExamples []string `json:"failure_examples" datastore:"failure_examples,noindex"` -} - // removes every container except itself (worker) -func shutdown(executionId, workflowId string) { - log.Printf("[INFO] Shutdown started") +func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason string, handleResultSend bool) { + log.Printf("[INFO] Shutdown (%s) started with reason %s", workflowExecution.Status, reason) + //reason := "Error in execution" + + sleepDuration := 1 + if handleResultSend && requestsSent < 2 { + data, err := json.Marshal(workflowExecution) + if err == nil { + sendResult(workflowExecution, data) + log.Printf("[WARNING] Sent shutdown update") + } else { + log.Printf("[WARNING] DIDNT send update") + } + + time.Sleep(time.Duration(sleepDuration) * time.Second) + } // Might not be necessary because of cleanupEnv hostconfig autoremoval if cleanupEnv == "true" && len(containerIds) > 0 { @@ -796,7 +128,20 @@ func shutdown(executionId, workflowId string) { log.Printf("[INFO] NOT cleaning up containers. IDS: %d, CLEANUP env: %s", len(containerIds), cleanupEnv) } - fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowId, executionId) + fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowExecution.Workflow.ID, workflowExecution.ExecutionId) + + path := fmt.Sprintf("?reason=%s", url.QueryEscape(reason)) + if len(nodeId) > 0 { + path += fmt.Sprintf("&node=%s", url.QueryEscape(nodeId)) + } + if len(environment) > 0 { + path += fmt.Sprintf("&env=%s", url.QueryEscape(environment)) + } + + //fmt.Println(url.QueryEscape(query)) + fullUrl += path + log.Printf("[INFO] Abort URL: %s", fullUrl) + req, err := http.NewRequest( "GET", fullUrl, @@ -839,7 +184,6 @@ func shutdown(executionId, workflowId string) { log.Printf("[INFO] Failed abort request: %s", err) } - sleepDuration := 1 log.Printf("[INFO] Finished shutdown (after %d seconds).", sleepDuration) // Allows everything to finish in subprocesses time.Sleep(time.Duration(sleepDuration) * time.Second) @@ -847,8 +191,9 @@ func shutdown(executionId, workflowId string) { } // Deploys the internal worker whenever something happens -func deployApp(cli *dockerclient.Client, image string, identifier string, env []string) error { +func deployApp(cli *dockerclient.Client, image string, identifier string, env []string, workflowExecution shuffle.WorkflowExecution) error { // form basic hostConfig + ctx := context.Background() hostConfig := &container.HostConfig{ LogConfig: container.LogConfig{ Type: "json-file", @@ -861,6 +206,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] } // form container id and use it as network source if it's not empty + containerId = getThisContainerId() if containerId != "" { hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) } else { @@ -872,13 +218,44 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] hostConfig.AutoRemove = true } + // FIXME: Add proper foldermounts here + //log.Printf("\n\nPRE FOLDERMOUNT\n\n") + //volumeBinds := []string{"/tmp/shuffle-mount:/rules"} + //volumeBinds := []string{"/tmp/shuffle-mount:/rules"} + volumeBinds := []string{} + if len(volumeBinds) > 0 { + log.Printf("[INFO] Setting up binds for container!") + hostConfig.Binds = volumeBinds + hostConfig.Mounts = []mount.Mount{} + for _, bind := range volumeBinds { + if !strings.Contains(bind, ":") || strings.Contains(bind, "..") || strings.HasPrefix(bind, "~") { + log.Printf("[WARNING] Bind %s is invalid.", bind) + continue + } + + log.Printf("[INFO] Appending bind %s", bind) + bindSplit := strings.Split(bind, ":") + sourceFolder := bindSplit[0] + destinationFolder := bindSplit[0] + hostConfig.Mounts = append(hostConfig.Mounts, mount.Mount{ + Type: mount.TypeBind, + Source: sourceFolder, + Target: destinationFolder, + }) + } + } else { + log.Printf("[WARNING] No mounted folders") + } + // hostConfig.Binds = volumeBinds + //} + config := &container.Config{ Image: image, Env: env, } cont, err := cli.ContainerCreate( - context.Background(), + ctx, config, hostConfig, nil, @@ -891,14 +268,86 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] return err } - err = cli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{}) + err = cli.ContainerStart(ctx, cont.ID, types.ContainerStartOptions{}) if err != nil { log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err) - //shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + //shutdown(workflowExecution, workflowExecution.Workflow.ID, true) return err } log.Printf("[INFO] Container %s was created for %s", cont.ID, identifier) + + // Waiting to see if it exits.. Stupid, but stable(r) + if workflowExecution.ExecutionSource != "default" { + log.Printf("[INFO] Handling NON-default execution source %s - NOT waiting and validating!", workflowExecution.ExecutionSource) + } else if workflowExecution.ExecutionSource == "default" { + time.Sleep(2 * time.Second) + + stats, err := cli.ContainerInspect(ctx, cont.ID) + if err != nil { + log.Printf("[ERROR] Failed getting container stats") + } else { + //log.Printf("[INFO] Info for container: %#v", stats) + //log.Printf("%#v", stats.Config) + //log.Printf("%#v", stats.ContainerJSONBase.State) + log.Printf("[INFO] EXECUTION STATUS: %s", stats.ContainerJSONBase.State.Status) + if stats.ContainerJSONBase.State.Status == "exited" { + logOptions := types.ContainerLogsOptions{ + ShowStdout: true, + } + + out, err := cli.ContainerLogs(ctx, cont.ID, logOptions) + if err != nil { + log.Printf("[INFO] Failed getting logs: %s", err) + } else { + log.Printf("IN ELSE FOR DEPLOY") + buf := new(strings.Builder) + io.Copy(buf, out) + logs := buf.String() + log.Printf("Logs: %s", logs) + + //log.Printf(logs) + // check errors + /* + if strings.Contains(logs, "Error") { + log.Printf("ERROR IN %s?", cont.ID) + log.Println(logs) + //return errors.New(fmt.Sprintf("ERROR FROM CONTAINER %s", cont.ID)) + } else { + log.Printf("NORMAL EXEC OF %s?", cont.ID) + } + */ + } + + log.Printf("ERROR IN CONTAINER DEPLOYMENT - ITS EXITED!") + + return errors.New(fmt.Sprintf(`{"success": false, "reason": "Container %s exited prematurely.","debug": "docker logs -f %s"}`, cont.ID, cont.ID)) + } + } + } + + /* + //log.Printf("%#v", stats.Config.Status) + //ContainerJSONtoConfig(cj dockType.ContainerJSON) ContainerConfig { + listOptions := types.ContainerListOptions{ + Filters: filters.Args{ + map[string][]string{"ancestor": {":"}}, + }, + } + containers, err := cli.ContainerList(ctx, listOptions) + */ + + //log.Printf("%#v", cont.Status) + //config := ContainerJSONtoConfig(stats) + //log.Printf("CONFIG: %#v", config) + + /* + logOptions := types.ContainerLogsOptions{ + ShowStdout: true, + } + + */ + containerIds = append(containerIds, cont.ID) return nil } @@ -937,7 +386,7 @@ func removeContainer(containername string) error { return nil } -func runFilter(workflowExecution WorkflowExecution, action Action) { +func runFilter(workflowExecution shuffle.WorkflowExecution, action shuffle.Action) { // 1. Get the parameter $.#.id if action.Label == "filter_cases" && len(action.Parameters) > 0 { if action.Parameters[0].Variant == "ACTION_RESULT" { @@ -953,7 +402,7 @@ func runFilter(workflowExecution WorkflowExecution, action Action) { } -func handleSubworkflowExecution(client *http.Client, workflowExecution WorkflowExecution, action Trigger, baseAction Action) error { +func handleSubworkflowExecution(client *http.Client, workflowExecution shuffle.WorkflowExecution, action shuffle.Trigger, baseAction shuffle.Action) error { apikey := "" workflowId := "" executionArgument := "" @@ -1005,14 +454,14 @@ func handleSubworkflowExecution(client *http.Client, workflowExecution WorkflowE } timeNow := time.Now().Unix() - //curaction := Action{ + //curaction := shuffle.Action{ // AppName: baseAction.AppName, // AppVersion: baseAction.AppVersion, // Label: baseAction.Label, // Name: baseAction.Name, // ID: baseAction.ID, //} - result := ActionResult{ + result := shuffle.ActionResult{ Action: baseAction, ExecutionId: workflowExecution.ExecutionId, Authorization: workflowExecution.Authorization, @@ -1065,7 +514,7 @@ func removeIndex(s []string, i int) []string { return s[:len(s)-1] } -func handleExecutionResult(workflowExecution WorkflowExecution) { +func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { if len(startAction) == 0 { startAction = workflowExecution.Start if len(startAction) == 0 { @@ -1149,7 +598,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { // care if it gets stuck in a loop. // FIXME: Force killing a worker should result in a notification somewhere if len(nextActions) == 0 { - log.Printf("[INFO] No next action. Finished? Result vs Actions: %d - %d", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)) + log.Printf("[INFO] No next action. Finished? Result vs shuffle.Actions: %d - %d", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)) exit := true for _, item := range workflowExecution.Results { if item.Status == "EXECUTING" { @@ -1165,7 +614,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { if exit && len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions) { log.Printf("Shutting down.") - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", true) } // Look for the NEXT missing action @@ -1266,7 +715,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { //visited = append(visited, action.ID) //executed = append(executed, action.ID) - trigger := Trigger{} + trigger := shuffle.Trigger{} for _, innertrigger := range workflowExecution.Workflow.Triggers { if innertrigger.ID == action.ID { trigger = innertrigger @@ -1275,18 +724,18 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { } // FIXME: Add startnode from frontend - action.Parameters = []WorkflowAppActionParameter{} + action.Parameters = []shuffle.WorkflowAppActionParameter{} for _, parameter := range trigger.Parameters { parameter.Variant = "STATIC_VALUE" action.Parameters = append(action.Parameters, parameter) } - action.Parameters = append(action.Parameters, WorkflowAppActionParameter{ + action.Parameters = append(action.Parameters, shuffle.WorkflowAppActionParameter{ Name: "source_workflow", Value: workflowExecution.Workflow.ID, }) - action.Parameters = append(action.Parameters, WorkflowAppActionParameter{ + action.Parameters = append(action.Parameters, shuffle.WorkflowAppActionParameter{ Name: "source_execution", Value: workflowExecution.ExecutionId, }) @@ -1309,7 +758,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { continue } else { log.Printf("Should stop after this iteration because it's user-input based. %#v", action) - trigger := Trigger{} + trigger := shuffle.Trigger{} for _, innertrigger := range workflowExecution.Workflow.Triggers { if innertrigger.ID == action.ID { trigger = innertrigger @@ -1402,12 +851,13 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { appname = strings.Replace(appname, ".", "-", -1) appversion = strings.Replace(appversion, ".", "-", -1) - image := fmt.Sprintf("%s:%s_%s", baseimagename, action.AppName, action.AppVersion) + image := fmt.Sprintf("%s:%s_%s", baseimagename, strings.ToLower(action.AppName), action.AppVersion) if strings.Contains(image, " ") { image = strings.ReplaceAll(image, " ", "-") } - identifier := fmt.Sprintf("%s_%s_%s_%s", appname, appversion, action.ID, workflowExecution.ExecutionId) + // Added UUID to identifier just in case + identifier := fmt.Sprintf("%s_%s_%s_%s_%s", appname, appversion, action.ID, workflowExecution.ExecutionId, uuid.NewV4()) if strings.Contains(identifier, " ") { identifier = strings.ReplaceAll(identifier, " ", "-") } @@ -1438,7 +888,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { } if len(action.Parameters) == 0 { - action.Parameters = []WorkflowAppActionParameter{} + action.Parameters = []shuffle.WorkflowAppActionParameter{} } if len(action.Errors) == 0 { @@ -1490,103 +940,127 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { } // Uses a few ways of getting / checking if an app is available - // 1. Try original - // 2. Go to lowercase + // 1. Try original with lowercase + // 2. Go to original // 3. Add remote repo location // 4. Actually download last repo images := []string{ - fmt.Sprintf("%s/%s:%s_%s", registryName, baseimagename, strings.ToLower(action.AppName), action.AppVersion), image, - fmt.Sprintf("%s:%s_%s", baseimagename, strings.ToLower(action.AppName), action.AppVersion), + fmt.Sprintf("%s:%s_%s", baseimagename, action.AppName, action.AppVersion), + fmt.Sprintf("%s/%s:%s_%s", registryName, baseimagename, strings.ToLower(action.AppName), action.AppVersion), } // If cleanup is set, it should run for efficiency pullOptions := types.ImagePullOptions{} if cleanupEnv == "true" { - err = deployApp(dockercli, images[0], identifier, env) + err = deployApp(dockercli, images[0], identifier, env, workflowExecution) if err != nil { + if strings.Contains(err.Error(), "exited prematurely") { + shutdown(workflowExecution, action.ID, err.Error(), true) + } + log.Printf("[WARNING] Failed CLEANUP execution. Downloading image remotely.") + image = images[2] reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) if err != nil { log.Printf("[ERROR] Failed getting %s. The couldn't be find locally, AND is missing.", image) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, action.ID, err.Error(), true) } buildBuf := new(strings.Builder) _, err = io.Copy(buildBuf, reader) if err != nil { log.Printf("[ERROR] Error in IO copy: %s", err) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, action.ID, err.Error(), true) } else { if strings.Contains(buildBuf.String(), "errorDetail") { log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, action.ID, err.Error(), true) } log.Printf("[INFO] Successfully downloaded %s", image) } - err = deployApp(dockercli, image, identifier, env) + err = deployApp(dockercli, image, identifier, env, workflowExecution) if err != nil { log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") + if strings.Contains(err.Error(), "exited prematurely") { + shutdown(workflowExecution, action.ID, err.Error(), true) + } + if strings.Contains(err.Error(), "No such image") { //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) log.Printf("[ERROR] Image doesn't exist. Shutting down") - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, action.ID, err.Error(), true) } } } } else { - err = deployApp(dockercli, image, identifier, env) + err = deployApp(dockercli, images[0], identifier, env, workflowExecution) if err != nil { + if strings.Contains(err.Error(), "exited prematurely") { + shutdown(workflowExecution, action.ID, err.Error(), true) + } + // Trying to replace with lowercase to deploy again. This seems to work with Dockerhub well. // FIXME: Should try to remotely download directly if this persists. - image = fmt.Sprintf("%s:%s_%s", baseimagename, strings.ToLower(action.AppName), action.AppVersion) + image = images[1] if strings.Contains(image, " ") { image = strings.ReplaceAll(image, " ", "-") } - err = deployApp(dockercli, image, identifier, env) + err = deployApp(dockercli, image, identifier, env, workflowExecution) if err != nil { - image = fmt.Sprintf("%s/%s:%s_%s", registryName, baseimagename, strings.ToLower(action.AppName), action.AppVersion) + if strings.Contains(err.Error(), "exited prematurely") { + shutdown(workflowExecution, action.ID, err.Error(), true) + } + + image = images[2] if strings.Contains(image, " ") { image = strings.ReplaceAll(image, " ", "-") } - err = deployApp(dockercli, image, identifier, env) + err = deployApp(dockercli, image, identifier, env, workflowExecution) if err != nil { - log.Printf("[WARNING] Failed deploying image THRICE. Attempting to download the latter as last resort.") + if strings.Contains(err.Error(), "exited prematurely") { + shutdown(workflowExecution, action.ID, err.Error(), true) + } + + log.Printf("[WARNING] Failed deploying image THREE TIMES. Attempting to download the latter as last resort.") reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) if err != nil { log.Printf("[ERROR] Failed getting %s. The couldn't be find locally, AND is missing.", image) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, action.ID, err.Error(), true) } buildBuf := new(strings.Builder) _, err = io.Copy(buildBuf, reader) if err != nil { log.Printf("[ERROR] Error in IO copy: %s", err) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, action.ID, err.Error(), true) } else { if strings.Contains(buildBuf.String(), "errorDetail") { log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, action.ID, err.Error(), true) } log.Printf("[INFO] Successfully downloaded %s", image) } - err = deployApp(dockercli, image, identifier, env) + err = deployApp(dockercli, image, identifier, env, workflowExecution) if err != nil { - log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") + if strings.Contains(err.Error(), "exited prematurely") { + shutdown(workflowExecution, action.ID, err.Error(), true) + } + if strings.Contains(err.Error(), "No such image") { //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) log.Printf("[ERROR] Image doesn't exist. Shutting down") - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, action.ID, err.Error(), true) } } } @@ -1626,7 +1100,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { if shutdownCheck { log.Println("[INFO] BREAKING BECAUSE RESULTS IS SAME LENGTH AS ACTIONS. SHOULD CHECK ALL RESULTS FOR WHETHER THEY'RE DONE") validateFinished(workflowExecution) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", true) } } @@ -1634,7 +1108,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { return } -func executionInit(workflowExecution WorkflowExecution) error { +func executionInit(workflowExecution shuffle.WorkflowExecution) error { parents = map[string][]string{} children = map[string][]string{} @@ -1676,10 +1150,10 @@ func executionInit(workflowExecution WorkflowExecution) error { } if trigger.ID == branch.SourceID { - log.Printf("[INFO] Trigger %s is the source!", trigger.AppName) + log.Printf("[INFO] shuffle.Trigger %s is the source!", trigger.AppName) sourceFound = true } else if trigger.ID == branch.DestinationID { - log.Printf("[INFO] Trigger %s is the destination!", trigger.AppName) + log.Printf("[INFO] shuffle.Trigger %s is the destination!", trigger.AppName) destinationFound = true } } @@ -1704,7 +1178,7 @@ func executionInit(workflowExecution WorkflowExecution) error { log.Printf("[INFO] NEXT ACTIONS: %#v\n\n", nextActions) */ - log.Printf("[INFO] Actions: %d + Special Triggers: %d", len(workflowExecution.Workflow.Actions), extra) + log.Printf("[INFO] shuffle.Actions: %d + Special shuffle.Triggers: %d", len(workflowExecution.Workflow.Actions), extra) onpremApps := []string{} toExecuteOnprem := []string{} for _, action := range workflowExecution.Workflow.Actions { @@ -1744,7 +1218,7 @@ func executionInit(workflowExecution WorkflowExecution) error { //reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) //if err != nil { // log.Printf("Failed getting %s. The app is missing or some other issue", image) - // shutdown(workflowExecution.ExecutionId) + // shutdown(workflowExecution) //} ////io.Copy(os.Stdout, reader) @@ -1755,14 +1229,14 @@ func executionInit(workflowExecution WorkflowExecution) error { return nil } -func handleExecution(client *http.Client, req *http.Request, workflowExecution WorkflowExecution) error { +func handleExecution(client *http.Client, req *http.Request, workflowExecution shuffle.WorkflowExecution) error { // if no onprem runs (shouldn't happen, but extra check), exit // if there are some, load the images ASAP for the app err := executionInit(workflowExecution) if err != nil { log.Printf("[INFO] Workflow setup failed: %s", workflowExecution.ExecutionId, err) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", true) } log.Printf("Startaction: %s", startAction) @@ -1798,7 +1272,11 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W if newresp.StatusCode != 200 { log.Printf("[ERROR] Bad statuscode: %d, %s", newresp.StatusCode, string(body)) - //shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + + if strings.Contains(string(body), "Workflowexecution is already finished") { + shutdown(workflowExecution, "", "", false) + } + time.Sleep(time.Duration(sleepTime) * time.Second) continue } @@ -1812,13 +1290,13 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" { log.Printf("[INFO] Workflow %s is finished. Exiting worker.", workflowExecution.ExecutionId) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", true) } log.Printf("[INFO] Status: %s, Results: %d, actions: %d", workflowExecution.Status, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extra) if workflowExecution.Status != "EXECUTING" { log.Printf("[WARNING] Exiting as worker execution has status %s!", workflowExecution.Status) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", true) } } @@ -1837,17 +1315,17 @@ func arrayContains(visited []string, id string) bool { return found } -func getResult(workflowExecution WorkflowExecution, id string) ActionResult { +func getResult(workflowExecution shuffle.WorkflowExecution, id string) shuffle.ActionResult { for _, actionResult := range workflowExecution.Results { if actionResult.Action.ID == id { return actionResult } } - return ActionResult{} + return shuffle.ActionResult{} } -func getAction(workflowExecution WorkflowExecution, id, environment string) Action { +func getAction(workflowExecution shuffle.WorkflowExecution, id, environment string) shuffle.Action { for _, action := range workflowExecution.Workflow.Actions { if action.ID == id { return action @@ -1856,7 +1334,7 @@ func getAction(workflowExecution WorkflowExecution, id, environment string) Acti for _, trigger := range workflowExecution.Workflow.Triggers { if trigger.ID == id { - return Action{ + return shuffle.Action{ ID: trigger.ID, AppName: trigger.AppName, Name: trigger.AppName, @@ -1867,12 +1345,12 @@ func getAction(workflowExecution WorkflowExecution, id, environment string) Acti } } - return Action{} + return shuffle.Action{} } -func runUserInput(client *http.Client, action Action, workflowId, workflowExecutionId, authorization string, configuration string) error { +func runUserInput(client *http.Client, action shuffle.Action, workflowId, workflowExecutionId, authorization string, configuration string) error { timeNow := time.Now().Unix() - result := ActionResult{ + result := shuffle.ActionResult{ Action: action, ExecutionId: workflowExecutionId, Authorization: authorization, @@ -1942,7 +1420,7 @@ func runTestExecution(client *http.Client, workflowId, apikey string) (string, s } log.Printf("[INFO] Test Body: %s", string(body)) - var workflowExecution WorkflowExecution + var workflowExecution shuffle.WorkflowExecution err = json.Unmarshal(body, &workflowExecution) if err != nil { log.Printf("Failed workflowExecution unmarshal: %s", err) @@ -1962,17 +1440,17 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { } //log.Printf("Got result: %s", string(body)) - var actionResult ActionResult + var actionResult shuffle.ActionResult err = json.Unmarshal(body, &actionResult) if err != nil { - log.Printf("Failed ActionResult unmarshaling: %s", err) + log.Printf("Failed shuffle.ActionResult unmarshaling: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } - // 1. Get the WorkflowExecution(ExecutionId) from the database - // 2. if ActionResult.Authentication != WorkflowExecution.Authentication -> exit + // 1. Get the shuffle.WorkflowExecution(ExecutionId) from the database + // 2. if shuffle.ActionResult.Authentication != shuffle.WorkflowExecution.Authentication -> exit // 3. Add to and update actionResult in workflowExecution // 4. Push to db // IF FAIL: Set executionstatus: abort or cancel @@ -2017,7 +1495,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { //if actionResult.Status == "WAITING" && actionResult.Action.AppName == "User Input" { // log.Printf("SHOULD WAIT A BIT AND RUN CLOUD STUFF WITH USER INPUT! WAITING!") - // var trigger Trigger + // var trigger shuffle.Trigger // err = json.Unmarshal([]byte(actionResult.Result), &trigger) // if err != nil { // log.Printf("Failed unmarshaling actionresult for user input: %s", err) @@ -2037,7 +1515,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { // actionResult.Result = fmt.Sprintf("Cloud error: %s", err) // workflowExecution.Results = append(workflowExecution.Results, actionResult) // workflowExecution.Status = "ABORTED" - // err = setWorkflowExecution(ctx, *workflowExecution, true) + // err = setshuffle.WorkflowExecution(ctx, *workflowExecution, true) // if err != nil { // log.Printf("Failed ") // } else { @@ -2055,7 +1533,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { // workflowExecution.Results = append(workflowExecution.Results, actionResult) // workflowExecution.Status = actionResult.Status - // err = setWorkflowExecution(ctx, *workflowExecution, true) + // err = setshuffle.WorkflowExecution(ctx, *workflowExecution, true) // if err != nil { // log.Printf("Failed ") // } else { @@ -2072,7 +1550,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { } -func findChildNodes(workflowExecution WorkflowExecution, nodeId string) []string { +func findChildNodes(workflowExecution shuffle.WorkflowExecution, nodeId string) []string { //log.Printf("\nNODE TO FIX: %s\n\n", nodeId) allChildren := []string{nodeId} @@ -2120,7 +1598,7 @@ func findChildNodes(workflowExecution WorkflowExecution, nodeId string) []string } // Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times -func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult ActionResult, resp http.ResponseWriter) { +func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult shuffle.ActionResult, resp http.ResponseWriter) { //log.Printf("IN WORKFLOWEXECUTION SUB!") // Should start a tx for the execution here workflowExecution, err := getWorkflowExecution(ctx, workflowExecutionId) @@ -2130,6 +1608,8 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution"}`))) return } + + log.Printf(`[INFO] Got result %s from %s`, actionResult.Status, actionResult.Action.ID) resultLength := len(workflowExecution.Results) dbSave := false setExecution := true @@ -2142,7 +1622,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl //} //key := datastore.NameKey("workflowexecution", workflowExecutionId, nil) - //workflowExecution := &WorkflowExecution{} + //workflowExecution := &shuffle.WorkflowExecution{} //if err := tx.Get(key, workflowExecution); err != nil { // log.Printf("[ERROR] tx.Get bug: %v", err) // tx.Rollback() @@ -2150,7 +1630,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting the workflow key"}`))) // return //} - actionResult.Action = Action{ + actionResult.Action = shuffle.Action{ AppName: actionResult.Action.AppName, AppVersion: actionResult.Action.AppVersion, Label: actionResult.Action.Label, @@ -2162,19 +1642,19 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl if actionResult.Status == "ABORTED" || actionResult.Status == "FAILURE" { //dbSave = true - newResults := []ActionResult{} + newResults := []shuffle.ActionResult{} childNodes := []string{} if workflowExecution.Workflow.Configuration.ExitOnError { - log.Printf("[WARNING] Actionresult is %s for node %s in %s. Should set workflowExecution and exit all running functions", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) + log.Printf("[WARNING] shuffle.Actionresult is %s for node %s in %s. Should set workflowExecution and exit all running functions", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) workflowExecution.Status = actionResult.Status workflowExecution.LastNode = actionResult.Action.ID // Find underlying nodes and add them } else { - log.Printf("[WARNING] Actionresult is %s for node %s in %s. Continuing anyway because of workflow configuration.", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) + log.Printf("[WARNING] shuffle.Actionresult is %s for node %s in %s. Continuing anyway because of workflow configuration.", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) // Finds ALL childnodes to set them to SKIPPED - childNodes = findChildNodes(*workflowExecution, actionResult.Action.ID) // Remove duplicates //log.Printf("CHILD NODES: %d", len(childNodes)) + childNodes = findChildNodes(*workflowExecution, actionResult.Action.ID) for _, nodeId := range childNodes { if nodeId == actionResult.Action.ID { continue @@ -2182,7 +1662,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // 1. Find the action itself // 2. Create an actionresult - curAction := Action{ID: ""} + curAction := shuffle.Action{ID: ""} for _, action := range workflowExecution.Workflow.Actions { if action.ID == nodeId { curAction = action @@ -2232,7 +1712,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } if !skipNodeAdd { - newResult := ActionResult{ + newResult := shuffle.ActionResult{ Action: curAction, ExecutionId: actionResult.ExecutionId, Authorization: actionResult.Authorization, @@ -2255,7 +1735,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Cleans up aborted, and always gives a result lastResult := "" - // type ActionResult struct { + // type shuffle.ActionResult struct { for _, result := range workflowExecution.Results { if actionResult.Action.ID == result.Action.ID { continue @@ -2288,6 +1768,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl for index, item := range workflowExecution.Results { if item.Action.ID == actionResult.Action.ID { found = true + if item.Status == actionResult.Status { skip = true } @@ -2302,8 +1783,8 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } else if found { // If result exists and execution variable exists, update execution value //log.Printf("Exec var backend: %s", workflowExecution.Results[outerindex].Action.ExecutionVariable.Name) - actionVarName := workflowExecution.Results[outerindex].Action.ExecutionVariable.Name // Finds potential execution arguments + actionVarName := workflowExecution.Results[outerindex].Action.ExecutionVariable.Name if len(actionVarName) > 0 { log.Printf("EXECUTION VARIABLE LOCAL: %s", actionVarName) for index, execvar := range workflowExecution.ExecutionVariables { @@ -2318,38 +1799,123 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl log.Printf("[INFO] Updating %s in workflow %s from %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, workflowExecution.Results[outerindex].Status, actionResult.Status) workflowExecution.Results[outerindex] = actionResult } else { - log.Printf("[INFO] Setting value of %s in workflow %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) workflowExecution.Results = append(workflowExecution.Results, actionResult) + log.Printf("[INFO] Setting value (1) of %s in execution %s to %s. New result length: %d", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status, len(workflowExecution.Results)) } } else { - log.Printf("[INFO] Setting value of %s in workflow %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) workflowExecution.Results = append(workflowExecution.Results, actionResult) + log.Printf("[INFO] Setting value (2) of %s in execution %s to %s. New result length: %d", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status, len(workflowExecution.Results)) + } + + if actionResult.Status == "SKIPPED" { + log.Printf("\n\n[INFO] Handling special case for SKIPPED!\n\n") + childNodes := findChildNodes(*workflowExecution, actionResult.Action.ID) + for _, nodeId := range childNodes { + if nodeId == actionResult.Action.ID { + continue + } + + // 1. Find the action itself + // 2. Create an actionresult + curAction := shuffle.Action{ID: ""} + for _, action := range workflowExecution.Workflow.Actions { + if action.ID == nodeId { + curAction = action + break + } + } + + if len(curAction.ID) == 0 { + log.Printf("Couldn't find subnode %s", nodeId) + continue + } + + resultExists := false + for _, result := range workflowExecution.Results { + if result.Action.ID == curAction.ID { + resultExists = true + break + } + } + + if !resultExists { + // Check parents are done here. Only add it IF all parents are skipped + skipNodeAdd := false + for _, branch := range workflowExecution.Workflow.Branches { + if branch.DestinationID == nodeId { + // If the branch's source node is NOT in childNodes, it's not a skipped parent + sourceNodeFound := false + for _, item := range childNodes { + if item == branch.SourceID { + sourceNodeFound = true + break + } + } + + if !sourceNodeFound { + log.Printf("[INFO] Not setting node %s to SKIPPED", nodeId) + skipNodeAdd = true + break + } + } + } + + if !skipNodeAdd { + newAction := shuffle.Action{ + AppName: curAction.AppName, + AppVersion: curAction.AppVersion, + Label: curAction.Label, + Name: curAction.Name, + ID: curAction.ID, + } + + newResult := shuffle.ActionResult{ + Action: newAction, + ExecutionId: actionResult.ExecutionId, + Authorization: actionResult.Authorization, + Result: "Skipped because of previous node", + StartedAt: 0, + CompletedAt: 0, + Status: "SKIPPED", + } + + workflowExecution.Results = append(workflowExecution.Results, newResult) + } + } + } } // FIXME: Have a check for skippednodes and their parents - for resultIndex, result := range workflowExecution.Results { - if result.Status != "SKIPPED" { - continue - } + /* + for resultIndex, result := range workflowExecution.Results { + if result.Status != "SKIPPED" { + continue + } - // Checks if all parents are skipped or failed. Otherwise removes them from the results - for _, branch := range workflowExecution.Workflow.Branches { - if branch.DestinationID == result.Action.ID { - for _, subresult := range workflowExecution.Results { - if subresult.Action.ID == branch.SourceID { - if subresult.Status != "SKIPPED" && subresult.Status != "FAILURE" { - log.Printf("SUBRESULT PARENT STATUS: %s", subresult.Status) - log.Printf("Should remove resultIndex: %d", resultIndex) + // Checks if all parents are skipped or failed. + // Otherwise removes them from the results + for _, branch := range workflowExecution.Workflow.Branches { + if branch.DestinationID == result.Action.ID { + for _, subresult := range workflowExecution.Results { + if subresult.Action.ID == branch.SourceID { + if subresult.Status != "SKIPPED" && subresult.Status != "FAILURE" { + //log.Printf("SUBRESULT PARENT STATUS: %s", subresult.Status) + //log.Printf("Should remove resultIndex: %d", resultIndex) - workflowExecution.Results = append(workflowExecution.Results[:resultIndex], workflowExecution.Results[resultIndex+1:]...) + // FIXME: Reinstate this? + //workflowExecution.Results = append(workflowExecution.Results[:resultIndex], workflowExecution.Results[resultIndex+1:]...) + _ = resultIndex - break + break + } } } } } } - } + + log.Printf("NEW LENGTH: %d", len(workflowExecution.Results)) + */ extraInputs := 0 for _, trigger := range workflowExecution.Workflow.Triggers { @@ -2438,7 +2004,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Result string `json:"result" datastore:"result,noindex"` // Arbitrary reduction size maxSize := 500000 - newResults := []ActionResult{} + newResults := []shuffle.ActionResult{} for _, item := range workflowExecution.Results { if len(item.Result) > maxSize { item.Result = "[ERROR] Result too large to handle (https://github.com/frikky/shuffle/issues/171)" @@ -2455,7 +2021,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Handled using cachhing, so actually pretty fast cacheKey := fmt.Sprintf("workflowexecution-%s", workflowExecution.ExecutionId) if value, found := requestCache.Get(cacheKey); found { - parsedValue := value.(*WorkflowExecution) + parsedValue := value.(*shuffle.WorkflowExecution) if len(parsedValue.Results) > 0 && len(parsedValue.Results) != resultLength { setExecution = false if attempts > 5 { @@ -2481,7 +2047,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl log.Printf("[INFO] Skipping setexec with status %s", workflowExecution.Status) // Just in case. Should MAYBE validate finishing another time as well. - // This fixes issues with e.g. Action -> Trigger -> Action. + // This fixes issues with e.g. shuffle.Action -> shuffle.Trigger -> shuffle.Action. handleExecutionResult(*workflowExecution) //validateFinished(workflowExecution) } @@ -2494,22 +2060,50 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl //resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } -func getWorkflowExecution(ctx context.Context, id string) (*WorkflowExecution, error) { +func getWorkflowExecution(ctx context.Context, id string) (*shuffle.WorkflowExecution, error) { //log.Printf("IN GET WORKFLOW EXEC!") cacheKey := fmt.Sprintf("workflowexecution-%s", id) if value, found := requestCache.Get(cacheKey); found { - parsedValue := value.(*WorkflowExecution) + parsedValue := value.(*shuffle.WorkflowExecution) //log.Printf("Found execution for id %s with %d results", parsedValue.ExecutionId, len(parsedValue.Results)) //validateFinished(*parsedValue) return parsedValue, nil } - return &WorkflowExecution{}, errors.New("No workflowexecution defined yet") + return &shuffle.WorkflowExecution{}, errors.New("No workflowexecution defined yet") } -func validateFinished(workflowExecution WorkflowExecution) { - log.Printf("[INFO] Status: %s, Actions: %d, Extra: %d, Results: %d\n", workflowExecution.Status, len(workflowExecution.Workflow.Actions), extra, len(workflowExecution.Results)) +func sendResult(workflowExecution shuffle.WorkflowExecution, data []byte) { + fullUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl) + req, err := http.NewRequest( + "POST", + fullUrl, + bytes.NewBuffer([]byte(data)), + ) + + if err != nil { + log.Printf("[ERROR] Failed creating finishing request: %s", err) + shutdown(workflowExecution, "", "", false) + } + + newresp, err := topClient.Do(req) + if err != nil { + log.Printf("[ERROR] Error running finishing request: %s", err) + shutdown(workflowExecution, "", "", false) + } + + body, err := ioutil.ReadAll(newresp.Body) + log.Printf("[INFO] BACKEND STATUS: %d", newresp.StatusCode) + if err != nil { + log.Printf("[ERROR] Failed reading body: %s", err) + } else { + log.Printf("[INFO] NEWRESP (from backend): %s", string(body)) + } +} + +func validateFinished(workflowExecution shuffle.WorkflowExecution) { + log.Printf("[INFO] VALIDATION. Status: %s, shuffle.Actions: %d, Extra: %d, Results: %d\n", workflowExecution.Status, len(workflowExecution.Workflow.Actions), extra, len(workflowExecution.Results)) //if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extra { if (len(environments) == 1 && requestsSent == 0 && len(workflowExecution.Results) >= 1) || (len(workflowExecution.Results) >= len(workflowExecution.Workflow.Actions) && len(workflowExecution.Workflow.Actions) > 0) { @@ -2520,34 +2114,10 @@ func validateFinished(workflowExecution WorkflowExecution) { data, err := json.Marshal(workflowExecution) if err != nil { log.Printf("[ERROR] Failed to unmarshal data for backend") - shutdown(workflowExecution.ExecutionId, "") + shutdown(workflowExecution, "", "", true) } - fullUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl) - req, err := http.NewRequest( - "POST", - fullUrl, - bytes.NewBuffer([]byte(data)), - ) - - if err != nil { - log.Printf("[ERROR] Failed creating finishing request: %s", err) - shutdown(workflowExecution.ExecutionId, "") - } - - newresp, err := topClient.Do(req) - if err != nil { - log.Printf("[ERROR] Error running finishing request: %s", err) - shutdown(workflowExecution.ExecutionId, "") - } - - body, err := ioutil.ReadAll(newresp.Body) - log.Printf("[INFO] BACKEND STATUS: %d", newresp.StatusCode) - if err != nil { - log.Printf("[ERROR] Failed reading body: %s", err) - } else { - log.Printf("[INFO] NEWRESP (from backend): %s", string(body)) - } + sendResult(workflowExecution, data) } } @@ -2560,10 +2130,10 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { return } - var actionResult ActionResult + var actionResult shuffle.ActionResult err = json.Unmarshal(body, &actionResult) if err != nil { - log.Printf("Failed ActionResult unmarshaling: %s", err) + log.Printf("Failed shuffle.ActionResult unmarshaling: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return @@ -2598,7 +2168,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { } -func setWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecution, dbSave bool) error { +func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.WorkflowExecution, dbSave bool) error { //log.Printf("IN SET WORKFLOW EXEC!") //log.Printf("\n\n\nRESULT: %s\n\n\n", workflowExecution.Status) if len(workflowExecution.ExecutionId) == 0 { @@ -2612,8 +2182,9 @@ func setWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecuti handleExecutionResult(workflowExecution) validateFinished(workflowExecution) if dbSave { - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", false) } + return nil } @@ -2646,7 +2217,7 @@ func getAvailablePort() (net.Listener, error) { //return fmt.Sprintf(":%d", port) } -func webserverSetup(workflowExecution WorkflowExecution) net.Listener { +func webserverSetup(workflowExecution shuffle.WorkflowExecution) net.Listener { hostname := getLocalIP() // FIXME: This MAY not work because of speed between first @@ -2654,7 +2225,7 @@ func webserverSetup(workflowExecution WorkflowExecution) net.Listener { listener, err := getAvailablePort() if err != nil { log.Printf("Failed to created listener: %s", err) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", true) } port := listener.Addr().(*net.TCPAddr).Port @@ -2717,14 +2288,17 @@ func main() { log.Printf("[INFO] Running normal execution with auth %s and ID %s", authorization, executionId) } + workflowExecution := shuffle.WorkflowExecution{ + ExecutionId: executionId, + } if len(authorization) == 0 { log.Println("[INFO] No AUTHORIZATION key set in env") - shutdown(executionId, "") + shutdown(workflowExecution, "", "", false) } if len(executionId) == 0 { log.Println("[INFO] No EXECUTIONID key set in env") - shutdown(executionId, "") + shutdown(workflowExecution, "", "", false) } data = fmt.Sprintf(`{"execution_id": "%s", "authorization": "%s"}`, executionId, authorization) @@ -2737,7 +2311,7 @@ func main() { if err != nil { log.Println("[ERROR] Failed making request builder for backend") - shutdown(executionId, "") + shutdown(workflowExecution, "", "", true) } topClient = client @@ -2765,7 +2339,6 @@ func main() { continue } - var workflowExecution WorkflowExecution err = json.Unmarshal(body, &workflowExecution) if err != nil { log.Printf("[ERROR] Failed workflowExecution unmarshal: %s", err) @@ -2795,12 +2368,13 @@ func main() { } log.Printf("Environments: %s. 1 = webserver, 0 or >1 = default", environments) - if len(environments) == 1 { //&& len(workflowExecution.Actions)+len(workflowExecution.Triggers) > 1 { + if len(environments) == 1 { //&& workflowExecution.ExecutionSource != "default" { + log.Printf("[INFO] Running OPTIMIZED execution (not manual)") listener := webserverSetup(workflowExecution) err := executionInit(workflowExecution) if err != nil { log.Printf("[INFO] Workflow setup failed: %s", workflowExecution.ExecutionId, err) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", true) } go func() { @@ -2813,13 +2387,16 @@ func main() { //wg := sync.WaitGroup{} //wg.Add(1) //wg.Wait() + } else { + log.Printf("[INFO] Running NON-OPTIMIZED execution for type %s with %d environments", workflowExecution.ExecutionSource, len(environments)) + } } if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" { log.Printf("[INFO] Workflow %s is finished. Exiting worker.", workflowExecution.ExecutionId) - shutdown(executionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", true) } if workflowExecution.Status == "EXECUTING" || workflowExecution.Status == "RUNNING" { @@ -2827,11 +2404,11 @@ func main() { err = handleExecution(client, req, workflowExecution) if err != nil { log.Printf("[INFO] Workflow %s is finished: %s", workflowExecution.ExecutionId, err) - shutdown(executionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", true) } } else { log.Printf("[INFO] Workflow %s has status %s. Exiting worker.", workflowExecution.ExecutionId, workflowExecution.Status) - shutdown(executionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, workflowExecution.Workflow.ID, "", true) } time.Sleep(time.Duration(sleepTime) * time.Second)