diff --git a/.env b/.env index a43c6669..45389f1f 100644 --- a/.env +++ b/.env @@ -39,4 +39,7 @@ SHUFFLE_PASS_WORKER_PROXY=TRUE SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io SHUFFLE_BASE_IMAGE_NAME=frikky -SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.0" +SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.3" + +# Used for auto-cleanup of containers. REALLY important at scale. +SHUFFLE_CONTAINER_AUTO_CLEANUP=false diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index b22e6fd7..f5675281 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -2,8 +2,8 @@ github: frikky patreon: # Replace with a single Patreon username -open_collective: # Replace with a single Open Collective username -ko_fi: # Replace with a single Ko-fi username +open_collective: shuffle +ko_fi: frikky tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry liberapay: # Replace with a single Liberapay username diff --git a/README.md b/README.md index 00aed0be..f9589f40 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,19 @@ Please consider [sponsoring](https://github.com/sponsors/frikky) the project if ## Website https://shuffler.io +## Contributors +![ICPL logo](https://github.com/frikky/Shuffle/blob/launch/frontend/src/assets/img/icpl_logo.png) + +**Shuffle** + + + + +[**App magicians**](https://github.com/frikky/shuffle-apps) + + + + ## License All modular information related to Shuffle will be under MIT (anyone can use it for whatever purpose), with Shuffle itself using AGPLv3. diff --git a/backend/app_sdk/Dockerfile b/backend/app_sdk/Dockerfile index 44294a5b..370c31f8 100644 --- a/backend/app_sdk/Dockerfile +++ b/backend/app_sdk/Dockerfile @@ -7,7 +7,7 @@ RUN mkdir /install WORKDIR /install COPY requirements.txt /requirements.txt -RUN pip install --prefix="/install" -r /requirements.txt +RUN pip3 install -r /requirements.txt FROM base diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index fffef6c3..e0c9ec6a 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -21,22 +21,31 @@ class AppBase: # apikey is for the user / org # authorization is for the specific workflow self.url = os.getenv("CALLBACK_URL", "https://shuffler.io") + self.base_url = os.getenv("BASE_URL", "") self.action = os.getenv("ACTION", "") self.authorization = os.getenv("AUTHORIZATION", "") self.current_execution_id = os.getenv("EXECUTIONID", "") self.full_execution = os.getenv("FULL_EXECUTION", "") + self.result_wrapper_count = 0 if isinstance(self.action, str): self.action = json.loads(self.action) + if len(self.base_url) == 0: + self.base_url = self.url + + # FIXME: Add more info like logs in here. + # Docker logs: https://forums.docker.com/t/docker-logs-inside-the-docker-container/68190/2 def send_result(self, action_result, headers, stream_path): if action_result["status"] == "EXECUTING": action_result["status"] = "FAILURE" # I wonder if this actually works self.logger.info("Before last stream result") + url = "%s%s" % (self.base_url, stream_path) + print("URL: %s" % url) try: - ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) + ret = requests.post(url, headers=headers, json=action_result) self.logger.info("Result: %d" % ret.status_code) if ret.status_code != 200: self.logger.info(ret.text) @@ -48,11 +57,311 @@ class AppBase: action_result["status"] = "FAILURE" action_result["result"] = "POST error: %s" % e self.logger.info("Before typeerror stream result") - ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) + ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) self.logger.info("Result: %d" % ret.status_code) if ret.status_code != 200: self.logger.info(ret.text) + async def cartesian_product(self, L): + if L: + return {(a, ) + b for a in L[0] for b in await self.cartesian_product(L[1:])} + else: + return {()} + + # Returns a list of all the executions to be done in the inner loop + # FIXME: Doesn't take into account whether you actually WANT to loop or not + # Check if the last part of the value is #? + async def get_param_multipliers(self, baseparams): + # Example: + # {'call': ['hello', 'hello4'], 'call2': ['hello2', 'hello3'], 'call3': '1'} + # + # Should become this because of pairs (all same-length arrays, PROBABLY indicates same source node's values. + # [ + # {'call': 'hello', 'call2': 'hello2', 'call3': '1'}, + # {'call': 'hello4', 'call2': 'hello3', 'call3': '1'} + # ] + # + # ---------------------------------------------------------------------- + # Example2: + # {'call': ['hello'], 'call2': ['hello2', 'hello3'], 'call3': '1'} + # + # Should become this because NOT pairs/triplets: + # [ + # {'call': 'hello', 'call2': 'hello2', 'call3': '1'}, + # {'call': 'hello', 'call2': 'hello3', 'call3': '1'} + # ] + # + # ---------------------------------------------------------------------- + # Example3: + # {'call': ['hello', 'hello2'], 'call2': ['hello3', 'hello4', 'hello5'], 'call3': '1'} + # + # Should become this because arrays are not same length, aka no pairs/triplets. This is the multiplier effect. 2x3 arrays = 6 iterations + # [ + # {'call': 'hello', 'call2': 'hello3', 'call3': '1'}, + # {'call': 'hello', 'call2': 'hello4', 'call3': '1'}, + # {'call': 'hello', 'call2': 'hello5', 'call3': '1'}, + # {'call': 'hello2', 'call2': 'hello3', 'call3': '1'}, + # {'call': 'hello2', 'call2': 'hello4', 'call3': '1'}, + # {'call': 'hello2', 'call2': 'hello5', 'call3': '1'} + # ] + # To achieve this, we'll do this: + # 1. For the first array, take the total amount(y) (2x3=6) and divide it by the current array (x): 2. x/y = 3. This means do 3 of each value + # 2. For the second array, take the total amount(y) (2x3=6) and divide it by the current array (x): 3. x/y = 2. + # 3. What does the 3rd array do? Same, but ehhh? + + paramlist = [] + listitems = [] + listlengths = [] + all_lists = [] + all_list_keys = [] + + #check_value = "$Filter_list_testing.wrapper.#.tmp" + #self.action = action + + loopnames = [] + for key, value in baseparams.items(): + check_value = "" + for param in self.action["parameters"]: + if param["name"] == key: + #print("PARAM: %s" % param) + check_value = param["value"] + # self.result_wrapper_count = 0 + + octothorpe_count = param["value"].count(".#") + if octothorpe_count > self.result_wrapper_count: + self.result_wrapper_count = octothorpe_count + print("NEW OCTOTHORPE WRAPPER: %d" % octothorpe_count) + + # This whole thing is hard. + # item = [{"data": "1.2.3.4", "dataType": "ip"}] + # $item = DONT loop items. + # $item.# = Loop items + # $item.#.data = Loop items + # With a single item, this is fine. + + # item = [{"list": [{"data": "1.2.3.4", "dataType": "ip"}]}] + # $item = DONT loop items + # $item.# = Loop items + # $item.#.list = DONT loop items + # $item.#.list.# = Loop items + # $item.#.list.#.data = Loop items + # If the item itself is a list.. hmm + + # FIXME: Check the above, and fix so that nested looped items can be + # Skipped if wanted + + print("\nCHECK: %s" % check_value) + should_merge = False + if "#" in check_value: + should_merge = True + + if isinstance(value, list): + if len(value) <= 1: + if len(value) == 1: + baseparams[key] = value[0] + + #if "#" in check_value: + # should_merge = True + else: + if not should_merge: + print("Adding WITHOUT looping list") + else: + if len(value) not in listlengths: + listlengths.append(len(value)) + + listitems.append( + { + key: len(value) + } + ) + + all_list_keys.append(key) + all_lists.append(baseparams[key]) + else: + print("%s is not a list: " % value) + + print("Listlengths: %s" % listlengths) + if len(listlengths) == 0: + print("NO multiplier. Running a single iteration.") + paramlist.append(baseparams) + elif len(listlengths) == 1: + print("NO MULTIPLIER NECESSARY. Length is %d" % len(listitems)) + + for item in listitems: + # This loops should always be length 1 + for key, value in item.items(): + if isinstance(value, int): + print("\nShould run key %s %d times from %s" % (key, value, baseparams[key])) + if len(paramlist) == value: + print("List ALREADY exists - just changing values") + for subloop in range(value): + baseitem = copy.deepcopy(baseparams) + paramlist[subloop][key] = baseparams[key][subloop] + else: + print("List DOESNT exist - ADDING values") + for subloop in range(value): + baseitem = copy.deepcopy(baseparams) + baseitem[key] = baseparams[key][subloop] + paramlist.append(baseitem) + + else: + print("Multipliers to handle: %s" % listitems) + newlength = 1 + for item in listitems: + for key, value in item.items(): + newlength = newlength * value + + print("Newlength of array: %d. Lists: %s" % (newlength, all_lists)) + # Get the cartesian product of the arrays + cartesian = await self.cartesian_product(all_lists) + newlist = [] + for item in cartesian: + newlist.append(list(item)) + + newobject = {} + for subitem in range(len(newlist)): + baseitem = copy.deepcopy(baseparams) + for key in range(len(newlist[subitem])): + baseitem[all_list_keys[key]] = newlist[subitem][key] + + paramlist.append(baseitem) + + #print("PARAMLIST: %s" % paramlist) + + #newlist[subitem[0]] + #if len(newlist) > 0: + # itemlength = len(newlist[0]) + + # How do we get it back, ordered? + #for item in cartesian: + #print("Listlengths: %s" % listlengths) + #paramlist = [baseparams] + + #print("[INFO] Return paramlist: %s" % paramlist) + return paramlist + + + # Runs recursed versions with inner loops and such + async def run_recursed_items(self, func, baseparams, loop_wrapper): + has_loop = False + + newparams = {} + for key, value in baseparams.items(): + if isinstance(value, list) and len(value) > 0: + print("In list check") + try: + value[0] = json.loads(value[0]) + except json.decoder.JSONDecodeError as e: + print("JSON casting error: %s" % e) + except TypeError as e: + print("TypeError: %s" % e) + + print("POST list check") + + if isinstance(value, list) and len(value) == 1 and isinstance(value[0], list): + try: + loop_wrapper[key] += 1 + except IndexError: + loop_wrapper[key] = 1 + except KeyError: + loop_wrapper[key] = 1 + + print("Key %s is a list: %s" % (key, value)) + newparams[key] = value[0] + has_loop = True + else: + print("Key %s is NOT a list within a list: %s" % (key, value)) + + newparams[key] = value + + results = [] + if has_loop: + print("[WARNING] Should run inner loop: %s" % newparams) + ret = await self.run_recursed_items(func, newparams, loop_wrapper) + else: + print("[INFO] Should run multiplier check with params (inner): %s" % newparams) + # 1. Find the loops that are required and create new multipliers + # If here: check for multipliers within this scope. + ret = [] + param_multiplier = await self.get_param_multipliers(newparams) + + print("[INFO] Multiplier length: %d" % len(param_multiplier)) + for subparams in param_multiplier: + try: + tmp = await func(**subparams) + except: + e = "" + try: + e = sys.exc_info()[1] + except: + print("Exc check fail: %s" % e) + pass + + tmp = "An error occured during execution: %s" % e + + print("RET from execution: %s" % ret) + new_value = tmp + if tmp == None: + new_value = "" + elif isinstance(tmp, dict): + new_value = json.dumps(tmp) + elif isinstance(tmp, list): + new_value = json.dumps(tmp) + #else: + #tmp = tmp.replace("\"", "\\\"", -1) + + try: + new_value = json.loads(new_value) + except json.decoder.JSONDecodeError as e: + pass + except TypeError as e: + pass + except: + pass + #print("Json: %s" % e) + #ret.append(tmp) + + #if self.result_wrapper_count > 0: + # ret.append("["*(self.result_wrapper_count-1)+new_value+"]"*(self.result_wrapper_count-1)) + #else: + ret.append(new_value) + + print("Ret length: %d" % len(ret)) + if len(ret) == 1: + ret = ret[0] + + print("Return from execution: %s" % ret) + if ret == None: + results.append("") + json_object = False + elif isinstance(ret, dict): + results.append(ret) + json_object = True + elif isinstance(ret, list): + results = ret + json_object = True + else: + ret = ret.replace("\"", "\\\"", -1) + + try: + results.append(json.loads(ret)) + json_object = True + except json.decoder.JSONDecodeError as e: + #print("Json: %s" % e) + results.append(ret) + except TypeError as e: + results.append(ret) + except: + results.append(ret) + + if len(results) == 1: + results = results[0] + + print("\nLOOP: %s\nRESULTS: %s" % (loop_wrapper, results)) + return results + + + # Things to consider for files: # - How can you download / stream a file? # - Can you decide if you want a stream or the files directly? @@ -73,7 +382,7 @@ class AppBase: for item in value: print("VALUE: %s" % item) if len(item) != 36: - print("Bad length for value") + print("Bad length for file value %s" % item) continue #return { # "filename": "", @@ -88,7 +397,7 @@ class AppBase: } ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers) - print("RET1: %s" % ret1.text) + print("RET1 (file get): %s" % ret1.text) if ret1.status_code != 200: returns.append({ "filename": "", @@ -99,7 +408,7 @@ class AppBase: content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"]) ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers) - print("Ret2: %s" % ret2.text) + print("RET2 (file get): %s" % ret2.text) if ret2.status_code == 200: tmpdata = ret1.json() returndata = { @@ -109,6 +418,8 @@ class AppBase: } returns.append(returndata) + print("RET3 (file get done)") + if len(returns) == 0: return { "success": False, @@ -130,6 +441,9 @@ class AppBase: "Authorization": "Bearer %s" % self.authorization } + if not isinstance(infiles, list): + infiles = [infiles] + create_path = "/api/v1/files/create?execution_id=%s" % full_execution["execution_id"] file_ids = [] for curfile in infiles: @@ -175,7 +489,6 @@ class AppBase: upload_path = "/api/v1/files/%s/upload?execution_id=%s" % (cur_id, full_execution["execution_id"]) print("Create path: %s" % create_path) - # FIXME: Typical failure here if data is returned badly formatted files={"shuffle_file": (filename, curfile["data"])} #open(filename,'rb')} @@ -187,8 +500,6 @@ class AppBase: return file_ids async def execute_action(self, action): - # FIXME - add request for the function STARTING here. Use "results stream" or something - # PAUSED, AWAITING_DATA, PENDING, COMPLETED, ABORTED, EXECUTING, SUCCESS, FAILURE # !!! Let this line stay - its used for some horrible codegeneration / stitching !!! # #STARTCOPY @@ -201,6 +512,8 @@ class AppBase: "started_at": int(time.time()), "status": "EXECUTING" } + + self.action = copy.deepcopy(action) self.logger.info("ACTION RESULT (start): %s", action_result) if len(self.action) == 0: @@ -208,11 +521,13 @@ class AppBase: action_result["result"] = "Error in setup ENV: ACTION not defined" self.send_result(action_result, headers, stream_path) return + if len(self.authorization) == 0: print("AUTHORIZATION env not defined") action_result["result"] = "Error in setup ENV: AUTHORIZATION not defined" self.send_result(action_result, headers, stream_path) return + if len(self.current_execution_id) == 0: print("EXECUTIONID env not defined") action_result["result"] = "Error in setup ENV: EXECUTIONID not defined" @@ -227,17 +542,19 @@ class AppBase: # Add async logger # self.console_logger.handlers[0].stream.set_execution_id() #self.logger.info("Before initial stream result") - try: - ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) - self.logger.info("Workflow: %d" % ret.status_code) - if ret.status_code != 200: - self.logger.info(ret.text) - except requests.exceptions.ConnectionError as e: - print("Connectionerror: %s" % e) - action_result["result"] = "Bad setup during startup: %s" % e - self.send_result(action_result, headers, stream_path) - return + # FIXME: Shouldn't skip this, but it's good for minimzing API calls + #try: + # ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) + # self.logger.info("Workflow: %d" % ret.status_code) + # if ret.status_code != 200: + # self.logger.info(ret.text) + #except requests.exceptions.ConnectionError as e: + # print("Connectionerror: %s" % e) + + # action_result["result"] = "Bad setup during startup: %s" % e + # self.send_result(action_result, headers, stream_path) + # return # Verify whether there are any parameters with ACTION_RESULT required # If found, we get the full results list from backend @@ -252,7 +569,7 @@ class AppBase: self.logger.info("Before FULLEXEC stream result") ret = requests.post( - "%s/api/v1/streams/results" % (self.url), + "%s/api/v1/streams/results" % (self.base_url), headers=headers, json=tmpdata ) @@ -260,8 +577,12 @@ class AppBase: if ret.status_code == 200: fullexecution = ret.json() else: - self.logger.info("Error: Data: ", ret.json()) - self.logger.info("Error with status code for results. Crashing because ACTION_RESULTS or WORKFLOW_VARIABLE can't be handled. Status: %d" % ret.status_code) + try: + self.logger.info("Error: Data: ", ret.json()) + self.logger.info("Error with status code for results. Crashing because ACTION_RESULTS or WORKFLOW_VARIABLE can't be handled. Status: %d" % ret.status_code) + except json.decoder.JSONDecodeError: + pass + action_result["result"] = "Bad result from backend: %d" % ret.status_code self.send_result(action_result, headers, stream_path) return @@ -283,7 +604,7 @@ class AppBase: self.full_execution = fullexecution - self.logger.info("AFTER FULLEXEC stream result") + self.logger.info("AFTER FULLEXEC stream result (init)") # Gets the value at the parenthesis level you want def parse_nested_param(string, level): @@ -357,11 +678,14 @@ class AppBase: if "len" in thistype or "length" in thistype or "lenght" in thistype: tmp = "" try: - tmpdata = data.replace("\'", "\"") tmp = json.loads(tmpdata) except: - print("Passing bug") - pass + try: + tmpdata = data.replace("\'", "\"") + tmp = json.loads(tmpdata) + except: + print("[ERROR] Parsing bug for length in app sdk") + pass if isinstance(tmp, list): return len(tmp) @@ -400,14 +724,14 @@ class AppBase: return tmp except IndexError as e: return default_error - + # Parses the INNER value and recurses until everything is done def parse_wrapper(data): try: if "(" not in data or ")" not in data: - return data + return (data, False) except TypeError: - return data + return (data, False) #print("Running %s" % data) @@ -422,7 +746,7 @@ class AppBase: break if not found: - return data + return (data, False) # Do stuff here. innervalue = parse_nested_param(data, maxDepth(data)-0) @@ -443,10 +767,11 @@ class AppBase: parsed_value = parse_type(innervalue[0], thistype.lower()) print("Parsed value from %s: %s" % (thistype, parsed_value)) - return parsed_value + return (parsed_value, True) print("DATA: %s\n" % data) - return parse_wrapper(data) + return (parse_wrapper(data)[0], True) + # Looks for parantheses to grab special cases within a string, e.g: # int(1) lower(HELLO) or length(what's the length) @@ -488,15 +813,20 @@ class AppBase: if len(newstring) > 0: newdata.append(newstring) - print("Newdata: ", newdata) parsedlist = [] non_string = False + parsed = False for item in newdata: ret = parse_wrapper(item) - if not isinstance(ret, str): + if not isinstance(ret[0], str): non_string = True - - parsedlist.append(ret) + + parsedlist.append(ret[0]) + if ret[1]: + parsed = True + + if not parsed: + return data if len(parsedlist) > 0 and not non_string: print("Returning parsed list: ", parsedlist) @@ -545,8 +875,7 @@ class AppBase: return newvalue, True elif len(actualitem) > 0: - # FIXME: This is absolutely not perfect. - print("In recursion v2: ", actualitem) + print("[INFO] In recursion v2: ", actualitem) is_loop = True newvalue = [] @@ -555,13 +884,14 @@ class AppBase: # Means it's a single item -> continue if seconditem == "": - print("In first - handling %s" % seconditem) + print("[INFO] In first - handling %s" % firstitem) tmpitem = basejson[int(firstitem)] try: newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:]) except IndexError: newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:]) else: + print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem)) if seconditem == "max": seconditem = len(basejson) if seconditem == "min": @@ -586,7 +916,6 @@ class AppBase: return newvalue, is_loop - # FIXME: Add specific loop for other indexes else: #print("BEFORE NORMAL VALUE: ", basejson, value) if len(value) == 0: @@ -618,14 +947,13 @@ class AppBase: actionname_lower = parsersplit[0][1:].lower() #Actionname: Start_node - - print(f"\nActionname: {actionname_lower}") + print(f"\n[INFO] Actionname: {actionname_lower}") # 1. Find the action baseresult = "" appendresult = "" - print("Parsersplit length: %d" % len(parsersplit)) + print("[INFO] Parsersplit length: %d" % len(parsersplit)) if (actionname_lower.startswith("exec ") or actionname_lower.startswith("webhook ") or actionname_lower.startswith("schedule ") or actionname_lower.startswith("userinput ") or actionname_lower.startswith("email_trigger ") or actionname_lower.startswith("trigger ")) and len(parsersplit) == 1: record = False for char in actionname_lower: @@ -690,29 +1018,32 @@ class AppBase: except KeyError as error: print(f"KeyError in JSON: {error}") - print(f"After first trycatch. Baseresult: ", baseresult) + print(f"[INFO] After first trycatch. Baseresult: ", baseresult) # 2. Find the JSON data if len(baseresult) == 0: return ""+appendresult, False - print("After second return") + print("[INFO] After second return") if len(parsersplit) == 1: return str(baseresult)+str(appendresult), False - baseresult = baseresult.replace("\'", "\"") baseresult = baseresult.replace(" True,", " true,") baseresult = baseresult.replace(" False", " false,") - print("After third parser return - Formatted: ", baseresult) + print("[INFP] After third parser return - Formatted: ", baseresult) basejson = {} try: basejson = json.loads(baseresult) except json.decoder.JSONDecodeError as e: - print("Parser issue with JSON: %s" % e) - return str(baseresult)+str(appendresult), False + try: + baseresult = baseresult.replace("\'", "\"") + basejson = json.loads(baseresult) + except json.decoder.JSONDecodeError as e: + print("Parser issue with JSON: %s" % e) + return str(baseresult)+str(appendresult), False - print("After fourth parser return as JSON") + print("[INFO] After fourth parser return as JSON") data, is_loop = recurse_json(basejson, parsersplit[1:]) parseditem = data @@ -866,21 +1197,39 @@ class AppBase: elif check.lower() == "contains": if destinationvalue.lower() in sourcevalue.lower(): return True + elif check.lower() == "contains_any_of": + newvalue = [destinationvalue.lower()] + if "," in destinationvalue: + newvalue = destinationvalue.split(",") + elif ", " in destinationvalue: + newvalue = destinationvalue.split(", ") + + for item in newvalue: + if not item: + continue + + if item.strip() in sourcevalue: + print("[INFO] Found %s in %s" % (item, sourcevalue)) + return True + + return False elif check.lower() == "larger than": try: - if sourcevalue.isdigit() and destinationvalue.isdigit(): + if str(sourcevalue).isdigit() and str(destinationvalue).isdigit(): if int(sourcevalue) > int(destinationvalue): return True + except AttributeError as e: - self.logger.error("Condition larger than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) + self.logger.error("[WARNING] Condition larger than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) return False elif check.lower() == "smaller than": try: - if sourcevalue.isdigit() and destinationvalue.isdigit(): + if str(sourcevalue).isdigit() and str(destinationvalue).isdigit(): if int(sourcevalue) < int(destinationvalue): return True + except AttributeError as e: - self.logger.error("Condition smaller than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) + self.logger.error("[WARNING] Condition smaller than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) return False else: self.logger.info("Condition: can't handle %s yet. Setting to true" % check) @@ -944,13 +1293,14 @@ class AppBase: "startswith", "endswith", "contains", + "contains_any_of", "re", "matches regex", ] # FIXME - what should I do here? if not condition["condition"]["value"] in available_checks: - self.logger.info("Skipping %s %s %s because %s is invalid." % (sourcevalue, condition["condition"]["value"], destinationvalue, condition["condition"]["value"])) + self.logger.warning("Skipping %s %s %s because %s is invalid." % (sourcevalue, condition["condition"]["value"], destinationvalue, condition["condition"]["value"])) continue #print(destinationvalue) @@ -981,14 +1331,14 @@ class AppBase: action_result["result"] = tmpresult action_result["status"] = "FAILURE" try: - ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) + ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) self.logger.info("Result: %d" % ret.status_code) if ret.status_code != 200: self.logger.info(ret.text) except requests.exceptions.ConnectionError as e: self.logger.exception(e) - print("\n\nRETURNING BECAUSE A BRANCH FAILED\n\n") + print("\n\nRETURNING BECAUSE A BRANCH FAILED: %s\n\n" % tmpresult) return # Replace name cus there might be issues @@ -1027,6 +1377,41 @@ class AppBase: pass #action["authentication"] + # Fixes OpenAPI body parameters for later. + newparams = [] + counter = -1 + bodyindex = -1 + for parameter in action["parameters"]: + counter += 1 + + if parameter["name"] == "body": + bodyindex = counter + #print("PARAM: %s" % parameter) + try: + values = parameter["value_replace"] + if values != None: + added = 0 + for val in values: + newparams.append({ + "name": val["key"], + "value": val["value"], + "variant": "STATIC_VALUE", + "id": "body_replacement", + }) + + print("Added param %s for body" % val["key"]) + added += 1 + + print("ADDED %d parameters for body" % added) + except KeyError as e: + print("KeyError body OpenAPI: %s" % e) + pass + + break + + for parameter in newparams: + action["parameters"].append(parameter) + # calltimes is used to handle forloops in the app itself. # 2 kinds of loop - one in gui with one app each, and one like this, # which is super fast, but has a bad overview (potentially good tho) @@ -1068,25 +1453,49 @@ class AppBase: print("Before first part in multiexec!") handled = False if len(actualitem[0]) > 2 and actualitem[0][1] == "SHUFFLE_NO_SPLITTER": - print("Pre replacement: %s" % actualitem[0][2]) + print("(1) Pre replacement: %s" % actualitem[0][2]) tmpitem = value replacement = actualitem[0][2] if replacement.startswith("\"") and replacement.endswith("\""): replacement = replacement[1:len(replacement)-1] - replacement = replacement.replace("\'", "\"", -1) print("POST replacement: %s" % replacement) - json_replacement = replacement + #json_replacement = tmpitem.replace(actualitem[0][0], replacement, 1) + #print("AFTER POST replacement: %s" % json_replacement) try: json_replacement = json.loads(replacement) except json.decoder.JSONDecodeError as e: - print("JSON error singular: %s" % e) + try: + replacement = replacement.replace("\'", "\"", -1) + json_replacement = json.loads(replacement) + except: + print("JSON error singular: %s" % e) if len(json_replacement) > minlength: minlength = len(json_replacement) + + # FIXME: Only do this IF they want to loop + new_replacement = [] + for i in range(len(json_replacement)): + if isinstance(json_replacement[i], dict) or isinstance(json_replacement[i], dict): + tmp_replacer = json.dumps(json_replacement[i]) + newvalue = tmpitem.replace(actualitem[0][0], tmp_replacer, 1) + else: + newvalue = tmpitem.replace(actualitem[0][0], json_replacement[i], 1) + try: + newvalue = json.loads(newvalue) + except json.decoder.JSONDecodeError as e: + print("DECODER ERROR: %s" % e) + pass + + new_replacement.append(newvalue) + + print("New replacement: %s" % new_replacement) + + # New tmpitem = tmpitem.replace(actualitem[0][0], replacement, 1) # This code handles files. @@ -1113,16 +1522,18 @@ class AppBase: print("(1) JSON ERROR IN FILE HANDLING: %s" % e) if not isfile: + print("Resultarray (NOT FILE): %s" % resultarray) params[parameter["name"]] = tmpitem - multi_parameters[parameter["name"]] = json_replacement + multi_parameters[parameter["name"]] = new_replacement else: - print("Resultarray: %s" % resultarray) + print("Resultarray (FILE): %s" % resultarray) params[parameter["name"]] = resultarray multi_parameters[parameter["name"]] = resultarray - multi_execution_lists.append(json_replacement) + multi_execution_lists.append(new_replacement) print("MULTI finished: %s" % json_replacement) else: + print("(2) Pre replacement: %s" % actualitem) # This is here to handle for loops within variables.. kindof # 1. Find the length of the longest array # 2. Build an array with the base values based on parameter["value"] @@ -1202,6 +1613,23 @@ class AppBase: print("Normal parsing (not looping) with data %s" % value) value = parse_wrapper_start(value) + if parameter["id"] == "body_replacement": + print("Should run body replacement in index %d with %s" % (bodyindex, parameter)) + try: + print("PREBODY: %s" % params["body"]) + params["body"] = params["body"].replace(parameter["name"], parameter["value"], -1) + print("POSTBODY: %s" % params["body"]) + except KeyError as e: + print("KEYERROR: %s" % e) + + #bodyindex = counter + continue + + #for parameter in action["parameters"]: + #if parameter["name"] == "body": + # print("PARAM: %s" % parameter) + #if param.id == "body_replacement": + print("POST data value: %s" % value) params[parameter["name"]] = value multi_parameters[parameter["name"]] = value @@ -1219,6 +1647,7 @@ class AppBase: print("SCHEMA ERROR IN FILE HANDLING: %s" % e) # Fix lists here + # FIXME: This doesn't really do anything anymore print("CHECKING multi execution list!") if len(multi_execution_lists) > 0: print("\n Multi execution list has more data: %d" % len(multi_execution_lists)) @@ -1230,11 +1659,11 @@ class AppBase: # FIXME: Subsub required?. Recursion! # Basically multiply what we have with the outer loop? # - if isinstance(listitem, list): - for subitem in listitem: - filteredlist.append(subitem) - else: - filteredlist.append(listitem) + #if isinstance(listitem, list): + # for subitem in listitem: + # filteredlist.append(subitem) + #else: + # filteredlist.append(listitem) #print("New list length: %d" % len(filteredlist)) if len(filteredlist) > 1: @@ -1249,27 +1678,34 @@ class AppBase: print("New multi execution length: %d\n" % tmplength) if not multiexecution: - print("APP_SDK DONE: Starting NORMAL execution of function") - print("Running with params (0): %s" % params) + #newparams.append({ + # "name": val["key"], + # "value": val["value"], + # "variant": "STATIC_VALUE", + # "id": "body_replacement", + #}) + + print("[INFO] APP_SDK DONE: Starting NORMAL execution of function") + print("[INFO] Running with params (0): %s" % params) newres = await func(**params) - print("Returned from execution.") + print("[INFO] Returned from execution:", newres) if isinstance(newres, tuple): - print("Handling return as tuple") + print("[INFO] Handling return as tuple") # Handles files. filedata = "" file_ids = [] print("TUPLE: %s" % newres[1]) if isinstance(newres[1], list): - print("HANDLING LIST FROM RET") + print("[INFO] HANDLING LIST FROM RET") file_ids = self.set_files(newres[1]) elif isinstance(newres[1], object): - print("Handling JSON from ret") + print("[INFO] Handling JSON from ret") file_ids = self.set_files([newres[1]]) elif isinstance(newres[1], str): - print("Handling STRING from ret") + print("[INFO] Handling STRING from ret") file_ids = self.set_files([newres[1]]) else: - print("NO FILES TO HANDLE") + print("[INFO] NO FILES TO HANDLE") tmp_result = { "result": newres[0], @@ -1278,7 +1714,7 @@ class AppBase: result = json.dumps(tmp_result) elif isinstance(newres, str): - print("Handling return as string") + print("[INFO] Handling return as string") result += newres else: try: @@ -1287,20 +1723,32 @@ class AppBase: result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) print("Can't handle type %s value from function" % (type(newres))) - print("POST NEWRES RESULT: ", result) + print("[INFO] POST NEWRES RESULT: ", result) else: - print("APP_SDK DONE: Starting MULTI execution (length: %d) with values %s" % (minlength, multi_parameters)) + print("[INFO] APP_SDK DONE: Starting MULTI execution (length: %d) with values %s" % (minlength, multi_parameters)) # 1. Use number of executions based on the arrays being similar # 2. Find the right value from the parsed multi_params - results = [] - json_object = False - for i in range(0, minlength): - # To be able to use the results as a list: - print("1: %s" % multi_parameters) - #baseparams = json.loads(json.dumps(multi_parameters)) - baseparams = copy.deepcopy(multi_parameters) - print("2: %s: %s" % (type(baseparams), baseparams)) + print("[INFO] Running WITHOUT outer loop") + json_object = False + results = await self.run_recursed_items(func, multi_parameters, {}) + if isinstance(results, dict) or isinstance(results, list): + json_object = True + + #for i in range(0, minlength): + # # To be able to use the results as a list: + # print("1: %s" % multi_parameters) + # #baseparams = json.loads(json.dumps(multi_parameters)) + # baseparams = copy.deepcopy(multi_parameters) + + # print("2: %s: %s" % (type(baseparams), baseparams)) + + # print("4") + # print("Running with params (1): %s" % baseparams) + + # results = await self.run_recursed_items(func, baseparams, {}) + # if isinstance(results, dict) or isinstance(results, list): + # json_object = True # {'call': ['GoogleSafebrowsing_2_0', 'VirusTotal_GetReport_3_0']} # 1. Check if list length is same as minlength @@ -1309,78 +1757,86 @@ class AppBase: # arraylength = 4 ["1", "2", "3", "4"] # minlength = 12 - 12/3 = 4 per item = ["1", "1", "1", "1", "2", "2", ...] - try: - firstlist = True - for key, value in baseparams.items(): - print("Itemtype: %s" % type(value)) - if isinstance(value, list): - try: - newvalue = value[i] - except IndexError: - pass + #try: + # firstlist = True + # for key, value in baseparams.items(): + # print("Itemtype: %s" % type(value)) + # if isinstance(value, list): + # try: + # newvalue = value[i] + # except IndexError: + # pass - if len(value) != minlength and len(value) > 0: - newarray = [] - print("VALUE: ", value) - additiontime = minlength/len(value) - print("Bad length for value: %d - should be %d. Additiontime: %d" % (len(value), minlength, additiontime)) - if firstlist: - print("Running normal list (FIRST)") - for subvalue in value: - for number in range(int(additiontime)): - newarray.append(subvalue) - else: - #print("Running secondary lists") - ## 1. Set up length of array - ## 2. Put values spread out - # FIXME: This works well, except if lists are same length - newarray = [""] * minlength + # if len(value) != minlength and len(value) > 0: + # newarray = [] + # print("VALUE: ", value) + # additiontime = minlength/len(value) + # print("Bad length for value: %d - should be %d. Additiontime: %d" % (len(value), minlength, additiontime)) + # if firstlist: + # print("Running normal list (FIRST)") + # for subvalue in value: + # for number in range(int(additiontime)): + # newarray.append(subvalue) + # else: + # #print("Running secondary lists") + # ## 1. Set up length of array + # ## 2. Put values spread out + # # FIXME: This works well, except if lists are same length + # newarray = [""] * minlength - cnt = 0 - for number in range(int(additiontime)): - for subvaluerange in range(len(value)): - # newlocation = number+(additiontime*subvaluerange) - # print("%d+(%d*%d) = %d. VAL: %s" % (number, additiontime, subvaluerange, newlocation, value[subvaluerange])) - # Reverse if same length? - if int(minlength/len(value)) == len(value): - tmp = int(len(value)-subvaluerange-1) - print("NEW: %d" % tmp) - newarray[cnt] = value[tmp] - else: - newarray[cnt] = value[subvaluerange] - cnt += 1 + # cnt = 0 + # for number in range(int(additiontime)): + # for subvaluerange in range(len(value)): + # # newlocation = number+(additiontime*subvaluerange) + # # print("%d+(%d*%d) = %d. VAL: %s" % (number, additiontime, subvaluerange, newlocation, value[subvaluerange])) + # # Reverse if same length? + # if int(minlength/len(value)) == len(value): + # tmp = int(len(value)-subvaluerange-1) + # print("NEW: %d" % tmp) + # newarray[cnt] = value[tmp] + # else: + # newarray[cnt] = value[subvaluerange] + # cnt += 1 - #print("Newarray =", newarray) - newvalue = newarray[i] - firstlist = False + # #print("Newarray =", newarray) + # newvalue = newarray[i] + # firstlist = False - baseparams[key] = newvalue + # baseparams[key] = newvalue - print("3") - except IndexError as e: - print("IndexError: %s" % e) - baseparams[key] = "IndexError: %s" % e - except KeyError as e: - print("KeyError: %s" % e) - baseparams[key] = "KeyError: %s" % e + # print("3") + #except IndexError as e: + # print("IndexError: %s" % e) + # baseparams[key] = "IndexError: %s" % e + #except KeyError as e: + # print("KeyError: %s" % e) + # baseparams[key] = "KeyError: %s" % e + #print("4") + #print("Running with params (1): %s" % baseparams) + #results = await self.run_recursed_items(func, baseparams, {}) + #if isinstance(results, dict) or isinstance(results, list): + # json_object = True - print("4") - print("Running with params (1): %s" % baseparams) - ret = await func(**baseparams) - print("Return from execution: %s" % ret) - if isinstance(ret, dict) or isinstance(ret, list): - results.append(ret) - json_object = True - else: - ret = ret.replace("\"", "\\\"", -1) + # Check the structure here. If "isloop", try to recurse? + # ret, is_loop = recurse_json(innervalue, parsersplit[outercnt+1:]) + #ret = await func(**baseparams) + #print("Return from execution: %s" % ret) + #if ret == None: + # results.append("") + # json_object = False + #elif isinstance(ret, dict) or isinstance(ret, list): + # results.append(ret) + # json_object = True + #else: + # ret = ret.replace("\"", "\\\"", -1) - try: - results.append(json.loads(ret)) - json_object = True - except json.decoder.JSONDecodeError as e: - #print("Json: %s" % e) - results.append(ret) + # try: + # results.append(json.loads(ret)) + # json_object = True + # except json.decoder.JSONDecodeError as e: + # #print("Json: %s" % e) + # results.append(ret) #print("Inner ret parsed: %s" % ret) @@ -1398,6 +1854,7 @@ class AppBase: result += item except json.decoder.JSONDecodeError as e: # Common nested issue which puts " around everything + print("Decodingerror: %s" % e) try: tmpitem = item.replace("\\\"", "\"", -1) json.loads(tmpitem) @@ -1411,7 +1868,7 @@ class AppBase: result = result[:-2] result += "]" else: - print("Normal result?") + print("Normal result - no list?") result = results print("RESULT: %s" % result) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 4532a20e..5f09b969 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.2 +VERSION=0.8.54 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION @@ -8,6 +8,7 @@ docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg. #docker push frikky/$NAME:$VERSION #docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION #docker push ghcr.io/frikky/$NAME:$VERSION +#docker tag ghcr.io/frikky/$NAME:$VERSION frikky/shuffle:app_sdk docker push frikky/shuffle:app_sdk docker push ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/app_sdk/requirements.txt b/backend/app_sdk/requirements.txt index 804abb1b..0da82340 100644 --- a/backend/app_sdk/requirements.txt +++ b/backend/app_sdk/requirements.txt @@ -1,2 +1,2 @@ -requests urllib3 +requests diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index a36ba78d..4b20d2fa 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -244,7 +244,7 @@ func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) { // This function generates the python code that's being used. // This is really meta when you program it. Handling parameters is hard here. -func makePythoncode(swagger *openapi3.Swagger, name, url, method string, parameters, optionalQueries, headers []string) (string, string) { +func makePythoncode(swagger *openapi3.Swagger, name, url, method string, parameters, optionalQueries, headers []string, fileField string) (string, string) { method = strings.ToLower(method) queryString := "" queryData := "" @@ -365,21 +365,39 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet preparedHeaders += "}" } + fileBalance := "" + fileAdder := `` + fileGrabber := `` + fileParameter := `` + if method == "post" && len(fileField) > 0 { + fileParameter = ", file_id" + fileGrabber = `filedata = self.get_file(file_id)` + + // This indentation is confusing (but correct) ROFL + fileAdder = fmt.Sprintf(`if not filedata["success"]: + return file_id+" is not a valid File ID" + files = {"%s": (filedata["filename"], filedata["data"])}`, fileField) + fileBalance = ", files=files" + } + // Extra param for url if it's changeable // Extra param for authentication scheme(s) // The last weird one is the body.. Tabs & spaces sucks. - data := fmt.Sprintf(` async def %s(self%s%s%s%s%s%s): + data := fmt.Sprintf(` async def %s(self%s%s%s%s%s%s%s): %s url=f"%s%s" %s %s %s %s - return requests.%s(url, headers=headers%s%s%s).text + %s + %s + return requests.%s(url, headers=headers%s%s%s%s).text `, functionname, authenticationParameter, urlParameter, + fileParameter, parameterData, queryString, bodyParameter, @@ -391,18 +409,20 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet authenticationSetup, queryData, bodyFormatter, + fileGrabber, + fileAdder, method, authenticationAddin, bodyAddin, verifyAddin, + fileBalance, ) - /* - if strings.Contains(functionname, "search") { - log.Println(data) - log.Printf("Queries: %s", queryString) - } - */ + if strings.Contains(functionname, "filescan") { + //log.Printf("FUNCTION: %s", data) + log.Println(data) + log.Printf("Queries: %s", queryString) + } //log.Printf(data) return functionname, data @@ -446,6 +466,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, api.Sharing = false api.Verified = false api.Tested = false + api.Invalid = false api.PrivateID = newmd5 api.Generated = true api.Activated = true @@ -638,10 +659,15 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, // This is the python code to be generated // Could just as well be go at this point lol pythonFunctions := []string{} + //Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` for actualPath, path := range swagger.Paths { actualPath = strings.Replace(actualPath, " ", "_", -1) //actualPath = strings.Replace(actualPath, ".", "", -1) actualPath = strings.Replace(actualPath, "\\", "", -1) + if !api.Invalid && strings.HasPrefix(actualPath, "tmp") { + log.Printf("[WARNING] Set api %s to invalid because of path %s", swagger.Info.Title, actualPath) + api.Invalid = true + } // FIXME: Handle everything behind questionmark (?) with dots as well. // https://godoc.org/github.com/getkin/kin-openapi/openapi3#PathItem @@ -846,10 +872,10 @@ def run(request): func deployAppToDatastore(ctx context.Context, workflowapp WorkflowApp) error { err := setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err != nil { - log.Printf("Failed setting workflowapp: %s", err) + log.Printf("[ERROR] Failed setting workflowapp: %s", err) return err } else { - log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion) + log.Printf("[INFO] Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion) } return nil @@ -948,6 +974,12 @@ func validateParameterName(name string) string { } } + newname = strings.ReplaceAll(newname, " ", "_") + newname = strings.ReplaceAll(newname, ",", "_") + newname = strings.ReplaceAll(newname, ".", "_") + newname = strings.ReplaceAll(newname, "|", "_") + newname = strings.ReplaceAll(newname, "-", "_") + return newname } @@ -1001,6 +1033,7 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ parsedName = strings.ReplaceAll(parsedName, ",", "_") parsedName = strings.ReplaceAll(parsedName, ".", "_") parsedName = strings.ReplaceAll(parsedName, "|", "_") + parsedName = strings.ReplaceAll(parsedName, "-", "_") parsedName = validateParameterName(parsedName) param.Value.Name = parsedName path.Connect.Parameters[counter].Value.Name = parsedName @@ -1076,7 +1109,7 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "connect", parameters, optionalQueries, headersFound) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "connect", parameters, optionalQueries, headersFound, "") if len(functionname) > 0 { action.Name = functionname @@ -1211,7 +1244,7 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "get", parameters, optionalQueries, headersFound) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "get", parameters, optionalQueries, headersFound, "") if len(functionname) > 0 { action.Name = functionname @@ -1344,7 +1377,7 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "head", parameters, optionalQueries, headersFound) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "head", parameters, optionalQueries, headersFound, "") if len(functionname) > 0 { action.Name = functionname @@ -1478,7 +1511,7 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound, "") if len(functionname) > 0 { action.Name = functionname @@ -1521,6 +1554,40 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo }, }) + fileField := "" + if path.Post.RequestBody != nil { + //log.Printf("DATA: %#v", + value := path.Post.RequestBody.Value + //log.Printf("VAL: %#v", value.Content) + if val, ok := value.Content["multipart/form-data"]; ok { + if val.Schema.Value != nil { + if innerval, ok := val.Schema.Value.Properties["fieldname"]; ok { + if extensionvalue, ok := innerval.Value.ExtensionProps.Extensions["value"]; ok { + fieldname := extensionvalue.(json.RawMessage) + newName := string(fmt.Sprintf("%s", string(fieldname))) + if newName[0] == 0x22 && newName[len(newName)-1] == 0x22 { + parsedName := newName[1 : len(newName)-1] + log.Printf("Parse name: %s", parsedName) + fileField = parsedName + + curParam := WorkflowAppActionParameter{ + Name: "file_id", + Description: "Files to be uploaded", + Multiline: false, + Required: true, + Schema: SchemaDefinition{ + Type: "string", + }, + } + + action.Parameters = append(action.Parameters, curParam) + } + } + } + } + } + } + headersFound := []string{} if len(path.Post.Parameters) > 0 { for counter, param := range path.Post.Parameters { @@ -1610,12 +1677,17 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "post", parameters, optionalQueries, headersFound) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "post", parameters, optionalQueries, headersFound, fileField) if len(functionname) > 0 { action.Name = functionname } + //log.Printf("PARAMS: %d", len(action.Parameters)) + //for _, param := range action.Parameters { + // log.Printf("%#v", param) + //} + return action, curCode } @@ -1743,7 +1815,7 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "patch", parameters, optionalQueries, headersFound) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "patch", parameters, optionalQueries, headersFound, "") if len(functionname) > 0 { action.Name = functionname @@ -1877,7 +1949,7 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "put", parameters, optionalQueries, headersFound) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "put", parameters, optionalQueries, headersFound, "") if len(functionname) > 0 { action.Name = functionname diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index b0398045..0a0b4bae 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -8,6 +8,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" @@ -125,12 +126,29 @@ func getParsedTarMemory(fs billy.Filesystem, tw *tar.Writer, baseDir, extra stri return err } + //log.Printf("FILENAME: %s", filename) readFile, err := ioutil.ReadAll(fileReader) if err != nil { log.Printf("Not file: %s", err) return err } + // Fixes issues with older versions of Docker and reference formats + // Specific to Shuffle rn. Could expand. + // FIXME: Seems like the issue was with multi-stage builds + /* + if filename == "Dockerfile" { + log.Printf("Should search and replace in readfile.") + + referenceCheck := "FROM frikky/shuffle:" + if strings.Contains(string(readFile), referenceCheck) { + log.Printf("SHOULD SEARCH & REPLACE!") + newReference := fmt.Sprintf("FROM registry.hub.docker.com/frikky/shuffle:") + readFile = []byte(strings.Replace(string(readFile), referenceCheck, newReference, -1)) + } + } + */ + //log.Printf("Filename: %s", filename) // FIXME - might need the folder from EXTRA here // Name has to be e.g. just "requirements.txt" @@ -156,8 +174,23 @@ func getParsedTarMemory(fs billy.Filesystem, tw *tar.Writer, baseDir, extra stri return nil } +/* +// Fixes App SDK issues.. meh +func fixTags(tags []string) []string { + checkTag := "frikky/shuffle" + newTags := []string{} + for _, tag := range tags { + if strings.HasPrefix(tag, checkTags) { + newTags.append(newTags, fmt.Sprintf("registry.hub.docker.com/%s", tag)) + } + + newTags.append(tag) + } +} +*/ + // Custom Docker image builder wrapper in memory -func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string) error { +func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string, downloadIfFail bool) error { ctx := context.Background() client, err := client.NewEnvClient() if err != nil { @@ -169,7 +202,7 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin tw := tar.NewWriter(buf) defer tw.Close() - log.Printf("Setting up memory build structure for folder: %s", dockerfileFolder) + log.Printf("[INFO] Setting up memory build structure for folder: %s", dockerfileFolder) err = getParsedTarMemory(fs, tw, dockerfileFolder, "") if err != nil { log.Printf("Tar issue: %s", err) @@ -197,17 +230,56 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin } // Build the actual image + log.Printf("[INFO] Building %s. This may take up to a few minutes.", dockerfileFolder) imageBuildResponse, err := client.ImageBuild( ctx, dockerFileTarReader, buildOptions, ) + + //log.Printf("Response: %#v", imageBuildResponse.Body) //log.Printf("IMAGERESPONSE: %#v", imageBuildResponse.Body) defer imageBuildResponse.Body.Close() - _, newerr := io.Copy(os.Stdout, imageBuildResponse.Body) + buildBuf := new(strings.Builder) + _, newerr := io.Copy(buildBuf, imageBuildResponse.Body) if newerr != nil { log.Printf("Failed reading Docker build STDOUT: %s", newerr) + } else { + log.Printf("STRING: %s", buildBuf.String()) + if strings.Contains(buildBuf.String(), "errorDetail") { + log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), strings.Join(tags, "\n")) + + // Handles pulling of the same image if applicable + // This fixes some issues with older versions of Docker which can't build + // on their own ( <17.05 ) + pullOptions := types.ImagePullOptions{} + downloaded := false + for _, image := range tags { + // Is this ok? Not sure. Tags shouldn't be controlled here prolly. + image = strings.ToLower(image) + + newImage := fmt.Sprintf("%s/%s", registryName, image) + log.Printf("[INFO] Pulling image %s", newImage) + reader, err := client.ImagePull(ctx, newImage, pullOptions) + if err != nil { + log.Printf("[ERROR] Failed getting image %s: %s", newImage, err) + continue + } + + // Attempt to retag the image to not contain registry... + + //newBuf := buildBuf + downloaded = true + io.Copy(os.Stdout, reader) + log.Printf("[INFO] Successfully downloaded and built %s", newImage) + } + + if !downloaded { + return errors.New(fmt.Sprintf("Failed to build / download images %s", strings.Join(tags, ","))) + } + //baseDockerName + } } if err != nil { @@ -272,9 +344,15 @@ func buildImage(tags []string, dockerfileFolder string) error { // Read the STDOUT from the build process defer imageBuildResponse.Body.Close() - _, err = io.Copy(os.Stdout, imageBuildResponse.Body) + buildBuf := new(strings.Builder) + _, err = io.Copy(buildBuf, imageBuildResponse.Body) if err != nil { return err + } else { + if strings.Contains(buildBuf.String(), "errorDetail") { + log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), strings.Join(tags, "\n")) + return errors.New(fmt.Sprintf("Failed building %s. Check backend logs for details. Most likely means you have an old version of Docker.", strings.Join(tags, ","))) + } } return nil @@ -424,7 +502,7 @@ func handleStopHookDocker(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() hook, err := getHook(ctx, fileId) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (stop docker): %s", fileId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -556,7 +634,7 @@ func handleStartHookDocker(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() hook, err := getHook(ctx, fileId) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (start docker): %s", fileId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -636,7 +714,7 @@ func handleStartHookDocker(resp http.ResponseWriter, request *http.Request) { } // FIXME - get some real data? - log.Printf("Successfully started %s-%s on port %s with filepath %s", image, fileId, port, filepath) + log.Printf("[INFO] Successfully started %s-%s on port %s with filepath %s", image, fileId, port, filepath) resp.WriteHeader(200) resp.Write([]byte(`{"success": true, "message": "Started webhook"}`)) return @@ -644,7 +722,7 @@ func handleStartHookDocker(resp http.ResponseWriter, request *http.Request) { // Checks if an image exists func imageCheckBuilder(images []string) error { - log.Printf("[FIXME] ImageNames to check: %#v", images) + //log.Printf("[FIXME] ImageNames to check: %#v", images) return nil ctx := context.Background() @@ -704,10 +782,115 @@ func hookTest() { returnHook, err := getHook(ctx, hook.Id) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (test): %s", hook.Id, err) } if len(returnHook.Id) > 0 { log.Printf("Success! - %s", returnHook.Id) } } + +//https://stackoverflow.com/questions/23935141/how-to-copy-docker-images-from-one-host-to-another-without-using-a-repository +func getDockerImage(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + // Just here to verify that the user is logged in + _, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("Api authentication failed in validate swagger: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + body, err := ioutil.ReadAll(request.Body) + if err != nil { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`)) + return + } + + type requestCheck struct { + Name string `datastore:"name" json:"name" yaml:"name"` + } + + //body = []byte(`swagger: "2.0"`) + //body = []byte(`swagger: '1.0'`) + //newbody := string(body) + //newbody = strings.TrimSpace(newbody) + //body = []byte(newbody) + //log.Println(string(body)) + //tmpbody, err := yaml.YAMLToJSON(body) + //log.Println(err) + //log.Println(string(tmpbody)) + + // This has to be done in a weird way because Datastore doesn't + // support map[string]interface and similar (openapi3.Swagger) + var version requestCheck + + err = json.Unmarshal(body, &version) + if err != nil { + resp.WriteHeader(422) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed JSON marshalling: %s"}`, err))) + return + } + + log.Printf("Image to load: %s", version.Name) + //cli, err := client.NewEnvClient() + //if err != nil { + // log.Println("Unable to create docker client") + // return err + //} + + dockercli, err := client.NewEnvClient() + if err != nil { + log.Printf("Unable to create docker client: %s", err) + resp.WriteHeader(422) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed JSON marshalling: %s"}`, err))) + return + } + + ctx := context.Background() + images, err := dockercli.ImageList(ctx, types.ImageListOptions{ + All: true, + }) + + img := types.ImageSummary{} + tagFound := "" + for _, image := range images { + for _, tag := range image.RepoTags { + log.Printf("Image: %s", tag) + + if strings.ToLower(tag) == strings.ToLower(version.Name) { + img = image + tagFound = tag + break + } + } + } + + if len(img.ID) == 0 { + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't find image %s"}`, version.Name))) + return + } + _ = tagFound + + /* + log.Printf("IMg: %#v", img) + pullOptions := types.ImagePullOptions{} + log.Printf("[INFO] Pulling image %s", image) + reader, err := dockercli.ImagePull(ctx, tag, pullOptions) + if err != nil { + log.Printf("[ERROR] Failed getting image %s: %s", image, err) + } + + io.Copy(os.Stdout, r) + */ + + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "message": "Downloading image %s"}`, version.Name))) +} diff --git a/backend/go-app/files.go b/backend/go-app/files.go index a2856e2b..0779a08c 100644 --- a/backend/go-app/files.go +++ b/backend/go-app/files.go @@ -42,6 +42,9 @@ type File struct { DownloadPath string `json:"download_path" datastore:"download_path"` Md5sum string `json:"md5_sum" datastore:"md5_sum"` Sha256sum string `json:"sha256_sum" datastore:"sha256_sum"` + FileSize int64 `json:"filesize" datastore:"filesize"` + Duplicate bool `json:"duplicate" datastore:"duplicate"` + Subflows []string `json:"subflows" datastore:"subflows"` } var basepath = os.Getenv("SHUFFLE_FILE_LOCATION") @@ -100,6 +103,51 @@ func fileExists(filename string) bool { return !info.IsDir() } +func handleGetFiles(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + // 1. Check user directly + // 2. Check workflow execution authorization + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("[INFO] INITIAL Api authentication failed in file LIST: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if user.Role != "admin" { + log.Printf("[AUTH] User isn't admin") + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Need to be admin"}`))) + return + } + + ctx := context.Background() + files, err := getAllFiles(ctx, user.ActiveOrg.Id) + if err != nil { + log.Printf("[ERROR] Failed to get files: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error getting files."}`))) + return + } + + log.Printf("Got %d files for org %s", len(files), user.ActiveOrg.Id) + newBody, err := json.Marshal(files) + if err != nil { + log.Printf("[ERROR] Failed marshaling files: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false, "reason": "Failed to marshal files"}`)) + return + } + + resp.WriteHeader(200) + resp.Write([]byte(newBody)) +} + func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -347,7 +395,7 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("\n\nUser is trying to download file %s\n\n", fileId) + log.Printf("\n\n[INFO] User is trying to download file %s\n\n", fileId) // 1. Check user directly // 2. Check workflow execution authorization @@ -417,8 +465,18 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) { Openfile, err := os.Open(downloadPath) defer Openfile.Close() //Close after function return if err != nil { + file.Status = "deleted" + err = setFile(ctx, *file) + if err != nil { + log.Printf("Failed setting file to uploading") + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false, "reason": "Failed setting file to uploading"}`)) + return + } + //File not found, send 404 - http.Error(resp, "File not found.", 404) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "File doesn't exist locally"}`)) return } @@ -552,6 +610,7 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) { var buf bytes.Buffer io.Copy(&buf, parsedFile) contents := buf.Bytes() + file.FileSize = int64(len(contents)) md5 := md5sum(contents) buf.Reset() @@ -642,7 +701,8 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) { // Loads of validation below if len(curfile.Filename) == 0 || len(curfile.OrgId) == 0 || len(curfile.WorkflowId) == 0 { - log.Printf("[ERROR] Missing field during upload.") + log.Printf("[ERROR] Missing field during fileupload. Required: filename, org_id, workflow_id") + log.Printf("INPUT: %s", string(body)) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Missing field. Required: filename, org_id, workflow_id"}`))) return @@ -715,6 +775,30 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) { fileId := uuid.NewV4().String() downloadPath := fmt.Sprintf("%s/%s", folderPath, fileId) + duplicateWorkflows := []string{} + for _, trigger := range workflow.Triggers { + if trigger.AppName == "Shuffle Workflow" && trigger.TriggerType == "SUBFLOW" { + for _, parameter := range trigger.Parameters { + if parameter.Name == "workflow" && len(parameter.Value) > 0 { + + found := false + for _, workflow := range duplicateWorkflows { + if workflow == parameter.Value { + found = true + break + } + } + + if !found { + duplicateWorkflows = append(duplicateWorkflows, parameter.Value) + } + + break + } + } + } + } + timeNow := time.Now().Unix() newFile := File{ Id: fileId, @@ -726,6 +810,7 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) { OrgId: curfile.OrgId, WorkflowId: curfile.WorkflowId, DownloadPath: downloadPath, + Subflows: duplicateWorkflows, } err = setFile(ctx, newFile) @@ -740,6 +825,7 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) { resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, fileId))) + } func getFile(ctx context.Context, id string) (*File, error) { @@ -754,6 +840,9 @@ func getFile(ctx context.Context, id string) (*File, error) { func setFile(ctx context.Context, file File) error { // clear session_token and API_token for user + timeNow := time.Now().Unix() + file.UpdatedAt = timeNow + k := datastore.NameKey("Files", file.Id, nil) if _, err := dbclient.Put(ctx, k, &file); err != nil { log.Println(err) @@ -762,3 +851,23 @@ func setFile(ctx context.Context, file File) error { return nil } + +func getAllFiles(ctx context.Context, orgId string) ([]File, error) { + var files []File + q := datastore.NewQuery("Files").Filter("org_id =", orgId).Order("-updated_at").Limit(100) + + _, err := dbclient.GetAll(ctx, q, &files) + if err != nil { + if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { + q = q.Limit(50) + _, err := dbclient.GetAll(ctx, q, &files) + if err != nil { + return []File{}, err + } + } else { + return []File{}, err + } + } + + return files, nil +} diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 553b8b3c..0bf0ccce 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -23,6 +23,7 @@ require ( github.com/gorilla/mux v1.7.4 github.com/h2non/filetype v1.0.12 github.com/opencontainers/go-digest v1.0.0-rc1 // indirect + github.com/patrickmn/go-cache v2.1.0+incompatible github.com/satori/go.uuid v1.2.0 golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 4fbd4b69..2d63be91 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -160,6 +160,8 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 64167166..b225de77 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -34,6 +34,11 @@ import ( "github.com/getkin/kin-openapi/openapi2" "github.com/getkin/kin-openapi/openapi2conv" "github.com/getkin/kin-openapi/openapi3" + /* + "github.com/frikky/kin-openapi/openapi2" + "github.com/frikky/kin-openapi/openapi2conv" + "github.com/frikky/kin-openapi/openapi3" + */ "github.com/google/go-github/v28/github" "golang.org/x/oauth2" @@ -65,6 +70,7 @@ import ( // "google.golang.org/appengine/memcache" // applog "google.golang.org/appengine/log" //cloudrun "google.golang.org/api/run/v1" + "github.com/patrickmn/go-cache" ) // This is used to handle onprem vs offprem databases etc @@ -72,6 +78,7 @@ var gceProject = "shuffle" var bucketName = "shuffler.appspot.com" var baseAppPath = "/home/frikky/git/shaffuru/tmp/apps" var baseDockerName = "frikky/shuffle" +var registryName = "registry.hub.docker.com" //var syncUrl = "http://192.168.102.54:5002" var syncUrl = "https://shuffler.io" @@ -79,6 +86,7 @@ var syncUrl = "https://shuffler.io" //var syncUrl = "http://localhost:5002" var dbclient *datastore.Client +var requestCache *cache.Cache type Userapi struct { Username string `datastore:"username"` @@ -108,6 +116,7 @@ type StatisticsItem struct { Total int64 `json:"total" datastore:"total"` Fieldname string `json:"field_name" datastore:"field_name"` Data []StatisticsData `json:"data" datastore:"data"` + OrgId string `json:"org_id" datastore:"org_id"` } // "Execution by status" @@ -610,13 +619,13 @@ func handleApiAuthentication(resp http.ResponseWriter, request *http.Request) (U apikey := request.Header.Get("Authorization") if len(apikey) > 0 { if !strings.HasPrefix(apikey, "Bearer ") { - log.Printf("Apikey doesn't start with bearer") + log.Printf("[WARNING] Apikey doesn't start with bearer") return User{}, errors.New("No bearer token for authorization header") } apikeyCheck := strings.Split(apikey, " ") if len(apikeyCheck) != 2 { - log.Printf("Invalid format for apikey.") + log.Printf("[WARNING] Invalid format for apikey.") return User{}, errors.New("Invalid format for apikey") } @@ -639,7 +648,7 @@ func handleApiAuthentication(resp http.ResponseWriter, request *http.Request) (U if len(Userdata.Username) > 0 { return Userdata, nil } else { - return Userdata, errors.New(fmt.Sprintf("User is invalid - no username found")) + return Userdata, errors.New(fmt.Sprintf("[WARNING] User is invalid - no username found")) } } @@ -1139,7 +1148,7 @@ func createNewUser(username, password, role, apikey string, org Org) error { neworg, err := getOrg(ctx, org.Id) if err == nil { - neworg.Users = append(neworg.Users, *newUser) + //neworg.Users = append(neworg.Users, *newUser) err = setOrg(ctx, *neworg, neworg.Id) if err != nil { log.Printf("Failed updating org with user %s", newUser.Username) @@ -1148,7 +1157,7 @@ func createNewUser(username, password, role, apikey string, org Org) error { } } - err = increaseStatisticsField(ctx, "successful_register", username, 1) + err = increaseStatisticsField(ctx, "successful_register", username, 1, org.Id) if err != nil { log.Printf("Failed to increase total apps loaded stats: %s", err) } @@ -1726,6 +1735,13 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { } } + // FIXME: Remove this dependency by updating users' orgs when org itself is updated + org, err := getOrg(ctx, userInfo.ActiveOrg.Id) + if err == nil { + userInfo.ActiveOrg = *org + userInfo.ActiveOrg.Users = []User{} + } + currentOrg, err := json.Marshal(userInfo.ActiveOrg) if err != nil { currentOrg = []byte("{}") @@ -2388,6 +2404,10 @@ func handleGetUsers(resp http.ResponseWriter, request *http.Request) { continue } + //for _, tmpUser := range newUsers { + // if tmpUser.Name + //} + item.Password = "" item.Session = "" item.VerificationToken = "" @@ -2470,7 +2490,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { if len(users) != 1 { log.Printf(`Found multiple or no users with the same username: %s: %d`, data.Username, len(users)) resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Found %d users with the same username: %s"}`, len(users), data.Username))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error: %d users with username %s"}`, len(users), data.Username))) return } @@ -2588,14 +2608,25 @@ func getOrg(ctx context.Context, id string) (*Org, error) { return curOrg, nil } -func setOrg(ctx context.Context, data Org, id string) error { +func setOrg(ctx context.Context, org Org, id string) error { // clear session_token and API_token for user + timeNow := int64(time.Now().Unix()) + if org.Created == 0 { + org.Created = timeNow + } + + org.Edited = timeNow + k := datastore.NameKey("Organizations", id, nil) - if _, err := dbclient.Put(ctx, k, &data); err != nil { - log.Println(err) + if _, err := dbclient.Put(ctx, k, &org); err != nil { + log.Printf("Failed setting org: %s", err) return err } + // FIXME: Make this update every user to have the correct org data. + //org = fixOrgUser(ctx, &org) + //_ = org + return nil } @@ -2610,6 +2641,23 @@ func getUser(ctx context.Context, id string) (*User, error) { return curUser, nil } +// Index = Username +func DeleteKeys(ctx context.Context, entity string, value []string) error { + // Non indexed User data + keys := []*datastore.Key{} + for _, item := range value { + keys = append(keys, datastore.NameKey(entity, item, nil)) + } + + err := dbclient.DeleteMulti(ctx, keys) + if err != nil { + log.Printf("Error deleting %s from %s: %s", value, entity, err) + return err + } + + return nil +} + // Index = Username func DeleteKey(ctx context.Context, entity string, value string) error { // Non indexed User data @@ -2703,6 +2751,75 @@ func setEnvironment(ctx context.Context, data *Environment) error { return nil } +func fixOrgUser(ctx context.Context, org *Org) *Org { + //found := false + //for _, id := range user.Orgs { + // if user.ActiveOrg.Id == id { + // found = true + // break + // } + //} + + //if !found { + // user.Orgs = append(user.Orgs, user.ActiveOrg.Id) + //} + + //// Might be vulnerable to timing attacks. + //for _, orgId := range user.Orgs { + // if len(orgId) == 0 { + // continue + // } + + // org, err := getOrg(ctx, orgId) + // if err != nil { + // log.Printf("Error getting org %s", orgId) + // continue + // } + + // orgIndex := 0 + // userFound := false + // for index, orgUser := range org.Users { + // if orgUser.Id == user.Id { + // orgIndex = index + // userFound = true + // break + // } + // } + + // if userFound { + // user.PrivateApps = []WorkflowApp{} + // user.Executions = ExecutionInfo{} + // user.Limits = UserLimits{} + // user.Authentication = []UserAuth{} + + // org.Users[orgIndex] = *user + // } else { + // org.Users = append(org.Users, *user) + // } + + // err = setOrg(ctx, *org, orgId) + // if err != nil { + // log.Printf("Failed setting org %s", orgId) + // } + //} + + return org +} + +// ListBooks returns a list of books, ordered by title. +func setUser(ctx context.Context, data *User) error { + data = fixUserOrg(ctx, data) + + // clear session_token and API_token for user + k := datastore.NameKey("Users", data.Id, nil) + if _, err := dbclient.Put(ctx, k, data); err != nil { + log.Println(err) + return err + } + + return nil +} + func fixUserOrg(ctx context.Context, user *User) *User { found := false for _, id := range user.Orgs { @@ -2758,33 +2875,18 @@ func fixUserOrg(ctx context.Context, user *User) *User { return user } -// ListBooks returns a list of books, ordered by title. -func setUser(ctx context.Context, data *User) error { - data = fixUserOrg(ctx, data) - - // clear session_token and API_token for user - k := datastore.NameKey("Users", data.Id, nil) - if _, err := dbclient.Put(ctx, k, data); err != nil { - log.Println(err) - return err - } - - return nil -} - // Used for testing only. Shouldn't impact production. func handleCors(resp http.ResponseWriter, request *http.Request) bool { - //allowedOrigins := "*" - allowedOrigins := "http://localhost:3000" + //allowedOrigins := "http://localhost:3000" + allowedOrigins := "http://localhost:3002" resp.Header().Set("Vary", "Origin") - resp.Header().Set("Access-Control-Allow-Headers", "Content-Type, Accept, X-Requested-With, remember-me") + resp.Header().Set("Access-Control-Allow-Headers", "Content-Type, Accept, X-Requested-With, remember-me, Authorization") resp.Header().Set("Access-Control-Allow-Methods", "POST, GET, PUT, DELETE, PATCH") resp.Header().Set("Access-Control-Allow-Credentials", "true") resp.Header().Set("Access-Control-Allow-Origin", allowedOrigins) if request.Method == "OPTIONS" { - resp.WriteHeader(200) resp.Write([]byte("OK")) return true @@ -3017,7 +3119,7 @@ func handleSetHook(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() _, err = getHook(ctx, workflowId) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (set): %s", workflowId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "Invalid ID"}`)) return @@ -3378,7 +3480,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { //log.Printf("HookID: %s", hookId) hook, err := getHook(ctx, hookId) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (callback): %s", hookId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -3391,7 +3493,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { //resp.WriteHeader(200) //resp.Write([]byte(`{"success": true}`)) if hook.Status == "stopped" { - log.Printf("Not running because hook status is stopped") + log.Printf("Not running %s because hook status is stopped", hook.Id) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "The webhook isn't running. Click start to start it"}`))) return @@ -3408,43 +3510,63 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { log.Printf("This should trigger in the cloud. Duplicate action allowed onprem.") } + type ExecutionStruct struct { + Start string `json:"start"` + ExecutionSource string `json:"execution_source"` + ExecutionArgument string `json:"execution_argument"` + } + + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Printf("Body data error: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + newBody := ExecutionStruct{ + Start: hook.Start, + ExecutionSource: "webhook", + ExecutionArgument: string(body), + } + + b, err := json.Marshal(newBody) + if err != nil { + log.Printf("Failed newBody marshaling: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + for _, item := range hook.Workflows { - log.Printf("Running webhook for workflow %s with startnode %s", item, hook.Start) + //log.Printf("Running webhook for workflow %s with startnode %s", item, hook.Start) workflow := Workflow{ ID: "", } - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Body data error: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } + //parsedBody := string(body) + //parsedBody = strings.Replace(parsedBody, "\"", "\\\"", -1) + //if len(parsedBody) > 0 { + // if string(parsedBody[0]) == `"` && string(parsedBody[len(parsedBody)-1]) == "\"" { + // parsedBody = parsedBody[1 : len(parsedBody)-1] + // } + //} - parsedBody := string(body) - parsedBody = strings.Replace(parsedBody, "\"", "\\\"", -1) - if len(parsedBody) > 0 { - if string(parsedBody[0]) == `"` && string(parsedBody[len(parsedBody)-1]) == "\"" { - parsedBody = parsedBody[1 : len(parsedBody)-1] - } - } - - bodyWrapper := fmt.Sprintf(`{"start": "%s", "execution_source": "webhook", "execution_argument": "%s"}`, hook.Start, string(parsedBody)) - if len(hook.Start) == 0 { - log.Printf("No start node for hook %s - running with workflow default.", hook.Id) - bodyWrapper = string(parsedBody) - } + //bodyWrapper := fmt.Sprintf(`{"start": "%s", "execution_source": "webhook", "execution_argument": "%s"}`, hook.Start, string(parsedBody)) + //if len(hook.Start) == 0 { + // log.Printf("No start node for hook %s - running with workflow default.", hook.Id) + // bodyWrapper = string(parsedBody) + //} newRequest := &http.Request{ Method: "POST", - Body: ioutil.NopCloser(strings.NewReader(bodyWrapper)), + Body: ioutil.NopCloser(bytes.NewReader(b)), } // OrgId: activeOrgs[0].Id, workflowExecution, executionResp, err := handleExecution(item, workflow, newRequest) if err == nil { - err = increaseStatisticsField(ctx, "total_webhooks_ran", workflowExecution.Workflow.ID, 1) + err = increaseStatisticsField(ctx, "total_webhooks_ran", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) if err != nil { log.Printf("Failed to increase total apps loaded stats: %s", err) } @@ -3613,7 +3735,7 @@ func handleNewHook(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } else { - log.Printf("Successfully set up cloud action schedule") + log.Printf("[INFO] Successfully set up cloud action schedule") } } @@ -3652,7 +3774,7 @@ func handleNewHook(resp http.ResponseWriter, request *http.Request) { return } - err = increaseStatisticsField(ctx, "total_workflow_triggers", requestdata.Workflow, 1) + err = increaseStatisticsField(ctx, "total_workflow_triggers", requestdata.Workflow, 1, user.ActiveOrg.Id) if err != nil { log.Printf("[INFO] Failed to increase total workflows: %s", err) } @@ -3699,7 +3821,7 @@ func sendHookResult(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() hook, err := getHook(ctx, workflowId) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (send): %s", workflowId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -3766,7 +3888,7 @@ func handleGetHook(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() hook, err := getHook(ctx, workflowId) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (get hook): %s", workflowId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -4448,7 +4570,7 @@ func handleGetallHooks(resp http.ResponseWriter, request *http.Request) { var allhooks []Hook _, err = dbclient.GetAll(ctx, q, &allhooks) if err != nil { - log.Printf("Failed getting workflows for user %s: %s", user.Username, err) + log.Printf("Failed getting hooks for user %s: %s", user.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -4686,7 +4808,7 @@ func getDocList(resp http.ResponseWriter, request *http.Request) { if len(item1) == 0 { resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No docs available."`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No docs available."}`))) return } @@ -4745,7 +4867,7 @@ func getDocs(resp http.ResponseWriter, request *http.Request) { location := strings.Split(request.URL.String(), "/") if len(location) != 5 { resp.WriteHeader(404) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"}`))) return } @@ -4769,7 +4891,7 @@ func getDocs(resp http.ResponseWriter, request *http.Request) { ) if err != nil { - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"}`))) resp.WriteHeader(404) //setBadMemcache(ctx, docPath) return @@ -4778,7 +4900,7 @@ func getDocs(resp http.ResponseWriter, request *http.Request) { newresp, err := client.Do(req) if err != nil { resp.WriteHeader(404) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"}`))) //setBadMemcache(ctx, docPath) return } @@ -4786,7 +4908,7 @@ func getDocs(resp http.ResponseWriter, request *http.Request) { body, err := ioutil.ReadAll(newresp.Body) if err != nil { resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't parse data"`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't parse data"}`))) //setBadMemcache(ctx, docPath) return } @@ -5790,7 +5912,7 @@ func getOpenapi(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("API LENGTH GET: %d, ID: %s", len(parsedApi.Body), id) + log.Printf("[INFO] API LENGTH GET: %d, ID: %s", len(parsedApi.Body), id) parsedApi.Success = true data, err := json.Marshal(parsedApi) @@ -5839,7 +5961,7 @@ func echoOpenapiData(resp http.ResponseWriter, request *http.Request) { req, err := http.NewRequest("GET", newbody, nil) if err != nil { - log.Printf("Requestbuilder err: %s", err) + log.Printf("[ERROR] Requestbuilder err: %s", err) resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Failed building request"}`)) return @@ -5848,16 +5970,18 @@ func echoOpenapiData(resp http.ResponseWriter, request *http.Request) { httpClient := &http.Client{} newresp, err := httpClient.Do(req) if err != nil { + log.Printf("[ERROR] Grabbing error: %s", err) resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed making request for data"`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed making remote request to get the data"}`))) return } defer newresp.Body.Close() urlbody, err := ioutil.ReadAll(newresp.Body) if err != nil { + log.Printf("[ERROR] URLbody error: %s", err) resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't get data from selected uri"`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't get data from selected uri"}`))) return } @@ -5917,8 +6041,11 @@ func handleSwaggerValidation(body []byte) (ParsedOpenApi, error) { if strings.HasPrefix(version.Swagger, "3.") || strings.HasPrefix(version.OpenAPI, "3.") { //log.Println("Handling v3 API") - swaggerv3, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(body) + swaggerLoader := openapi3.NewSwaggerLoader() + swaggerLoader.IsExternalRefsAllowed = true + swaggerv3, err := swaggerLoader.LoadSwaggerFromData(body) if err != nil { + log.Printf("Failed parsing OpenAPI: %s", err) return ParsedOpenApi{}, err } @@ -6028,6 +6155,8 @@ func validateSwagger(resp http.ResponseWriter, request *http.Request) { // support map[string]interface and similar (openapi3.Swagger) var version versionCheck + log.Printf("API length SET: %d", len(string(body))) + isJson := false err = json.Unmarshal(body, &version) if err != nil { @@ -6035,25 +6164,30 @@ func validateSwagger(resp http.ResponseWriter, request *http.Request) { err = yaml.Unmarshal(body, &version) if err != nil { log.Printf("Yaml error (3): %s", err) - //resp.WriteHeader(422) - //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed reading openapi to json and yaml: %s"}`, err))) - //return + resp.WriteHeader(422) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed reading openapi to json and yaml. Is version defined?: %s"}`, err))) + return } else { - log.Printf("Successfully parsed YAML!") + log.Printf("[INFO] Successfully parsed YAML (3)!") } } else { isJson = true - log.Printf("Successfully parsed JSON!") + log.Printf("[INFO] Successfully parsed JSON!") } if len(version.SwaggerVersion) > 0 && len(version.Swagger) == 0 { version.Swagger = version.SwaggerVersion } + log.Printf("[INFO] Version: %#v", version) + log.Printf("[INFO] OpenAPI: %s", version.OpenAPI) if strings.HasPrefix(version.Swagger, "3.") || strings.HasPrefix(version.OpenAPI, "3.") { - log.Println("Handling v3 API") - swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(body) + log.Println("[INFO] Handling v3 API") + swaggerLoader := openapi3.NewSwaggerLoader() + swaggerLoader.IsExternalRefsAllowed = true + swagger, err := swaggerLoader.LoadSwaggerFromData(body) if err != nil { + log.Printf("[WARNING] Failed to convert v3 API: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return @@ -6063,16 +6197,22 @@ func validateSwagger(resp http.ResponseWriter, request *http.Request) { hasher.Write(body) idstring := hex.EncodeToString(hasher.Sum(nil)) - log.Printf("Swagger v3 validation success with ID %s!", idstring) - log.Printf("Paths: %d", len(swagger.Paths)) + log.Printf("Swagger v3 validation success with ID %s and %d paths!", idstring, len(swagger.Paths)) if !isJson { - log.Printf("FIXME: NEED TO TRANSFORM FROM YAML TO JSON for %s", idstring) + log.Printf("[INFO] NEED TO TRANSFORM FROM YAML TO JSON for %s", idstring) } + swaggerdata, err := json.Marshal(swagger) + if err != nil { + log.Printf("Failed unmarshaling v3 data: %s", err) + resp.WriteHeader(422) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed marshalling swaggerv3 data: %s"}`, err))) + return + } parsed := ParsedOpenApi{ ID: idstring, - Body: string(body), + Body: string(swaggerdata), } ctx := context.Background() @@ -6083,6 +6223,8 @@ func validateSwagger(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed reading openapi2: %s"}`, err))) return } + + log.Printf("[INFO] Successfully set OpenAPI with ID %s", idstring) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, idstring))) return @@ -6117,7 +6259,7 @@ func validateSwagger(resp http.ResponseWriter, request *http.Request) { swaggerdata, err := json.Marshal(swaggerv3) if err != nil { - log.Printf("Failed unmarshaling v3 data: %s", err) + log.Printf("Failed unmarshaling v3 from v2 data: %s", err) resp.WriteHeader(422) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed marshalling swaggerv3 data: %s"}`, err))) return @@ -6170,6 +6312,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { return } + log.Printf("[INFO] SETTING APP TO LIVE!!!") user, err := handleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in verify swagger: %s", err) @@ -6231,17 +6374,25 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { // Test = client side with fetch? ctx := context.Background() - - swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(body) + swaggerLoader := openapi3.NewSwaggerLoader() + swaggerLoader.IsExternalRefsAllowed = true + swagger, err := swaggerLoader.LoadSwaggerFromData(body) if err != nil { - log.Printf("Swagger validation error: %s", err) + log.Printf("[ERROR] Swagger validation error: %s", err) resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Failed verifying openapi"}`)) return } + if swagger.Info == nil { + log.Printf("[ERORR] Info is nil?: %#v", swagger) + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false, "reason": "Info not parsed"}`)) + return + } + if strings.Contains(swagger.Info.Title, " ") { - strings.Replace(swagger.Info.Title, " ", "", -1) + swagger.Info.Title = strings.Replace(swagger.Info.Title, " ", "_", -1) } basePath, err := buildStructure(swagger, newmd5) @@ -6263,7 +6414,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { // FIXME: CHECK IF SAME NAME AS NORMAL APP // Can't overwrite existing normal app - workflowApps, err := getAllWorkflowApps(ctx) + workflowApps, err := getAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting all workflow apps from database to verify: %s", err) resp.WriteHeader(401) @@ -6318,7 +6469,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { if err != nil { log.Printf("Failed adding app to db: %s", err) resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed adding app to db"}`)) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed adding app to db: %s"}`, err))) return } @@ -6344,13 +6495,13 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { // 3. Zip and stream it directly in the directory _, err = streamZipdata(ctx, identifier, stitched, "requests\nurllib3") if err != nil { - log.Printf("Zipfile error: %s", err) + log.Printf("[ERROR] Zipfile error: %s", err) resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Failed to build zipfile"}`)) return } - log.Printf("Successfully stitched ZIPFILE for %s", identifier) + log.Printf("[INFO] Successfully stitched ZIPFILE for %s", identifier) // 4. Upload as cloud function - this apikey is specifically for cloud functions rofl //environmentVariables := map[string]string{ @@ -6369,9 +6520,9 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { // 4. Build the image locally. // FIXME: Should be moved to a local docker registry dockerLocation := fmt.Sprintf("%s/Dockerfile", basePath) - log.Printf("Dockerfile: %s", dockerLocation) + log.Printf("[INFO] Dockerfile: %s", dockerLocation) - versionName := fmt.Sprintf("%s_%s", strings.ReplaceAll(api.Name, " ", "-"), api.AppVersion) + versionName := fmt.Sprintf("%s_%s", strings.ToLower(strings.ReplaceAll(api.Name, " ", "-")), api.AppVersion) dockerTags := []string{ fmt.Sprintf("%s:%s", baseDockerName, identifier), fmt.Sprintf("%s:%s", baseDockerName, versionName), @@ -6379,15 +6530,15 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { err = buildImage(dockerTags, dockerLocation) if err != nil { - log.Printf("Docker build error: %s", err) + log.Printf("[ERROR] Docker build error: %s", err) resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error in Docker build"}`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error in Docker build: %s"}`, err))) return } found := false foundNumber := 0 - log.Printf("Checking for api with ID %s", newmd5) + log.Printf("[INFO] Checking for api with ID %s", newmd5) for appCounter, app := range user.PrivateApps { if app.ID == api.ID { found = true @@ -6413,25 +6564,25 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { err = setUser(ctx, &user) if err != nil { - log.Printf("Failed adding verification for user %s: %s", user.Username, err) + log.Printf("[ERROR] Failed adding verification for user %s: %s", user.Username, err) resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Failed updating user"}`))) return } - log.Printf("DO I REACH HERE WHEN SAVING?") + //log.Printf("DO I REACH HERE WHEN SAVING?") parsed := ParsedOpenApi{ ID: newmd5, Body: string(body), } - log.Printf("API LENGTH: %d, ID: %s", len(parsed.Body), newmd5) + log.Printf("[INFO] API LENGTH: %d, ID: %s", len(parsed.Body), newmd5) // FIXME: Might cause versioning issues if we re-use the same!! // FIXME: Need a way to track different versions of the same app properly. // Hint: Save API.id somewhere, and use newmd5 to save latest version err = setOpenApiDatastore(ctx, newmd5, parsed) if err != nil { - log.Printf("Failed saving to datastore: %s", err) + log.Printf("[ERROR] Failed saving to datastore: %s", err) resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%"}`, err))) } @@ -6439,16 +6590,19 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { // Backup every single one setOpenApiDatastore(ctx, api.ID, parsed) - err = increaseStatisticsField(ctx, "total_apps_created", newmd5, 1) + err = increaseStatisticsField(ctx, "total_apps_created", newmd5, 1, user.ActiveOrg.Id) if err != nil { log.Printf("Failed to increase success execution stats: %s", err) } - err = increaseStatisticsField(ctx, "openapi_apps_created", newmd5, 1) + err = increaseStatisticsField(ctx, "openapi_apps_created", newmd5, 1, user.ActiveOrg.Id) if err != nil { log.Printf("Failed to increase success execution stats: %s", err) } + cacheKey := fmt.Sprintf("workflowapps-sorted") + requestCache.Delete(cacheKey) + resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, api.ID))) } @@ -6536,6 +6690,9 @@ func handleAppHotload(location string, forceUpdate bool) error { return err } + cacheKey := fmt.Sprintf("workflowapps-sorted") + requestCache.Delete(cacheKey) + return nil } @@ -6662,7 +6819,7 @@ func handleCloudJob(job CloudSyncJob) error { } workflowExecution.Status = "EXECUTING" - err = setWorkflowExecution(ctx, *workflowExecution) + err = setWorkflowExecution(ctx, *workflowExecution, true) if err != nil { return err } @@ -6715,7 +6872,7 @@ func handleCloudJob(job CloudSyncJob) error { workflowExecution.Results = newResults workflowExecution.Status = "ABORTED" - err = setWorkflowExecution(ctx, *workflowExecution) + err = setWorkflowExecution(ctx, *workflowExecution, true) if err != nil { return err } @@ -6831,7 +6988,7 @@ func remoteOrgJobHandler(org Org, interval int) error { func runInit(ctx context.Context) { // Setting stats for backend starts (failure count as well) log.Printf("Starting INIT setup") - err := increaseStatisticsField(ctx, "backend_executions", "", 1) + err := increaseStatisticsField(ctx, "backend_executions", "", 1, "") if err != nil { log.Printf("Failed increasing local stats: %s", err) } @@ -6846,6 +7003,8 @@ func runInit(ctx context.Context) { log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy) } + requestCache = cache.New(5*time.Minute, 10*time.Minute) + /* proxyUrl, err := url.Parse(httpProxy) if err != nil { @@ -7038,7 +7197,14 @@ func runInit(ctx context.Context) { } } } else { - log.Printf("Found %d users.", len(users)) + if len(users) < 5 && len(users) > 0 { + for _, user := range users { + log.Printf("Username: %s, role: %s", user.Username, user.Role) + } + } else { + log.Printf("Found %d users.", len(users)) + } + if len(activeOrgs) == 1 && len(users) > 0 { for _, user := range users { if user.ActiveOrg.Id == "" && len(user.Username) > 0 { @@ -7092,23 +7258,31 @@ func runInit(ctx context.Context) { // Fixing workflows to have real activeorg IDs if len(activeOrgs) == 1 { - q := datastore.NewQuery("workflow") + q := datastore.NewQuery("workflow").Limit(35) var workflows []Workflow _, err = dbclient.GetAll(ctx, q, &workflows) if err != nil { log.Printf("Error getting workflows in runinit: %s", err) } else { updated := 0 + timeNow := time.Now().Unix() for _, workflow := range workflows { + setLocal := false if workflow.ExecutingOrg.Id == "" || len(workflow.OrgId) == 0 { workflow.OrgId = activeOrgs[0].Id workflow.ExecutingOrg = activeOrgs[0] + setLocal = true + } else if workflow.Edited == 0 { + workflow.Edited = timeNow + setLocal = true + } + if setLocal { err = setWorkflow(ctx, workflow, workflow.ID) if err != nil { log.Printf("Failed setting workflow in init: %s", err) } else { - log.Printf("Fixed workflow %s to have the right org.", workflow.ID) + log.Printf("Fixed workflow %s to have the right info.", workflow.ID) updated += 1 } } @@ -7269,9 +7443,25 @@ func runInit(ctx context.Context) { // Getting apps to see if we should initialize a test log.Printf("Getting remote workflow apps") - workflowapps, err := getAllWorkflowApps(ctx) + workflowapps, err := getAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting apps (runInit): %s", err) + } else if err == nil && len(workflowapps) > 0 { + //getAllWorkflowApps(ctx context.Context) ([]WorkflowApp, error) { + var allworkflowapps []WorkflowApp + q := datastore.NewQuery("workflowapp") + _, err := dbclient.GetAll(ctx, q, &allworkflowapps) + if err == nil { + for _, workflowapp := range allworkflowapps { + if workflowapp.Edited == 0 { + err = setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) + if err == nil { + log.Printf("Updating time for workflowapp %s:%s", workflowapp.Name, workflowapp.AppVersion) + } + } + } + } + } else if err == nil && len(workflowapps) == 0 { log.Printf("Downloading default workflow apps") fs := memfs.New() @@ -7352,7 +7542,7 @@ func runInit(ctx context.Context) { workflowLocation := os.Getenv("SHUFFLE_DOWNLOAD_WORKFLOW_LOCATION") if len(workflowLocation) > 0 { log.Printf("Downloading WORKFLOWS from %s if no workflows - EXTRA workflows", workflowLocation) - q := datastore.NewQuery("workflow") + q := datastore.NewQuery("workflow").Limit(35) var workflows []Workflow _, err = dbclient.GetAll(ctx, q, &workflows) if err != nil { @@ -7361,20 +7551,25 @@ func runInit(ctx context.Context) { if len(workflows) == 0 { username := os.Getenv("SHUFFLE_DOWNLOAD_WORKFLOW_USERNAME") password := os.Getenv("SHUFFLE_DOWNLOAD_WORKFLOW_PASSWORD") - err = loadGithubWorkflows(workflowLocation, username, password, "", os.Getenv("SHUFFLE_DOWNLOAD_WORKFLOW_BRANCH")) + orgId := "" + if len(activeOrgs) > 0 { + orgId = activeOrgs[0].Id + } + + err = loadGithubWorkflows(workflowLocation, username, password, "", os.Getenv("SHUFFLE_DOWNLOAD_WORKFLOW_BRANCH"), orgId) if err != nil { log.Printf("Failed to upload workflows from github: %s", err) } else { - log.Printf("Finished downloading workflows from github!") + log.Printf("[INFO] Finished downloading workflows from github!") } } else { - log.Printf("Skipping because there are %d workflows already", len(workflows)) + log.Printf("[INFO] Skipping because there are %d workflows already", len(workflows)) } } } - log.Printf("Finished INIT") + log.Printf("[INFO] Finished INIT") } func handleVerifyCloudsync(orgId string) (SyncFeatures, error) { @@ -7540,10 +7735,11 @@ func handleEditOrg(resp http.ResponseWriter, request *http.Request) { } type ReturnData struct { - Image string `json:"image" datastore:"image"` - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` - OrgId string `json:"org_id" datastore:"org_id"` + Image string `json:"image" datastore:"image"` + Name string `json:"name" datastore:"name"` + Description string `json:"description" datastore:"description"` + OrgId string `json:"org_id" datastore:"org_id"` + Defaults Defaults `json:"defaults" datastore:"defaults"` } var tmpData ReturnData @@ -7614,6 +7810,8 @@ func handleEditOrg(resp http.ResponseWriter, request *http.Request) { org.Image = tmpData.Image org.Name = tmpData.Name org.Description = tmpData.Description + org.Defaults = tmpData.Defaults + //log.Printf("Org: %#v", org) err = setOrg(ctx, *org, org.Id) if err != nil { @@ -7914,6 +8112,182 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { resp.Write(respBody) } +//func handleEditOrg(resp http.ResponseWriter, request *http.Request) { +// cors := handleCors(resp, request) +// if cors { +// return +// } +// +// user, err := handleApiAuthentication(resp, request) +// if err != nil { +// log.Printf("Api authentication failed in cloud setup: %s", err) +// resp.WriteHeader(401) +// resp.Write([]byte(`{"success": false}`)) +// return +// } +// +// ctx := context.Background() +// if user.Role != "admin" { +// /* +// log.Printf("User: %s", user.Role) +// dbclient, err := getDatastoreClient(ctx, gceProject) +// if err != nil { +// log.Printf("Err1: %s", err) +// } +// +// user.Role = "admin" +// key := datastore.NameKey("Users", strings.ToLower(user.Username), nil) +// if _, err := dbclient.Put(ctx, key, &user); err != nil { +// log.Printf("Err2: %s", err) +// } +// */ +// +// log.Printf("Not admin, can't edit org.") +// resp.WriteHeader(401) +// resp.Write([]byte(`{"success": false, "reason": "Not admin"}`)) +// return +// } +// +// body, err := ioutil.ReadAll(request.Body) +// if err != nil { +// resp.WriteHeader(401) +// resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`)) +// return +// } +// +// type ReturnData struct { +// Image string `json:"image" datastore:"image"` +// Name string `json:"name" datastore:"name"` +// Description string `json:"description" datastore:"description"` +// OrgId string `json:"org_id" datastore:"org_id"` +// SubscriptionId string `json:"subscription_id" datastore:"subscription_id"` +// Action string `json:"action" datastore:"action"` +// } +// +// var tmpData ReturnData +// err = json.Unmarshal(body, &tmpData) +// if err != nil { +// log.Printf("Failed unmarshalling test: %s", err) +// resp.WriteHeader(401) +// resp.Write([]byte(`{"success": false}`)) +// return +// } +// +// var fileId string +// location := strings.Split(request.URL.String(), "/") +// if location[1] == "api" { +// if len(location) <= 4 { +// log.Printf("Path too short: %d", len(location)) +// resp.WriteHeader(401) +// resp.Write([]byte(`{"success": false}`)) +// return +// } +// +// fileId = location[4] +// } +// +// if tmpData.OrgId != user.ActiveOrg.Id || fileId != user.ActiveOrg.Id { +// log.Printf("User can't edit the org. Not part of ORG: %s vs %s", tmpData.OrgId, user.ActiveOrg.Id) +// resp.WriteHeader(401) +// resp.Write([]byte(`{"success": false, "No permission to edit this org"}`)) +// return +// } +// +// org, err := getOrg(ctx, tmpData.OrgId) +// if err != nil { +// log.Printf("Organization doesn't exist: %s", err) +// resp.WriteHeader(401) +// resp.Write([]byte(`{"success": false}`)) +// return +// } +// +// admin := false +// userFound := false +// for _, inneruser := range org.Users { +// if inneruser.Id == user.Id { +// userFound = true +// if inneruser.Role == "admin" { +// admin = true +// } +// +// break +// } +// } +// +// if !userFound { +// log.Printf("User %s doesn't exist in organization for edit %s", user.Id, org.Id) +// resp.WriteHeader(401) +// resp.Write([]byte(`{"success": false}`)) +// return +// } +// +// if !admin { +// log.Printf("User %s doesn't have edit rights to %s", user.Id, org.Id) +// resp.WriteHeader(401) +// resp.Write([]byte(`{"success": false}`)) +// return +// } +// +// if tmpData.Image != org.Image { +// org.Image = tmpData.Image +// } +// +// if tmpData.Name != org.Name { +// org.Name = tmpData.Name +// } +// +// if tmpData.Description != org.Description { +// org.Description = tmpData.Description +// } +// +// if len(tmpData.SubscriptionId) > 0 { +// log.Printf("Should update subscription %s with action %s if it exists", tmpData.SubscriptionId, tmpData.Action) +// found := false +// foundIndex := 0 +// for index, sub := range org.Subscriptions { +// if tmpData.SubscriptionId == sub.Reference { +// found = true +// foundIndex = index +// } +// } +// +// if !found { +// log.Printf("Couldn't find sub %s in org %s", tmpData.SubscriptionId, tmpData.OrgId) +// resp.WriteHeader(401) +// resp.Write([]byte(`{"success": false}`)) +// return +// } +// +// if tmpData.Action == "cancel" { +// _, err := sub.Cancel(tmpData.SubscriptionId, nil) +// //log.Printf("Ret: %#v", subReturn) +// if err != nil { +// log.Printf("Failed canceling sub %s.", tmpData.SubscriptionId) +// resp.WriteHeader(401) +// resp.Write([]byte(`{"success": false}`)) +// return +// } else { +// log.Printf("Successfully canceled sub %s in org %s", tmpData.SubscriptionId, tmpData.OrgId) +// timeNow := time.Now().Unix() +// org.Subscriptions[foundIndex].Active = false +// org.Subscriptions[foundIndex].CancellationDate = timeNow +// } +// } +// } +// +// //log.Printf("Org: %#v", org) +// err = setOrg(ctx, *org, org.Id) +// if err != nil { +// log.Printf("User %s doesn't have edit rights to %s", user.Id, org.Id) +// resp.WriteHeader(401) +// resp.Write([]byte(`{"success": false}`)) +// return +// } +// +// resp.WriteHeader(200) +// resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Successfully updated org"}`))) +//} + func initHandlers() { var err error ctx := context.Background() @@ -7990,6 +8364,7 @@ func initHandlers() { r.HandleFunc("/api/v1/apps/authentication", getAppAuthentication).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps/authentication", addAppAuthentication).Methods("PUT", "OPTIONS") + r.HandleFunc("/api/v1/apps/authentication/{appauthId}/config", setAuthenticationConfig).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/authentication/{appauthId}", deleteAppAuthentication).Methods("DELETE", "OPTIONS") @@ -8042,6 +8417,9 @@ func initHandlers() { r.HandleFunc("/api/v1/orgs/{orgId}", handleEditOrg).Methods("POST", "OPTIONS") //r.HandleFunc("/api/v1/orgs/{orgId}", handleEditOrg).Methods("POST", "OPTIONS") + // Docker orborus specific + //r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "OPTIONS") + // Important for email, IDS etc. Create this by: // PS: For cloud, this has to use cloud storage. // https://developer.box.com/reference/get-files-id-content/ @@ -8051,6 +8429,7 @@ func initHandlers() { r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}", handleGetFileMeta).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS") + r.HandleFunc("/api/v1/files", handleGetFiles).Methods("GET", "OPTIONS") http.Handle("/", r) } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 7372121b..0dc28d3a 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -36,6 +36,9 @@ import ( //"google.golang.org/appengine/memcache" //"cloud.google.com/go/firestore" // "google.golang.org/api/option" + + "github.com/patrickmn/go-cache" + "google.golang.org/api/iterator" ) var localBase = "http://localhost:5001" @@ -87,13 +90,13 @@ type SyncFeatures struct { type SyncData struct { Active bool `json:"active" datastore:"active"` - Type string `json:"type" datastore:"type"` - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` - Limit int64 `json:"limit" datastore:"limit"` - StartDate int64 `json:"start_date" datastore:"start_date"` - EndDate int64 `json:"end_date" datastore:"end_date"` - DataCollection int64 `json:"data_collection" datastore:"data_collection"` + Type string `json:"type,omitempty" datastore:"type"` + Name string `json:"name,omitempty" datastore:"name"` + Description string `json:"description,omitempty" datastore:"description"` + Limit int64 `json:"limit,omitempty" datastore:"limit"` + StartDate int64 `json:"start_date,omitempty" datastore:"start_date"` + EndDate int64 `json:"end_date,omitempty" datastore:"end_date"` + DataCollection int64 `json:"data_collection,omitempty" datastore:"data_collection"` } type SyncConfig struct { @@ -103,17 +106,41 @@ type SyncConfig struct { // Role is just used for feedback for a user type Org struct { - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` - Image string `json:"image" datastore:"image,noindex"` - Id string `json:"id" datastore:"id"` - Org string `json:"org" datastore:"org"` - Users []User `json:"users" datastore:"users"` - Role string `json:"role" datastore:"role"` - Roles []string `json:"roles" datastore:"roles"` - CloudSync bool `json:"cloud_sync" datastore:"CloudSync"` - SyncConfig SyncConfig `json:"sync_config" datastore:"sync_config"` - SyncFeatures SyncFeatures `json:"sync_features" datastore:"sync_features"` + Name string `json:"name" datastore:"name"` + Description string `json:"description" datastore:"description"` + Image string `json:"image" datastore:"image,noindex"` + Id string `json:"id" datastore:"id"` + Org string `json:"org" datastore:"org"` + Users []User `json:"users" datastore:"users"` + Role string `json:"role" datastore:"role"` + Roles []string `json:"roles" datastore:"roles"` + CloudSync bool `json:"cloud_sync" datastore:"CloudSync"` + SyncConfig SyncConfig `json:"sync_config" datastore:"sync_config"` + SyncFeatures SyncFeatures `json:"sync_features" datastore:"sync_features"` + Subscriptions []PaymentSubscription `json:"subscriptions" datastore:"subscriptions"` + Created int64 `json:"created" datastore:"created"` + Edited int64 `json:"edited" datastore:"edited"` + Defaults Defaults `json:"defaults" datastore:"defaults"` +} + +type PaymentSubscription struct { + Active bool `json:"active" datastore:"active"` + Startdate int64 `json:"startdate" datastore:"startdate"` + CancellationDate int64 `json:"cancellationdate" datastore:"cancellationdate"` + Enddate int64 `json:"enddate" datastore:"enddate"` + Name string `json:"name" datastore:"name"` + Recurrence string `json:"recurrence" datastore:"recurrence"` + Reference string `json:"reference" datastore:"reference"` + Level string `json:"level" datastore:"level"` + Amount string `json:"amount" datastore:"amount"` + Currency string `json:"currency" datastore:"currency"` +} + +type Defaults struct { + AppDownloadRepo string `json:"app_download_repo" datastore:"app_download_repo"` + AppDownloadBranch string `json:"app_download_branch" datastore:"app_download_branch"` + WorkflowDownloadRepo string `json:"workflow_download_repo" datastore:"workflow_download_repo"` + WorkflowDownloadBranch string `json:"workflow_download_branch" datastore:"workflow_download_branch"` } type AppAuthenticationStorage struct { @@ -126,6 +153,9 @@ type AppAuthenticationStorage struct { WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"` NodeCount int64 `json:"node_count" datastore:"node_count"` OrgId string `json:"org_id" datastore:"org_id"` + Created int64 `json:"created" datastore:"created"` + Edited int64 `json:"edited" datastore:"edited"` + Defined bool `json:"defined" datastore:"defined"` } type AuthenticationUsage struct { @@ -146,6 +176,7 @@ type WorkflowApp struct { Downloaded bool `json:"downloaded" yaml:"downloaded" required:false datastore:"downloaded"` Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` + Invalid bool `json:"invalid" yaml:"invalid" required:false datastore:"invalid"` Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"` Owner string `json:"owner" datastore:"owner" yaml:"owner"` @@ -163,6 +194,9 @@ type WorkflowApp struct { Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"` Tags []string `json:"tags" yaml:"tags" required:false datastore:"activated"` Categories []string `json:"categories" yaml:"categories" required:false datastore:"categories"` + Created int64 `json:"created" datastore:"created"` + Edited int64 `json:"edited" datastore:"edited"` + LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` } type WorkflowAppActionParameter struct { @@ -180,6 +214,12 @@ type WorkflowAppActionParameter struct { Tags []string `json:"tags" datastore:"tags" yaml:"tags"` Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` SkipMulticheck bool `json:"skip_multicheck" datastore:"skip_multicheck" yaml:"skip_multicheck"` + ValueReplace []Valuereplace `json:"value_replace" datastore:"value_replace,noindex" yaml:"value_replace,omitempty"` +} + +type Valuereplace struct { + Key string `json:"key" datastore:"key" yaml:"key"` + Value string `json:"value" datastore:"value" yaml:"value"` } type SchemaDefinition struct { @@ -254,29 +294,29 @@ type Action struct { Errors []string `json:"errors" datastore:"errors"` ID string `json:"id" datastore:"id"` IsValid bool `json:"is_valid" datastore:"is_valid"` - IsStartNode bool `json:"isStartNode" datastore:"isStartNode"` - Sharing bool `json:"sharing" datastore:"sharing"` - PrivateID string `json:"private_id" datastore:"private_id"` - Label string `json:"label" datastore:"label"` - SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` - Environment string `json:"environment" datastore:"environment"` + IsStartNode bool `json:"isStartNode,omitempty" datastore:"isStartNode"` + Sharing bool `json:"sharing,omitempty" datastore:"sharing"` + PrivateID string `json:"private_id,omitempty" datastore:"private_id"` + Label string `json:"label,omitempty" datastore:"label"` + SmallImage string `json:"small_image,omitempty" datastore:"small_image,noindex" required:false yaml:"small_image"` + LargeImage string `json:"large_image,omitempty" datastore:"large_image,noindex" yaml:"large_image" required:false` + Environment string `json:"environment,omitempty" datastore:"environment"` Name string `json:"name" datastore:"name"` Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` ExecutionVariable struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` + Description string `json:"description,omitempty" datastore:"description,noindex"` + ID string `json:"id,omitempty" datastore:"id"` + Name string `json:"name,omitempty" datastore:"name"` + Value string `json:"value,omitempty" datastore:"value,noindex"` } `json:"execution_variable,omitempty" datastore:"execution_variable,omitempty"` Position struct { - X float64 `json:"x" datastore:"x"` - Y float64 `json:"y" datastore:"y"` - } `json:"position"` - Priority int `json:"priority" datastore:"priority"` + X float64 `json:"x,omitempty" datastore:"x"` + Y float64 `json:"y,omitempty" datastore:"y"` + } `json:"position,omitempty"` + Priority int `json:"priority,omitempty" datastore:"priority"` AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` - Example string `json:"example" datastore:"example"` - AuthNotRequired bool `json:"auth_not_required" datastore:"auth_not_required" yaml:"auth_not_required"` + Example string `json:"example,omitempty" datastore:"example"` + AuthNotRequired bool `json:"auth_not_required,omitempty" datastore:"auth_not_required" yaml:"auth_not_required"` } // Added environment for location to execute @@ -339,6 +379,9 @@ type Workflow struct { ExitOnError bool `json:"exit_on_error" datastore:"exit_on_error"` StartFromTop bool `json:"start_from_top" datastore:"start_from_top"` } `json:"configuration,omitempty" datastore:"configuration"` + Created int64 `json:"created" datastore:"created"` + Edited int64 `json:"edited" datastore:"edited"` + LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` Errors []string `json:"errors,omitempty" datastore:"errors"` Tags []string `json:"tags,omitempty" datastore:"tags"` ID string `json:"id" datastore:"id"` @@ -364,6 +407,7 @@ type Workflow struct { Value string `json:"value" datastore:"value,noindex"` } `json:"execution_variables,omitempty" datastore:"execution_variables"` ExecutionEnvironment string `json:"execution_environment" datastore:"execution_environment"` + PreviouslySaved bool `json:"first_save" datastore:"first_save"` } type ActionResult struct { @@ -416,7 +460,7 @@ type AppExecutionExample struct { // This might be... a bit off, but that's fine :) // This might also be stupid, as we want timelines and such // Anyway, these are super basic stupid stats. -func increaseStatisticsField(ctx context.Context, fieldname, id string, amount int64) error { +func increaseStatisticsField(ctx context.Context, fieldname, id string, amount int64, orgId string) error { // 1. Get current stats // 2. Increase field(s) @@ -437,6 +481,7 @@ func increaseStatisticsField(ctx context.Context, fieldname, id string, amount i if strings.Contains(fmt.Sprintf("%s", err), "entity") { statisticsItem = StatisticsItem{ Total: amount, + OrgId: orgId, Fieldname: fieldname, Data: []StatisticsData{ newData, @@ -459,21 +504,23 @@ func increaseStatisticsField(ctx context.Context, fieldname, id string, amount i statisticsItem.Data = append(statisticsItem.Data, newData) // New struct, to not add body, author etc - if _, err := dbclient.Put(ctx, key, &statisticsItem); err != nil { - log.Printf("Error stats to %s: %s", fieldname, err) - return err - } + // FIXME - reintroduce + //if _, err := dbclient.Put(ctx, key, &statisticsItem); err != nil { + // log.Printf("Error stats to %s: %s", fieldname, err) + // return err + //} //log.Printf("Stats: %#v", statisticsItem) return nil } -func setWorkflowQueue(ctx context.Context, executionRequests ExecutionRequestWrapper, id string) error { - key := datastore.NameKey("workflowqueue", id, nil) +func setWorkflowQueue(ctx context.Context, executionRequest ExecutionRequest, env string) error { + orgKey := fmt.Sprintf("workflowqueue-%s", env) + key := datastore.NameKey(orgKey, executionRequest.ExecutionId, nil) // New struct, to not add body, author etc - if _, err := dbclient.Put(ctx, key, &executionRequests); err != nil { + if _, err := dbclient.Put(ctx, key, &executionRequest); err != nil { log.Printf("Error adding workflow queue: %s", err) return err } @@ -481,14 +528,37 @@ func setWorkflowQueue(ctx context.Context, executionRequests ExecutionRequestWra return nil } +// +//func setWorkflowQueue(ctx context.Context, executionRequests ExecutionRequestWrapper, id string) error { +// key := datastore.NameKey("workflowqueue", id, nil) +// +// // New struct, to not add body, author etc +// if _, err := dbclient.Put(ctx, key, &executionRequests); err != nil { +// log.Printf("Error adding workflow queue: %s", err) +// return err +// } +// +// return nil +//} + func getWorkflowQueue(ctx context.Context, id string) (ExecutionRequestWrapper, error) { - key := datastore.NameKey("workflowqueue", id, nil) - workflows := ExecutionRequestWrapper{} - if err := dbclient.Get(ctx, key, &workflows); err != nil { + orgId := fmt.Sprintf("workflowqueue-%s", id) + q := datastore.NewQuery(orgId).Limit(10) + executions := []ExecutionRequest{} + _, err := dbclient.GetAll(ctx, q, &executions) + if err != nil { return ExecutionRequestWrapper{}, err } - return workflows, nil + return ExecutionRequestWrapper{Data: executions}, nil + + //key := datastore.NameKey("workflowqueue", id, nil) + //executions := ExecutionRequestWrapper{} + //if err := dbclient.Get(ctx, key, &workflows); err != nil { + // return ExecutionRequestWrapper{}, err + //} + + //return workflows, nil } //func setWorkflowqueuetest(id string) { @@ -624,9 +694,9 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque } if len(executionRequests.Data) == 0 { - log.Printf("No requests to fix. Why did this request occur?") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Some error"}`))) + log.Printf("[INFO] No requests to handle from queue") + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Nothing in queue"}`))) return } @@ -650,41 +720,47 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque } if len(removeExecutionRequests.Data) == 0 { - log.Printf("No requests to fix remove") + log.Printf("No requests to fix remove from DB") resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Some removal error"}`))) return } // remove items from DB - var newExecutionRequests ExecutionRequestWrapper - for _, execution := range executionRequests.Data { - found := false - for _, removeExecution := range removeExecutionRequests.Data { - if removeExecution.ExecutionId == execution.ExecutionId && removeExecution.WorkflowId == execution.WorkflowId { - found = true - break - } - } - - if !found { - newExecutionRequests.Data = append(newExecutionRequests.Data, execution) - } + parsedId := fmt.Sprintf("workflowqueue-%s", id) + ids := []string{} + for _, execution := range removeExecutionRequests.Data { + ids = append(ids, execution.ExecutionId) } + err = DeleteKeys(ctx, parsedId, ids) + if err != nil { + log.Printf("[ERROR] Failed deleting %d execution keys for org %s", len(ids), id) + } else { + //log.Printf("[INFO] Deleted %d keys from org %s", len(ids), parsedId) + } + + //var newExecutionRequests ExecutionRequestWrapper + //for _, execution := range executionRequests.Data { + // found := false + // for _, removeExecution := range removeExecutionRequests.Data { + // if removeExecution.ExecutionId == execution.ExecutionId && removeExecution.WorkflowId == execution.WorkflowId { + // found = true + // break + // } + // } + + // if !found { + // newExecutionRequests.Data = append(newExecutionRequests.Data, execution) + // } + //} + // Push only the remaining to the DB (remove) - if len(executionRequests.Data) != len(newExecutionRequests.Data) { - err := setWorkflowQueue(ctx, newExecutionRequests, id) - if err != nil { - log.Printf("Fail: %s", err) - } - } - - //newjson, err := json.Marshal(removeExecutionRequests) - //if err != nil { - // resp.WriteHeader(401) - // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflow execution"}`))) - // return + //if len(executionRequests.Data) != len(newExecutionRequests.Data) { + // err := setWorkflowQueue(ctx, newExecutionRequests, id) + // if err != nil { + // log.Printf("Fail: %s", err) + // } //} resp.WriteHeader(200) @@ -760,7 +836,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflowExecution, err := getWorkflowExecution(ctx, actionResult.ExecutionId) if err != nil { - log.Printf("Failed getting execution (streamresult) %s: %s", actionResult.ExecutionId, err) + //log.Printf("Failed getting execution (streamresult) %s: %s", actionResult.ExecutionId, err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key or execution_id might not exist."}`))) return @@ -835,6 +911,57 @@ func findChildNodes(workflowExecution WorkflowExecution, nodeId string) []string return newNodes } +// Checks if data is sent from Worker >0.8.51, which sends a full execution +// instead of individial results +func validateNewWorkerExecution(body []byte) error { + //type WorkflowExecution struct { + //} + + ctx := context.Background() + var execution WorkflowExecution + err := json.Unmarshal(body, &execution) + if err != nil { + log.Printf("[WARNING] Failed execution unmarshaling: %s", err) + return err + } + + baseExecution, err := getWorkflowExecution(ctx, execution.ExecutionId) + if err != nil { + log.Printf("[ERROR] Failed getting execution (workflowqueue) %s: %s", execution.ExecutionId, err) + return err + } + + if baseExecution.Authorization != execution.Authorization { + return errors.New("Bad authorization when validating execution") + } + + // used to validate if it's actually the right marshal + if len(baseExecution.Workflow.Actions) != len(execution.Workflow.Actions) { + return errors.New(fmt.Sprintf("Bad length of actions (probably normal app): %d", len(execution.Workflow.Actions))) + } + + if len(baseExecution.Workflow.Triggers) != len(execution.Workflow.Triggers) { + return errors.New(fmt.Sprintf("Bad length of trigger: %d (probably normal app)", len(execution.Workflow.Triggers))) + } + + // FIXME: Add extra here + //executionLength := len(baseExecution.Workflow.Actions) + //if executionLength != len(execution.Results) { + // return errors.New(fmt.Sprintf("Bad length of actions vs results: want: %d have: %d", executionLength, len(execution.Results))) + //} + + //log.Printf("\n\nSHOULD SET BACKEND DATA FOR EXEC \n\n") + err = setWorkflowExecution(ctx, execution, true) + if err == nil { + log.Printf("[INFO] Set workflowexecution based on new worker (>0.8.53) for execution %s", baseExecution.ExecutionId) + //log.Printf("[INFO] Successfully set the execution to wait.") + } else { + log.Printf("[WARNING] Failed to set the execution to wait.") + } + + return nil +} + func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -849,6 +976,16 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { return } + //log.Printf("Actionresult unmarshal: %s", string(body)) + err = validateNewWorkerExecution(body) + if err == nil { + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Success"}`))) + return + } else { + //log.Printf("[WARNING] Failed to handle new execution variant: %s", err) + } + var actionResult ActionResult err = json.Unmarshal(body, &actionResult) if err != nil { @@ -867,14 +1004,14 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflowExecution, err := getWorkflowExecution(ctx, actionResult.ExecutionId) if err != nil { - log.Printf("Failed getting execution (workflowqueue) %s: %s", actionResult.ExecutionId, err) + log.Printf("[ERROR] Failed getting execution (workflowqueue) %s: %s", actionResult.ExecutionId, err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution ID %s because it doesn't exist."}`, actionResult.ExecutionId))) return } if workflowExecution.Authorization != actionResult.Authorization { - log.Printf("Bad authorization key when updating node (workflowQueue) %s. Want: %s, Have: %s", actionResult.ExecutionId, workflowExecution.Authorization, actionResult.Authorization) + log.Printf("[INFO] Bad authorization key when updating node (workflowQueue) %s. Want: %s, Have: %s", actionResult.ExecutionId, workflowExecution.Authorization, actionResult.Authorization) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key"}`))) return @@ -882,7 +1019,6 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { if workflowExecution.Status == "FINISHED" { log.Printf("Workflowexecution is already FINISHED. No further action can be taken") - resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is already finished because of %s with status %s"}`, workflowExecution.LastNode, workflowExecution.Status))) return @@ -925,9 +1061,9 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { actionResult.Result = fmt.Sprintf("Cloud error: %s", err) workflowExecution.Results = append(workflowExecution.Results, actionResult) workflowExecution.Status = "ABORTED" - err = setWorkflowExecution(ctx, *workflowExecution) + err = setWorkflowExecution(ctx, *workflowExecution, true) if err != nil { - log.Printf("Failed ") + log.Printf("Failed to set execution during wait") } else { log.Printf("Successfully set the execution to waiting.") } @@ -943,7 +1079,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { workflowExecution.Results = append(workflowExecution.Results, actionResult) workflowExecution.Status = actionResult.Status - err = setWorkflowExecution(ctx, *workflowExecution) + err = setWorkflowExecution(ctx, *workflowExecution, true) if err != nil { log.Printf("Failed ") } else { @@ -960,38 +1096,50 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { // Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult ActionResult, resp http.ResponseWriter) { // Should start a tx for the execution here - tx, err := dbclient.NewTransaction(ctx) + workflowExecution, err := getWorkflowExecution(ctx, workflowExecutionId) if err != nil { - log.Printf("client.NewTransaction: %v", err) + log.Printf("[ERROR] Failed getting execution cache: %s", err) resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed creating transaction"}`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution"}`))) return } + resultLength := len(workflowExecution.Results) + dbSave := false + setExecution := true + //tx, err := dbclient.NewTransaction(ctx) + //if err != nil { + // log.Printf("client.NewTransaction: %v", err) + // resp.WriteHeader(401) + // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed creating transaction"}`))) + // return + //} - key := datastore.NameKey("workflowexecution", workflowExecutionId, nil) - workflowExecution := &WorkflowExecution{} - if err := tx.Get(key, workflowExecution); err != nil { - log.Printf("tx.Get bug: %v", err) - tx.Rollback() - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting the workflow key"}`))) - return - } + //key := datastore.NameKey("workflowexecution", workflowExecutionId, nil) + //workflowExecution := &WorkflowExecution{} + //if err := tx.Get(key, workflowExecution); err != nil { + // log.Printf("[ERROR] tx.Get bug: %v", err) + // tx.Rollback() + // resp.WriteHeader(401) + // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting the workflow key"}`))) + // return + //} if actionResult.Status == "ABORTED" || actionResult.Status == "FAILURE" { - log.Printf("Actionresult is %s. Should set workflowExecution and exit all running functions", actionResult.Status) + dbSave = true newResults := []ActionResult{} childNodes := []string{} if workflowExecution.Workflow.Configuration.ExitOnError { + log.Printf("[WARNING] Actionresult is %s for node %s in %s. Should set workflowExecution and exit all running functions", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) workflowExecution.Status = actionResult.Status workflowExecution.LastNode = actionResult.Action.ID // Find underlying nodes and add them } else { + log.Printf("[WARNING] Actionresult is %s for node %s in %s. Continuing anyway because of workflow configuration.", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) // Finds ALL childnodes to set them to SKIPPED childNodes = findChildNodes(*workflowExecution, actionResult.Action.ID) // Remove duplicates - log.Printf("CHILD NODES: %d", len(childNodes)) + //log.Printf("CHILD NODES: %d", len(childNodes)) for _, nodeId := range childNodes { if nodeId == actionResult.Action.ID { continue @@ -1012,40 +1160,57 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl continue } - // Check parents are done here. Only add it IF all parents are skipped - skipNodeAdd := false - for _, branch := range workflowExecution.Workflow.Branches { - if branch.DestinationID == nodeId { - // If the branch's source node is NOT in childNodes, it's not a skipped parent - sourceNodeFound := false - for _, item := range childNodes { - if item == branch.SourceID { - sourceNodeFound = true - break - } - } - - if !sourceNodeFound { - log.Printf("Not setting node %s to SKIPPED", nodeId) - skipNodeAdd = true - break - } + resultExists := false + for _, result := range workflowExecution.Results { + if result.Action.ID == curAction.ID { + resultExists = true + break } } - if !skipNodeAdd { - newResult := ActionResult{ - Action: curAction, - ExecutionId: actionResult.ExecutionId, - Authorization: actionResult.Authorization, - Result: "Skipped because of previous node", - StartedAt: 0, - CompletedAt: 0, - Status: "SKIPPED", + if !resultExists { + // Check parents are done here. Only add it IF all parents are skipped + skipNodeAdd := false + for _, branch := range workflowExecution.Workflow.Branches { + if branch.DestinationID == nodeId { + // If the branch's source node is NOT in childNodes, it's not a skipped parent + sourceNodeFound := false + for _, item := range childNodes { + if item == branch.SourceID { + sourceNodeFound = true + break + } + } + + if !sourceNodeFound { + log.Printf("Not setting node %s to SKIPPED", nodeId) + skipNodeAdd = true + break + } + } } - newResults = append(newResults, newResult) - increaseStatisticsField(ctx, "workflow_execution_actions_skipped", workflowExecution.Workflow.ID, 1) + if !skipNodeAdd { + newAction := Action{ + AppName: curAction.AppName, + AppVersion: curAction.AppVersion, + Label: curAction.Label, + Name: curAction.Name, + ID: curAction.ID, + } + newResult := ActionResult{ + Action: newAction, + ExecutionId: actionResult.ExecutionId, + Authorization: actionResult.Authorization, + Result: "Skipped because of previous node", + StartedAt: 0, + CompletedAt: 0, + Status: "SKIPPED", + } + + newResults = append(newResults, newResult) + //increaseStatisticsField(ctx, "workflow_execution_actions_skipped", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) + } } } } @@ -1054,9 +1219,13 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl lastResult := "" // type ActionResult struct { for _, result := range workflowExecution.Results { + if actionResult.Action.ID == result.Action.ID { + continue + } + if result.Status == "EXECUTING" { result.Status = actionResult.Status - result.Result = "Aborted because of error in another node" + result.Result = "Aborted because of error in another node (2)" } if len(result.Result) > 0 { @@ -1070,15 +1239,15 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl workflowExecution.Results = newResults if workflowExecution.Status == "ABORTED" { - err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1) - if err != nil { - log.Printf("Failed to increase aborted execution stats: %s", err) - } + //err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) + //if err != nil { + // log.Printf("Failed to increase aborted execution stats: %s", err) + //} } else if workflowExecution.Status == "FAILURE" { - err = increaseStatisticsField(ctx, "workflow_executions_failure", workflowExecution.Workflow.ID, 1) - if err != nil { - log.Printf("Failed to increase failure execution stats: %s", err) - } + //err = increaseStatisticsField(ctx, "workflow_executions_failure", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) + //if err != nil { + // log.Printf("Failed to increase failure execution stats: %s", err) + //} } } @@ -1087,17 +1256,24 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Find the appropriate action if len(workflowExecution.Results) > 0 { // FIXME + skip := false found := false outerindex := 0 for index, item := range workflowExecution.Results { if item.Action.ID == actionResult.Action.ID { found = true + if item.Status == actionResult.Status { + skip = true + } + outerindex = index break } } - if found { + if skip { + //log.Printf("Both are %s. Skipping this node", item.Status) + } else if found { // If result exists and execution variable exists, update execution value //log.Printf("Exec var backend: %s", workflowExecution.Results[outerindex].Action.ExecutionVariable.Name) actionVarName := workflowExecution.Results[outerindex].Action.ExecutionVariable.Name @@ -1113,14 +1289,14 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } } - log.Printf("[INFO] Updating %s in %s from %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, workflowExecution.Results[outerindex].Status, actionResult.Status) + log.Printf("[INFO] Updating %s in workflow %s from %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, workflowExecution.Results[outerindex].Status, actionResult.Status) workflowExecution.Results[outerindex] = actionResult } else { - log.Printf("[INFO] Setting value of %s in %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) + log.Printf("[INFO] Setting value of %s in workflow %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) workflowExecution.Results = append(workflowExecution.Results, actionResult) } } else { - log.Printf("[INFO] Setting value of %s in %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) + log.Printf("[INFO] Setting value of %s in workflow %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) workflowExecution.Results = append(workflowExecution.Results, actionResult) } @@ -1150,16 +1326,19 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } extraInputs := 0 - for _, result := range workflowExecution.Results { - if result.Action.Name == "User Input" && result.Action.AppName == "User Input" { - log.Printf("Found User Input node - prepare cloud?") + for _, trigger := range workflowExecution.Workflow.Triggers { + if trigger.Name == "User Input" && trigger.AppName == "User Input" { + extraInputs += 1 + } else if trigger.Name == "Shuffle Workflow" && trigger.AppName == "Shuffle Workflow" { extraInputs += 1 } } - //log.Printf("LENGTH: %d - %d", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)) + //log.Printf("EXTRA: %d", extraInputs) + //log.Printf("LENGTH: %d - %d", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extraInputs) if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extraInputs { + //log.Printf("\nIN HERE WITH RESULTS %d vs %d\n", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extraInputs) finished := true lastResult := "" @@ -1203,6 +1382,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl _ = skippedNodes if finished { + dbSave = true log.Printf("[INFO] Execution of %s finished.", workflowExecution.ExecutionId) //log.Println("Might be finished based on length of results and everything being SUCCESS or FINISHED - VERIFY THIS. Setting status to finished.") @@ -1213,10 +1393,10 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl workflowExecution.LastNode = actionResult.Action.ID } - err = increaseStatisticsField(ctx, "workflow_executions_success", workflowExecution.Workflow.ID, 1) - if err != nil { - log.Printf("Failed to increase success execution stats: %s", err) - } + //err = increaseStatisticsField(ctx, "workflow_executions_success", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) + //if err != nil { + // log.Printf("Failed to increase success execution stats: %s", err) + //} // Handles extra statistics stuff when it's done // Does autocomplete magic with JSON @@ -1234,6 +1414,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl tmpJson, err := json.Marshal(workflowExecution) if err == nil { if len(tmpJson) >= 1048487 { + dbSave = true log.Printf("[ERROR] Result length is too long! Need to reduce result size") // Result string `json:"result" datastore:"result,noindex"` @@ -1252,37 +1433,82 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } } - // Transactions: https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-go - // Prevents timing issues - //ExecutionId - if _, err := tx.Put(key, workflowExecution); err != nil { - tx.Rollback() - log.Printf("[ERROR] tx.Put bug: %v", err) + // Validating that action results hasn't changed + // Handled using cachhing, so actually pretty fast + cacheKey := fmt.Sprintf("workflowexecution-%s", workflowExecution.ExecutionId) + if value, found := requestCache.Get(cacheKey); found { + parsedValue := value.(*WorkflowExecution) + if len(parsedValue.Results) > 0 && len(parsedValue.Results) != resultLength { + setExecution = false + if attempts > 5 { + //log.Printf("\n\nSkipping execution input - %d vs %d. Attempts: (%d)\n\n", len(parsedValue.Results), resultLength, attempts) + } - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution actionresult: %s"}`, err))) - return + attempts += 1 + if len(workflowExecution.Results) <= len(workflowExecution.Workflow.Actions) { + runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp) + return + } + } } - if _, err = tx.Commit(); err != nil { - if attempts >= 5 { - log.Printf("[ERROR] QUITTING: tx.Commit %d: %v", attempts, err) - tx.Rollback() - workflowExecution.Status = "ABORTED" - setWorkflowExecution(ctx, *workflowExecution) - + if setExecution || workflowExecution.Status == "FINISHED" || workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" { + err = setWorkflowExecution(ctx, *workflowExecution, dbSave) + if err != nil { resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution actionresult: %s"}`, err))) return } - - log.Printf("[WARNING] tx.Commit %d: %v", attempts, err) - - attempts += 1 - runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp) - return + } else { + log.Printf("Skipping setexec with status %s", workflowExecution.Status) } + //ExecutionId + // Transactions: https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-go + // Prevents timing issues + //if _, err := tx.Put(key, workflowExecution); err != nil { + // log.Printf("[ERROR] tx.Put error: %v", err) + // err = tx.Rollback() + // if err != nil { + // log.Printf("[ERROR] Rollback error (3): %s", err) + // } + + // resp.WriteHeader(401) + // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution actionresult: %s"}`, err))) + // return + //} + + //if _, err = tx.Commit(); err != nil { + // err = tx.Rollback() + // if err != nil { + // log.Printf("[ERROR] Rollback error expected ? (1): %s", err) + // } + + // if attempts >= 7 { + // log.Printf("[ERROR] QUITTING: tx.Commit %d: %v", attempts, err) + + // workflowExecution.Status = "ABORTED" + // setWorkflowExecution(ctx, *workflowExecution, true) + + // resp.WriteHeader(401) + // resp.Write([]byte(`{"success": false}`)) + // return + // } + + // if attempts > 3 { + // log.Printf("[WARNING] tx.Commit %d: %v", attempts, err) + // } + + // attempts += 1 + // runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp) + // return + //} else { + // //if grpc.Code(err) == codes.Aborted { + // // return nil, ErrConcurrentTransaction + // //} + // //t.id = nil // mark the transaction as expired + //} + resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } @@ -1367,7 +1593,7 @@ func handleExecutionStatistics(execution WorkflowExecution) { log.Printf("[INFO] Added %d exampleresults to backend", successful) } else { - log.Printf("[INFO] No example results necessary to be added for execution %s", execution.ExecutionId) + //log.Printf("[INFO] No example results necessary to be added for execution %s", execution.ExecutionId) } } @@ -1406,13 +1632,26 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) { log.Printf("[INFO] Getting workflows (ADMIN) for organization %s", user.ActiveOrg.Id) } + q = q.Order("-edited") + var workflows []Workflow _, err = dbclient.GetAll(ctx, q, &workflows) if err != nil { - log.Printf("Failed getting workflows for user %s: %s", user.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return + if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { + q = q.Limit(35) + _, err = dbclient.GetAll(ctx, q, &workflows) + if err != nil { + log.Printf("Failed getting workflows for user %s: %s", user.Username, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + } else { + log.Printf("Failed getting workflows for user %s: %s", user.Username, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } } if len(workflows) == 0 { @@ -1485,13 +1724,13 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { user.ActiveOrg.Users = []User{} workflow.ExecutingOrg = user.ActiveOrg workflow.OrgId = user.ActiveOrg.Id + //log.Printf("TRIGGERS: %d", len(workflow.Triggers)) ctx := context.Background() - log.Printf("Saved new workflow %s with name %s", workflow.ID, workflow.Name) - err = increaseStatisticsField(ctx, "total_workflows", workflow.ID, 1) - if err != nil { - log.Printf("Failed to increase total workflows stats: %s", err) - } + //err = increaseStatisticsField(ctx, "total_workflows", workflow.ID, 1, workflow.OrgId) + //if err != nil { + // log.Printf("Failed to increase total workflows stats: %s", err) + //} if len(workflow.Actions) == 0 { workflow.Actions = []Action{} @@ -1513,15 +1752,16 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { action.IsValid = true } + action.LargeImage = "" newActions = append(newActions, action) } // Initialized without functions = adding a hello world node. if len(newActions) == 0 { - log.Printf("APPENDING NEW APP FOR NEW WORKFLOW") + //log.Printf("APPENDING NEW APP FOR NEW WORKFLOW") // Adds the Testing app if it's a new workflow - workflowapps, err := getAllWorkflowApps(ctx) + workflowapps, err := getAllWorkflowApps(ctx, 500) if err == nil { // FIXME: Add real env envName := "Shuffle" @@ -1545,8 +1785,8 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { Environment: envName, Parameters: []WorkflowAppActionParameter{}, Position: struct { - X float64 "json:\"x\" datastore:\"x\"" - Y float64 "json:\"y\" datastore:\"y\"" + X float64 "json:\"x,omitempty\" datastore:\"x\"" + Y float64 "json:\"y,omitempty\" datastore:\"y\"" }{X: 449.5, Y: 446}, Priority: 0, Errors: []string{}, @@ -1566,18 +1806,71 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { } } } else { - log.Printf("Has %d actions already", len(newActions)) + log.Printf("[INFO] Has %d actions already", len(newActions)) + // FIXME: Check if they require authentication and if they exist locally + //log.Printf("\n\nSHOULD VALIDATE AUTHENTICATION") + //AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"` + //allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) + //if err == nil { + // log.Printf("AUTH: %#v", allAuths) + // for _, action := range newActions { + // log.Printf("ACTION: %#v", action) + // } + //} } + workflow.Actions = []Action{} for _, item := range workflow.Actions { + oldId := item.ID + sourceIndexes := []int{} + destinationIndexes := []int{} + for branchIndex, branch := range workflow.Branches { + if branch.SourceID == oldId { + sourceIndexes = append(sourceIndexes, branchIndex) + } + + if branch.DestinationID == oldId { + destinationIndexes = append(destinationIndexes, branchIndex) + } + } + item.ID = uuid.NewV4().String() + for _, index := range sourceIndexes { + workflow.Branches[index].SourceID = item.ID + } + + for _, index := range destinationIndexes { + workflow.Branches[index].DestinationID = item.ID + } + newActions = append(newActions, item) } newTriggers := []Trigger{} for _, item := range workflow.Triggers { - item.Status = "uninitialized" + oldId := item.ID + sourceIndexes := []int{} + destinationIndexes := []int{} + for branchIndex, branch := range workflow.Branches { + if branch.SourceID == oldId { + sourceIndexes = append(sourceIndexes, branchIndex) + } + + if branch.DestinationID == oldId { + destinationIndexes = append(destinationIndexes, branchIndex) + } + } + item.ID = uuid.NewV4().String() + for _, index := range sourceIndexes { + workflow.Branches[index].SourceID = item.ID + } + + for _, index := range destinationIndexes { + workflow.Branches[index].DestinationID = item.ID + } + + item.Status = "uninitialized" newTriggers = append(newTriggers, item) } @@ -1587,11 +1880,13 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { newSchedules = append(newSchedules, item) } + timeNow := int64(time.Now().Unix()) workflow.Actions = newActions workflow.Triggers = newTriggers workflow.Schedules = newSchedules workflow.IsValid = true workflow.Configuration.ExitOnError = false + workflow.Created = timeNow workflowjson, err := json.Marshal(workflow) if err != nil { @@ -1609,6 +1904,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { return } + log.Printf("Saved new workflow %s with name %s", workflow.ID, workflow.Name) //memcacheName := fmt.Sprintf("%s_workflows", user.Username) //memcache.Delete(ctx, memcacheName) @@ -1686,7 +1982,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { } } - err = increaseStatisticsField(ctx, "total_workflow_triggers", workflow.ID, -1) + err = increaseStatisticsField(ctx, "total_workflow_triggers", workflow.ID, -1, workflow.OrgId) if err != nil { log.Printf("Failed to increase total workflows: %s", err) } @@ -1702,7 +1998,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { return } - err = increaseStatisticsField(ctx, "total_workflows", fileId, -1) + err = increaseStatisticsField(ctx, "total_workflows", fileId, -1, workflow.OrgId) if err != nil { log.Printf("Failed to increase total workflows: %s", err) } @@ -1841,9 +2137,10 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { err = json.Unmarshal([]byte(body), &workflow) //log.Printf(string(body)) if err != nil { - log.Printf("Failed workflow unmarshaling: %s", err) + log.Printf(string(body)) + log.Printf("[ERROR] Failed workflow unmarshaling: %s", err) resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } @@ -1899,8 +2196,143 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { newActions = append(newActions, action) } - workflow.Actions = newActions + if !workflow.PreviouslySaved { + log.Printf("[WORKFLOW INIT] NOT PREVIOUSLY SAVED - SET ACTION AUTH!") + //AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"` + workflowapps, apperr := getAllWorkflowApps(ctx, 500) + allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) + if err == nil && len(workflowapps) > 0 && apperr == nil { + log.Printf("Setting actions") + actionFixing := []Action{} + appsAdded := []string{} + for _, action := range newActions { + setAuthentication := false + if len(action.AuthenticationId) > 0 { + //found := false + authenticationFound := false + for _, auth := range allAuths { + if auth.Id == action.AuthenticationId { + authenticationFound = true + break + } + } + + if !authenticationFound { + setAuthentication = true + } + } else { + // FIXME: 1. Validate if the app needs auth + // 1. Validate if auth for the app exists + // var appAuth AppAuthenticationStorage + setAuthentication = true + + //App WorkflowApp `json:"app" datastore:"app,noindex"` + } + + if setAuthentication { + authSet := false + for _, auth := range allAuths { + if !auth.Active { + continue + } + + if !auth.Defined { + continue + } + + if auth.App.Name == action.AppName { + //log.Printf("FOUND AUTH FOR APP %s: %s", auth.App.Name, auth.Id) + action.AuthenticationId = auth.Id + authSet = true + break + } + } + + // FIXME: Only o this IF there isn't another one for the app already + if !authSet { + //log.Printf("Validate if the app NEEDS auth or not") + outerapp := WorkflowApp{} + for _, app := range workflowapps { + if app.Name == action.AppName { + outerapp = app + break + } + } + + if len(outerapp.ID) > 0 && outerapp.Authentication.Required { + found := false + for _, auth := range allAuths { + if auth.App.ID == outerapp.ID { + found = true + break + } + } + + for _, added := range appsAdded { + if outerapp.ID == added { + found = true + } + } + + // FIXME: Add app auth + if !found { + timeNow := int64(time.Now().Unix()) + authFields := []AuthenticationStore{} + for _, param := range outerapp.Authentication.Parameters { + authFields = append(authFields, AuthenticationStore{ + Key: param.Name, + Value: "", + }) + } + + appAuth := AppAuthenticationStorage{ + Active: true, + Label: fmt.Sprintf("default_%s", outerapp.Name), + Id: uuid.NewV4().String(), + App: outerapp, + Fields: authFields, + Usage: []AuthenticationUsage{}, + WorkflowCount: 0, + NodeCount: 0, + OrgId: user.ActiveOrg.Id, + Created: timeNow, + Edited: timeNow, + } + + err = setWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id) + if err != nil { + log.Printf("Failed setting appauth for with name %s", appAuth.Label) + } else { + appsAdded = append(appsAdded, outerapp.ID) + } + } + + action.Errors = append(action.Errors, "Requires authentication") + action.IsValid = false + workflow.IsValid = false + } + + //outerapp.Authentication.Required + // Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"` + //workflowapps, apperr := getAllWorkflowApps(ctx, 100) + } + } + + actionFixing = append(actionFixing, action) + } + + newActions = actionFixing + } else { + log.Printf("Err: %s - %s", err, apperr) + //workflowapps, apperr := getAllWorkflowApps(ctx, 100) + //allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) + } + + workflow.PreviouslySaved = true + } + + workflow.Actions = newActions newTriggers := []Trigger{} for _, trigger := range workflow.Triggers { log.Printf("Trigger %s: %s", trigger.TriggerType, trigger.Status) @@ -1914,6 +2346,43 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } else if schedule.Id == "" { trigger.Status = "stopped" } + } else if trigger.TriggerType == "SUBFLOW" { + for index, param := range trigger.Parameters { + if len(param.Value) == 0 && param.Name != "argument" { + log.Printf("Param: %#v", param) + if param.Name == "user_apikey" { + apikey := "" + if len(user.ApiKey) > 0 { + apikey = user.ApiKey + } else { + user, err = generateApikey(ctx, user) + if err != nil { + workflow.IsValid = false + workflow.Errors = []string{"Trigger is missing a parameter: %s", param.Name} + + log.Printf("No type specified for user input node") + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Trigger %s is missing the parameter %s"}`, trigger.Label, param.Name))) + return + } + + apikey = user.ApiKey + } + + log.Printf("[INFO] Set apikey in subflow trigger for user during save") + trigger.Parameters[index].Value = apikey + } else { + + workflow.IsValid = false + workflow.Errors = []string{"Trigger is missing a parameter: %s", param.Name} + + log.Printf("No type specified for user input node") + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Trigger %s is missing the parameter %s"}`, trigger.Label, param.Name))) + return + } + } + } } else if trigger.TriggerType == "WEBHOOK" && trigger.Status != "uninitialized" { hook, err := getHook(ctx, trigger.ID) if err != nil { @@ -2001,11 +2470,11 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } if len(workflow.ExecutionVariables) > 0 { - log.Printf("Found %d execution variable(s)", len(workflow.ExecutionVariables)) + log.Printf("[INFO] Found %d execution variable(s)", len(workflow.ExecutionVariables)) } if len(workflow.WorkflowVariables) > 0 { - log.Printf("Found %d workflow variable(s)", len(workflow.WorkflowVariables)) + log.Printf("[INFO] Found %d workflow variable(s)", len(workflow.WorkflowVariables)) } // FIXME - do actual checks ROFL @@ -2071,7 +2540,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { //if item, err := memcache.Get(ctx, memcacheName); err == memcache.ErrCacheMiss { // // Not in cache // log.Printf("Apps not in cache.") - workflowApps, err = getAllWorkflowApps(ctx) + workflowApps, err = getAllWorkflowApps(ctx, 100) if err != nil { log.Printf("Failed getting all workflow apps from database: %s", err) resp.WriteHeader(401) @@ -2198,9 +2667,9 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // Check to see if the action is valid if curappaction.Name != action.Name { - log.Printf("Appaction %s doesn't exist.", action.Name) + log.Printf("[ERROR] Action %s in app %s doesn't exist.", action.Name, curapp.Name) resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Action %s in app %s doesn't exist"}`, action.Name, curapp.Name))) return } @@ -2253,7 +2722,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { workflow.Actions = newActions workflow.IsValid = true - log.Printf("Tags: %#v", workflow.Tags) + log.Printf("[INFO] Tags: %#v", workflow.Tags) // FIXME: Is this too drastic? May lead to issues in the future. // Should maybe make a copy for the old org. @@ -2274,7 +2743,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { totalOldActions := len(tmpworkflow.Actions) totalNewActions := len(workflow.Actions) - err = increaseStatisticsField(ctx, "total_workflow_actions", workflow.ID, int64(totalNewActions-totalOldActions)) + err = increaseStatisticsField(ctx, "total_workflow_actions", workflow.ID, int64(totalNewActions-totalOldActions), workflow.OrgId) if err != nil { log.Printf("Failed to change total actions data: %s", err) } @@ -2289,7 +2758,9 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { Errors: workflow.Errors, } - log.Printf("Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId) + cacheKey := fmt.Sprintf("workflowapps-sorted") + requestCache.Delete(cacheKey) + log.Printf("[INFO] Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId) resp.WriteHeader(200) newBody, err := json.Marshal(returndata) if err != nil { @@ -2369,7 +2840,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflowExecution, err := getWorkflowExecution(ctx, executionId) if err != nil { - log.Printf("Failed getting execution (abort) %s: %s", executionId, err) + log.Printf("[ERROR] Failed getting execution (abort) %s: %s", executionId, err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution ID %s because it doesn't exist (abort)."}`, executionId))) return @@ -2402,7 +2873,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { return } } else { - log.Printf("[INFO] API key to abort/finish execution %s is correct.", executionId) + //log.Printf("[INFO] API key to abort/finish execution %s is correct.", executionId) } if workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" || workflowExecution.Status == "FINISHED" { @@ -2423,7 +2894,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { for _, result := range workflowExecution.Results { if result.Status == "EXECUTING" { result.Status = "ABORTED" - result.Result = "Aborted because of error in another node" + result.Result = "Aborted because of error in another node (1)" } if len(result.Result) > 0 { @@ -2438,7 +2909,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { workflowExecution.Result = lastResult } - err = setWorkflowExecution(ctx, *workflowExecution) + err = setWorkflowExecution(ctx, *workflowExecution, true) if err != nil { log.Printf("Error saving workflow execution for updates when aborting %s: %s", topic, err) resp.WriteHeader(401) @@ -2446,7 +2917,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { return } - err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1) + err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) if err != nil { log.Printf("Failed to increase aborted execution stats: %s", err) } @@ -2524,13 +2995,35 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if len(workflow.Actions) == 0 { workflow.Actions = []Action{} + } else { + newactions := []Action{} + for _, action := range workflow.Actions { + action.LargeImage = "" + action.SmallImage = "" + newactions = append(newactions, action) + //log.Printf("ACTION: %#v", action) + } + + workflow.Actions = newactions } + if len(workflow.Branches) == 0 { workflow.Branches = []Branch{} } if len(workflow.Triggers) == 0 { workflow.Triggers = []Trigger{} + } else { + newtriggers := []Trigger{} + for _, trigger := range workflow.Triggers { + trigger.LargeImage = "" + trigger.SmallImage = "" + newtriggers = append(newtriggers, trigger) + //log.Printf("ACTION: %#v", trigger) + } + + workflow.Triggers = newtriggers } + if len(workflow.Errors) == 0 { workflow.Errors = []string{} } @@ -2558,16 +3051,19 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if request.Method == "POST" { body, err := ioutil.ReadAll(request.Body) if err != nil { - log.Printf("Failed request POST read: %s", err) + log.Printf("[ERROR] Failed request POST read: %s", err) return WorkflowExecution{}, "Failed getting body", err } // This one doesn't really matter. - log.Printf("Running POST execution with data %s", body) + log.Printf("[INFO] Running POST execution with body of length %d", len(string(body))) + if len(string(body)) < 50 { + log.Printf("Body: %s", string(body)) + } var execution ExecutionRequest err = json.Unmarshal(body, &execution) if err != nil { - log.Printf("Failed execution POST unmarshaling - continuing anyway: %s", err) + log.Printf("[WARNING] Failed execution POST unmarshaling - continuing anyway: %s", err) //return WorkflowExecution{}, "", err } @@ -2662,7 +3158,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } oldExecution.Results = newResults - err = setWorkflowExecution(ctx, *oldExecution) + err = setWorkflowExecution(ctx, *oldExecution, true) if err != nil { log.Printf("Error saving workflow execution actionresult setting: %s", err) return WorkflowExecution{}, fmt.Sprintf("Failed setting workflowexecution actionresult in execution: %s", err), err @@ -2770,7 +3266,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf allAuths := []AppAuthenticationStorage{} for _, action := range workflowExecution.Workflow.Actions { - action.LargeImage = "" + //action.LargeImage = "" if action.ID == workflowExecution.Start { startFound = true } @@ -2821,6 +3317,11 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf action.Parameters = newParams } + action.LargeImage = "" + if len(action.Label) == 0 { + action.Label = action.ID + } + //log.Printf("LABEL: %s", action.Label) newActions = append(newActions, action) // If the node is NOT found, it's supposed to be set to SKIPPED, @@ -2850,9 +3351,18 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf continue } - log.Printf("[WARNING] Set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID) + //log.Printf("[WARNING] Set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID) + curaction := Action{ + AppName: action.AppName, + AppVersion: action.AppVersion, + Label: action.Label, + Name: action.Name, + ID: action.ID, + } + //action + //curaction.Parameters = [] defaultResults = append(defaultResults, ActionResult{ - Action: action, + Action: curaction, ExecutionId: workflowExecution.ExecutionId, Authorization: workflowExecution.Authorization, Result: "Skipped because it's not under the startnode", @@ -2865,7 +3375,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } for _, trigger := range workflowExecution.Workflow.Triggers { - log.Printf("ID: %s vs %s", trigger.ID, workflowExecution.Start) + //log.Printf("[INFO] ID: %s vs %s", trigger.ID, workflowExecution.Start) if trigger.ID == workflowExecution.Start { if trigger.AppName == "User Input" { startFound = true @@ -2968,7 +3478,21 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf return WorkflowExecution{}, "Failed building missing Docker images", err } - err = setWorkflowExecution(ctx, workflowExecution) + //b, err := json.Marshal(workflowExecution) + //if err == nil { + // log.Printf("%s", string(b)) + // log.Printf("LEN: %d", len(string(b))) + // //workflowExecution.ExecutionOrg.SyncFeatures = Org{} + //} + + workflowExecution.Workflow.ExecutingOrg = Org{ + Id: workflowExecution.Workflow.ExecutingOrg.Id, + } + workflowExecution.Workflow.Org = []Org{ + workflowExecution.Workflow.ExecutingOrg, + } + //Org []Org `json:"org,omitempty" datastore:"org"` + err = setWorkflowExecution(ctx, workflowExecution, true) if err != nil { log.Printf("Error saving workflow execution for updates %s: %s", topic, err) return WorkflowExecution{}, "Failed getting workflowexecution", err @@ -2978,6 +3502,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf // FIXME - add specifics to executionRequest, e.g. specific environment (can run multi onprem) if onpremExecution { // FIXME - tmp name based on future companyname-companyId + // This leads to issues with overlaps. Should set limits and such instead for _, environment := range environments { log.Printf("[INFO] Execution: %s should execute onprem with execution environment \"%s\"", workflowExecution.ExecutionId, environment) @@ -2988,19 +3513,19 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf Environments: environments, } - executionRequestWrapper, err := getWorkflowQueue(ctx, environment) - if err != nil { - executionRequestWrapper = ExecutionRequestWrapper{ - Data: []ExecutionRequest{executionRequest}, - } - } else { - executionRequestWrapper.Data = append(executionRequestWrapper.Data, executionRequest) - } + //executionRequestWrapper, err := getWorkflowQueue(ctx, environment) + //if err != nil { + // executionRequestWrapper = ExecutionRequestWrapper{ + // Data: []ExecutionRequest{executionRequest}, + // } + //} else { + // executionRequestWrapper.Data = append(executionRequestWrapper.Data, executionRequest) + //} //log.Printf("Execution request: %#v", executionRequest) - err = setWorkflowQueue(ctx, executionRequestWrapper, environment) + err = setWorkflowQueue(ctx, executionRequest, environment) if err != nil { - log.Printf("Failed adding to db: %s", err) + log.Printf("[ERROR] Failed adding execution to db: %s", err) } } } @@ -3029,10 +3554,10 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } } - err = increaseStatisticsField(ctx, "workflow_executions", workflow.ID, 1) - if err != nil { - log.Printf("Failed to increase stats execution stats: %s", err) - } + //err = increaseStatisticsField(ctx, "workflow_executions", workflow.ID, 1, workflowExecution.ExecutionOrg) + //if err != nil { + // log.Printf("Failed to increase stats execution stats: %s", err) + //} return workflowExecution, "", nil } @@ -3791,15 +4316,22 @@ func getSpecificWorkflow(resp http.ResponseWriter, request *http.Request) { resp.Write(body) } -func setWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecution) error { +func setWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecution, dbSave bool) error { + //log.Printf("\n\n\nRESULT: %s\n\n\n", workflowExecution.Status) if len(workflowExecution.ExecutionId) == 0 { log.Printf("Workflowexeciton executionId can't be empty.") return errors.New("ExecutionId can't be empty.") } - key := datastore.NameKey("workflowexecution", workflowExecution.ExecutionId, nil) + cacheKey := fmt.Sprintf("workflowexecution-%s", workflowExecution.ExecutionId) + requestCache.Set(cacheKey, &workflowExecution, cache.DefaultExpiration) + if !dbSave && workflowExecution.Status == "EXECUTING" && len(workflowExecution.Results) > 1 { + //log.Printf("[WARNING] SHOULD skip DB saving for execution") + return nil + } // New struct, to not add body, author etc + key := datastore.NameKey("workflowexecution", workflowExecution.ExecutionId, nil) if _, err := dbclient.Put(ctx, key, &workflowExecution); err != nil { log.Printf("Error adding workflow_execution: %s", err) return err @@ -3809,8 +4341,25 @@ func setWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecuti } func getWorkflowExecution(ctx context.Context, id string) (*WorkflowExecution, error) { - key := datastore.NameKey("workflowexecution", strings.ToLower(id), nil) workflowExecution := &WorkflowExecution{} + cacheKey := fmt.Sprintf("workflowexecution-%s", id) + if value, found := requestCache.Get(cacheKey); found { + parsedValue := value.(*WorkflowExecution) + //log.Printf("Found execution for id %s with %d results", parsedValue.ExecutionId, len(parsedValue.Results)) + return parsedValue, nil + + //log.Printf("[INFO] FOUND key %s with value length %d", cacheKey, len(parsedValue)) + //err := json.Unmarshal([]byte(parsedValue), &workflowExecution) + //if err == nil { + // log.Printf("SHOULD RETURN CACHED EXECUTION of length %d", len(parsedValue)) + //} else { + // log.Printf("Failed unmarshalling cached value: %s", err) + //} + } else { + //log.Printf("[ERROR] Couldn't find key %s", cacheKey) + } + + key := datastore.NameKey("workflowexecution", strings.ToLower(id), nil) if err := dbclient.Get(ctx, key, workflowExecution); err != nil { return &WorkflowExecution{}, err } @@ -3823,6 +4372,7 @@ func getApp(ctx context.Context, id string) (*WorkflowApp, error) { workflowApp := &WorkflowApp{} if err := dbclient.Get(ctx, key, workflowApp); err != nil { return &WorkflowApp{}, err + } return workflowApp, nil @@ -3863,13 +4413,14 @@ func getAllWorkflows(ctx context.Context, orgId string) ([]Workflow, error) { } func setExampleresult(ctx context.Context, result AppExecutionExample) error { - key := datastore.NameKey("example_result", result.ExampleId, nil) + // FIXME: Reintroduce this for stats + //key := datastore.NameKey("example_result", result.ExampleId, nil) - // New struct, to not add body, author etc - if _, err := dbclient.Put(ctx, key, &result); err != nil { - log.Printf("Error adding workflow: %s", err) - return err - } + //// New struct, to not add body, author etc + //if _, err := dbclient.Put(ctx, key, &result); err != nil { + // log.Printf("Error adding workflow: %s", err) + // return err + //} return nil } @@ -3877,6 +4428,7 @@ func setExampleresult(ctx context.Context, result AppExecutionExample) error { // Hmm, so I guess this should use uuid :( // Consistency PLX func setWorkflow(ctx context.Context, workflow Workflow, id string) error { + workflow.Edited = int64(time.Now().Unix()) key := datastore.NameKey("workflow", id, nil) // New struct, to not add body, author etc @@ -3973,7 +4525,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { log.Printf("ID: %s", fileId) app, err := getApp(ctx, fileId) if err != nil { - log.Printf("Error getting app %s: %s", app.Name, err) + log.Printf("Error getting app (delete) %s: %s", fileId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -3990,15 +4542,18 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": false}`)) return } else { + log.Printf("App to be deleted is private") private = true } - q := datastore.NewQuery("workflow").Filter("org_id = ", user.ActiveOrg.Id) + // FIXME: Make workflows track themself INSIDE apps, or with a reference + q := datastore.NewQuery("workflow").Filter("org_id = ", user.ActiveOrg.Id).Limit(30) var workflows []Workflow _, err = dbclient.GetAll(ctx, q, &workflows) if err != nil { + log.Printf("Failed getting related workflows for the app: %s", err) resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "}`)) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } @@ -4057,7 +4612,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { // Not really deleting it, just removing from user cache if private { - log.Printf("Deleting private app") + log.Printf("[INFO] Deleting private app") var privateApps []WorkflowApp for _, item := range user.PrivateApps { if item.ID == fileId { @@ -4070,14 +4625,14 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { user.PrivateApps = privateApps err = setUser(ctx, &user) if err != nil { - log.Printf("Failed removing %s app for user %s: %s", app.Name, user.Username, err) + log.Printf("[ERROR]Failed removing %s app for user %s: %s", app.Name, user.Username, err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": true"}`))) return } } - log.Printf("Deleting public app") + log.Printf("[INFO] Deleting public app") err = DeleteKey(ctx, "workflowapp", fileId) if err != nil { log.Printf("Failed deleting workflowapp") @@ -4086,10 +4641,13 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { return } - err = increaseStatisticsField(ctx, "total_apps_deleted", fileId, 1) + err = increaseStatisticsField(ctx, "total_apps_deleted", fileId, 1, user.ActiveOrg.Id) if err != nil { log.Printf("Failed to increase total apps loaded stats: %s", err) } + cacheKey := fmt.Sprintf("workflowapps-sorted") + requestCache.Delete(cacheKey) + //err = memcache.Delete(request.Context(), sessionToken) resp.WriteHeader(200) resp.Write([]byte(`{"success": true}`)) @@ -4124,7 +4682,7 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() app, err := getApp(ctx, fileId) if err != nil { - log.Printf("Error getting app: %s", app.Name) + log.Printf("Error getting app (app config): %s", fileId) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -4164,6 +4722,161 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { resp.Write(data) } +func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + user, userErr := handleApiAuthentication(resp, request) + if userErr != nil { + log.Printf("Api authentication failed in get all apps: %s", userErr) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if user.Role != "admin" { + log.Printf("[WARNING] User isn't admin during auth edit config") + resp.WriteHeader(409) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Must be admin to perform this action"}`))) + return + } + + var fileId string + location := strings.Split(request.URL.String(), "/") + if location[1] == "api" { + if len(location) <= 5 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fileId = location[5] + } + + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Printf("Error with body read: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + type configAuth struct { + Id string `json:"id"` + Action string `json:"action"` + } + + var config configAuth + err = json.Unmarshal(body, &config) + if err != nil { + log.Printf("Failed unmarshaling (appauth): %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if config.Id != fileId { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Bad ID match"}`)) + return + } + + ctx := context.Background() + auth, err := getWorkflowAppAuthDatastore(ctx, fileId) + if err != nil { + log.Printf("Authget error: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": ":("}`)) + return + } + + if auth.OrgId != user.ActiveOrg.Id { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "User can't edit this org"}`)) + return + } + + if config.Action == "assign_everywhere" { + log.Printf("Should set authentication config") + q := datastore.NewQuery("workflow").Filter("org_id =", user.ActiveOrg.Id) + q = q.Order("-edited").Limit(35) + + var workflows []Workflow + _, err = dbclient.GetAll(ctx, q, &workflows) + if err != nil { + log.Printf("Getall error in auth update: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Failed getting workflows to update"}`)) + return + } + + // FIXME: Add function to remove auth from other auth's + actionCnt := 0 + workflowCnt := 0 + authenticationUsage := []AuthenticationUsage{} + for _, workflow := range workflows { + newActions := []Action{} + edited := false + usage := AuthenticationUsage{ + WorkflowId: workflow.ID, + Nodes: []string{}, + } + + for _, action := range workflow.Actions { + if action.AppName == auth.App.Name { + action.AuthenticationId = auth.Id + + edited = true + actionCnt += 1 + usage.Nodes = append(usage.Nodes, action.ID) + } + + newActions = append(newActions, action) + } + + workflow.Actions = newActions + if edited { + //auth.Usage = usage + authenticationUsage = append(authenticationUsage, usage) + err = setWorkflow(ctx, workflow, workflow.ID) + if err != nil { + log.Printf("Failed setting (authupdate) workflow: %s", err) + continue + } + + workflowCnt += 1 + } + } + + //Usage []AuthenticationUsage `json:"usage" datastore:"usage"` + log.Printf("[INFO] Found %d workflows, %d actions", workflowCnt, actionCnt) + if actionCnt > 0 && workflowCnt > 0 { + auth.WorkflowCount = int64(workflowCnt) + auth.NodeCount = int64(actionCnt) + auth.Usage = authenticationUsage + auth.Defined = true + + err = setWorkflowAppAuthDatastore(ctx, *auth, auth.Id) + if err != nil { + log.Printf("Failed setting appauth: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Failed setting app auth for all workflows"}`)) + return + } else { + // FIXME: Remove ALL workflows from other auths using the same + } + } + } + + resp.WriteHeader(200) + resp.Write([]byte(`{"success": true}`)) + //var config configAuth + + //log.Printf("Should set %s +} + func addAppAuthentication(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -4195,11 +4908,43 @@ func addAppAuthentication(resp http.ResponseWriter, request *http.Request) { return } + ctx := context.Background() if len(appAuth.Id) == 0 { appAuth.Id = uuid.NewV4().String() + } else { + auth, err := getWorkflowAppAuthDatastore(ctx, appAuth.Id) + if err == nil { + // OrgId string `json:"org_id" datastore:"org_id"` + if auth.OrgId != user.ActiveOrg.Id { + log.Printf("[WARNING] User isn't a part of the right org during auth edit") + resp.WriteHeader(409) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": ":("}`))) + return + } + + if user.Role != "admin" { + log.Printf("[WARNING] User isn't admin during auth edit") + resp.WriteHeader(409) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": ":("}`))) + return + } + + if !auth.Active { + log.Printf("[WARNING] Auth isn't active for edit") + resp.WriteHeader(409) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't update an inactive auth"}`))) + return + } + + if auth.App.Name != appAuth.App.Name { + log.Printf("[WARNING] User tried to modify auth") + resp.WriteHeader(409) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad app configuration: need to specify correct name"}`))) + return + } + } } - ctx := context.Background() if len(appAuth.Label) == 0 { resp.WriteHeader(409) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Label can't be empty"}`))) @@ -4214,12 +4959,33 @@ func addAppAuthentication(resp http.ResponseWriter, request *http.Request) { return } + // FIXME: Doens't validate Org app, err := getApp(ctx, appAuth.App.ID) if err != nil { - log.Printf("Failed finding app %s while setting auth.", appAuth.App.ID) - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return + log.Printf("[WARNING] Failed finding app %s while setting auth. Finding it by looping apps.", appAuth.App.ID) + workflowapps, err := getAllWorkflowApps(ctx, 500) + if err != nil { + resp.WriteHeader(409) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + + foundIndex := -1 + for i, workflowapp := range workflowapps { + if workflowapp.Name == appAuth.App.Name { + foundIndex = i + break + } + } + + if foundIndex >= 0 { + log.Printf("[INFO] Found app %s by looping auth", appAuth.App.ID) + } else { + log.Printf("[ERROR] Failed finding app %s which has auth after looping", appAuth.App.ID) + resp.WriteHeader(409) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } } // Check if the items are correct @@ -4239,7 +5005,9 @@ func addAppAuthentication(resp http.ResponseWriter, request *http.Request) { } } + //appAuth.LargeImage = "" appAuth.OrgId = user.ActiveOrg.Id + appAuth.Defined = true err = setWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id) if err != nil { log.Printf("Failed setting up app auth %s: %s", appAuth.Id, err) @@ -4393,7 +5161,7 @@ func updateWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() app, err := getApp(ctx, fileId) if err != nil { - log.Printf("Error getting app: %s (update app)", app.Name) + log.Printf("Error getting app (update app): %s", fileId) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -4444,6 +5212,9 @@ func updateWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { return } + cacheKey := fmt.Sprintf("workflowapps-sorted") + requestCache.Delete(cacheKey) + log.Printf("Changed workflow app %s", app.ID) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) @@ -4455,7 +5226,8 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { return } - // FIXME - set this to be per user IF logged in, as there might exist private and public + // FIXME - set this to be per user IF logged in, + // as there might exist private and public //memcacheName := "all_apps" ctx := context.Background() @@ -4463,10 +5235,11 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { // FIXME - need to be logged in? user, userErr := handleApiAuthentication(resp, request) if userErr != nil { - log.Printf("Api authentication failed in get all apps: %s", userErr) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return + log.Printf("Continuing with apps even without auth") + //log.Printf("Api authentication failed in get all apps: %s", userErr) + //resp.WriteHeader(401) + //resp.Write([]byte(`{"success": false}`)) + //return } //if item, err := memcache.Get(ctx, memcacheName); err == memcache.ErrCacheMiss { @@ -4497,7 +5270,7 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { // return //} - workflowapps, err := getAllWorkflowApps(ctx) + workflowapps, err := getAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting apps (getworkflowapps): %s", err) resp.WriteHeader(401) @@ -4507,33 +5280,49 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { //log.Printf("Length: %d", len(workflowapps)) // FIXME - this is really garbage, but is here to protect again null values etc. - newapps := []WorkflowApp{} - baseApps := []WorkflowApp{} - for _, workflowapp := range workflowapps { - if !workflowapp.Activated && workflowapp.Generated { - continue - } + newapps := workflowapps + /* + skipApps := []string{"Shuffle Subflow"} + newapps := []WorkflowApp{} + baseApps := []WorkflowApp{} + for _, workflowapp := range workflowapps { + //if !workflowapp.Activated && workflowapp.Generated { + // continue + //} - if workflowapp.Owner != user.Id && user.Role != "admin" && !workflowapp.Sharing { - continue - } + if workflowapp.Owner != user.Id && user.Role != "admin" && !workflowapp.Sharing { + continue + } - //workflowapp.Environment = "cloud" - newactions := []WorkflowAppAction{} - for _, action := range workflowapp.Actions { - //action.Environment = workflowapp.Environment - if len(action.Parameters) == 0 { - action.Parameters = []WorkflowAppActionParameter{} + continueOuter := false + for _, skip := range skipApps { + if workflowapp.Name == skip { + continueOuter = true + break + } + } + + if continueOuter { + continue + } + + //workflowapp.Environment = "cloud" + newactions := []WorkflowAppAction{} + for _, action := range workflowapp.Actions { + //action.Environment = workflowapp.Environment + if len(action.Parameters) == 0 { + action.Parameters = []WorkflowAppActionParameter{} + } + + newactions = append(newactions, action) + } + + workflowapp.Actions = newactions + newapps = append(newapps, workflowapp) + baseApps = append(baseApps, workflowapp) } - - newactions = append(newactions, action) - } - - workflowapp.Actions = newactions - newapps = append(newapps, workflowapp) - baseApps = append(baseApps, workflowapp) - } + */ if len(user.PrivateApps) > 0 { found := false @@ -4674,7 +5463,7 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) { // FIXME - continue the search here with github repos etc. // Caching might be smart :D ctx := context.Background() - workflowapps, err := getAllWorkflowApps(ctx) + workflowapps, err := getAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Error: Failed getting workflowapps: %s", err) resp.WriteHeader(401) @@ -4906,7 +5695,7 @@ func deployWebhookFunction(ctx context.Context, name, localization, applocation return nil } -func loadGithubWorkflows(url, username, password, userId, branch string) error { +func loadGithubWorkflows(url, username, password, userId, branch, orgId string) error { fs := memfs.New() log.Printf("Starting load of %s with branch %s", url, branch) @@ -4943,7 +5732,7 @@ func loadGithubWorkflows(url, username, password, userId, branch string) error { _ = r log.Printf("Starting workflow folder iteration") - iterateWorkflowGithubFolders(fs, dir, "", "", userId) + iterateWorkflowGithubFolders(fs, dir, "", "", userId, orgId) } else if strings.Contains(url, "s3") { //https://docs.aws.amazon.com/sdk-for-go/api/service/s3/ @@ -5018,7 +5807,7 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { } // Field3 = branch - err = loadGithubWorkflows(tmpBody.URL, tmpBody.Field1, tmpBody.Field2, user.Id, tmpBody.Field3) + err = loadGithubWorkflows(tmpBody.URL, tmpBody.Field1, tmpBody.Field2, user.Id, tmpBody.Field3, user.ActiveOrg.Id) if err != nil { log.Printf("Failed to update workflows: %s", err) resp.WriteHeader(401) @@ -5098,9 +5887,12 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { return } - // Field1 & 2 can be a lot of things.. + // Field1 & 2 can be a lot of things. + // Field1 = Username + // Field2 = Password type tmpStruct struct { URL string `json:"url"` + Branch string `json:"branch"` Field1 string `json:"field_1"` Field2 string `json:"field_2"` ForceUpdate bool `json:"force_update"` @@ -5123,6 +5915,10 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { URL: tmpBody.URL, } + if len(tmpBody.Branch) > 0 && tmpBody.Branch != "master" && tmpBody.Branch != "main" { + cloneOptions.ReferenceName = plumbing.ReferenceName(tmpBody.Branch) + } + // FIXME: Better auth. if len(tmpBody.Field1) > 0 && len(tmpBody.Field2) > 0 { cloneOptions.Auth = &http2.BasicAuth{ @@ -5182,7 +5978,7 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error { ctx := context.Background() - workflowapps, err := getAllWorkflowApps(ctx) + workflowapps, err := getAllWorkflowApps(ctx, 500) appCounter := 0 if err != nil { log.Printf("Failed to get existing generated apps") @@ -5308,6 +6104,9 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, log.Printf("Failed uploading openapi to datastore in loop: %s", err) continue } + + cacheKey := fmt.Sprintf("workflowapps-sorted") + requestCache.Delete(cacheKey) } } else { //log.Printf("Skipped upload of %s (%s)", api.Name, api.ID) @@ -5326,7 +6125,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, } // Onlyname is used to -func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, userId string) error { +func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname, userId, orgId string) error { var err error for _, file := range dir { @@ -5345,7 +6144,7 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra } // Go routine? Hmm, this can be super quick I guess - err = iterateWorkflowGithubFolders(fs, dir, tmpExtra, "", userId) + err = iterateWorkflowGithubFolders(fs, dir, tmpExtra, "", userId, orgId) if err != nil { continue } @@ -5377,13 +6176,41 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra workflow.Owner = userId } + workflow.ID = uuid.NewV4().String() + workflow.OrgId = orgId + workflow.ExecutingOrg = Org{ + Id: orgId, + } + + workflow.Org = append(workflow.Org, Org{ + Id: orgId, + }) + workflow.IsValid = false + workflow.Errors = []string{"Imported, not locally saved. Save before using."} + + /* + // Find existing similar ones + q = datastore.NewQuery("workflow").Filter("org_id =", user.ActiveOrg.Id).Filter("name", workflow.name) + var workflows []Workflow + _, err = dbclient.GetAll(ctx, q, &workflows) + if err == nil { + log.Printf("Failed getting workflows for user %s: %s", user.Username, err) + if len(workflows) == 0 { + resp.WriteHeader(200) + resp.Write([]byte("[]")) + return + } + } + */ + ctx := context.Background() err = setWorkflow(ctx, workflow, workflow.ID) if err != nil { log.Printf("Failed setting (download) workflow: %s", err) continue } - log.Printf("Uploaded workflow %s for user %s!", filename, userId) + + log.Printf("Uploaded workflow %s for user %s and org %s!", filename, userId, orgId) } } } @@ -5461,6 +6288,8 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } } + //log.Printf("HANDLING DOCKER FILEREADER - SEARCH&REPLACE?") + appfileData, err := ioutil.ReadAll(fileReader) if err != nil { log.Printf("Failed reading %s: %s", fullPath, err) @@ -5517,11 +6346,11 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin newName = strings.ReplaceAll(newName, " ", "-") tags := []string{ - fmt.Sprintf("%s:%s_%s", baseDockerName, newName, workflowapp.AppVersion), + fmt.Sprintf("%s:%s_%s", baseDockerName, strings.ToLower(newName), workflowapp.AppVersion), } if len(allapps) == 0 { - allapps, err = getAllWorkflowApps(ctx) + allapps, err = getAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting apps to verify: %s", err) continue @@ -5591,7 +6420,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } if len(appendParams) > 0 { - log.Printf("Appending %d params to the START of %s", len(appendParams), action.Name) + log.Printf("[AUTH] Appending %d params to the START of %s", len(appendParams), action.Name) workflowapp.Actions[index].Parameters = append(appendParams, workflowapp.Actions[index].Parameters...) } @@ -5606,10 +6435,10 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin if len(removeApps) > 0 { for _, item := range removeApps { - log.Printf("Removing duplicate: %s", item) + log.Printf("[WARNING] Removing duplicate: %s", item) err = DeleteKey(ctx, "workflowapp", item) if err != nil { - log.Printf("Failed deleting %s", item) + log.Printf("[ERROR] Failed deleting duplicate %s: %s", item, err) } } } @@ -5627,12 +6456,12 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin continue } - err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1) + err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1, "") if err != nil { log.Printf("Failed to increase total apps created stats: %s", err) } - err = increaseStatisticsField(ctx, "total_apps_loaded", workflowapp.ID, 1) + err = increaseStatisticsField(ctx, "total_apps_loaded", workflowapp.ID, 1, "") if err != nil { log.Printf("Failed to increase total apps loaded stats: %s", err) } @@ -5674,28 +6503,28 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin if len(extra) == 0 { log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst)) for _, item := range buildLaterFirst { - err = buildImageMemory(fs, item.Tags, item.Extra) + err = buildImageMemory(fs, item.Tags, item.Extra, true) if err != nil { log.Printf("Failed image build memory: %s", err) } else { if len(item.Tags) > 0 { - log.Printf("Successfully built image %s", item.Tags[0]) + log.Printf("[INFO] Successfully built image %s", item.Tags[0]) } else { - log.Printf("Successfully built Docker image") + log.Printf("[INFO] Successfully built Docker image") } } } - log.Printf("Starting build of %d skipped docker images", len(buildLaterList)) + log.Printf("[INFO] Starting build of %d skipped docker images", len(buildLaterList)) for _, item := range buildLaterList { - err = buildImageMemory(fs, item.Tags, item.Extra) + err = buildImageMemory(fs, item.Tags, item.Extra, true) if err != nil { - log.Printf("Failed image build memory: %s", err) + log.Printf("[INFO] Failed image build memory: %s", err) } else { if len(item.Tags) > 0 { - log.Printf("Successfully built image %s", item.Tags[0]) + log.Printf("[INFO] Successfully built image %s", item.Tags[0]) } else { - log.Printf("Successfully built Docker image") + log.Printf("[INFO] Successfully built Docker image") } } } @@ -5738,7 +6567,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - allapps, err := getAllWorkflowApps(ctx) + allapps, err := getAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting apps to verify: %s", err) resp.WriteHeader(401) @@ -5790,6 +6619,8 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { } //memcache.Delete(ctx, "all_apps") + cacheKey := fmt.Sprintf("workflowapps-sorted") + requestCache.Delete(cacheKey) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) @@ -5831,7 +6662,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := getWorkflow(ctx, fileId) if err != nil { - log.Printf("Failed getting the workflow locally (get executions): %s", err) + log.Printf("Failed getting the workflow %s locally (get executions): %s", fileId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -5850,10 +6681,21 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { var workflowExecutions []WorkflowExecution _, err = dbclient.GetAll(ctx, q, &workflowExecutions) if err != nil { - log.Printf("Error getting workflowexec: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting all workflowexecutions for %s"}`, fileId))) - return + if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { + q = datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(15) + _, err = dbclient.GetAll(ctx, q, &workflowExecutions) + if err != nil { + log.Printf("Error getting workflowexec (2): %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting all workflowexecutions for %s"}`, fileId))) + return + } + } else { + log.Printf("Error getting workflowexec: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting all workflowexecutions for %s"}`, fileId))) + return + } } if len(workflowExecutions) == 0 { @@ -5889,16 +6731,102 @@ func getAllSchedules(ctx context.Context, orgId string) ([]ScheduleOld, error) { return schedules, nil } -func getAllWorkflowApps(ctx context.Context) ([]WorkflowApp, error) { - var allworkflowapps []WorkflowApp - q := datastore.NewQuery("workflowapp").Limit(50) +//FIXME: Add cursor +func getAllWorkflowApps(ctx context.Context, maxLen int) ([]WorkflowApp, error) { + var apps []WorkflowApp + query := datastore.NewQuery("workflowapp").Order("-edited").Limit(20) + //query := datastore.NewQuery("workflowapp").Order("-edited").Limit(40) - _, err := dbclient.GetAll(ctx, q, &allworkflowapps) - if err != nil { - return []WorkflowApp{}, err + cacheKey := fmt.Sprintf("workflowapps-sorted-%d", maxLen) + if value, found := requestCache.Get(cacheKey); found { + parsedValue := value.(*[]WorkflowApp) + log.Printf("[INFO] Returning %d apps from cache", len(*parsedValue)) + return *parsedValue, nil } - return allworkflowapps, nil + cursorStr := "" + + // NOT BEING UPDATED + // FIXME: Update the app with the correct actions. HOW DOES THIS WORK?? + // Seems like only actions are wrong. Could get the app individually. + // Guessing it's a memory issue. + //Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` + //errors.New(nil) + var err error + for { + it := dbclient.Run(ctx, query) + //_, err = it.Next(&app) + for { + var app WorkflowApp + _, err := it.Next(&app) + if err != nil { + break + } + + found := false + //log.Printf("ACTIONS: %d - %s", len(app.Actions), app.Name) + for _, innerapp := range apps { + if innerapp.Name == app.Name { + found = true + break + } + } + + if !found { + apps = append(apps, app) + } + } + + if err != iterator.Done { + //log.Printf("[INFO] Failed fetching results: %v", err) + //break + } + + // Get the cursor for the next page of results. + nextCursor, err := it.Cursor() + if err != nil { + log.Printf("Cursorerror: %s", err) + break + } else { + //log.Printf("NEXTCURSOR: %s", nextCursor) + nextStr := fmt.Sprintf("%s", nextCursor) + if cursorStr == nextStr { + break + } + + cursorStr = nextStr + query = query.Start(nextCursor) + //cursorStr = nextCursor + //break + } + + if len(apps) > maxLen { + break + } + } + + if len(apps) > 20 { + log.Printf("[INFO] Setting %d apps in cache", len(apps)) + requestCache.Set(cacheKey, &apps, cache.DefaultExpiration) + } + + //var allworkflowapps []WorkflowApp + //_, err := dbclient.GetAll(ctx, query, &allworkflowapps) + //if err != nil { + // if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { + // //datastore.NewQuery("workflowapp").Limit(30).Order("-edited") + // query = datastore.NewQuery("workflowapp").Order("-edited").Limit(25) + // //q := q.Limit(25) + // _, err := dbclient.GetAll(ctx, query, &allworkflowapps) + // if err != nil { + // return []WorkflowApp{}, err + // } + // } else { + // return []WorkflowApp{}, err + // } + //} + + return apps, nil } func getAllWorkflowAppAuth(ctx context.Context, OrgId string) ([]AppAuthenticationStorage, error) { @@ -5913,12 +6841,31 @@ func getAllWorkflowAppAuth(ctx context.Context, OrgId string) ([]AppAuthenticati return allworkflowapps, nil } +func getWorkflowAppAuthDatastore(ctx context.Context, id string) (*AppAuthenticationStorage, error) { + + key := datastore.NameKey("workflowappauth", id, nil) + appAuth := &AppAuthenticationStorage{} + // New struct, to not add body, author etc + if err := dbclient.Get(ctx, key, appAuth); err != nil { + return &AppAuthenticationStorage{}, err + } + + return appAuth, nil +} + func setWorkflowAppAuthDatastore(ctx context.Context, workflowappauth AppAuthenticationStorage, id string) error { + timeNow := int64(time.Now().Unix()) + if workflowappauth.Created == 0 { + workflowappauth.Created = timeNow + } + + workflowappauth.Edited = timeNow + key := datastore.NameKey("workflowappauth", id, nil) // New struct, to not add body, author etc if _, err := dbclient.Put(ctx, key, &workflowappauth); err != nil { - log.Printf("Error adding workflow app: %s", err) + log.Printf("Error adding workflow app auth: %s", err) return err } @@ -5928,6 +6875,12 @@ func setWorkflowAppAuthDatastore(ctx context.Context, workflowappauth AppAuthent // Hmm, so I guess this should use uuid :( // Consistency PLX func setWorkflowAppDatastore(ctx context.Context, workflowapp WorkflowApp, id string) error { + timeNow := int64(time.Now().Unix()) + if workflowapp.Created == 0 { + workflowapp.Created = timeNow + } + + workflowapp.Edited = timeNow key := datastore.NameKey("workflowapp", id, nil) // New struct, to not add body, author etc @@ -5976,7 +6929,7 @@ func handleStopHook(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() hook, err := getHook(ctx, fileId) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (stop): %s", fileId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -6059,7 +7012,7 @@ func handleDeleteHook(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() hook, err := getHook(ctx, fileId) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (delete): %s", fileId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -6073,10 +7026,10 @@ func handleDeleteHook(resp http.ResponseWriter, request *http.Request) { } if len(hook.Workflows) > 0 { - err = increaseStatisticsField(ctx, "total_workflow_triggers", hook.Workflows[0], -1) - if err != nil { - log.Printf("Failed to increase total workflows: %s", err) - } + //err = increaseStatisticsField(ctx, "total_workflow_triggers", hook.Workflows[0], -1, user.ActiveOrg.Id) + //if err != nil { + // log.Printf("Failed to increase total workflows: %s", err) + //} } hook.Status = "stopped" @@ -6200,7 +7153,7 @@ func handleStartHook(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() hook, err := getHook(ctx, fileId) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (start): %s", fileId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return diff --git a/backend/tests/execute.sh b/backend/tests/execute.sh index a4584bbe..ef852bbb 100644 --- a/backend/tests/execute.sh +++ b/backend/tests/execute.sh @@ -1,5 +1,26 @@ #!/bin/sh -curl -XPOST http://localhost:5001/api/v1/workflows/1d9d8ce2-566e-4c3f-8a37-5d6c7d2000b5/execute -d '{"execution_argument":""}' -H "Authorization: Bearer 144308d0-6aab-4d4f-8bb2-75189281ee26" - - -curl -XPOST http://localhost:5001/api/v1/workflows/1d9d8ce2-566e-4c3f-8a37-5d6c7d2000b5/execute -d '{"execution_argument":""}' -H "Authorization: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjYwZjQwNjBlNThkNzVmZDNmNzBiZWZmODhjNzk0YTc3NTMyN2FhMzEiLCJ0eXAiOiJKV1QifQ.eyJhdWQiOiJodHRwczovL3NodWZmbGVyLmlvL2FwaS92MS93b3JrZmxvd3MvMWQ5ZDhjZTItNTY2ZS00YzNmLThhMzctNWQ2YzdkMjAwMGI1L2V4ZWN1dGUiLCJhenAiOiIxMDMwNzY3ODIwNjE0MjQ2MTg0MjIiLCJlbWFpbCI6InNjaGVkdWxlckBzaHVmZmxlLTI0MTUxNy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJleHAiOjE1NjU1Mjc1NTEsImlhdCI6MTU2NTUyMzk1MSwiaXNzIjoiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tIiwic3ViIjoiMTAzMDc2NzgyMDYxNDI0NjE4NDIyIn0.r0EDq9fjhf_5CPTiltyfk_L3uYJp577Uy0yYPcCAl2nv50_z_oUtbWGBpQLL8gcj-NGd3g4E52Qur8k6hCMIQweLS6WAb1279vGffEoCNDfkWb3Oy-yJGP1kzwLvqFJqnHLkSWYXNWvSyWnEimW8Rryx_m1BXS5wcA8l4NIr83kS7fPZrTwjnwFSeGSThwk91DVARzapQb8r0GEgOUyHZ1aBXnV98mikzSUt-5xFKe9eMdD22YJAj0Ru-DxAxs5nOqghX4PMRysWjshjOMrlR1piPWxqAmewp8YKZDCQ5gXskpeAFBDoULT971Wsx_NCohnJsFqx1JfPS9ZYMTW2oQ" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" diff --git a/docker-compose.yml b/docker-compose.yml index 5ed77582..addef547 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.3 + image: ghcr.io/frikky/shuffle-frontend:0.8.56 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.3 + image: ghcr.io/frikky/shuffle-backend:0.8.56 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -45,7 +45,7 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: ghcr.io/frikky/shuffle-orborus:0.8.0 + image: ghcr.io/frikky/shuffle-orborus:0.8.5 container_name: shuffle-orborus hostname: shuffle-orborus networks: @@ -53,8 +53,8 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock environment: - - SHUFFLE_APP_SDK_VERSION=0.8.0 - - SHUFFLE_WORKER_VERSION=0.8.0 + - SHUFFLE_APP_SDK_VERSION=0.8.51 + - SHUFFLE_WORKER_VERSION=0.8.54 - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} @@ -66,6 +66,7 @@ services: - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} + - CLEANUP=${SHUFFLE_CONTAINER_AUTO_CLEANUP} restart: unless-stopped database: #build: ./backend/database diff --git a/frontend/Dockerfile b/frontend/Dockerfile index cc791b82..f48b048d 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -8,7 +8,8 @@ ENV PATH /usr/src/app/node_modules/.bin:$PATH COPY package.json /usr/src/app/package.json -RUN npm install --verbose +#RUN npm install --verbose +RUN yarn install # copy only required files to not trigger rebuilding every time COPY ./certs /usr/src/app/certs/ @@ -19,9 +20,9 @@ COPY ./*.json /usr/src/app/ # There were issues with the webpack installer from package.json RUN rm -rf /usr/src/app/node_modules/webpack -RUN npm install webpack@4.42.0 +#RUN yarn add webpack@4.42.0 -RUN npm run-script build +RUN yarn build # Production environment FROM nginx:latest diff --git a/frontend/package-lock.json b/frontend/package-lock.json index bd6df28b..e5e02309 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,6 +1,6 @@ { "name": "shuffler", - "version": "0.6.0", + "version": "0.8.53", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -336,6 +336,7 @@ "@babel/highlight": "^7.10.1" } }, + "@babel/core": {}, "@babel/helper-function-name": { "version": "7.10.1", "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.10.1.tgz", @@ -2566,37 +2567,6 @@ "@babel/helper-plugin-utils": "^7.10.1" } }, - "@babel/plugin-transform-runtime": { - "version": "7.9.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.9.0.tgz", - "integrity": "sha512-pUu9VSf3kI1OqbWINQ7MaugnitRss1z533436waNXp+0N3ur3zfut37sXiQMxkuCF4VUjwZucen/quskCh7NHw==", - "requires": { - "@babel/helper-module-imports": "^7.8.3", - "@babel/helper-plugin-utils": "^7.8.3", - "resolve": "^1.8.1", - "semver": "^5.5.1" - }, - "dependencies": { - "@babel/helper-module-imports": { - "version": "7.10.1", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.10.1.tgz", - "integrity": "sha512-SFxgwYmZ3HZPyZwJRiVNLRHWuW2OgE5k2nrVs6D9Iv4PPnXVffuEHy83Sfx/l4SqF+5kyJXjAyUmrG7tNm+qVg==", - "requires": { - "@babel/types": "^7.10.1" - } - }, - "@babel/types": { - "version": "7.10.2", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.10.2.tgz", - "integrity": "sha512-AD3AwWBSz0AWF0AkCN9VPiWrvldXq+/e3cHa4J89vo4ymjz1XwrBFFVZmkJTsQIPNk+ZVomPSXUJqq8yyjZsng==", - "requires": { - "@babel/helper-validator-identifier": "^7.10.1", - "lodash": "^4.17.13", - "to-fast-properties": "^2.0.0" - } - } - } - }, "@babel/plugin-transform-shorthand-properties": { "version": "7.10.1", "resolved": "https://registry.npmjs.org/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.10.1.tgz", @@ -3393,6 +3363,16 @@ "react-is": "^16.7.0" } }, + "jss": { + "dependencies": { + "warning": {} + } + }, + "jss-nested": { + "dependencies": { + "warning": {} + } + }, "popper.js": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/popper.js/-/popper.js-1.16.1.tgz", @@ -4088,11 +4068,6 @@ } } }, - "acorn-jsx": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.2.0.tgz", - "integrity": "sha512-HiUX/+K2YpkpJ+SzBffkM/AQ2YE03S0U1kjTLVpoJdhZMOWy8qvXVN9JdLqv2QsaQ6MPYQIuNmwD8zOiYUofLQ==" - }, "acorn-walk": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-6.2.0.tgz", @@ -4706,6 +4681,7 @@ "slash": "^2.0.0" }, "dependencies": { + "@babel/core": {}, "ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", @@ -4747,25 +4723,6 @@ } } }, - "babel-loader": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-8.1.0.tgz", - "integrity": "sha512-7q7nC1tYOrqvUrN3LQK4GwSk/TQorZSOlO9C+RZDZpODgyN4ZlCqE5q9cDsyWOliN+aU9B4JX01xK9eJXowJLw==", - "requires": { - "find-cache-dir": "^2.1.0", - "loader-utils": "^1.4.0", - "mkdirp": "^0.5.3", - "pify": "^4.0.1", - "schema-utils": "^2.6.5" - }, - "dependencies": { - "pify": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", - "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" - } - } - }, "babel-plugin-dynamic-import-node": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz", @@ -4971,6 +4928,202 @@ "babel-plugin-transform-react-remove-prop-types": "0.4.24" }, "dependencies": { + "@babel/code-frame": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.12.11.tgz", + "integrity": "sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw==", + "requires": { + "@babel/highlight": "^7.10.4" + } + }, + "@babel/core": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.9.0.tgz", + "integrity": "sha512-kWc7L0fw1xwvI0zi8OKVBuxRVefwGOrKSQMvrQ3dW+bIIavBY3/NpXmpjMy7bQnLgwgzWQZ8TlM57YHpHNHz4w==", + "requires": { + "@babel/code-frame": "^7.8.3", + "@babel/generator": "^7.9.0", + "@babel/helper-module-transforms": "^7.9.0", + "@babel/helpers": "^7.9.0", + "@babel/parser": "^7.9.0", + "@babel/template": "^7.8.6", + "@babel/traverse": "^7.9.0", + "@babel/types": "^7.9.0", + "convert-source-map": "^1.7.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.1", + "json5": "^2.1.2", + "lodash": "^4.17.13", + "resolve": "^1.3.2", + "semver": "^5.4.1", + "source-map": "^0.5.0" + } + }, + "@babel/generator": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.12.11.tgz", + "integrity": "sha512-Ggg6WPOJtSi8yYQvLVjG8F/TlpWDlKx0OpS4Kt+xMQPs5OaGYWy+v1A+1TvxI6sAMGZpKWWoAQ1DaeQbImlItA==", + "requires": { + "@babel/types": "^7.12.11", + "jsesc": "^2.5.1", + "source-map": "^0.5.0" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, + "@babel/helper-annotate-as-pure": { + "version": "7.12.10", + "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.12.10.tgz", + "integrity": "sha512-XplmVbC1n+KY6jL8/fgLVXXUauDIB+lD5+GsQEh6F6GBF1dq1qy4DP4yXWzDKcoqXB3X58t61e85Fitoww4JVQ==", + "requires": { + "@babel/types": "^7.12.10" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, + "@babel/helper-create-regexp-features-plugin": { + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.12.7.tgz", + "integrity": "sha512-idnutvQPdpbduutvi3JVfEgcVIHooQnhvhx0Nk9isOINOIGYkZea1Pk2JlJRiUnMefrlvr0vkByATBY/mB4vjQ==", + "requires": { + "@babel/helper-annotate-as-pure": "^7.10.4", + "regexpu-core": "^4.7.1" + } + }, + "@babel/helper-define-map": { + "version": "7.10.5", + "resolved": "https://registry.npmjs.org/@babel/helper-define-map/-/helper-define-map-7.10.5.tgz", + "integrity": "sha512-fMw4kgFB720aQFXSVaXr79pjjcW5puTCM16+rECJ/plGS+zByelE8l9nCpV1GibxTnFVmUuYG9U8wYfQHdzOEQ==", + "requires": { + "@babel/helper-function-name": "^7.10.4", + "@babel/types": "^7.10.5", + "lodash": "^4.17.19" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, + "@babel/helper-function-name": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.12.11.tgz", + "integrity": "sha512-AtQKjtYNolKNi6nNNVLQ27CP6D9oFR6bq/HPYSizlzbp7uC1M59XJe8L+0uXjbIaZaUJF99ruHqVGiKXU/7ybA==", + "requires": { + "@babel/helper-get-function-arity": "^7.12.10", + "@babel/template": "^7.12.7", + "@babel/types": "^7.12.11" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, + "@babel/helper-get-function-arity": { + "version": "7.12.10", + "resolved": "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.12.10.tgz", + "integrity": "sha512-mm0n5BPjR06wh9mPQaDdXWDoll/j5UpCAPl1x8fS71GHm7HA6Ua2V4ylG1Ju8lvcTOietbPNNPaSilKj+pj+Ag==", + "requires": { + "@babel/types": "^7.12.10" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, + "@babel/helper-member-expression-to-functions": { + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.12.7.tgz", + "integrity": "sha512-DCsuPyeWxeHgh1Dus7APn7iza42i/qXqiFPWyBDdOFtvS581JQePsc1F/nD+fHrcswhLlRc2UpYS1NwERxZhHw==", + "requires": { + "@babel/types": "^7.12.7" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, "@babel/helper-module-imports": { "version": "7.10.1", "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.10.1.tgz", @@ -4979,6 +5132,106 @@ "@babel/types": "^7.10.1" } }, + "@babel/helper-optimise-call-expression": { + "version": "7.12.10", + "resolved": "https://registry.npmjs.org/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.12.10.tgz", + "integrity": "sha512-4tpbU0SrSTjjt65UMWSrUOPZTsgvPgGG4S8QSTNHacKzpS51IVWGDj0yCwyeZND/i+LSN2g/O63jEXEWm49sYQ==", + "requires": { + "@babel/types": "^7.12.10" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, + "@babel/helper-replace-supers": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.12.11.tgz", + "integrity": "sha512-q+w1cqmhL7R0FNzth/PLLp2N+scXEK/L2AHbXUyydxp828F4FEa5WcVoqui9vFRiHDQErj9Zof8azP32uGVTRA==", + "requires": { + "@babel/helper-member-expression-to-functions": "^7.12.7", + "@babel/helper-optimise-call-expression": "^7.12.10", + "@babel/traverse": "^7.12.10", + "@babel/types": "^7.12.11" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, + "@babel/helper-split-export-declaration": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.12.11.tgz", + "integrity": "sha512-LsIVN8j48gHgwzfocYUSkO/hjYAOJqlpJEc7tGXcIm4cubjVUf8LGW6eWRyxEu7gA25q02p0rQUWoCI33HNS5g==", + "requires": { + "@babel/types": "^7.12.11" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, + "@babel/highlight": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.10.4.tgz", + "integrity": "sha512-i6rgnR/YgPEQzZZnbTHHuZdlE8qyoBNalD6F+q4vAFlcMEcqmkoG+mPqJYJCo63qPf74+Y1UZsl3l6f7/RIkmA==", + "requires": { + "@babel/helper-validator-identifier": "^7.10.4", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + } + } + }, + "@babel/parser": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.12.11.tgz", + "integrity": "sha512-N3UxG+uuF4CMYoNj8AhnbAcJF0PiuJ9KHuy1lQmkYsxTer/MAH9UBNHsBoAX/4s6NvlDD047No8mYVGGzLL4hg==" + }, "@babel/plugin-proposal-class-properties": { "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.8.3.tgz", @@ -5006,6 +5259,31 @@ "@babel/plugin-syntax-numeric-separator": "^7.8.3" } }, + "@babel/plugin-proposal-object-rest-spread": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.12.1.tgz", + "integrity": "sha512-s6SowJIjzlhx8o7lsFx5zmY4At6CTtDvgNQDdPzkBQucle58A6b/TTeEBYtyDgmcXjUTM+vE8YOGHZzzbc/ioA==", + "requires": { + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/plugin-syntax-object-rest-spread": "^7.8.0", + "@babel/plugin-transform-parameters": "^7.12.1" + }, + "dependencies": { + "@babel/helper-plugin-utils": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", + "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" + }, + "@babel/plugin-transform-parameters": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.12.1.tgz", + "integrity": "sha512-xq9C5EQhdPK23ZeCdMxl8bbRnAgHFrw5EOC3KJUsSylZqdkCaFEXxGSBuTSObOpiiHHNyb82es8M1QYgfQGfNg==", + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + } + } + } + }, "@babel/plugin-proposal-optional-chaining": { "version": "7.9.0", "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.9.0.tgz", @@ -5015,6 +5293,83 @@ "@babel/plugin-syntax-optional-chaining": "^7.8.0" } }, + "@babel/plugin-syntax-async-generators": { + "dependencies": { + "@babel/core": {} + } + }, + "@babel/plugin-syntax-json-strings": { + "dependencies": { + "@babel/core": {} + } + }, + "@babel/plugin-syntax-jsx": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.12.1.tgz", + "integrity": "sha512-1yRi7yAtB0ETgxdY9ti/p2TivUxJkTdhu/ZbF9MshVGqOx1TdB3b7xCXs49Fupgg50N45KcAsRP/ZqWjs9SRjg==", + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "dependencies": { + "@babel/helper-plugin-utils": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", + "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" + } + } + }, + "@babel/plugin-syntax-optional-catch-binding": { + "dependencies": { + "@babel/core": {} + } + }, + "@babel/plugin-transform-classes": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-classes/-/plugin-transform-classes-7.12.1.tgz", + "integrity": "sha512-/74xkA7bVdzQTBeSUhLLJgYIcxw/dpEpCdRDiHgPJ3Mv6uC11UhjpOhl72CgqbBCmt1qtssCyB2xnJm1+PFjog==", + "requires": { + "@babel/helper-annotate-as-pure": "^7.10.4", + "@babel/helper-define-map": "^7.10.4", + "@babel/helper-function-name": "^7.10.4", + "@babel/helper-optimise-call-expression": "^7.10.4", + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/helper-replace-supers": "^7.12.1", + "@babel/helper-split-export-declaration": "^7.10.4", + "globals": "^11.1.0" + }, + "dependencies": { + "@babel/helper-plugin-utils": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", + "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" + } + } + }, + "@babel/plugin-transform-destructuring": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.12.1.tgz", + "integrity": "sha512-fRMYFKuzi/rSiYb2uRLiUENJOKq4Gnl+6qOv5f8z0TZXg3llUwUhsNNwrwaT/6dUhJTzNpBr+CUvEWBtfNY1cw==", + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "dependencies": { + "@babel/helper-plugin-utils": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", + "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" + } + } + }, + "@babel/plugin-transform-named-capturing-groups-regex": { + "dependencies": { + "@babel/core": {} + } + }, + "@babel/plugin-transform-new-target": { + "dependencies": { + "@babel/core": {} + } + }, "@babel/plugin-transform-react-display-name": { "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.8.3.tgz", @@ -5023,6 +5378,37 @@ "@babel/helper-plugin-utils": "^7.8.3" } }, + "@babel/plugin-transform-react-jsx": { + "dependencies": { + "@babel/core": {} + } + }, + "@babel/plugin-transform-react-jsx-self": { + "dependencies": { + "@babel/core": {} + } + }, + "@babel/plugin-transform-react-jsx-source": { + "dependencies": { + "@babel/core": {} + } + }, + "@babel/plugin-transform-regenerator": { + "dependencies": { + "@babel/core": {} + } + }, + "@babel/plugin-transform-runtime": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.9.0.tgz", + "integrity": "sha512-pUu9VSf3kI1OqbWINQ7MaugnitRss1z533436waNXp+0N3ur3zfut37sXiQMxkuCF4VUjwZucen/quskCh7NHw==", + "requires": { + "@babel/helper-module-imports": "^7.8.3", + "@babel/helper-plugin-utils": "^7.8.3", + "resolve": "^1.8.1", + "semver": "^5.5.1" + } + }, "@babel/preset-env": { "version": "7.9.0", "resolved": "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.9.0.tgz", @@ -5088,6 +5474,63 @@ "invariant": "^2.2.2", "levenary": "^1.1.1", "semver": "^5.5.0" + }, + "dependencies": { + "@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-transform-named-capturing-groups-regex": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.12.1.tgz", + "integrity": "sha512-tB43uQ62RHcoDp9v2Nsf+dSM8sbNodbEicbQNA53zHz8pWUhsgHSJCGpt7daXxRydjb0KnfmB+ChXOv3oADp1Q==", + "requires": { + "@babel/helper-create-regexp-features-plugin": "^7.12.1" + } + }, + "@babel/plugin-transform-new-target": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.12.1.tgz", + "integrity": "sha512-+eW/VLcUL5L9IvJH7rT1sT0CzkdUTvPrXC2PXTn/7z7tXLBuKvezYbGdxD5WMRoyvyaujOq2fWoKl869heKjhw==", + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "dependencies": { + "@babel/helper-plugin-utils": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", + "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" + } + } + }, + "@babel/plugin-transform-regenerator": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.12.1.tgz", + "integrity": "sha512-gYrHqs5itw6i4PflFX3OdBPMQdPbF4bj2REIUxlMRUFk0/ZOAIpDFuViuxPjUL7YC8UPnf+XG7/utJvqXdPKng==", + "requires": { + "regenerator-transform": "^0.14.2" + } + } } }, "@babel/preset-react": { @@ -5101,6 +5544,95 @@ "@babel/plugin-transform-react-jsx-development": "^7.9.0", "@babel/plugin-transform-react-jsx-self": "^7.9.0", "@babel/plugin-transform-react-jsx-source": "^7.9.0" + }, + "dependencies": { + "@babel/helper-module-imports": { + "version": "7.12.5", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.12.5.tgz", + "integrity": "sha512-SR713Ogqg6++uexFRORf/+nPXMmWIn80TALu0uaFb+iQIUoR7bOC7zBWyzBs5b3tBBJXuyD0cRu1F15GyzjOWA==", + "requires": { + "@babel/types": "^7.12.5" + } + }, + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/plugin-transform-react-display-name": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.12.1.tgz", + "integrity": "sha512-cAzB+UzBIrekfYxyLlFqf/OagTvHLcVBb5vpouzkYkBclRPraiygVnafvAoipErZLI8ANv8Ecn6E/m5qPXD26w==", + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "dependencies": { + "@babel/helper-plugin-utils": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", + "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" + } + } + }, + "@babel/plugin-transform-react-jsx": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.12.12.tgz", + "integrity": "sha512-JDWGuzGNWscYcq8oJVCtSE61a5+XAOos+V0HrxnDieUus4UMnBEosDnY1VJqU5iZ4pA04QY7l0+JvHL1hZEfsw==", + "requires": { + "@babel/helper-annotate-as-pure": "^7.12.10", + "@babel/helper-module-imports": "^7.12.5", + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/plugin-syntax-jsx": "^7.12.1", + "@babel/types": "^7.12.12" + }, + "dependencies": { + "@babel/helper-plugin-utils": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", + "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" + } + } + }, + "@babel/plugin-transform-react-jsx-self": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.12.1.tgz", + "integrity": "sha512-FbpL0ieNWiiBB5tCldX17EtXgmzeEZjFrix72rQYeq9X6nUK38HCaxexzVQrZWXanxKJPKVVIU37gFjEQYkPkA==", + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "dependencies": { + "@babel/helper-plugin-utils": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", + "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" + } + } + }, + "@babel/plugin-transform-react-jsx-source": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.12.1.tgz", + "integrity": "sha512-keQ5kBfjJNRc6zZN1/nVHCd6LLIHq4aUKcVnvE/2l+ZZROSbqoiGFRtT5t3Is89XJxBQaP7NLZX2jgGHdZvvFQ==", + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "dependencies": { + "@babel/helper-plugin-utils": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", + "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" + } + } + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } } }, "@babel/runtime": { @@ -5111,6 +5643,66 @@ "regenerator-runtime": "^0.13.4" } }, + "@babel/template": { + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.12.7.tgz", + "integrity": "sha512-GkDzmHS6GV7ZeXfJZ0tLRBhZcMcY0/Lnb+eEbXDBfCAcZCjrZKe6p3J4we/D24O9Y8enxWAg1cWwof59yLh2ow==", + "requires": { + "@babel/code-frame": "^7.10.4", + "@babel/parser": "^7.12.7", + "@babel/types": "^7.12.7" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, + "@babel/traverse": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.12.12.tgz", + "integrity": "sha512-s88i0X0lPy45RrLM8b9mz8RPH5FqO9G9p7ti59cToE44xFm1Q+Pjh5Gq4SXBbtb88X7Uy7pexeqRIQDDMNkL0w==", + "requires": { + "@babel/code-frame": "^7.12.11", + "@babel/generator": "^7.12.11", + "@babel/helper-function-name": "^7.12.11", + "@babel/helper-split-export-declaration": "^7.12.11", + "@babel/parser": "^7.12.11", + "@babel/types": "^7.12.12", + "debug": "^4.1.0", + "globals": "^11.1.0", + "lodash": "^4.17.19" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, "@babel/types": { "version": "7.10.2", "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.10.2.tgz", @@ -5121,10 +5713,76 @@ "to-fast-properties": "^2.0.0" } }, + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "requires": { + "color-convert": "^1.9.0" + } + }, + "babel-loader": {}, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, "regenerator-runtime": { "version": "0.13.5", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.5.tgz", "integrity": "sha512-ZS5w8CpKFinUzOwW3c83oPeVXoNsrLsaCoLtJvAClH135j/R77RuymhiSErhm2lKcwSCIpmvIWSbDkIfAqKQlA==" + }, + "regexpu-core": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-4.7.1.tgz", + "integrity": "sha512-ywH2VUraA44DZQuRKzARmw6S66mr48pQVva4LBeRhcOltJ6hExvWly5ZjFLYo67xbIxb6W1q4bAGtgfEl20zfQ==", + "requires": { + "regenerate": "^1.4.0", + "regenerate-unicode-properties": "^8.2.0", + "regjsgen": "^0.5.1", + "regjsparser": "^0.6.4", + "unicode-match-property-ecmascript": "^1.0.4", + "unicode-match-property-value-ecmascript": "^1.2.0" + } + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "requires": { + "has-flag": "^3.0.0" + } } } }, @@ -6401,6 +7059,17 @@ "tar-pack": "3.4.1", "tmp": "0.0.33", "validate-npm-package-name": "3.0.0" + }, + "dependencies": { + "cross-spawn": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz", + "integrity": "sha1-e5JHYhwjrf3ThWAEqCPL45dCTUE=", + "requires": { + "lru-cache": "^4.0.1", + "which": "^1.2.9" + } + } } }, "create-react-class": { @@ -6422,15 +7091,6 @@ "warning": "^4.0.3" } }, - "cross-spawn": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz", - "integrity": "sha1-e5JHYhwjrf3ThWAEqCPL45dCTUE=", - "requires": { - "lru-cache": "^4.0.1", - "which": "^1.2.9" - } - }, "crypto-browserify": { "version": "3.12.0", "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz", @@ -8204,6 +8864,13 @@ "acorn": "^7.1.1", "acorn-jsx": "^5.2.0", "eslint-visitor-keys": "^1.1.0" + }, + "dependencies": { + "acorn-jsx": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.1.tgz", + "integrity": "sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==" + } } }, "esprima": { @@ -11669,24 +12336,6 @@ "resolved": "https://registry.npmjs.org/jss-global/-/jss-global-3.0.0.tgz", "integrity": "sha512-wxYn7vL+TImyQYGAfdplg7yaxnPQ9RaXY/cIA8hawaVnmmWxDHzBK32u1y+RAvWboa3lW83ya3nVZ/C+jyjZ5Q==" }, - "jss-nested": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/jss-nested/-/jss-nested-6.0.1.tgz", - "integrity": "sha512-rn964TralHOZxoyEgeq3hXY8hyuCElnvQoVrQwKHVmu55VRDd6IqExAx9be5HgK0yN/+hQdgAXQl/GUrBbbSTA==", - "requires": { - "warning": "^3.0.0" - }, - "dependencies": { - "warning": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/warning/-/warning-3.0.0.tgz", - "integrity": "sha1-MuU3fLVy3kqwR1O9+IIcAe1gW3w=", - "requires": { - "loose-envify": "^1.0.0" - } - } - } - }, "jss-plugin-camel-case": { "version": "10.1.1", "resolved": "https://registry.npmjs.org/jss-plugin-camel-case/-/jss-plugin-camel-case-10.1.1.tgz", @@ -12182,19 +12831,6 @@ "resolved": "https://registry.npmjs.org/material-ui-nested-menu-item/-/material-ui-nested-menu-item-1.0.2.tgz", "integrity": "sha512-LZb8xI0FrAI/A3P2vT3CB9bmSoOFWOK0dikTc1t9VvEpp1a8hZkbVUz7VhETnoLUYu3NXCkgulmXcl3zitqI9A==" }, - "material-ui-pickers": { - "version": "2.2.4", - "resolved": "https://registry.npmjs.org/material-ui-pickers/-/material-ui-pickers-2.2.4.tgz", - "integrity": "sha512-QCQh08Ylmnt+o4laW+rPs92QRAcESv3sPXl50YadLm++rAZAXAOh3K8lreGdynCMYFgZfdyu81Oz9xzTlAZNfw==", - "requires": { - "@types/react-text-mask": "^5.4.3", - "clsx": "^1.0.2", - "react-event-listener": "^0.6.6", - "react-text-mask": "^5.4.3", - "react-transition-group": "^2.5.3", - "tslib": "^1.9.3" - } - }, "md5-file": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/md5-file/-/md5-file-4.0.0.tgz", @@ -12277,6 +12913,24 @@ "warning": "^4.0.1" }, "dependencies": { + "jss-nested": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/jss-nested/-/jss-nested-6.0.1.tgz", + "integrity": "sha512-rn964TralHOZxoyEgeq3hXY8hyuCElnvQoVrQwKHVmu55VRDd6IqExAx9be5HgK0yN/+hQdgAXQl/GUrBbbSTA==", + "requires": { + "warning": "^3.0.0" + }, + "dependencies": { + "warning": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/warning/-/warning-3.0.0.tgz", + "integrity": "sha1-MuU3fLVy3kqwR1O9+IIcAe1gW3w=", + "requires": { + "loose-envify": "^1.0.0" + } + } + } + }, "react-transition-group": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-2.9.0.tgz", @@ -12339,6 +12993,37 @@ } } }, + "jss-nested": { + "dependencies": { + "warning": {} + } + }, + "material-ui-pickers": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/material-ui-pickers/-/material-ui-pickers-2.2.4.tgz", + "integrity": "sha512-QCQh08Ylmnt+o4laW+rPs92QRAcESv3sPXl50YadLm++rAZAXAOh3K8lreGdynCMYFgZfdyu81Oz9xzTlAZNfw==", + "requires": { + "@types/react-text-mask": "^5.4.3", + "clsx": "^1.0.2", + "react-event-listener": "^0.6.6", + "react-text-mask": "^5.4.3", + "react-transition-group": "^2.5.3", + "tslib": "^1.9.3" + }, + "dependencies": { + "react-transition-group": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-2.9.0.tgz", + "integrity": "sha512-+HzNTCHpeQyl4MJ/bdE0u6XRMe9+XG/+aL4mCxVN4DnPBQ0/5bfHWPDuOZUzYdMj94daZaZdCCc1Dzt9R/xSSg==", + "requires": { + "dom-helpers": "^3.4.0", + "loose-envify": "^1.4.0", + "prop-types": "^15.6.2", + "react-lifecycles-compat": "^3.0.4" + } + } + } + }, "moment": { "version": "2.24.0", "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", @@ -12393,7 +13078,8 @@ "loose-envify": "^1.4.0", "prop-types": "^15.6.2" } - } + }, + "sass-loader": {} } }, "mdn-data": { @@ -13806,6 +14492,19 @@ "requires": { "postcss": "^7.0.2", "postcss-selector-parser": "^6.0.2" + }, + "dependencies": { + "postcss-selector-parser": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.4.tgz", + "integrity": "sha512-gjMeXBempyInaBqpp8gODmwZ52WaYsVOsfr4L4lDQ7n3ncD6mEyySiDtgzCT+NYC0mmeOLvtsF8iaEf0YT6dBw==", + "requires": { + "cssesc": "^3.0.0", + "indexes-of": "^1.0.1", + "uniq": "^1.0.1", + "util-deprecate": "^1.0.2" + } + } } }, "postcss-browser-comments": { @@ -13826,6 +14525,17 @@ "postcss-value-parser": "^4.0.2" }, "dependencies": { + "postcss-selector-parser": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.4.tgz", + "integrity": "sha512-gjMeXBempyInaBqpp8gODmwZ52WaYsVOsfr4L4lDQ7n3ncD6mEyySiDtgzCT+NYC0mmeOLvtsF8iaEf0YT6dBw==", + "requires": { + "cssesc": "^3.0.0", + "indexes-of": "^1.0.1", + "uniq": "^1.0.1", + "util-deprecate": "^1.0.2" + } + }, "postcss-value-parser": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.1.0.tgz", @@ -14889,13 +15599,12 @@ } }, "react": { - "version": "16.10.2", - "resolved": "https://registry.npmjs.org/react/-/react-16.10.2.tgz", - "integrity": "sha512-MFVIq0DpIhrHFyqLU0S3+4dIcBhhOvBE8bJ/5kHPVOVaGdo0KuiQzpcjCPsf585WvhypqtrMILyoE2th6dT+Lw==", + "version": "17.0.1", + "resolved": "https://registry.npmjs.org/react/-/react-17.0.1.tgz", + "integrity": "sha512-lG9c9UuMHdcAexXtigOZLX8exLWkW0Ku29qPRU8uhF2R9BN96dLCt0psvzPLlHc5OWkgymP3qwTRgbnw5BKx3w==", "requires": { "loose-envify": "^1.1.0", - "object-assign": "^4.1.1", - "prop-types": "^15.6.2" + "object-assign": "^4.1.1" } }, "react-alert": { @@ -15306,14 +16015,24 @@ } }, "react-dom": { - "version": "16.10.2", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-16.10.2.tgz", - "integrity": "sha512-kWGDcH3ItJK4+6Pl9DZB16BXYAZyrYQItU4OMy0jAkv5aNqc+mAKb4TpFtAteI6TJZu+9ZlNhaeNQSVQDHJzkw==", + "version": "17.0.1", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-17.0.1.tgz", + "integrity": "sha512-6eV150oJZ9U2t9svnsspTMrWNyHc6chX0KzDeAOXftRa8bNeOKTTfCJ7KorIwenkHd2xqVTBTCZd79yk/lx/Ug==", "requires": { "loose-envify": "^1.1.0", "object-assign": "^4.1.1", - "prop-types": "^15.6.2", - "scheduler": "^0.16.2" + "scheduler": "^0.20.1" + }, + "dependencies": { + "scheduler": { + "version": "0.20.1", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.20.1.tgz", + "integrity": "sha512-LKTe+2xNJBNxu/QhHvDR14wUXHRQbVY5ZOYpOGWRzhydZUqrLb2JBvLPY7cAqFmqrWuDED0Mjk7013SZiOz6Bw==", + "requires": { + "loose-envify": "^1.1.0", + "object-assign": "^4.1.1" + } + } } }, "react-draggable": { @@ -15584,6 +16303,49 @@ "workbox-webpack-plugin": "4.3.1" }, "dependencies": { + "@babel/core": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.9.0.tgz", + "integrity": "sha512-kWc7L0fw1xwvI0zi8OKVBuxRVefwGOrKSQMvrQ3dW+bIIavBY3/NpXmpjMy7bQnLgwgzWQZ8TlM57YHpHNHz4w==", + "requires": { + "@babel/code-frame": "^7.8.3", + "@babel/generator": "^7.9.0", + "@babel/helper-module-transforms": "^7.9.0", + "@babel/helpers": "^7.9.0", + "@babel/parser": "^7.9.0", + "@babel/template": "^7.8.6", + "@babel/traverse": "^7.9.0", + "@babel/types": "^7.9.0", + "convert-source-map": "^1.7.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.1", + "json5": "^2.1.2", + "lodash": "^4.17.13", + "resolve": "^1.3.2", + "semver": "^5.4.1", + "source-map": "^0.5.0" + }, + "dependencies": { + "@babel/code-frame": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.12.11.tgz", + "integrity": "sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw==", + "requires": { + "@babel/highlight": "^7.10.4" + } + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + }, + "source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=" + } + } + }, "@babel/generator": { "version": "7.11.4", "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.11.4.tgz", @@ -15887,6 +16649,46 @@ "resolve": "^1.12.0" } }, + "babel-loader": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-8.1.0.tgz", + "integrity": "sha512-7q7nC1tYOrqvUrN3LQK4GwSk/TQorZSOlO9C+RZDZpODgyN4ZlCqE5q9cDsyWOliN+aU9B4JX01xK9eJXowJLw==", + "requires": { + "find-cache-dir": "^2.1.0", + "loader-utils": "^1.4.0", + "mkdirp": "^0.5.3", + "pify": "^4.0.1", + "schema-utils": "^2.6.5" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "ajv-keywords": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==" + }, + "schema-utils": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.1.tgz", + "integrity": "sha512-SHiNtMOUGWBQJwzISiVYKu82GiV4QYGePp3odlY1tuKO7gPtphAT5R/py0fA6xtbgLL/RvtJZnU9b8s0F1q0Xg==", + "requires": { + "@types/json-schema": "^7.0.5", + "ajv": "^6.12.4", + "ajv-keywords": "^3.5.2" + } + } + } + }, "cacache": { "version": "12.0.4", "resolved": "https://registry.npmjs.org/cacache/-/cacache-12.0.4.tgz", @@ -15982,6 +16784,11 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" + }, "schema-utils": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-1.0.0.tgz", @@ -16115,6 +16922,19 @@ "classnames": "^2.2.6", "prop-types": "^15.7.2", "react-transition-group": "^2.6.1" + }, + "dependencies": { + "react-transition-group": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-2.9.0.tgz", + "integrity": "sha512-+HzNTCHpeQyl4MJ/bdE0u6XRMe9+XG/+aL4mCxVN4DnPBQ0/5bfHWPDuOZUzYdMj94daZaZdCCc1Dzt9R/xSSg==", + "requires": { + "dom-helpers": "^3.4.0", + "loose-envify": "^1.4.0", + "prop-types": "^15.6.2", + "react-lifecycles-compat": "^3.0.4" + } + } } }, "react-transition-group": { @@ -16852,15 +17672,6 @@ "xmlchars": "^2.1.1" } }, - "scheduler": { - "version": "0.16.2", - "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.16.2.tgz", - "integrity": "sha512-BqYVWqwz6s1wZMhjFvLfVR5WXP7ZY32M/wYPo04CcuPM7XZEbV2TBNW7Z0UkguPTl0dWMA59VbNXxK6q+pHItg==", - "requires": { - "loose-envify": "^1.1.0", - "object-assign": "^4.1.1" - } - }, "schema-utils": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index a4c40596..bb24de65 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,13 +1,13 @@ { "name": "shuffler", "homepage": "https://shuffler.io", - "version": "0.6.0", + "version": "0.8.53", "private": true, "dependencies": { "@material-ui/core": "^4.5.2", "@material-ui/icons": "^4.5.1", "@material-ui/styles": "^4.5.2", - "@use-it/interval": "^0.1.3", + "@use-it/interval": "^1.0.0", "babel-eslint": "^10.1.0", "class-transformer": "^0.3.1", "create-react-app": "^2.0.3", @@ -32,7 +32,7 @@ "md5-file": "^4.0.0", "mdbreact": "^4.21.1", "moment": "~2.20.1", - "react": "^16.10.2", + "react": "^16.14.0", "react-alert": "^5.5.0", "react-alert-template-basic": "^1.0.0", "react-beforeunload": "^2.2.1", @@ -40,7 +40,7 @@ "react-cookie": "^4.0.1", "react-cytoscapejs": "^1.2.0", "react-device-detect": "^1.9.10", - "react-dom": "^16.10.2", + "react-dom": "^16.14.0", "react-draggable": "^3.3.2", "react-dropzone": "^10.1.10", "react-ga": "^2.7.0", diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 5bdda5b9..0864b984 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -142,7 +142,7 @@ const App = (message, props) => { } /> } /> } /> - } /> + } /> } /> } /> { window.location.pathname = "/docs/about" }} /> diff --git a/frontend/src/assets/img/icpl_logo.png b/frontend/src/assets/img/icpl_logo.png new file mode 100644 index 00000000..00567778 Binary files /dev/null and b/frontend/src/assets/img/icpl_logo.png differ diff --git a/frontend/src/components/OrgHeader.js b/frontend/src/components/OrgHeader.js index 7e242f50..5d86745a 100644 --- a/frontend/src/components/OrgHeader.js +++ b/frontend/src/components/OrgHeader.js @@ -3,9 +3,15 @@ import { makeStyles } from '@material-ui/styles'; import { useTheme } from '@material-ui/core/styles'; import Tooltip from '@material-ui/core/Tooltip'; +import Grid from '@material-ui/core/Grid'; import Button from '@material-ui/core/Button'; import TextField from '@material-ui/core/TextField'; +import Typography from '@material-ui/core/Typography'; import { useAlert } from "react-alert"; +import IconButton from '@material-ui/core/IconButton'; +import ExpandLessIcon from '@material-ui/icons/ExpandLess'; +import ExpandMoreIcon from '@material-ui/icons/ExpandMore'; +import SaveIcon from '@material-ui/icons/Save'; const useStyles = makeStyles({ notchedOutline: { @@ -23,11 +29,17 @@ const OrgHeader = (props) => { const classes = useStyles() var upload = "" + const defaultBranch = "master" const [orgName, setOrgName] = React.useState(selectedOrganization.name) const [orgDescription, setOrgDescription] = React.useState(selectedOrganization.description) + const [appDownloadUrl, setAppDownloadUrl] = React.useState(selectedOrganization.defaults === undefined ? "https://github.com/frikky/shuffle-apps" : selectedOrganization.defaults.app_download_repo === undefined || selectedOrganization.defaults.app_download_repo.length === 0 ? "https://github.com/frikky/shuffle-apps" : selectedOrganization.defaults.app_download_repo) + const [appDownloadBranch, setAppDownloadBranch] = React.useState(selectedOrganization.defaults === undefined ? defaultBranch : selectedOrganization.defaults.app_download_branch === undefined || selectedOrganization.defaults.app_download_branch.length === 0 ? defaultBranch : selectedOrganization.defaults.app_download_branch) + const [workflowDownloadUrl, setWorkflowDownloadUrl] = React.useState(selectedOrganization.defaults === undefined ? "https://github.com/frikky/shuffle-apps" : selectedOrganization.defaults.workflow_download_repo === undefined || selectedOrganization.defaults.workflow_download_repo.length === 0 ? "https://github.com/frikky/shuffle-workflows" : selectedOrganization.defaults.workflow_download_repo) + const [workflowDownloadBranch, setWorkflowDownloadBranch] = React.useState(selectedOrganization.defaults === undefined ? defaultBranch : selectedOrganization.defaults.workflow_download_branch === undefined || selectedOrganization.defaults.workflow_download_branch.length === 0 ? defaultBranch : selectedOrganization.defaults.workflow_download_branch) const [file, setFile] = React.useState("") const [fileBase64, setFileBase64] = React.useState(selectedOrganization.image) + const [expanded, setExpanded] = React.useState(false) if (file !== "") { const img = document.getElementById('logo') @@ -55,12 +67,13 @@ const OrgHeader = (props) => { } } - const handleEditOrg = (name, description, orgId, image) => { + const handleEditOrg = (name, description, orgId, image, defaults) => { const data = { "name": name, "description": description, "org_id": orgId, "image": image, + "defaults": defaults, } const url = globalUrl + `/api/v1/orgs/${selectedOrganization.id}`; @@ -102,9 +115,14 @@ const OrgHeader = (props) => { style={{ width: 150, height: 55, flex: 1 }} variant="contained" color="primary" - onClick={() => handleEditOrg(orgName, orgDescription, selectedOrganization.id, selectedOrganization.image)} + onClick={() => handleEditOrg(orgName, orgDescription, selectedOrganization.id, selectedOrganization.image, { + "app_download_repo": appDownloadUrl, + "app_download_branch": appDownloadBranch, + "workflow_download_repo": workflowDownloadUrl, + "workflow_download_branch": workflowDownloadBranch, + })} > - Save Changes + var imageData = file.length > 0 ? file : fileBase64 @@ -113,7 +131,7 @@ const OrgHeader = (props) => { return (
- +
0 ? null : "1px solid #f85a3e", cursor: "pointer", backgroundColor: imageData !== undefined && imageData.length > 0 ? null : theme.palette.inputColor, maxWidth: 174, maxHeight: 174}} onClick={() => {upload.click()}}> upload = ref} onChange={editHeaderImage} /> {imageInfo} @@ -188,9 +206,151 @@ const OrgHeader = (props) => {
{orgSaveButton}
-
-
+ + +
+ { + setExpanded(!expanded) + }}> + {expanded ? + + : + + } + + {expanded ? + + + + + App Download URL + + { + setAppDownloadUrl(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style:{ + color: "white", + }, + }} + /> + + + + + + App Download Branch + + { + setAppDownloadBranch(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style:{ + color: "white", + }, + }} + /> + + + + + + Workflow Download URL + + { + setWorkflowDownloadUrl(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style:{ + color: "white", + }, + }} + /> + + + + + + Workflow Download Branch + + { + setWorkflowDownloadBranch(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style:{ + color: "white", + }, + }} + /> + + + {/* + + {expanded ? + + : + + } + + */} + + : + null + } +
) } diff --git a/frontend/src/defaultCytoscapeStyle.js b/frontend/src/defaultCytoscapeStyle.js index b41a048f..e0732bac 100644 --- a/frontend/src/defaultCytoscapeStyle.js +++ b/frontend/src/defaultCytoscapeStyle.js @@ -74,6 +74,8 @@ const data = [{ 'shape': 'octagon', 'border-color': 'orange', 'background-color': '#213243', + 'background-width': '100%', + 'background-height': '100%', }, }, { diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index ad857ad6..f590ad60 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -1,4 +1,4 @@ -import React, { useEffect} from 'react'; +import React, { useState } from 'react'; import { makeStyles } from '@material-ui/styles'; import {Link} from 'react-router-dom'; @@ -31,6 +31,11 @@ import { useTheme } from '@material-ui/core/styles'; import HandlePayment from './HandlePayment' import OrgHeader from '../components/OrgHeader' +import EditIcon from '@material-ui/icons/Edit'; +import SelectAllIcon from '@material-ui/icons/SelectAll'; +import OpenInNewIcon from '@material-ui/icons/OpenInNew'; +import CloudDownloadIcon from '@material-ui/icons/CloudDownload'; +import DescriptionIcon from '@material-ui/icons/Description'; import PolymerIcon from '@material-ui/icons/Polymer'; import CheckCircleIcon from '@material-ui/icons/CheckCircle'; import CloseIcon from '@material-ui/icons/Close'; @@ -59,6 +64,7 @@ const Admin = (props) => { const theme = useTheme(); const classes = useStyles(); const [firstRequest, setFirstRequest] = React.useState(true); + const [orgRequest, setOrgRequest] = React.useState(true); const [modalUser, setModalUser] = React.useState({}); const [modalOpen, setModalOpen] = React.useState(false); @@ -78,11 +84,13 @@ const Admin = (props) => { const [environments, setEnvironments] = React.useState([]); const [authentication, setAuthentication] = React.useState([]); const [schedules, setSchedules] = React.useState([]) + const [files, setFiles] = React.useState([]) const [selectedUser, setSelectedUser] = React.useState({}) const [newPassword, setNewPassword] = React.useState(""); const [selectedUserModalOpen, setSelectedUserModalOpen] = React.useState(false) const [selectedAuthentication, setSelectedAuthentication] = React.useState({}) const [selectedAuthenticationModalOpen, setSelectedAuthenticationModalOpen] = React.useState(false) + const [authenticationFields, setAuthenticationFields] = React.useState([]) const [showArchived, setShowArchived] = React.useState(false) const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" @@ -272,9 +280,70 @@ const Admin = (props) => { }) } - - - + const saveAuthentication = (authentication) => { + const data = authentication + const url = globalUrl + '/api/v1/apps/authentication'; + + fetch(url, { + mode: 'cors', + method: 'PUT', + body: JSON.stringify(data), + credentials: 'include', + crossDomain: true, + withCredentials: true, + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + }) + .then(response => + response.json().then(responseJson => { + if (responseJson["success"] === false) { + alert.error("Failed changing authentication") + } else { + //alert.success("Successfully password!") + setSelectedUserModalOpen(false) + getAppAuthentication() + } + }), + ) + .catch(error => { + alert.error("Err: " + error.toString()) + }); + } + + const editAuthenticationConfig = (id) => { + const data = { + "id": id, + "action": "assign_everywhere", + } + const url = globalUrl + '/api/v1/apps/authentication/'+id+"/config"; + + fetch(url, { + mode: 'cors', + method: 'POST', + body: JSON.stringify(data), + credentials: 'include', + crossDomain: true, + withCredentials: true, + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + }) + .then(response => + response.json().then(responseJson => { + if (responseJson["success"] === false) { + alert.error("Failed overwriting appauth in workflows") + } else { + alert.success("Successfully updated auth everywhere!") + setSelectedUserModalOpen(false) + getAppAuthentication() + } + }), + ) + .catch(error => { + alert.error("Err: " + error.toString()) + }); + } const onPasswordChange = () => { const data = { "username": selectedUser.username, "newpassword": newPassword } @@ -296,7 +365,7 @@ const Admin = (props) => { if (responseJson["success"] === false) { alert.error("Failed setting new password") } else { - alert.success("Successfully password!") + alert.success("Successfully updated password!") setSelectedUserModalOpen(false) } }), @@ -466,6 +535,33 @@ const Admin = (props) => { }) } + const flushQueue = (name) => { + // Just use this one? + const url = globalUrl + '/api/v1/flush_queue'; + fetch(url, { + method: 'DELETE', + credentials: "include", + headers: { + 'Content-Type': 'application/json', + }, + }) + .then(response => + response.json().then(responseJson => { + if (responseJson["success"] === false) { + alert.error(responseJson.reason) + getEnvironments() + } else { + setLoginInfo("") + setModalOpen(false) + getEnvironments() + } + }), + ) + .catch(error => { + console.log("Error when deleting: ", error) + }) + } + const deleteEnvironment = (name) => { // FIXME - add some check here ROFL alert.info("Deleting environment " + name) @@ -550,6 +646,78 @@ const Admin = (props) => { }); } + const getFiles = () => { + fetch(globalUrl + "/api/v1/files", { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for apps :O!") + return + } + + return response.json() + }) + .then((responseJson) => { + //console.log(responseJson) + setFiles(responseJson) + }) + .catch(error => { + alert.error(error.toString()) + }); + } + + const downloadFile = (file) => { + fetch(globalUrl + "/api/v1/files/"+file.id+"/content", { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for apps :O!") + return "" + } + + return response.text() + }) + .then((respdata) => { + if (respdata.length === 0) { + alert.error("Failed getting file") + return + } + + var blob = new Blob( [ respdata ], { + type: 'application/octet-stream' + }) + + var url = URL.createObjectURL( blob ) + var link = document.createElement( 'a' ) + link.setAttribute( 'href', url ) + link.setAttribute( 'download', `${file.filename}` ) + var event = document.createEvent( 'MouseEvents' ) + event.initMouseEvent( 'click', true, true, window, 1, 0, 0, 0, 0, false, false, false, false, 0, null) + link.dispatchEvent( event ) + + //return response.json() + }) + .then((responseJson) => { + //console.log(responseJson) + //setSchedules(responseJson) + }) + .catch(error => { + alert.error(error.toString()) + }); + } + const getSchedules = () => { fetch(globalUrl + "/api/v1/workflows/schedules", { method: 'GET', @@ -596,6 +764,7 @@ const Admin = (props) => { .then((responseJson) => { if (responseJson.success) { //console.log(responseJson.data) + console.log(responseJson) setAuthentication(responseJson.data) } else { alert.error("Failed getting authentications") @@ -705,6 +874,48 @@ const Admin = (props) => { }); } + const setConfig = (event, newValue) => { + if (newValue === 1) { + getUsers() + } else if (newValue === 2) { + getAppAuthentication() + } else if (newValue === 3) { + getEnvironments() + } else if (newValue === 4) { + getSchedules() + } else if (newValue === 5) { + getFiles() + } else if (newValue === 6) { + getOrgs() + } + + if (newValue === 6) { + console.log("Should get apps for categories.") + } + + const views = { + 0: "organization", + 1: "users", + 2: "app_auth", + 3: "environments", + 4: "schedules", + 5: "files", + 6: "categories", + } + + //var theURL = window.location.pathname + //FIXME: Add url edits + //var theURL = window.location + //theURL.replace(`/${views[curTab]}`, `/${views[newValue]}`) + //window.history.pushState({"html":response.html,"pageTitle":response.pageTitle},"", urlPath); + + //console.log(newpath) + //window.location.pathame = newpath + + setModalUser({}) + setCurTab(newValue) + } + if (firstRequest) { setFirstRequest(false) @@ -720,19 +931,20 @@ const Admin = (props) => { "app_auth": 2, "environments": 3, "schedules": 4, - "categories": 5, + "files": 5, } if (props.match.params.key !== undefined) { const tmpitem = views[props.match.params.key] if (tmpitem !== undefined) { - setCurTab(tmpitem) + //setCurTab(tmpitem) + setConfig("", tmpitem) } } } - if (selectedOrganization.id === undefined && userdata !== undefined && userdata.active_org !== undefined) { - //setSelectedOrganization(userdata.active_org) + if (selectedOrganization.id === undefined && userdata !== undefined && userdata.active_org !== undefined && orgRequest) { + setOrgRequest(false) handleGetOrg(userdata.active_org.id) } @@ -820,8 +1032,8 @@ const Admin = (props) => { }); } - const editAuthenticationModal = - { setSelectedAuthenticationModalOpen(false) }} PaperProps={{ @@ -833,56 +1045,71 @@ const Admin = (props) => { }, }} > - Edit authentication + Edit authentication for {selectedAuthentication.app.name} ({selectedAuthentication.label}) -
- setNewPassword(e.target.value)} - /> - -
- - - + {selectedAuthentication.fields.map((data, index) => { + return ( +
+ {data.key} + { + authenticationFields[index].value = e.target.value + setAuthenticationFields(authenticationFields) + }} + /> +
+ ) + })} + + + + + : null const editUserModal = { }, }} > - Edit user +
{ })} - {isCloud && selectedOrganization.subscriptions !== null && selectedOrganization.subscriptions.length > 0 ? + {isCloud && selectedOrganization.subscriptions !== undefined && selectedOrganization.subscriptions !== null && selectedOrganization.subscriptions.length > 0 ?
Your subscription{selectedOrganization.subscriptions.length > 1 ? "s" : ""} @@ -1429,6 +1656,7 @@ const Admin = (props) => {
: null + const filesView = curTab === 5 ? +
+
+

Files

+ Files from Workflows. Learn more +
+ + + + + + + + + + + + {files === undefined || files === null ? null : files.map((file, index) => { + var bgColor = "#27292d" + if (index % 2 === 0) { + bgColor = "#1f2023" + } + + return ( + + + + + + + + + + + style={{minWidth: 100, maxWidth: 100, overflow: "hidden"}} + /> + + + + + { + downloadFile(file) + }}> + + + + style={{minWidth: 75, maxWidth: 75, overflow: "hidden"}} + /> + {/* + + + + */} + + ) + })} + +
+ : null + const schedulesView = curTab === 4 ?
@@ -1562,8 +1902,13 @@ const Admin = (props) => { /> {schedules === undefined || schedules === null ? null : schedules.map((schedule, index) => { + var bgColor = "#27292d" + if (index % 2 === 0) { + bgColor = "#1f2023" + } + return ( - + {schedule.seconds} seconds} @@ -1673,35 +2018,53 @@ const Admin = (props) => {
: null + const updateAppAuthentication = (field) => { + setSelectedAuthenticationModalOpen(true) + setSelectedAuthentication(field) + //{selectedAuthentication.fields.map((data, index) => { + var newfields = [] + for (var key in field.fields) { + newfields.push({ + "key": field.fields[key].key, + "value": "", + }) + } + setAuthenticationFields(newfields) + } + const authenticationView = curTab === 2 ?

App Authentication

Control the authentication options for individual apps. Actions can be destructive! - . Learn more +  Learn more
+ { /> {authentication === undefined ? null : authentication.map((data, index) => { + var bgColor = "#27292d" + if (index % 2 === 0) { + bgColor = "#1f2023" + } + return ( - + - style={{minWidth: 150, maxWidth: 150}} + style={{minWidth: 75, maxWidth: 75}} /> + { @@ -1741,16 +2113,43 @@ const Admin = (props) => { style={{minWidth: 200, maxWidth: 200, overflow: "hidden"}} /> - + + ) @@ -1820,6 +2219,11 @@ const Admin = (props) => { return null } + //var bgColor = "#27292d" + //if (index % 2 === 0) { + // bgColor = "#1f2023" + //} + return ( { style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}} > + {/**/} {
: null - const organizationsTab = curTab === 6 ? + const organizationsTab = curTab === 7 ?

Organizations

@@ -1941,7 +2346,7 @@ const Admin = (props) => {
: null - const hybridTab = curTab === 5 ? + const hybridTab = curTab === 6 ?

Hybrid

@@ -1983,48 +2388,11 @@ const Admin = (props) => { // primary={environment.Registered ? "true" : "false"} - const setConfig = (event, newValue) => { - if (newValue === 1) { - getUsers() - } else if (newValue === 2) { - getAppAuthentication() - } else if (newValue === 3) { - getEnvironments() - } else if (newValue === 4) { - getSchedules() - } else if (newValue === 6) { - getOrgs() - } - - if (newValue === 6) { - console.log("Should get apps for categories.") - } - - const views = { - 0: "organization", - 1: "users", - 2: "app_auth", - 3: "environments", - 4: "schedules", - 5: "categories", - } - - //var theURL = window.location.pathname - //FIXME: Add url edits - //var theURL = window.location - //theURL.replace(`/${views[curTab]}`, `/${views[newValue]}`) - //window.history.pushState({"html":response.html,"pageTitle":response.pageTitle},"", urlPath); - - //console.log(newpath) - //window.location.pathame = newpath - - setModalUser({}) - setCurTab(newValue) - } + const iconStyle = {marginRight: 10} const data = -
+
{ aria-label="disabled tabs example" > Organization/> - {isCloud ? null : Users />} + Users /> {isCloud ? null : App Authentication/>} {isCloud ? null : Environments/>} {isCloud ? null : Schedules />} + {isCloud ? null : Files />} {window.location.protocol == "http:" && window.location.port === "3000" ? Hybrid/> : null} {window.location.protocol == "http:" && window.location.port === "3000" ? Organizations/> : null} {window.location.protocol === "http:" && window.location.port === "3000" ? Categories/> : null} @@ -2049,6 +2418,7 @@ const Admin = (props) => { {usersView} {environmentView} {schedulesView} + {filesView} {hybridTab} {organizationsTab}
diff --git a/frontend/src/views/AdminSetup.jsx b/frontend/src/views/AdminSetup.jsx index d79c01f6..275cd5b3 100644 --- a/frontend/src/views/AdminSetup.jsx +++ b/frontend/src/views/AdminSetup.jsx @@ -6,11 +6,6 @@ import TextField from '@material-ui/core/TextField'; import Button from '@material-ui/core/Button'; import Paper from '@material-ui/core/Paper'; -const hrefStyle = { - color: "white", - textDecoration: "none" -} - const bodyDivStyle = { margin: "auto", marginTop: "100px", @@ -35,7 +30,7 @@ const useStyles = makeStyles({ }); const AdminAccount = props => { - const { globalUrl, isLoaded, isLoggedIn, setIsLoggedIn, setCookie, } = props; + const { globalUrl, isLoaded, isLoggedIn, } = props; const [username, setUsername] = useState(""); const [password, setPassword] = useState(""); diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index e47c7c6c..7ce3669f 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -19,6 +19,7 @@ import Select from '@material-ui/core/Select'; import MenuItem from '@material-ui/core/MenuItem'; import Divider from '@material-ui/core/Divider'; import Dialog from '@material-ui/core/Dialog'; +import Modal from '@material-ui/core/Modal'; import DialogActions from '@material-ui/core/DialogActions'; import DialogTitle from '@material-ui/core/DialogTitle'; import InputLabel from '@material-ui/core/InputLabel'; @@ -37,8 +38,15 @@ import Switch from '@material-ui/core/Switch'; import ReactJson from 'react-json-view' import { useBeforeunload } from 'react-beforeunload'; import NestedMenuItem from "material-ui-nested-menu-item"; +import Fade from '@material-ui/core/Fade'; import ArrowUpwardIcon from '@material-ui/icons/ArrowUpward'; +import VisibilityIcon from '@material-ui/icons/Visibility'; +import DoneIcon from '@material-ui/icons/Done'; +import CloseIcon from '@material-ui/icons/Close'; +import ErrorIcon from '@material-ui/icons/Error'; +import FindReplaceIcon from '@material-ui/icons/FindReplace'; +import ArrowLeftIcon from '@material-ui/icons/ArrowLeft'; import CachedIcon from '@material-ui/icons/Cached'; import AddIcon from '@material-ui/icons/Add'; import DirectionsRunIcon from '@material-ui/icons/DirectionsRun'; @@ -75,6 +83,7 @@ import cxtmenu from 'cytoscape-cxtmenu'; import { w3cwebsocket as W3CWebSocket } from "websocket"; import { useAlert } from "react-alert"; +import { validateJson } from "./Workflows"; import { GetParsedPaths } from "./Apps"; const surfaceColor = "#27292D" @@ -105,13 +114,14 @@ const splitter = "|~|" //const referenceUrl = window.location.origin+"/api/v1/hooks/" const AngularWorkflow = (props) => { - const { globalUrl, isLoggedIn, isLoaded } = props; + const { globalUrl, isLoggedIn, isLoaded, userdata } = props; const referenceUrl = globalUrl+"/api/v1/hooks/" const alert = useAlert() const borderRadius = 3 const [bodyWidth, bodyHeight] = useWindowSize(); const appBarSize = 74 + var to_be_copied = "" const [cystyle, ] = useState(cytoscapestyle) const [cy, setCy] = React.useState() @@ -119,9 +129,12 @@ const AngularWorkflow = (props) => { const [currentView, setCurrentView] = React.useState(0) const [triggerAuthentication, setTriggerAuthentication] = React.useState({}) const [triggerFolders, setTriggerFolders] = React.useState([]) + const [workflows, setWorkflows] = React.useState([]) const [showEnvironment, setShowEnvironment] = React.useState(false) const [workflow, setWorkflow] = React.useState({}); + const [userSettings, setUserSettings] = React.useState({}); + const [subworkflow, setSubworkflow] = React.useState({}); const [leftViewOpen, setLeftViewOpen] = React.useState(true); const [leftBarSize, setLeftBarSize] = React.useState(350) const [executionText, setExecutionText] = React.useState(""); @@ -140,12 +153,21 @@ const AngularWorkflow = (props) => { const [requiresAuthentication, setRequiresAuthentication] = React.useState(false) const [rightSideBarOpen, setRightSideBarOpen] = React.useState(false) const [showSkippedActions, setShowSkippedActions] = React.useState(false) + const [lastExecution, setLastExecution] = React.useState("") + + const [selectedResult, setSelectedResult] = React.useState({}) + const [codeModalOpen, setCodeModalOpen] = React.useState(false); const [variableAnchorEl, setVariableAnchorEl] = React.useState(null) const [sourceValue, setSourceValue] = React.useState({}) const [destinationValue, setDestinationValue] = React.useState({}) const [conditionValue, setConditionValue] = React.useState({}) + const [dragging, setDragging] = React.useState(false) + const [dragPosition, setDragPosition] = React.useState({ + x: 0, + y: 0, + }) // Trigger stuff const [selectedTrigger, setSelectedTrigger] = React.useState({}); @@ -192,7 +214,7 @@ const AngularWorkflow = (props) => { const cloudSyncEnabled = props.userdata !== undefined && props.userdata.active_org !== null && props.userdata.active_org !== undefined ? props.userdata.active_org.cloud_sync === true : false //const triggerEnvironments = cloudSyncEnabled ? ["cloud", "onprem"] : environments const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" - const triggerEnvironments = isCloud ? ["cloud"] : ["cloud", "onprem"] + const triggerEnvironments = isCloud ? ["cloud"] : ["onprem", "cloud"] const unloadText = 'Are you sure you want to leave without saving (CTRL+S)?' useBeforeunload(() => { @@ -210,6 +232,98 @@ const AngularWorkflow = (props) => { } }) + const getAvailableWorkflows = (trigger_index) => { + fetch(globalUrl+"/api/v1/workflows", { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for workflows :O!") + return + } + return response.json() + }) + .then((responseJson) => { + if (responseJson !== undefined) { + setWorkflows(responseJson) + + if (trigger_index > -1) { + const trigger = workflow.triggers[trigger_index] + if (trigger.parameters.length >= 3) { + for (var key in trigger.parameters) { + const param = trigger.parameters[key] + if (param.name === "workflow") { + const sub = responseJson.find(data => data.id === param.value) + if (sub !== undefined && subworkflow.id !== sub.id) { + setSubworkflow(sub) + } + } + } + } + } + } + }) + .catch(error => { + alert.error(error.toString()) + }); + } + + const generateApikey = () => { + fetch(globalUrl+"/api/v1/generateapikey", { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for WORKFLOW EXECUTION :O!") + } + + return response.json() + }) + .then((responseJson) => { + setUserSettings(responseJson) + }) + .catch(error => { + console.log(error) + }); + } + + const getSettings = () => { + fetch(globalUrl+"/api/v1/getsettings", { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for WORKFLOW EXECUTION :O!") + } + + return response.json() + }) + .then((responseJson) => { + if (responseJson.apikey === undefined || responseJson.apikey.length === 0 || responseJson.apikey === null) { + generateApikey() + } + setUserSettings(responseJson) + }) + .catch(error => { + console.log(error) + }); + } + const setNewAppAuth = (appAuthData) => { console.log("DAta: ", appAuthData) fetch(globalUrl+"/api/v1/apps/authentication", { @@ -262,6 +376,7 @@ const AngularWorkflow = (props) => { // FIXME: Sort this by time setWorkflowExecutions(responseJson) } + //alert.info("Loaded executions") //setWorkflowExecutions(responseJson) }) .catch(error => { @@ -402,7 +517,7 @@ const AngularWorkflow = (props) => { if (!visited.includes(item.action.label)) { if (executionRunning) { - alert.show("WAITING FOR "+item.action.label+" with result "+item.result) + //alert.show("WAITING FOR "+item.action.label+" with result "+item.result) visited.push(item.action.label) setVisited(visited) } @@ -424,7 +539,7 @@ const AngularWorkflow = (props) => { if (!visited.includes(item.action.label)) { if (executionRunning) { - alert.show("Success in node "+item.action.label) + //alert.show("Success in node "+item.action.label) //+" with result "+item.result) visited.push(item.action.label) setVisited(visited) @@ -459,7 +574,9 @@ const AngularWorkflow = (props) => { currentnode.addClass('failure-highlight') if (!visited.includes(item.action.label)) { - alert.error("Success for "+item.action.label+" with result "+item.result) + if (!item.action.result.includes("failed condition")) { + alert.error("Error for "+item.action.label+" with result "+item.result) + } visited.push(item.action.label) setVisited(visited) } @@ -499,9 +616,11 @@ const AngularWorkflow = (props) => { getWorkflowExecution(props.match.params.key) } else if (responseJson.status === "FINISHED") { + console.log("STOPPING BECAUSE ITS OVAH!") setExecutionRunning(false) stop() getWorkflowExecution(props.match.params.key) + setUpdate(Math.random()) } } @@ -566,7 +685,19 @@ const AngularWorkflow = (props) => { // Override just in this place curworkflowAction.errors = [] curworkflowAction.isValid = true + + // Cleans up OpenAPI items + var newparams = [] + for (var key in curworkflowAction.parameters) { + const thisitem = curworkflowAction.parameters[key] + if (thisitem.name.startsWith("${") && thisitem.name.endsWith("}")) { + continue + } + newparams.push(thisitem) + } + + curworkflowAction.parameters = newparams newActions.push(curworkflowAction) } else if (type === "TRIGGER") { //console.log("TRIGGER") @@ -614,7 +745,7 @@ const AngularWorkflow = (props) => { } else { success = true if (responseJson.errors !== undefined) { - console.log(responseJson) + //console.log(responseJson) workflow.errors = responseJson.errors if (responseJson.errors.length === 0) { workflow.isValid = true @@ -818,7 +949,16 @@ const AngularWorkflow = (props) => { }) .then((responseJson) => { if (responseJson.success) { - setAppAuthentication(responseJson.data) + var newauth = [] + for (var key in responseJson.data) { + if (responseJson.data[key].defined === false) { + continue + } + + newauth.push(responseJson.data[key]) + } + + setAppAuthentication(newauth) } else { alert.error("Failed getting authentications") } @@ -845,7 +985,7 @@ const AngularWorkflow = (props) => { return response.json() }) - .then((responseJson) => { + .then((responseJson) => { // FIXME - handle versions on left bar //handleAppVersioning(responseJson) //var tmpapps = [] @@ -959,14 +1099,20 @@ const AngularWorkflow = (props) => { setRightSideBarOpen(true) setLastSaved(false) - const triggercheck = workflow.triggers.find(trigger => trigger.id === event.target.data()["source"]) - if (triggercheck === undefined) { - setSelectedEdgeIndex(workflow.branches.findIndex(data => data.id === event.target.data()["id"])) - setSelectedEdge(event.target.data()) + /* + // Used to not be able to edit trigger-based branches. + const triggercheck = workflow.triggers.find(trigger => trigger.id === event.target.data()["source"]) + if (triggercheck === undefined) { + */ + setSelectedEdgeIndex(workflow.branches.findIndex(data => data.id === event.target.data()["id"])) + setSelectedEdge(event.target.data()) + /* } else { //alert.info("Can't edit branches from triggers") + console.log("IN HERE: !", triggercheck) } + */ setSelectedAction({}) setSelectedTrigger({}) @@ -1034,7 +1180,7 @@ const AngularWorkflow = (props) => { setSelectedApp(curapp) } - if (environments !== undefined) { + if (environments !== undefined && environments !== null) { var env = environments.find(a => a.Name === curaction.environment) if (!env || env === undefined) { env = environments[defaultEnvironmentIndex] @@ -1070,11 +1216,16 @@ const AngularWorkflow = (props) => { } else if (data.type === "TRIGGER") { //console.log("Should handle trigger "+data.triggertype) //console.log(data) - - setSelectedTriggerIndex(workflow.triggers.findIndex(a => a.id === data.id)) + const trigger_index = workflow.triggers.findIndex(a => a.id === data.id) + setSelectedTriggerIndex(trigger_index) setSelectedTrigger(data) setSelectedActionName(data.name) setSelectedActionEnvironment(data.env) + + if (data.app_name === "Shuffle Workflow") { + getAvailableWorkflows(trigger_index) + getSettings() + } } else { alert.error("Can't handle "+data.type) } @@ -1131,8 +1282,8 @@ const AngularWorkflow = (props) => { // FIXME - do this for both actions and other types? targetnode = workflow.triggers.findIndex(data => data.id === edge.target) if (targetnode !== -1) { - if (workflow.triggers[targetnode].app_name !== "User Input") { - + if (workflow.triggers[targetnode].app_name === "User Input" || workflow.triggers[targetnode].app_name === "Shuffle Workflow") { + } else { alert.error("Can't have triggers as target of branch") event.target.remove() found = true @@ -1214,12 +1365,13 @@ const AngularWorkflow = (props) => { if (workflow.start === data.id && workflow.actions.length > 1) { // FIXME - should check branches connected to startnode, as picking random // might just be confusing - cy.nodes().forEach(function( ele ) { + cy.nodes().some(function( ele ) { if (ele.id() !== workflow.start && ele.data()["label"] !== undefined) { alert.success("Changed startnode to "+ele.data()["label"]) ele.data("isStartNode", true) workflow.start = ele.id() - return true + //throw BreakException + return false } }); } @@ -1240,13 +1392,13 @@ const AngularWorkflow = (props) => { // CTRL = 17 //console.log(event.keyCode) switch( event.keyCode ) { - case 27: - console.log("ESCAPE") - break; + case 27: + console.log("ESCAPE") + break; case 46: - removeNode() + //removeNode() console.log("DELETE") - break; + break; case 38: console.log("UP") break; @@ -1286,11 +1438,11 @@ const AngularWorkflow = (props) => { } break; case 70: - if (previouskey === 17) { - event.preventDefault() - cy.fit(null, 50) - } - break; + //if (previouskey === 17) { + // event.preventDefault() + // cy.fit(null, 50) + //} + //break; case 65: // As a poweruser myself, I found myself hitting this a few // too many times to just edit text. Need a better bind @@ -1382,6 +1534,14 @@ const AngularWorkflow = (props) => { getAppAuthentication() getEnvironments() getWorkflowExecution(props.match.params.key) + getAvailableWorkflows(-1) + getSettings() + + const cursearch = typeof window === 'undefined' || window.location === undefined ? "" : window.location.search + const tmpView = new URLSearchParams(cursearch).get("view") + if (tmpView !== undefined && tmpView !== null && tmpView === "executions") { + setExecutionModalOpen(true) + } return } @@ -1722,8 +1882,8 @@ const AngularWorkflow = (props) => { const paperVariableStyle = { borderRadius: borderRadius, - minHeight: "50px", - maxHeight: "50px", + minHeight: 50, + maxHeight: 50, minWidth: "100%", maxWidth: "100%", marginTop: "5px", @@ -1764,7 +1924,7 @@ const AngularWorkflow = (props) => { return (
- What are WORKFLOW variables? + What are WORKFLOW variables? {workflow.workflow_variables === null ? null : workflow.workflow_variables.map(variable=> { return ( @@ -1831,7 +1991,7 @@ const AngularWorkflow = (props) => { }}>New workflow variable
- What are EXECUTION variables? + What are EXECUTION variables? {workflow.execution_variables === null || workflow.execution_variables === undefined ? null : workflow.execution_variables.map(variable=> { return ( @@ -1980,10 +2140,23 @@ const AngularWorkflow = (props) => { "large_image": 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABkCAIAAAD/gAIDAAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAABmJLR0QA/wD/AP+gvaeTAAAAB3RJTUUH4wYNAxEP4A5uKQAAGipJREFUeNrtXHt4lNWZf8853zf3SSZDEgIJJtxCEnLRLSkXhSKgTcEL6yLK1hZWWylVbO1q7SKsSu3TsvVZqF2g4haoT2m9PIU+gJVHtFa5NQRD5FICIUAumBAmc81cvss5Z/845MtkAskEDJRu3r8Y8n3nfc/vvOe9zyDOOQxScoRvtAA3Ew2C1Q8aBKsfNAhWP2gQrH7QIFj9oEGw+kGDYPWDBsHqBw2C1Q+SbrQAPSg+/ULoRkvTjf4uwOKMAeeAEMI4AaBuf7rRhG5kIs05Zxxh1AUQ5yymUkVFgLBFxhZzbw///wGLUyZ2zikLn2oIVJ3o+NtZ5Xyb5u/gmgYAyCTLLqdlRKajaFRqeZFtTA7C+BJk5MZo2Y0Ai3EOHGGshyIX393btnNv5FQjjSoIYyQRRDBgdOkxyriuc8aJzeIozMu4d2rG16YQm4UzhtANULHrDRZnDGHMGW/b9lHzxh3RxlZslrHFjDAG4JxziBcHAUIIAHHGWFRhqmYblZ3z7bmZc24HAM75dTZk1xUsThkiWPn84umVv/btrSF2K7aYOGPA+pYBYQQIs5hCo8qQGRNGP/9vpky3WPAfECyxseCntSef+6Xq8UupDk4Z9Jc7QohgzR+yDMsY9/OlzpIx1xOv6wSW2JJvb03tM78AxrHVzHWaiAJGAMA5F4p2KYzgnHMG3WVEEqGRGDbJhWt+kFpedN3wuh5gCTsVPHzyb9/9L845Nkmcsm5CEMw0yiIxzhg2ycgkAQemqFyniBBiMyOJXOYVRcNmuXjDMntBnmBx84PFOSCktvmOLHxR9fiJ1Ry/bYQxZ0wPhk3pLtek4tQJhda84cRpA8Y0bzBS3xz4tDZYXav5QlKKXTwcjxcNxyy3ZJVufkFKtQtG/whg1f77Lzy7K+U0Z/ztQwTTiIJN0rAFdw97+G5TRlrihjkAAuVzT8tbu1ve3s01SmzdsZaI5g1m/cudY158/Doo18CCJTbg2V158plfXLLocUjpoYhtdE7+T5bYx+WKaJNR2mW8GOMciERESBWuPXdq2brIuRbJYU3QTRqOFq37oWtSyUDjNZBHwQFhzCk9v3knkqV4Iy2QcpaMKfnf5fZxuZxSRhlgREwykSVMCCaEyLJkkhHGlFKuU3tBXvH/LncU5Okd0QRzzjk/v2kngAjKBpAGULPEOXv/8umJ7/+3lGLvUgeMuKKZhrpLNv6nKcPFKeUIYYxVVa2srKyurm5ra+Ocp6enl5WVTZo0yW63M8YQ40giyueeo4+u1HwhJEtG2IEwohFl/Gv/kTqhcECVa8CrDhd3HUhw/MCBUzbquYXxSFVVVb322mv19fWcc0IIAFBKt2/fnp2dvWjRorvuuosBA52ah6ePfPYbtc++KpnkrmNGiGm65739qRMKYSAt8ICBxTnCWPd3hGrqsNXEO2N0RLAeDKffNTHtjjLOmEBq+/bta9askSQpJSUFRKjVeafa29tXrlx57ty5b3/72wwYMDZkZrl76q3eTw5LDptwjpxxYjEFDp2gkRixWQbOLQ6UxooNh+sa1Yu++CvDOUeEDJ03AwAYYxjjysrKNWvW2Gw2i8VCKaWUMsYYY+LfJpPJ7Xa/8cYbW7duxRgzygAga96M7k6TI5OstHgi9ecN1jcTWOI6hOuamKp12V2EWEyz5g1LuTUfAIgkqaq6YcMGSZIwxoyxnssI4FJSUjZv3nz+/HkiSxwg5bYCS3YGUzUDMoQRjamR+maD9U0FFgAAKOfb4j8ijLiq2gvzsNlENR0A/vrXv545c8ZqtV4WqUuwcy5JUiAQePfddwGA6TpxWO1jb2GKGuf+EHDeye6m0ywEAKB6At19E+KM20ZmCwwA4NChQ8ncGsaY2WyuqakBAIwwAFhGZHLGoOsuckBIC3R08b6ZwAIEACyqAEJxJ80BITnNCSJPBmhpaSGEJIMXIcTr9YZCIRFkSSkO4Im4cI32uc7fJ1gCMdTjUvD4/I5Smnwk2R3TnvhybJYHdDcDBxYHAOKwivwu/r/VNp+xc5fL1Yu1iidKqdvtdjgcwBgA6MEIksilAjQAAAIO8pDUK+D4dw4WBwAwZ6bF6xFwQBIJ1zaI3QFAfn5+Msol4vuioiKEEKUMAMKnGvVAB4sqAIAIRhghgq25WWAsfTOBBQAA1lHZ8QER54xYzKEjdbHzF4kkAcC0adNSU1P7xIsxJsvytGnToNPYDX/kazmP3WcdORwY1/whzRfEZpM9PxcGMkMcqAheSOwoHNmtSMAByUT1Bi5s+yj3yflU1YYPH37fffdt3rw5IyND07TLLoUxjkQixcXFZWVlAIAJBoC020vTbi/llEbPtgQ/O+X75DAgZM0bBgBxd/MmAUtIbBuTYxuT03H8LLaZRbGYMyY5bK1v7U6/a5J93C3A2KJFi86ePfvxxx+73W6EEO8kYyURZ/l8Pr/f73K5OOcIIc4YcECECBZZ/zIjsU49AERefPHFAVpatFH1QNi3t4ZYLV1FAkJoROk4Vp9RMRmbTRjgjqlTFUU5fvx4OBxmjBFCJEmKx0uW5ZaWFoTQhAkTRJKEuspeHBgDQNehDD+AYCEEgJBleMbFXQeYonRFp5xjsxxrbus4cTZ9ZjkyyRjQpMmTJk2a5HA4CCHBYDAYDJrNXb17zrnZbD516tTkyZPdbrdQrk4uCGE80JWsAQdLtOYlp412RPz7jxK7pas/yDmxWiKnmwNVf3NNKpZTHUyn6RkZEyZMqKiomDlzJqX02LFjstwVNxFCOjo6gsHgV77ylXiwricNJFidymUvHOn96FPNF8Iy6YqBOCdWS6y5zbPrgGVYhn3sCM454xwB2Gy2iRMnyrJ84MABi8Ui7iPn3GKxnD59urCwMCcnh4kO/j8SWIAQZ4xYTObsjIvv7sMmU7euKufEYqJR5eJ7+yOnmx35t5jcKUh08TkvLS2tra09d+6c2Ww2Klyapn3++ecVFRX4RkwgDXyvDWPOmHvabTmL7lG9ASSR+L9yypAkSU57+wdVNQ8tC59sAIRwZ1S5cOFCWe6qiDLG7Hb70aNH33vvPQCgdMDd3/UGS+AFnOc+9VBGxRTN40+skXMOCBBBriml9nG5wAEwEuWtwsLCmTNnhkIhUWgWeFkslt///vfBYDDJDPwmA6sTMzT25e+4Z5TTmJI43kcZtphzn3jwEnaXHkcAsGDBApfLpeu6+CjcYlNT0zvvvCOwS2DSM0z7uwNLVIF7ExEhTimxmrMeuJPTbrYZEaIHw8Mevts2dgSnzIi/EUKU0pycnHvuuaejo8MwUowxh8Oxffv2pqYmQohRiTbsmiDOuZAqyUR9wMHinAuMMMaEkN7dEyKEa3rz5h0ovm6DEI0ptlHZ2YvuATFXFC8cxgDw4IMPZmdnK4piKJckScFg8Le//a1AhxAiwlRKaSwWi8ViQhOFVBhjQ85rBOvq0x3hvAkhuq7X1tZ++OGHxcXFM2fOFBF2IqyUIYJb3v4gWH1STnMa2SLCiCvaiMUPSE5bz2EYsf/U1NSHHnpo9erVZrNZGHVKqcPh+Oijj2bNmjV06NCqqqqzZ8+2trYGAgFFUcRVTU1NzcrKys/PLykpycvLEwbusrIlT1fTZBVGAWOsKMr777+/Y8eOc+fOeb3emTNnrlq16jICcQ4IKa3tRx75T70jiiQiDJPoS6fdXlr0Pz/svX+l6/rSpUtPnz4dX60XKqZpWjgcRgiJrodgLVRJ3EGbzTZ69OhZs2bNmjXL4XCI168Osn6DZSB18ODB1157ra6uzmw2WywW4dc3bNhg5LpdrzCGMD790uutf/hIdjl5nMvnjJX8eoWjaOSlSeTLkUB///79K1asEN3pS6IjZGi3IVjXxhASMlBKFUVRVTU7O3vBggWzZ88mhFydivXvBYECY2z9+vXPPfdcY2Ojy+Uym82Ct8fjOXnyJHSv/wqkAlV/a9uxV0qxG0hdsuvzZjqKRl6aXL6SiBhzzqdMmTJ58uR4S28cSbyNN8joPAKA1Wp1uVxer/fnP//5s88+29zcjDG+ijCtH2AJ4SKRyIoVK7Zs2eJwOERb1HBDlFLRgOl2whhzxhrXvgPxCQpCLKZYc4flPHYf9LDrl2UNAN/85jeFCvd3kwI4WZbdbndNTc3SpUurqqqEJx0QsARSsVhs+fLl+/btGzJkiGh/xj9gsVg+++wzADBiSGHIW9/5MFBdSzq77QIdqqgjFv+z5HJyyvrstosYNT8/v6KioqOjw1i/60jifJ+4gD0dNOdc13Wn0xmLxZ5//vmPP/64v3j17xquWrWqqqoqLS1N1/WEzXDOI5GI1+v1+/1CMuAcEay2+Zp/vZ3YrF1ICbs+pSzz3qnimWRYGzGqqKkaKAhQdF0PhUJ+vz8cDiuKEovFxEdVVRMgEyomy/JPfvKTQ4cOCfuV5PaTMvDCJG3ZsuVXv/qV2+1OQIoQEo1GAWDOnDmPPPJIenr6pWImZYjg+pc3try9W3aldLPrlJX8erlj/Khe7HpPopQSQt56661169aJthBCKBqNapqWlZVVXFxcVFSUk5PjcDgope3t7bW1tZWVlWfPnrVarbIsx4MiOiB2u/2Xv/zl8OHDk6z59A2WQOrUqVPf+973hJLHvyJqdaNGjXr66adLSkqEQgHnopETrD55bPFPsVmOL5NqvmD2wntGPvP1/k4Ziy0pivLEE080NzcDgKIo48ePv/fee6dMmeJ0Onu+oqrqn//8502bNnk8HgFivOShUKi8vHzVqlVJgtW3rGKVTZs2xWKxhNwVYxwIBO666661a9eWlJRQTeeMI4wRJqK60LD2HR7fuUGIKYr1lqycbyVl13tKIvr43/jGN/x+//Dhw1944YVXX331q1/9ak+kdF3XNE2W5YqKivXr1992220i9zYeoJQ6nc7Kyspdu3aJlfsWoHfNEmpVXV397LPP2my2BE0OBoMPP/zwkiVLOOeMUiJJwHnH8TO+/UeiDa1Ki6fj+JluI3oEa/6O/Je/k3nftGsZXldVdffu3VOnTk1JSaGUCn1vbW09d+5cOBx2OBy5ublZWVnQOYQjYteXXnpp37594hUDfVVVc3Jy1q9fbzKZ+tSvpNKdnTt3JgBPCAkEAnPnzl2yZAljDBgnktRx4lzDq28Gqk4wRUUYIUnCVnPcMCPWO6JpU0oy75uWvF2/LMmyPGfOHM650J36+vrNmzfX1NSIfgfG2OFwFBUVPfzww7feeit0th2XL1/+gx/8oK6uzkgDhAc/c+bM/v37p0+f3idYvUksIvW2trbDhw8b5V2hU+FwuLS0dOmTS4WRwhK5+O7eo4te8h84hq0mOS1FSnUQmxl6qO2wf60A0ZK5NtJ1Xdd1WZb37Nnz5JNP7tmzR6QQTqfTbrfrun7w4MGnn3769ddfN3Jsi8Xy/e9/32QyJUQ8CKEPP/wwGaa9gSUWPXLkiNfrja9YiqTs8ccfl2SJ6RQT4v3o01PPr0cESyl2YJxTyinrhghCTNflNKejIA/6b60SSKQ4Aqkf//jHACDmK1knIYQcDofD4di0adPatWtF2EUp7RmpCeWqra31er0iALpKsAQdO3as2wsYh8Ph8vLykpISRimRJc0XPLPqDWySESH8ijEeRwhxnTJNv/yfe6WEhzVNa2xsXLt27cqVKyVJkiSpZ2wpUov09PS33nrrk08+MZz47Nmz7Xa78bw4eK/XW1dXB32NWPZms0QW1tDQEN/yFI7jjjvuABGgE3Lhjx/Hmi/IQ1J76wlzQBLR/aHQkTpLdgZw0HTtpZdeunDhgqGz8ZoLcSMLRj3PCFxCodCFCxcikYhwgldyZAJok8n05ptvTps2Texi9OjRY8eOPXbsmOGvEEK6rp85c2bixIlXCZYR1Hi93viIgVJqs9ny8/MBQMQH/n2fYbOczHcGOQf/viMZX5sCCBBAQ0NDY2OjyMMNXC4rSYJUGGNZlsVESe8cRc3+9OnTtbW1BQUFlFJJkvLz82tqarpVaxFqaWnpU/4+vGEsFotGo0aiLyyl3W53uVwAgDGm4ZjS6kXdu+1XQh+bpGhDi1iIcS5JktVqFT4brnwFeiIoVCbJtE7U3c6ePVtQUCBOJTs7O1EwjEWWdk2hA6XUaBYYS4uzvfSRMZ5kbsUBEKLRGNN0LEuqoookySif94JyUuv3SqFQyPh3zwhWdCT7BKsPA08Iib+D4hCi0WhHR4f4KDltl8rEffo3BMA5sVmQLAFAIBgIh8M9HRBOmvrVkbbZbMa/e842iX31uUgfmmWxWGw2WyAQiIcvHA6fP38+JyeH6TqRZcf40aGj9cRm4dDbvUAIMVW3jc4RW2xtbY1EIglZAQBEo9FkWvPC5ffp7MWTsizn5nbNuXk8noS3OOd2ux3iCor9A0v4HbPZ7Ha7m5ubDdcrzNaRI0cmTpwoBhIzKiZf+MOfk/i2M0IYDZn5ZfHh5MmT8ZUWQ+hx48bFB8BXIoxxXV2dqMD08rDwUSNGjCgoKIDOQlt9fX38W8K/Z2RkwLWEDmJUatSoUdXV1cauGGMmk+ngwYOLFi2SZZkzlvJP49Lvnti2Y4+c7uJXCKOQLGntAff0L6XdUSbKMtXV1fGBrrAamZmZr7zyitVq7f2ERU6za9eun/70p737REJIJBKZO3euLMuiwhMMBk+cOGHMTxjcher1cUJ9PlFaWhpflhH6X19ff+DAAQBgjAPAyB9+016Qp3mDSJa6RecIxE9baN6AbXTOmBWPcgCEUV1d3fHjx+NrxMJnFRUVWa1WsfleYlShCxUVFXPnzvV4PKKv01OnJElqb2+/884777//fuP/9+3b19raGn9OIhgaM2YMXIuBFxKUlZVlZmYmXBlRC1RVlUiEMyanOYvW/tBVXqRe9NGoIhwfIMQp18NRzRtMu71s/Gv/Ycp0i2+hbNu2LRqNxhdMBARixBbiAtFeiHP+1FNPPfDAA+3t7bFYTJRMBWGMNU3zeDzTp09ftmyZWJ8QEovFtm7dmqDRqqrm5uaOHDkS+mqR9TZyJA7QarU2NzcfO3ZM3A7oHDg4f/68pmnl5eWMMQQgOW0Zs6eYh6Vr7UE9GGaKyhmT7NaU0rF5Tz2Uu/QhyWnTNU2S5YMHD77++uvxpt0olSxevFiSJKOL1QsZKnb77bePGDGiqanJ4/FEIhHRkWaMDRs27Fvf+tbixYvFRJy4uZs2bfrLX/5idA+h01/df//9ZWVlotrTC9Ok6lmnT59eunRpwkIY446OjieeeGLevHmMMc4YxgRhxClTWjxaewAINg8dYspwAQCjjDEqyXJTU9Mzzzzj9/vjj5cQ4vf7n3zyyfnz5wvL0jtSRtdPhKaiXFVbW1tfX9/R0WGz2UaOHFlYWGhcc1HS2r1796pVq4wjN0CXJGn9+vXJFJeTLSuvXr1627ZtLpcrvnIGAOFweMGCBY899pjwL1TTESGks1bFAZiuIwBECELoxIkTK1eu9Hg88dbKMO3r1693OBy9SywagoSQ999/v7m5+ZFHHjFKLglnKXA0ksrdu3e/8sorwrolHNK8efOWLl2aTNu1b7CE9F6v97vf/a7P54uvBwlRgsFgUVHRwoULy8vLeyqFeP3ixYtbt27dtm2bqAvHx1aijvjCCy/MmDGjd4kNULZs2bJx40Zd10ePHj1//vzp06dbLBYDSsFRHJ5o3/3mN795++23E+IykT87nc5169YZTZZrBctQrn379okGekLZRLhnSunYsWPLy8sLCgoyMzOtVquu636/v6Gh4bPPPqupqfH5fA6HI+FLmGLAfc6cOc8991zvSInNqKq6Zs2anTt3ulwukUsoipKXlzdt2rSJEyfm5eWJ2FLI3NLScuDAgR07djQ0NIgUJ0HsQCCwbNmyu+++O8lufrKzDmK53/3ud+vWrXO73QkJneAUi8UURcEYm81mSZIYY6qqappGCLFarT2rTuIrl+PHj+8zthJ/8vl8K1eurK6uNqyBuGLCqJvN5vT09IyMDNHF8Xq9LS0twWDQYrGIznkC6/b29vnz5yd5AfsHloHXhg0b3njjjbS0tJ5lOSF6fMXOqED1fFiSJL/fn5+fv2rVqoTR9ssiFYlEHn/88aampiFDhqiq2pMvY0zTNF3XjWkRWZbFmfVk3d7ePmvWrBUrViTjeQ3qx7SyiCQmTJhgNpsPHDggiko9k6wEX9MTJoGg1+udMGHCyy+/LPS0l7MVcJtMJrvdXl1dLZQoIaMULAghpk4yRmsSWAOAz+ebPXv2j370I/FM8mD1e+RIuPa9e/euXr3a4/E4nU5xqsmsI2AKh8MA8OCDDz722GOiUZzMLRD6dfz48Z/97GeNjY0pKSn9moQRrCORCAAsWrTo61//ekI9dkDAMvDyeDwbN2784IMPFEWx2Wwi9rvs3RQC6bouClhlZWWPPvpoaWmpuC/JiytgDYVCGzdu/NOf/iT678LrXbZUD3GWQdjT4uLiJUuWFBcX95f11YMFnTOSCKFTp0798Y9/rKysbG9vFwGeMcoCnbM+uq5zzlNSUkpLS++9994vf/nLQhmvQlzjrZMnT7755ptVVVWhUEiWZXHv4hcUcZamaaqqSpI0ZsyYBx54YMaMGcKKXafJP4PEYRpW4PDhw0ePHhXzkiKSEG4xLS1txIgR48eP/9KXvjRs2LCEF6+Rb1NT0549ew4dOtTY2BgMBjVNM7YjSZLNZhs6dGhxcfHUqVNLS0tFw+JaWF/rD/f0jJ5VVY1EIrquY4ytVqvVau3l4S+Kr8/na2lpuXjxomhKWywWt9udlZU1dOhQw9KL0P9amH4xv3IkRIFOO9pzY+I8v/CvJiWzskh6vpAT+uJ/Eqqngf9i178S0wQbb1RyvkAuN/S3lW82uvE/hX0T0SBY/aBBsPpBg2D1gwbB6gcNgtUPGgSrHzQIVj9oEKx+0CBY/aD/A/ORNiwv2PAfAAAAJXRFWHRkYXRlOmNyZWF0ZQAyMDE5LTA2LTEzVDAzOjE3OjE2LTA0OjAwj3mANAAAACV0RVh0ZGF0ZTptb2RpZnkAMjAxOS0wNi0xM1QwMzoxNzoxNS0wNDowMM/MIhUAAAAASUVORK5CYII=', "is_valid": true, "label": "Webhook", - "environment": "cloud", + "environment": "onprem", "description": "Simple HTTP webhook", "long_description": "Execute a workflow with an unauthicated POST request", }, + { + "name": "Shuffle Workflow", + "type": "TRIGGER", + "status": "uninitialized", + "trigger_type": "SUBFLOW", + "errors": null, + "large_image": 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAK4AAACuCAYAAACvDDbuAAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAACE4AAAhOAFFljFgAAAAB3RJTUUH5AsGCjIrX+G1HgAAMc5JREFUeNrtfW2stll11rWec78jztjOaIFAES2R0kGpP0q1NFKIiX/EpNE0/GgbRUzUaERj8CM2qUZN+s+ayA9jTOyY2JqItU2jnabBRKCJQKExdqRQPoZgUiylgFPnhXnfc87yx3N/7PW97uc8Z/5wdjLvnGff+2PttddeH9fe974JzXT1tx8F3/s6cD09BvAbAHwfgDcBeBLAKwH6ZjAmAOC1Fh3/x+DjDwpaJ4DHeqr+2qioL9qcf4DZ6cPWBWtaOKJtfkYACwI1bbaPYfwbfaZfr96Wx5Khtl+ioR1nbA3a3LbHRxzUxcKTgndbugTwHED/B8CvAfjvAD4I4H8dLqbnLx8+wBNPfQ6dRFWBy79/D7gkgPhbALwNwNsB/HEALwVwETGGG4w5DpzWXI5IcgXPKcuBALCTN7bD+YJyx5W2r2hbBZf8xRnQxwCIM75Ynox9EM/s5Vxow/Zd3sTKhHW5hC/zQr5ixm8R6KMA/iPATz94ePnFe4/cw+P/5jPIUiq4l3/3Hpj5UQL+LEB/DUeBfaQirr3i2RmsnollCsRDh1nuyo/pO2rnjA0zg3n7JSaYR/5TWHerF42v0mYJfY7Aj1lHoY/HppWKaY7zugs/CGjVXfi+ds2i/kMAHwPjXzDwcwA9/8RTn0aUXMG9fvdL8PDe13FxOb0OwI8AeDtAj1pKvNWm3QOvGykUbpnKxInBB+UqbVm5B4acbn0tuLW29OvdZGxevSEvFdrCivDpWnoUWLd9xn0AP8PAP2bmTx0OBzz+E58yJQ864+G77+Hzv/m7cHF1760AfgrAO4zQcs747Qe5TGfWQqsH7ra3/mZTJJ5YXpvcIXQsXGeYcXBVfxufL7RBPVPE4R/PKiFqamVO0q/yh3cJLU4WWh7+zdKjAH6YQP/+QIe3fO3qs/i/7/xDObcevvseplc9xNVvTH8GwHsAvCbwrVZt6mqFhnsQ+rWBP0uGcRGTC22UBhP+GFg2FoxtyNvhl471cgsSj01o58z9KFyLHn39eMDwhPbUJwB4FuB3PXj4xH955N5X8PhTn9GtAtd/7xFcXV2BDoe3gPkpuELrRbCKURvjrF/KurqOkGPGWeG5JfTANNkR+oa/fqzPGwygxla5TdR1D0R96YOnwVgieJVrsPEujkUqhREHqM8CeCdA7weusfi9BwDgf/h7cX19DSJ6HZj/OSJNuxKwMI22ToX9dkaZCa1h/NhSEIm7dQM4RzLXZ5zS6IxNMKijZUv6CFZo81YEbdypYVwLR2hJWC8x4LStFu+M0DIKt7AOMl8D4J8B/DqA8KW/dPRaDwBwdf93MPuxPwrguxANjG22pN3zaQEwqepRIKEZQLY0h0KhmtR+aeEeJBozNZHLwMil1h+vQ5Pv09rYdllQCj3gCgEYf7es1/CcM6ENhR7qmWe98wU5yMsbAfwIMz86Xb3qmHP5d+6BjjbshwD61zg6x+GATZRc4LRLMLb5tPuiX+m7BQzIBr/DRMULq3YPDFyW1t34EtHtj42GOUAudAZjJd1ywhu/XznOhC9oYNAdf32se0Qb/gqAnwSAwzGIppcD9C6MQuswMxZaHz3Qfq0bAUfC7g7AEQrWzW51S6E1OUEfBU8WpvfrUj42l1lwhDZK2vWxtoCtlRoQlK2e1u6x0A7tzchH6Eu7gZgWWtEeADwK0F8F8FKAcJixqbcB/MZogkafj4Iy7uAzjbIEHM7kRK3CzxN+VRkgjfUHjWlgoQry41G+HKFraKPctSB2FQOPdRH2Mf6WCE7ATxCtfVaavJzXxXGiRn3ysxfYT1ra7wHoT4OBAxE9BtAPAHRPMImle+D6R5awY2gpcEx3cKxWncBLBb0eEyv0oMRZq7rRxFHIePkz0UZzH2PwJ4kht+ERgt3qkioq6XPdMvYESvZpsd1kLJr8aFwO3/XYVPseOvEIgLeD8NjEjDcA+O6tFRaa1dUK6eaB61zrug7ntp5S/43NfC6jZGahXJzKFnUQ5wcaGGoNeSVaHoX74ga/kWuWj81UCefMBHCCd4QhRKmgSlb/H/Nd5avm3LXUJu+PAfjOCcCbAbxMF+W4YsCohkbmG9T1J4YCuMwXWuMNnyoUgQl08u25iKXwPl864YGif+u3os3rbwGaBxm00Ib6LXxnV1/Ewd5Gei20M9dexsCbJwDfi/WUVzq4gdtGIBe3LYmuo/ajnSndu55sI7Dkn76a+zXoQVdr2ejfRr5x3aWsCWzF2GhsdOblEKw0cdCyn4TmtIvEiuToRqT0NB93zAVwQeDvmQA8WZr3NSPsYP3HmN/OwE7Km58wmGcNGwqTo2nTiVL1FxW0MLm3sSFNaNgfGxpJCDwD7OyaRXzhvTxlGXTbICl0m3hj5L4+pUeRaFqRv/GEQU9OYLyy7LSLtc55HW1kEYosmHLIWZhO5BExP/ecq52LSgSZHZ+2D79l5ldMbgbwYwjkDW26j3he3TnLFwtxZ/WnLlgyNlFERloEfOsE0DfLajuCFdNxL4jjgVlLDOkyoTo70NRE48TuCcRUsO2XyVIaiHWDqUC4ncUsYa/aXeC2MG19S1XQ5Ut3MVeadqX7myZgfN3Gq2j4gtHviwch/LaQuOPcdqPzHeiGItwlo+pDmviNB0WEvndvXqM4vENox3mLNwc0S6jwlfJAjLPxB/Ot8HmHBit74U4a497k0mAbNCqsNPMdn2wRHq7qbYPItxFtfR/O60BWhmYPV7TlMgsUaOnFQWcxSz0UoBq/Q58YUO02qanf6dNaZIz8Bw79gdACmLVtixCGMM2clWv5w1QzGVaj0LJaw0GPZZuIxdgHNxCAcmwRX/x+rSceWKBhJuMD6rH5HnmpnaGw37HpHRj0xkWyspLjtHDlWvF96gLcTBiO9wXl3B0tJ3Bw/dmob91edDRwPwSkf9tXZsKxrZPefSm00khMdb1tJ8AxoQ1tqRQC5XVl3mbp9vDUE0Dze4VS3dYC2ibboCm0+iMpThvkkdYoHiFu393of843zqjyyzKGM9TCL5ECrf7C8Z9izbK6rs9eCS1XizpZVNliueGiXZuNtG085zzZjg2nHJ+o57sJjdKcXD+Yyhjvuy/CbW6cVQ3py3z12XzHC8M+CMdXBLFjXs0bD8rLeafrjnT23xQOshoLclRwjU0NmiRTNPApB+F2ykH7xn0KhDuBvMgELKpHE13nDI1o2yLe7sbE1kwYLLpu084glm1dwQ6ulcBoe7r+vlU2SaGhXwFvejzReYpHgQIYJGwrq4IzkkI7+DWp31euqB7eas1RtiI6JqkwkQLSUyaq4XOiLbTzUf3lcYU81CQ3+EkIi3b84SzmCOZbLpCuFRE/j/txsk93vqeeAEREe5qrP9AQRJ8Zy06ZmL5Tg7NIE+YYar64YlMY8G8eehQongbnlb6923Y2jlwYWzzJ0QMqfX0Jh/nEFZrPqdl5R58Ce9AUvN6xyaKNDM6rNV7oNhU+I8J6dvYXoRM7fh34MA1ivfEde1lC+9TE7xHaBuQ1jtNpyPJqoG9KGp9j3f4WsEEdUthrZFuH8bF2lw59r76bFwYa8mgi75zYlS9NZERj19sf8Rg8we4GgK1g7ySh9ZCXY75ldQtVYYCIVlTBHyyhKbT+nVA1YewyvR8l06AtupsLcR9x35v7EqQCUdn63ulWNGgbaege69Sk0w0WpKGlEe/sFtpt3tZgTPi4edDgE7H8m2sGn3Gr4IU4puLw0CvvmdzVg5RBUkVfT+DjPGsJgrouOoIiU9alcDEiVSQplNcW2pq+vTBqNgyG2oCIg4G6sapTVYVkP2YAMljJfO6GULH4p8e89WyEM8Yc7hna2GFFQv5l7kWF07q3y3jQWm9Bem8Ld11Cb/vdXzQ25vCan9LB14yXTNkc/0UPmF550X0iQDK8FQ7Uhg96fh8Acw3p1kTvRkifeVHdTENJfzblnRjb8acbKLrY+vKYmCoBMHXJLzf01UGP5vyonHBv3UBsvG4D/bk5lvVRhagxzTih0rTgFYMn5NpkqD+yO/C5KaobjiPBMlvXfBrmngYLLWNb/XZeeKMFz0c/+q6FLeP4tM68LQ06GDtX7Uuqt7nz5sxLsT88uUxuDLyOdvWwx2FUg5bReD+YInjbPTSu7gxYd5vZCXk1gsQsws5vNJT8qfkR9733FR/eRl4qpZHH7hrI+q0s1fE5T1mnQw75g+35bsbEZ2mYlE0bsWo3G5Q9RtcW2k5wejKkJAstOsxObKWIim3pxF9PhZbrJjnUyI12OqjKMH8e1jK27bgKpgMBi+WRsh9wSKe8oRWgzUzDqpi+9mmlUvBab7pG7dvJ7gut9IcbxKd998/xSquXIg+Khl3vEyrejchBUpc6xxrXXF6KNT8Swkr0OJ581oGcu2NE+YpNB5tMTmf8/ux1DuXkfHFpcfPcAGd4HJvvLfiLBpaPa5uDvsU9RYGweDJAlw7fp6qxHiGxCV0ibM4HQHLFBuUi6Kj0t26iLQHvyiYhQDs2F3bdVrk4lhwJbRSXyPwQQ434XObFUF4vWByUDW+SIpTjyAOn7nAe14M8dGjkNBS4DKvK3+kPdfBRM3iH0m5dGupHEaxDolLzta+/+LSp8Om8obBLX4Z9A7teukx9S4+AQGjVgvabHJ5pVCXh+9rUcQNCMn7dHCAjfD3za8H6IDBTg+KhrG2sF+xEAw1TFv0Gq31REAVOOwS0FvLa+q98WuspBvGFENo8PqgRII+WPCmLsPGGvHJzEfJlg8oFCbbHGme7XUBjw1qRrfsRYjBQNbE6iEsYzaTfHbTlAmY7whBuXui8FipybFGAsKd8LspTCEnAIyZ22dwJzy6wbtbRlk7gJECx9RHp0tHRxG0IBidP+WkYRYD1cduaK/NvGtiubOoEfyv03ZAKhdQKmwYV5Rx/UY6tNz6XvlaQGFnXnE9S+FpWRLoWBQ90sDgK3yqxrm+4KSqwo+TKfgf65ufmWKP0UfY79DTSFpVT6EEXy+R5YGY4RpjGFmkdsdXOOmolWxccbP2OdUeKttj4GJhyo67t0zTnlTOraeRlzJdFsC23imA3c602RTLMPUVlTnNJBrlYX90xZpq6GlOJUbid5+N9emBpP0rOaHoJcDEFuLFleLTKrYviGOXsWw4u7ST/Va/zuaacxzo+77LAjoqgBpBRPK6vwA9fSOYghqySFArtXkRKdzgu3UXjqmsT9qMHpXvASTMR0eNvVjTSARd/6q/j4vV/Eri+Slts99lzunakOqg8uY2gWU/3e20RHXD5uWfwwk//OPjBCwhx4E5glgmeypNaPkAsliKsTxFuaZoxScrpK3zGUmgrl6PWFPIHASAcXvkdOLz2TRVn71KQmK+BwwTghVLTDnaHpLLqdOR9tSfGgpc/Mmh0EtHgCTtitaatzGNU0TNTfe1zlxppp4Vh7etUUJuDLDWFdrawFChUWk+HUaZV87xMmHTE2twiHeuwzr1Lt5fiQCyB4qK6zNkdb5TUhxODqzJTqvFMcEvh86huWKQBmY2Hze+07S0mDvSWB5U1tPQCDlvX1V8Yoi4vjqDenJBaOrit0V9REiqLyLa7NwaEL4Q2vXt3qdtGPe5SmsIg/IQXNof67hFYsVsa7w3Ux2cJwXlcZojvKiggKf0Cufwz3DkKGMCZlh8F/uwIwDd6quaISp5vCse5qsB0Yv0F/xWftV+x9RudDhNCK/7eEUyNxOQ7Ttos5OXu0jlTB1/va1ovrsnqjruYNaFmA6IYlBqYW451zk5Tk8Ex0cDutO6Zk9kBLcvFAjvnFRivZ5GFkgz2s6YSQy335v3dFdIZEbzFGvt2/BuXi4Q7DXzzFPE9trBBOa9MZ2Oi8R0Oj+jgJhu98jKfVhz6gTH72Vbf2ISnZcNlfyew50x7b7AZKmLX+4S24zWYchcQAd5dcURIXk8vj571Btbf/vOK3Pm5L0rycNVobnmskriDiWwMdU0sJRWWd2bk+EjguNbXSNyDIdLrH9+TBI7nqHonxDbk4Q4NO1Ny4bAEjpqFirrtBW3lb8Z47ohsY9oednfEvPMDUYQZ1FWBWHU6S5dbzkfcxWbnSI1zJEviolxxyivavrVtB8k71pgTMFYOxiEyY58pwPf6jBw2J+6U7k3TuEWk8dLx+ZZ3qj+cHZjJ6kV5E3eFlhuENAIxdzcsrTs/Ni7NGcT2+grg65u3c440mhOdT065KlFR9uICuLr02rVWlAPLuoumBqKQ3Uau8p17FXLCQtiq2gLmTNN2NyVk2Zu6Cpcf/Clc/ur7ALpo0AP4t6En5VUd82g+sL+4PhEP/Mr2uZRx/7DS9vgAfu7L80Hy0j1guCe1egF66IaKxWXPxsR8Sb8smTHvjJrWY6pugfeYmn66+t8fx+WvPA06TL4gbpliu3Ecb32IRJZtXaIcRNUCYhTaSfTrngdwNwWInEU7jndFAChUIIFm30iTAknQ97jFi4+Tc9zJBgSsmQonh5yaJGY2FLwAPbCHeW7Boz0cQIcL4HDh45HmxkSPB0UQKyZtmWWqv7dwrCuEcKvtwFFCMrX26vJ97VsIloHUG9v+R23pbjqVwG/5YgKqY41DBwuj7AJzGU8LAal7EJiZVSFEk8trF2dIm78cCK071oW5QoMEvJOkqy+lR2SJgfvxBbPXiNSE5uBKyDutkCJIqrbA6WcDUmQiyneCs7Bipv6BfMUO0EccxNksvapT9+BMWJgfE9UTu2g7jmdYjc3i112Xa6TR7GRqi6QEY+Fp+/yAE8+IPivBczeuakHc5r3y648p/FzUJkjeu7Gx0K4LOjLxQRAiNELRx1mTcGW28Yd9pmas4AsKCxRpT0FqT4BOWZDjhpBsq9lnZeIrZVNesbr1HV56pxGYTbBywohQ31kV5bGjUXTZlqlpJk9AS03epW0ru9w0IF4ND8av8xtumVNfa1p3XHL/seJ9KXQFfZlCGrD5egv5+FteCGLgCO3TeUmuWPIIdH0x8cTRfKqPW9smWxjTCbKCuMINYhX5KZqAcnzWIg20qXxDjm/lHEL0JSFN96C1mCOrQKuFLpAdjPyfbAeD898izmFuK4o9Rq8tedyLcXbTOkt9xtvtacs/UX7VJt0DKf7i4Gjx8tiM49Z1DoEHfWY46kgXcaIp3f5UG9GOncP7paVJE21NGXJtEK0UUdePX2j44ZupPav7hNS1IthxpxrUW7EE+J88cqEip1RyU84Si2wIx8DLBv8CLd1VXJsVCfoI+BsEYi6C4tel7X5cV3DSQElhdm45n4g1Ih/a70ysKXdj98EYVPJW+x6hHVsrN00CBMCMN/HFxwB6hBGzG8pl9zY4qwRveeaf6pvJKQO1nqxE1mA4HVbTajoooZ2EUS1nX/Z7a27uOoAdzCyQh8bmQjG2xI1wrNRYrGW52lirU44TtKkQ2s7mwkhf5PdOsVZtmZn5Sp4OHpncDngC487mMYQ4KHklg9XTdWliYSQR2+ZB0RbMdPtQAxlci1rLKo3s1h1GYMe+zlZxTerA33pc0+LDSfA/rqg+Xkdm3YVXhOo/wgGYeuTVb2N+RSKvz3G7YHQdc/eFUrooXHCDAg141NDm1YSLdhN0JKqr+oiVD3mN1Is+fT/MHkSaRHtNWzwS3UMeOrBKB/iv+jkhLZPBIodqYbIMiYsM7Svfub1BcNLGzI68arGQp2UdPsocmv+h3thi+dXoRvFJVH9gIbTTMqMljrkpc13tXFpWdauCyyCKseNg5uCKU18QjPWgi2GYfbO/YboMXF/b8luddZNBaMjVRBBwuOjhtNHcBgtq7HgQRgt5qfZEkJkEp+o8bi5kPkwiHqosqoDaYwwq+zFVjsSfX2iFFwD4GiW8wfIaF698LS7+8Jsh5mKHD3z5yQ/j6vOfAOgQ8nAMgOVpMMbFK16D6Q1/wnF3yPwh2wFAB1z/9hfw4H98ALi8NMW3biKjE+O0i6p1XBqreUP0xruVf6s7+QJrG26ZwbHTcrAAXBPimKRbEVrLIAO1JSfQ+PoaF9/2nXjJD/4ocDic1PPXfvKf4upzHwcudP0GFHh9jcMffD1+9w//g+PbDCeky4//Mh4+8yHw5aXfJ2tsV9EXIkdH18J5vmnc3a6fLDN1TFSKRxa+0dISe5/piSL3+Z82dHIrqUY3bgzPWf8avlAkFuHmBNig0YWiFG2OVzXAB1FdZzIDaxEJ9tzJ6OMGqyHoL/R7XJyR6vYAjCft02+AMW73/fQEVVn/uU3XZUujXyyD6AULumFi/3eJK3OsyMpLDkN0w8iUP9EsBddVf63BztXdhesxIPCHZaUMrKWzCA8LaVgyT8NGb546sNV5FwsvwxVdH5dDrt2D+BUFtjwUbH7B3dXQDOdzUYuap6heITAcqfjlN9vmqDIPzu9zyBA1csxjYd5vKkiqnWo3jdOfp42fNwO2BGI5xFkpk6BsBuct8QRV5TaeiC1fqS39igkL1CmhPk7bwIK5tj+npgFZSCAsB/g/L03BhCmorubUri43dcuty+e6cc6eNL/+lPHSaX8+ZEPR86Fyx+9zBrz8FiDzoGWR1Vv7pmbJXanlTwPBwfjbcLI3fq7RfART8nmszrG14pBTgh6sCFekbdPdujWPTL2hI89pnMZJc1d3yJ1I2BXYXL3FuzbS39GJBf/8KX+d/OZpxLqFwC7jDybgVpT9Nmr1UC6oJJiCLzvduCGSO7uwDkt58iqvv/24zW4VO2BzoivJC44Cxh7/i8DwU1O0qBbzuYzz9hbJqCyMUBga41k6L0VzUgpF9MnOTlFuRdnkZZNIi9z5bU4pvLNuGxpVrj7S1ooQoVcOE9CJ4gdYbVsxtzJz0qc9jnOHS3NC8ncjqWnpbkaHFUnfvIssxSvV2nIa163AocDGFj7ii3+TjRRG63+sUWjNEneglQmpvs5y1mQ1G9f0vQh0iSTxzDM5t6sSGHcJqyCxUjQKGTBZTdor3TS5jjnFqrzUtC7G19Va5D70txvPJDiOOYy12Yuw6eBrIxFQrPSdRYDrceZCp9xDjuqX/vAsq9H6lG1Opi02MrNqvhyjbRG3q24TpD5bWg6HtOg7F44rCIgx00UprijDbfAhGBMhQTdE3S2v9ua6QJFRogwQTXVjO7RlgAB4A8vqxsjG+TWeRje697+eRW54+C/DSGnByG/RPeF4e/UEiznwqLG55CI3od9LgLhmNN8N24ZURo7WN01nuRDajFGLCbg+8Y7b+fKtFPLKFuPZUh/8jwGym/RNwHb/kIIBGoulinVKoU14GtSd7ENZaHWu14MdcaS3VOK2tixci4wpzHj43/4dLp/5ALbLmeO63iRcP/s/gcNht0+7oBx8dQV+8LXjsca90nQ4zBcrd/o9e1wWpcC5VUXU+1tj1TbGu+X5lwYWdSf4V0GuRfWB4Ay2iffWe2aFkgnzBnb16x8Bf/LDCoVYmiOXGZLXF1gPcQd9aFrWrMMBV5/8CO6/52/ktAbBCoFw9YXPOmd5Ny2VL6gI49yXvPibQxW4GFPfpO8S2i3QpLRckOff1jiLQXoe1gkkfNijZ35HO8VBGTueA4hWXHkT3uCAOob2j1YkhByLLWwARLj+6hdx/eXfdPsJA9mRL3RA69WdMu/0tFrTpdlw3nKsdcO+e66Fvyh7rgVDXXq3vt3qz+YgGD5x6dah/j30YffmvbrLatpo1Y2vGxWBGRtpTDHodG9+aI9o2Xs0def1FFih2rL45N0CmqAtJhGNW9ARfZ4FJspft9m3BZzzZHKqJlyLOjSfl0S48nj8vwL9wwBQC7yvKbPrMDeoS/Ub9TPk2T161Bo5YmGV5ui+tkDnS5pvIU+40UYxXh8ma/JPfy7K36UrBE+UGb8RkPulegBSlvLBH4VPrg+zYRAwl0z5mLYlb6Ex78NObGgGKwB/VnljF2tbmvCzCfJsp/iQN2kX/RBUtQMxXwEkFnLrTAdnUJelZaasML8lLKLyskDP1Atgk3wXaQh0Gguj5GdzXMZdDHlTjy/i/bkw3YV/qZuntnDnp1QpC13XFqWThBYgdZA8cq5Lnw8r422yvotdcbHvtpmWZKISgV99rvYrOclFHZ2Ay+ujsSD7KQDpX6S0yFmsrHYEmaXViMscXYUMgG+B4165xmoMB7/Vt4FCRZvXbsftUUFii0bkGqWhZdnmyrKNHafzpkZkD6B1VoQjNxQtgc++/TbtvfZeRJEZ1uhuke7Aaf0+paUqNk7i5Ne1B46ixUdCyrk1Dvt7wXOlP4tAC3eRh5ukOsbolDsWphle14vZi/H3xwNTR42PWmGErWKh3XyXLcDRAtXwS+sR+o92atpjlsR592CKfvuB7zwg7qSrvAi4rW2b5vnKfVpRp9xcILMoRTnJU+GZuZtyTr+tu8O0j1muPDO/qgXfzIjdLitIkjFZCgOrhibzhdY30fvcl3k8K2GBZUghp4VLowq5QRLOqu5f3UKp8tzxKr60kKZx76IKUIeegu+c2YrWd0s0NfllcjzS/+JgjTx4g+3eIt4N+IRfAKSLKumT92pWb8GQL3CnJKvyQz75c2eElhG9RFpq6YpA+UAKbnJuIStjBJS93EbddGLDMRSDj9oL4Bm3nZweWvhUCJPFnmv6trrRpsm5Exk6yw9kD6q1di2a+G7EvCU42x6sGnsFlesI2W+cvPoVVBbirBEDPOb14ajAHCkkzDODEqddGS4bNJpnxMd3R9i3JanGFfIsZC9oV8xjZB+uFp1INmGQvzlnzjMbEKLBefail797guF+gyCbHNpAZlcbRXWvr4931ApOI8CF9UJ0R7ihFbSV0QLqaz9SP3aeUxj8Q+rUvbo6/RyyOw+BW9KuL+auA+2IOZBGWvDEJcL9eroRnqY2s/3UsEYsCNXACRdPvgmHl38b9Hlct1rrHbUYCajK5HzSwyE9U41ARo2DrzG99o/i3PeSbG5CIxBTgpLPXTfAzdGcRcYmXWDwb4ZzmbWW6Puluq71hKFXnq7PAA6Ee2/9Qdx70/cPgvuNlujGgjvGeSPq03LzjPDlMcK4C3pi8LxiKlNUaLvLqY+FyucFQ90jcOOo4kHw0Mbx/6ddrHyX5jR65M0dMZvqAHP5/lp49LEMbmmVeOHj5js3EYHeAfIaPciZkgjt7YbU36Bpnv82etB9R2zInnUlVy6Ek7dq6kHDi50zAnqXfHQGoOu26ozkSXRD0kZNn/Uu1WlRWh1lkyizRCDX3bAaB/aT80nZya/XiXxjJnRYleyGpXAUtRfLXWqlHcB/L5Cy9Vec2zbnfPi4R8sk+s1WlQrEOCnjdjpDVL16lo79GxN3aVdad74cJkdy0TH77nkOBdon7mXkvky50ObQBDp1h/r9vf2t36Xp9cPVuEu3mMjMGwlXcrb6PQUS7wdU9WrXIvmWL8QqjJ3xXL0fR3oaMrENoP9ttbt0QvJ2LllOvNnFbymzxL0QD+2xAgcmHuvw1PAr4vfJbiS0dd2YKTfHL+/SmAj9eKJ39gDq9VenrtnK1e5gsilD8b0KQWPbg14U2tSUAvEYNyGTIO4unSVFgVhnzp3dsBXyGjVLsr0uMrqvtzfO49oD4GgLbcAR9cyerlk0tbvaXWm+S6el7vmEZl77XgVb36/nTfSxnvu5qK0fGp/MmXkwtfO8gDAnPLRRQi93wnsLqZrbIHFdJBRaRv0GtuM2qvO44/8cX6PyS5Phx+ZCHwvcw7w7t+HGaVE23H81HlE527YDLEjFRsFJPkmfzGOsFwhZh9vqxphgBuZ7uAJtG+JYtESuJmgVfbj174T2HKnYxSw1LcuvNrGoT6QwTOsjn3pl07S82Kjrn3L2IA7EHIKxqdsYG3SPRdoPWt+lGyT7xkNXq7JVVPJgVnE0sefu+cpwuR+XCc4XZqqNBSC/0bEj8OFlIDJ/YOj6ctrVp3/leE1ncqA6hvI6TMvGVrWzb21Z3JKKsyM37JMOuPz8Z44H0oVw9NyD9GxDFoSFPDOK0GwGj/TR//vLv59DQkr4w9uWRa2lF4c8K6cG5zNquKazPCrnoSMFncM4e8FmBg9ltAV0sNGAjOCl0nF8K2s77/gJ4ppCm30adyUggNmau6zVYlmPNVrwQQT7ElgAzOVzISFD3rLaKK4YDtinj4FrDuqPSLeqLfrwQsptr5lVW3ldVucxCMM1QQPg7vcZv2kxVzVvZA5v3w7XE7DUILIt3s62GA5XcQQX870iPsn8o4NoFpsQDJ7cK3Jkx6Qrjvu/icC75HBWLjchsztjGGV2YMZ6ngnOGJUfvxzlztMIqyHz6lFUT/JnH44qlFvvpNd6AqHNkyGvruen0UXI3bdtvtmRvcVdnESlUMWfvCMm6uZ3lHmMECsvcvxdobW6pqFRQgy6EWFvvFEWavzT9+cXC9bb1w/orHYUt6/qKIOXugebenY9FGdXN5qLhjbvuW3H/02iTnl+IHv/vaqblHOI7n62KZrchc5E1gbbiiRRNKmqtbWpQBXHQhsqAmdc4vscaCoBtQUrXZmknouze23HvDlVaA3SNCweQHx1J3eY8w2rmAH55kJhAptlxzG4roFvCaQyXkNU8ttPUhqsCPqCnz0TD/s2NO2oW5fxlUrCg0xoOYItc6ENfFoz1qk78ErT5IRl9eWfvkanuC1OmwySw5SIcWXkXy2oRIjcuyTyic0DG6uhFkext5hlv/3Loz3+YbfQxlVtmanUKumKo5CTp3w+lbJBeIyuAqkWXrinrh5jN9DzSjjXiwZ13Wi8c8NMFQwl+VwJT+r69ayjba7fln86bN5jZmf5GMZ1BDRgqC7TC4oGrDBqLkx5ENfSZEs57izGLVATweIJmx8bfb0AJr/tMuNJ5voUfJkFMnXzBFRHbUujK/jf8iUQR5sLQL2iduehqfW03x0cU8+0ES8ckJp2jxlNEQAJlxk59YUvX4VCaBvBaXpFVGVgK/dFWTkZypGD9aoOqeZvfD3WNjZ5k8YShnMkEAgYJ9GD+M0HS7Rvkhz4zvFzdwutaobMg5U+9sa2PKlhK3KfEjfoW4EJHdCfpgV9+oKxwfvwysAw5R5I5MYVSIueue6LCoijoQ6Lyvi4oUNegOd6Qs2qcwBA55BGM0r2XIsGTjuYMss8t+Jm6rfFGOO0YaC38KelaUctTXRUI1RdpMFqbHv5WV4G4vNp89d990NNFCUtIY6nHOsqNyB2p9MiQszLyjVHxkHqCC1gX3HKTagp5wcbQ2c0quGgjy7y4tPn+5sNVcub67M/EGsIbYbccPEmrxqb2hgaJq1WmGMwaz6J6lcOGb86zCl/q12fouxp6IEtVwcqETmVY2jLxNF/2LcJ5FrCV1nCqF46/nQxD3UDyxe0PY5LKoF8nKxyVsENjyYmGxOzrY/HPgws51OCTggl2voizlyjAznUrgWH9c3ECkrZ/BH3MeQTl+V810dqMQr63xk8V0Lb8qWbFplFrVWYPfdoyZqi7+JWK3muk6yYGBskZBdLSB+R1ORsfbMX6JDXXs8L8nxGiz4EC1kILSGyQDQWEO2J7PaNidLnrmOF7ZmPyMzNBOZzmY/+QSDJk5AfXiOcNAZ9BZNlitfRkX6KyngEGU3WInDUIONc+99bMFqWI0EuNa3jV4lyhTZq0rf8RSyyKQBffSsieJlZL6ctN7glp73RJdz5jpgZm2SUn7hWNpNfwB/sYodjE4pQW0jhc828yYtRioLeWeVxVS5gpis3DaHtuxZrPrnCk9FYmeDdmwu++6ELcTZnSXtmzqtxNhds6ztnq+4TkFcHPTArO9aCTp4RoIYJjWErlEwvv7JZCm3fhJ7yNaPR+sjhnCq0Td64b+uqPjIoUJaLrxSrNi+GPAWHeW5n1/zUvJilliXE45sZBZscmZ5BZbauEtp8oR2H38CCozG69TzzcQKEOAiFBphLt2Qpy8VWrJM3uC7U1YT52Gb/0muLg2aEqd7amtyCutNU024d6zxPM7hCq3SjcRGqcxGDUJg+GVrJu7FCuo3rCN24sMpjjWpREUax7sB5nibrfUdh4V8PspJCRuDsfbkCg03RA5E/fibW1A9YO0UNtiPH4v2i7sDCa5fqk/2CXqlVXIE3UuyD6F0s08uMg5UxK/Wlxdgq2nz68jew4/qrkst5T2E9r6gbz/gLMuPdklIfl9bBe5VjgaegvYh5MvKsUgqz1ZZB9W398FwoVo05093Zxo2cOmvm/brj0x6PEJzUyi3DtqC6vno8F6JMIbTZ+GXbGw+mkBO7fLIof5+JryPsnFF7gpX0CtSkzxijVf16xmmsX/qMQQDY0bauJjMNBYEyHNOdIw/bcDsBZ8RTtMous+YfawxNVMzQxcy3iQiYRyFWKOolgyZHcHwm135fLwj1eOcFVAvwn/KzCnR8gXVHnJvfHCXYAz/2gj5/jHtchAUrOmrcoYAcfRKEEYy2zBkcEqbA9wgLVfFcZ/cmEj7ONCZKoS3fesC2kE2T7YVAkdKOK0D6tL4q9WmOg8Rc0+blghT5tY2+R5dLfnWnq2nZtFnQLOsujNJASSsgdH23bWcnrdsx8UG5BVLysu3Au0KbT05nYlcCFF/aboUl1H9Gti677WXta6ssEIohw7YpFRwN53GdYCAf2M39XCHwHQCeDb4784EonZwZElu/t5WOTQrEqD2daFk9oKDZtgUy/ZS3dCMO/jJ+jrQRa7e2hx4ZXobnaROBN9LvoC/OopoAXIIxpUFHAN7XsEdnhUZ5eRnuV5y1ZeWLBeEIY7jnzCl7U6Gdf0eQV5ZPQwGLzHTgPFosoC+0wYLSC/r4sJYd1/oot1NSd1y0zpxdHgA8d2w0GlhsCuRfEewBP18xz/axMZ7nNvyF0lssIRkZ1rk8iwPFRHkPYwj4NxLlOiGGaIuMpH0kmtzPQNiWdAcJYrFxxr+YfkWjJc+Mfa3/3IEZv7H3vqqRp2SYpDpZH6uJZDhCZ/sdEQv5wFtQjosQuiCUCy2ixewRa8/ECkZ59A35yv1xaLJjI+6V8zjLoULab/lyTZto25U3azmphvM3K75wAOgTLnFqtbsKAN69UORU2Lcw9MAic5lpDO4gDxFyACzXTTngm9Vmo5lmh3chjWpij7zMLchSrwerWeHpnG3YFnzVR65pBzZIMczmzX1s6Pu1A4APA7jKGHXUeYGab0SstbZUddXV+sb7TBi/CV3CVEcTr9nDMy4HFPRBQYaD8Rr/rYRAywJaWQhjTJXQulrSKZng7Azz6v92FLYKonn1u/2YiHEF4EMHAn4JwG/FBEUaj+zvYWLW1Q2ludgbvhRanmdQQyB9rBEZzVxKR3O8ngVK3YN1/N7mRwO6YjC5k0/HxZ5gkQwCU+MMiWMhfZ82Qh4CFyT2V49VmRqLkoCjrP7SAUS/CtAvr4GOE+XJoLow0yxXysznAJ9T7c0Vwy+kN7QFZYzjufmhORY/EvPo+EpyCVKtTVCc7E+w4HliKa0f1W3MWRQPsF6w6U3jqBfGrjw3JvgoM545gPE8mN8LxoNocLnQFnDH8ssPxByKOyvWlouRh5hWGv72t+0bWnChL1MWHP1sa1pd2qOPwhKu41j4rzov89tn14yzulkfFca9WYFLAD9NRM8fZkjlaRx9XctQn0lu3qKfdc3+5XBd2CbUZrLkslgCwdmHta59SH5WsFU4aVEZO7ZxA6TCacfnDApe1fEie5lVBn9LeQ5cpnL8s1xwVVfw5KMAfh4ADnOlLzHwrxi4bwfgMckhJAs4TIphoRwHdXwj2d8wAznjN01U+F8O43P8sjM2OGX8Z8ZUG9dH9rm4dey2nfuQIX3JKa8URKuEFqXAj4/vM/AeBr5IBByeeOrTy9OfAei9pR82dqCYZyanEgrIumpkoXlaJmVDAJzJJfU7G0fEdAfyEmPL2ks1rWcJrCJwr6hqjaFvQTxNLV/0TXiX+bWVe9Dj3VjwPwH4WQD41v/8i8dL776ObwJA9wH8GAMfy5kcd+r6X1FafdJIKGKNuQWLlfn1gg2s4L2ll1RTW6Qr+mRnoTi02LFFlksHQImpToSdh/pwSgu+WKwVRojduMT0MXCogNY4a1NTLNr6GDP+CQH3n3v+nnz61Xd8+/LrrQB+AqDXpIxyfKN1chvf/PUu+ognCCzQgNY7Xr7QRs90v8fxlJp2HrbPF8GTpY9OkGQ02UlCa8txUVeMKhfaekE6/cS8dy4hWX9+lhnvPBwOH7i6usSrn34fgOGa0Sf+7afAIDz+4PB+AO8C8GwdcMgVqwjjsG6FZdp8oQp9sY2F9hSIZuxlG5ERMvL4IngyttLBMmMPKaT3JkJr8yIUB0i36Dlvk5FAlcbUrQv/WQb+5u/7PS/9wNX11Sq07ii+8hdfh/uPXuKx56e3APhxAG/0iNv9JUqhRU6AnsLgD4rRQfQ7lzEaUNG3LKiBmuAyDB+yCgPUQlv6Vxu5ykJaH8WbsC7ZcqG75QqtHZuwSJlPvwVa5PLeH8PHGHj3l164fv+3PHLAq3/+F6PRbekr73gSwBUIh28H8I8A/DkAjw7E2deSuCeM7st/RV0ZOPRMqDTTidlTTOfhSYyoKNaFyEND4LEtqj3tj30QgPSbG87RwsUvzXmJVaBqly4P4lKrJ4X2PoD3MuPHDtPFr189eIhX/8L7vFbj9NW/8B0A+DEQfT8Yf4tB30XAPUFv6LcMmqGCyhqarL7bNvBpgfaiCieoEDyfxtpXX2jr+fkxpJSMzfjgvsaLFufGu+a8MVRn8sulqZV6AOAjAP4lM/8sge6/SmnZYhZk+vKffz2mi4e4up5eTsDbGPgBAN8N4GUAXaSabCX+mFdrS8uA3uZF4NNWgZhiXh1MxYuqDsJk/djS5P2uNLbeGJH5oZuWfIjF8qTBl4WfVSDKuALoSzgK7H8A8PQ1X//2vekeXvFzTyNLpeAu6Ss/9CToEYAv8RgIfwRE3wfG9wJ4EsArADwOfTU/R8zqRNdorvbAp638PqeN3L2ING34HlXabw/Os2UCgZeaPPE3e3i0zA/NfHjB4XZ7m1JUDwH8DhhfANEnAP4QGB8E8AxNh+cvn7/EH3jff0Un/X9D3uNHk45pqgAAACV0RVh0ZGF0ZTpjcmVhdGUAMjAyMC0xMS0wNlQxMDo1MDo1NSswMTowMKO0v5oAAAAldEVYdGRhdGU6bW9kaWZ5ADIwMjAtMTEtMDZUMTA6NTA6NDMrMDE6MDB9kzKCAAAAAElFTkSuQmCC', + "is_valid": true, + "label": "Subflow", + "environment": "onprem", + "description": "Control another workflow", + "long_description": "Execute another workflow from this workflow", + }, { "name": "User Input", "type": "TRIGGER", @@ -2007,7 +2180,7 @@ const AngularWorkflow = (props) => { "large_image": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABkCAAAAABVicqIAAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAAAmJLR0QA/4ePzL8AAAAJcEhZcwAACxMAAAsTAQCanBgAAAAHdElNRQfjCB8QNSt2pVcCAAAIxUlEQVRo3u2aa4xV1RXH/2vtfWeAYQbBBwpUBqGkjQLKw4LQ1NqmDy2RxmKJSGxiaatttbWhabQPSFuT2i+NqdFKbVqjjDZGYxqgxFbbWgHlNSkBChQj8ii+ZV4Mc/be/34459w5995zH4zYpA3709x7z9m/vdbae6/XCPH+D/0vMM5AzkDOQBoY9hSfJ4n4khCISGMvySlcKww0UvYNpAFdNAxhEAXQc/T1g2/2RFJoOW/8OeNHAaDXepwGIYEGOL5146a9/z5R/LJp7KRp86+4UOJf3yskUKVv/ZN/OQoAogIIQQYAaJ117aL2ehjWHcEF7lsxEQK1RgeNLaLGKgSti5919L76DPUhzvPAV0cAanL3khgDyMf+GOjCUCHB8Z0VI6GFGsYVq8Cnt1cXpg7Eez7ZXiKEiKgxqqqSFUcx7M5+uqFAHLuWQwYRYqzNfsjAjWLmdka5Kqu5u5ztvOkfNoTko4oHICNHtzSHqK+rywMwCEyZruX+ZV5zLFcL4uzTy7qtT24RZUDh4ivmTL2wdbgN/mTvkZd3bO58F2KKTwT+aGXIu2uq6yribxQmMYRRYMZPO8uVfmTNouGx4QFALW6OQqX5q0McV8MkR0wNdOEzAyRd5H2Ih3cuMHD3N0ZB0+ea8MUBhoYhER9GcimJUXz0eTJEvvx9H/nAA19pQrwFRApY6iueqgZxXF9ItCsWZz0Y6KocNu8Ct8xBqjKL2+kbg3juH5vao4D5/6SrcgRiDPu/J4nYanF/+XnJhwT2z0v8mRjc0l/jyojldvz9qHhRotL81zJKPsTxjoShBj+uefklq4q4KRXd4NKeUuMjn7E21ZXFPVWOcdkY4PaxScRgcUepKHmQwJ5Liqta1RiDjLi5LaaImL+VUPIgjj+LlSUWt1Saw3vvK7cpGbEjsb7BfJdVWA4k8Nj5UAHEYt5JNiZHTFmZWNLgt1lRciCOK2FFAEHLjvLdGHhi+eev/8J112ytOA0MwX08VrPi0uzBr4QEvj4BEhvwbkYVv3adC4HiDznOw3P7SKgAMOjI/F7p8AI6DlsCUDf9NlTGB9oMaw3yAgd1l92exqSrs8FppSBuNgwAUTxeudrA7gugUKzNc4OBr10YvyzmpUF9VkhCbN4qAYCGuYvz1pveaHkeSPx5X4MAoPonUHRVOZCnYeOfbxWfM1F/BIJVInXFstFOABDrnWEVCI1/BgGA+kmfy5mJ6Pf5q0tEmXAtFACxcweqQrBnFwIAwWdH+0qdCOqF8tcjAKDBCwhVIS9GhgCIhVVmqRnYKha0J4Z+oWi3Sqm3xSsO4y8fSoYkvnV+bHp09qVGKZuHBjuT72eOCQ3mOGVjbiLvkWPIhwA9h0EAgukYYtllNrwA1BP7qkCI195EbJIZQ4MIJoyGAFAcqirJWz0ggICx+ecNI5sACFxVyLjk6sOR9Dsbrx+gAEDQ4xACQnsWSEr65uAwAmH1PSbUs5M/j6bv2XSO9LLoSyLXEaOgjWa3pRqXtuSv3hLIu7CuRwHAjetKfjFvjxgQFlrAUOgbMbxyruqYpmSKgYy6gm5dYk2/gBA2DcADII5/UgBqE2GOT3tqOCuFCrWz6+wqSHr+LqP28tkUk3YP3tqBPRdAIZj5HENuNOa5EAaAxQ3pa4gd7mNGraqKDKYXoiKipoCp+zOeNrB7AgQQmGUH6XN9ypUJZHnqcpCEAB2an3gWcNG+rHsKfOccWADG4Oyf91XGfYH+kgTyw9R5Iw001qjmeCiDSbvLXGC4pxWqcTo6fV2FzjwPnYXYzT9YBmHEx4ya8j1r0b67Ml751xKBUYEayOL99CUYx01ITvz6UnWlGtPSjK+Ai/ZUunIXuGE21ApgDNpW9ZbozPGXsZfH8AMlhk8ppnRTWkzZmxcueMeT950LNRCxig894TM7w/FGGACKD/aloVcmWonYYTIFH7GYvK9KZu48jy63MCqiRnDN7mIkF9jVDgVgcF3x5WxIVLrHCphSjRHXWzYuiFNSNWi+N5XFcV186Vr8ohgZlsRdA+wwYlJdTd7L2ulVeGgcjAGkGb9KH3W8KTGJbCkqsTS4i7hGjYEILCbVZCQ6+3oTjLH41ODWezX1JtOiog7LIsiIHUaNqEX7rjoMMjjP7VdBTWFTuuiId8PGBv3u4PvlYWpsfYuJ9Rmxzvyjk/Ht9NnAYx+AojxMrYiFI3YYg8m7G2GQdI5v/eRQqpiId8IKAItP1EwdIj5e3x5ZnYXidJ7bW9KsY01mhpwkKOKvX2qYQTKkSWUI0VVpEnRZ7SSI9GTdnDpvRFyVuHODh+ukcyy9vxvmRVwTuyOxWFAvMS0XyzeaYm9sjXM5Eft8ydrqQTx39IdGhBngtrGIfYXFd+oXC0qW9xDuyvWypSNE3DhY9pje1UDZI8NYrYovdderSjjHx9riEwLBsL83VMApMh6BqsWcnTWF8Y4nVkBt6iEeaKwUlbycuDGD1ntdmZfNIgI3zoLR1EN8q9GiWsx4NHFiRvGRZ8ngqpQHv9wEW4xIbwwNlwdJz4Nj0kaRGshn1p4k6SJXVujcdWsb1CYBtcWSUyl0koFrmpEWIo0CF6/aVm6Zw49cMywtgYsYi5tdzoavVXwOuuGGtwsuU3y2H55/+ZT21hE2hP7eI/s7X+w8DjFJCVyUIb/4XLOM7s3+pVuSMrqARjwBtI5qbeZAb/fxAMBISDppYtzIB5bmltHrNQR6bwNsMbQULekBZD6INZi1o8p5qt/a2DC1rD8jqqpamiEZg+HfPzG01gYZHLt/0AYxtZo0RoGrO4fcpCHpPF/5ZhugNie9E1FrAblyHd9Du4mxg335rilp4yyrN2MNBK1LnvM1a8cNtwBP/OmpP78aLz4WiHF3pm3u1Ysm1mkBnkozs3vbxk17jmabmRfNmD9vwmlqZsYLhwHQe/SNV97ojrTQcv74MRPaAIRwutqyCYdl853uBnM6LdNqxPvUKh/y+P/594UzkDOQ/3HIfwCAE6puXSx5zQAAACV0RVh0ZGF0ZTpjcmVhdGUAMjAxOS0wOC0zMVQxNjo1Mzo0My0wNDowMGtSg1gAAAAldEVYdGRhdGU6bW9kaWZ5ADIwMTktMDgtMzFUMTY6NTM6NDMtMDQ6MDAaDzvkAAAAAElFTkSuQmCC", "label": "Schedule", "is_valid": true, - "environment": "cloud", + "environment": "onprem", "long_description": "Create a schedule based on cron", }, { @@ -2108,6 +2281,8 @@ const AngularWorkflow = (props) => { return } + const triggerLabel = getNextActionName(data.name) + newNodeId = uuid.v4() const newposition = { "x": e.pageX-cycontainer.offsetLeft, @@ -2125,7 +2300,7 @@ const AngularWorkflow = (props) => { id_: newNodeId, _id_: newNodeId, id: newNodeId, - label: data.label, + label: triggerLabel, type: data.type, is_valid: true, trigger_type: data.trigger_type, @@ -2177,7 +2352,7 @@ const AngularWorkflow = (props) => { data: newcybranch, } - if (data.name !== "User Input") { + if (data.name !== "User Input" && data.name !== "Shuffle Workflow") { //workflow.branches.push(newbranch) cy.add(edgeToBeAdded) } @@ -2225,13 +2400,21 @@ const AngularWorkflow = (props) => { var newAppPopup = false + /* + FIXME: Add auth. + selectedAction.selectedAuthentication = e.target.value + selectedAction.authentication_id = e.target.value.id + setSelectedAction(selectedAction) + setUpdate(Math.random()) + */ + const newAppData = { app_name: app.name, app_version: app.app_version, app_id: app.id, sharing: app.sharing, private_id: app.private_id, - environment: environments[defaultEnvironmentIndex].Name, + environment: environments === null ? "cloud" : environments[defaultEnvironmentIndex].Name, errors: [], id_: newNodeId, _id_: newNodeId, @@ -2398,7 +2581,7 @@ const AngularWorkflow = (props) => { InputProps={{ style:{ color: "white", - minHeight: "50px", + minHeight: 50, marginLeft: "5px", maxWidth: "95%", fontSize: "1em", @@ -2418,6 +2601,11 @@ const AngularWorkflow = (props) => { ) })} {filteredApps.filter(innerapp => !internalIds.includes(innerapp.id)).map((app, index) => { + if (app.invalid) { + return null + } + + console.log("APP: ", app) return( ) @@ -2431,9 +2619,10 @@ const AngularWorkflow = (props) => { const getNextActionName = (appName) => { var highest = "" //label = name + _number - for (var key in workflow.actions) { - const item = workflow.actions[key] - if (item.app_name === appName) { + const allitems = workflow.actions.concat(workflow.triggers) + for (var key in allitems) { + const item = allitems[key] + if (item.app_name === appName && item.label !== undefined && item.label !== null) { var number = item.label.split("_") if (isNaN(number[-1]) && parseInt(number[number.length-1]) > highest) { highest = number[number.length-1] @@ -2637,7 +2826,6 @@ const AngularWorkflow = (props) => { foundResult.result = foundResult.result.split(" None").join(" \"None\"") foundResult.result = foundResult.result.split(" False").join(" false") foundResult.result = foundResult.result.split(" True").join(" true") - foundResult.result = foundResult.result.split("\'").join("\"") var jsonvalid = true try { @@ -2646,7 +2834,15 @@ const AngularWorkflow = (props) => { jsonvalid = false } } catch (e) { - jsonvalid = false + try { + foundResult.result = foundResult.result.split("\'").join("\"") + const tmp = String(JSON.parse(foundResult.result)) + if (!foundResult.result.includes("{") && !foundResult.result.includes("[")) { + jsonvalid = false + } + } catch (e) { + jsonvalid = false + } } // Finds the FIRST json only @@ -2670,7 +2866,37 @@ const AngularWorkflow = (props) => { } }) - const changeActionParameter = (event, count) => { + const changeActionParameter = (event, count, data) => { + if (data.name.startsWith("${") && data.name.endsWith("}")) { + // PARAM FIX - Gonna use the ID field, even though it's a hack + const paramcheck = selectedAction.parameters.find(param => param.name === "body") + if (paramcheck !== undefined) { + if (paramcheck["value_replace"] === undefined || paramcheck["value_replace"] === null) { + paramcheck["value_replace"] = [{ + "key": data.name, + "value": event.target.value, + }] + + } else { + const subparamindex = paramcheck["value_replace"].findIndex(param => param.key === data.name) + if (subparamindex === -1) { + paramcheck["value_replace"].push({ + "key": data.name, + "value": event.target.value, + }) + } else { + paramcheck["value_replace"][subparamindex]["value"] = event.target.value + } + } + //console.log("PARAM: ", paramcheck) + + selectedActionParameters[count]["value_replace"] = paramcheck + selectedAction.parameters[count]["value_replace"] = paramcheck + setSelectedAction(selectedAction) + return + } + } + if (event.target.value[event.target.value.length-1] === "$") { if (!showDropdown) { setShowAutocomplete(false) @@ -2717,6 +2943,7 @@ const AngularWorkflow = (props) => { } } + //console.log("CURSTRING: ", curstring) if (curstring.length > 0 && actionlist !== null) { // Search back in the action list curstring = curstring.split(" ").join("_").toLowerCase() @@ -2837,7 +3064,7 @@ const AngularWorkflow = (props) => { if (Object.getOwnPropertyNames(selectedAction).length > 0 && selectedActionParameters.length > 0) { return (
- Arguments + Parameters {selectedActionParameters.map((data, count) => { if (data.variant === "") { data.variant = "STATIC_VALUE" @@ -2863,7 +3090,7 @@ const AngularWorkflow = (props) => { multiline = true } - if (data.value !== undefined && data.value.startsWith("{") && data.value.endsWith("}")) { + if (data.value !== undefined && data.value !== null && data.value.startsWith("{") && data.value.endsWith("}")) { multiline = true } @@ -2872,14 +3099,85 @@ const AngularWorkflow = (props) => { placeholder = data.example } + if (data.name.startsWith("${") && data.name.endsWith("}")) { + const paramcheck = selectedAction.parameters.find(param => param.name === "body") + if (paramcheck !== undefined && paramcheck !== null) { + if (paramcheck["value_replace"] !== undefined && paramcheck["value_replace"] !== null) { + //console.log("IN THE VALUE REPLACE: ", paramcheck["value_replace"]) + const subparamindex = paramcheck["value_replace"].findIndex(param => param.key === data.name) + if (subparamindex !== -1) { + data.value = paramcheck["value_replace"][subparamindex]["value"] + } + } + } + } + + var disabled = false + var rows = "5" + var openApiHelperText = "This is an OpenAPI specific field" + if (selectedApp.generated && selectedApp.activated && data.name === "body") { + const regex = /\${(\w+)}/g + const found = placeholder.match(regex) + if (found === null) { + //setExtraBodyFields([]) + } else { + rows = "1" + disabled = true + openApiHelperText = "OpenAPI spec: fill the following fields." + //console.log("SHOULD ADD TO selectedActionParameters!: ", found, selectedActionParameters) + var changed = false + for (var specKey in found) { + const tmpitem = found[specKey] + var skip = false + for (var innerkey in selectedActionParameters) { + if (selectedActionParameters[innerkey].name === tmpitem) { + skip = true + break + } + } + + if (skip) { + //console.log("SKIPPING ", tmpitem) + continue + } + + changed = true + selectedActionParameters.push({ + action_field: "", + configuration: false, + description: "Generated by OpenAPI body example", + example: "", + id: "", + multiline: false, + name: tmpitem, + options: null, + required: false, + schema: {type: "string"}, + skip_multicheck: false, + tags: null, + value: "", + variant: "STATIC_VALUE", + }) + } + + if (changed) { + setSelectedActionParameters(selectedActionParameters) + } + + return + } + } + + //console.log("Data: ", data) var datafield = { }} fullWidth multiline={multiline} - rows="5" + rows={rows} color="primary" defaultValue={data.value} type={placeholder.includes("***") ? "password" : "text"} placeholder={placeholder} onChange={(event) => { - changeActionParameter(event, count) + changeActionParameter(event, count, data) }} + helperText={selectedApp.generated && selectedApp.activated && data.name === "body" ? + + {openApiHelperText} + + : + data.name.startsWith("${") && data.name.endsWith("}") ? + + OpenAPI helperfield + + : + null + } onBlur={(event) => { // Super basic check //if (event.target.value.startsWith("{")) { @@ -2929,7 +3239,7 @@ const AngularWorkflow = (props) => { InputProps={{ style:{ color: "white", - minHeight: "50px", + minHeight: 50, marginLeft: "5px", maxWidth: "95%", fontSize: "1em", @@ -2958,7 +3268,7 @@ const AngularWorkflow = (props) => { type={"text"} placeholder={"The file ID to get"} onChange={(event) => { - changeActionParameter(event, count) + changeActionParameter(event, count, data) }} onBlur={(event) => { }} @@ -2967,7 +3277,7 @@ const AngularWorkflow = (props) => { //datafield = `SHOW FILES FROM OTHER NODES? Filename: ${selectedActionParameters[count].value}` /* if (selectedActionParameters[count].value != fileId) { - changeActionParameter(fileId, count) + changeActionParameter(fileId, count, data) setUpdate(Math.random()) } @@ -2981,7 +3291,7 @@ const AngularWorkflow = (props) => { } } - changeActionParameter(e, count) + changeActionParameter(e, count, data) } datafield = @@ -2995,7 +3305,7 @@ const AngularWorkflow = (props) => { fullWidth onChange={(e) => { console.log("VAL: ", e.target.value) - changeActionParameter(e, count) + changeActionParameter(e, count, data) setUpdate(Math.random()) }} style={{backgroundColor: surfaceColor, color: "white", height: "50px"}} @@ -3048,7 +3358,7 @@ const AngularWorkflow = (props) => { color: "white", marginLeft: "5px", maxWidth: "95%", - height: "50px", + height: 50, fontSize: "1em", }, }} @@ -3058,7 +3368,7 @@ const AngularWorkflow = (props) => { helperText={
Example: $.body will get "data" from {'{"body": "data"}'}
} placeholder="Action variable ($.)" onChange={(event) => { - changeActionParameter(event, count) + changeActionParameter(event, count, data) }} />
@@ -3139,6 +3449,40 @@ const AngularWorkflow = (props) => { toComplete += values[key].autocomplete } + // Handles the fields under OpenAPI body to be parsed. + if (data.name.startsWith("${") && data.name.endsWith("}")) { + // PARAM FIX - Gonna use the ID field, even though it's a hack + const paramcheck = selectedAction.parameters.find(param => param.name === "body") + if (paramcheck !== undefined) { + if (paramcheck["value_replace"] === undefined) { + paramcheck["value_replace"] = [{ + "key": data.name, + "value": toComplete, + }] + + } else { + const subparamindex = paramcheck["value_replace"].findIndex(param => param.key === data.name) + if (subparamindex === -1) { + paramcheck["value_replace"].push({ + "key": data.name, + "value": toComplete, + }) + } else { + paramcheck["value_replace"][subparamindex]["value"] += toComplete + } + } + + selectedActionParameters[count]["value_replace"] = paramcheck + selectedAction.parameters[count]["value_replace"] = paramcheck + setSelectedAction(selectedAction) + setUpdate(Math.random()) + + setShowDropdown(false) + setMenuPosition(null) + return + } + } + selectedActionParameters[count].value += toComplete selectedAction.parameters[count].value = selectedActionParameters[count].value setSelectedAction(selectedAction) @@ -3153,134 +3497,132 @@ const AngularWorkflow = (props) => { } return ( -
- { - handleMenuClose() - }} - open={!!menuPosition} - style={{ - border: `2px solid #f85a3e`, - color: "white", - marginTop: 2, - }} - > - {actionlist.map(innerdata => { - const icon = innerdata.type === "action" ? : innerdata.type === "workflow_variable" || innerdata.type === "execution_variable" ? : + { + handleMenuClose() + }} + open={!!menuPosition} + style={{ + border: `2px solid #f85a3e`, + color: "white", + marginTop: 2, + }} + > + {actionlist.map(innerdata => { + const icon = innerdata.type === "action" ? : innerdata.type === "workflow_variable" || innerdata.type === "execution_variable" ? : - const handleExecArgumentHover = (inside) => { - var exec_text_field = document.getElementById("execution_argument_input_field") - if (exec_text_field !== null) { - if (inside) { - exec_text_field.style.border = "2px solid #f85a3e" - } else { - exec_text_field.style.border = "" - } + const handleExecArgumentHover = (inside) => { + var exec_text_field = document.getElementById("execution_argument_input_field") + if (exec_text_field !== null) { + if (inside) { + exec_text_field.style.border = "2px solid #f85a3e" + } else { + exec_text_field.style.border = "" } + } - // Also doing arguments - if (workflow.triggers !== undefined && workflow.triggers !== null && workflow.triggers.length > 0) { - for (var key in workflow.triggers) { - const item = workflow.triggers[key] + // Also doing arguments + if (workflow.triggers !== undefined && workflow.triggers !== null && workflow.triggers.length > 0) { + for (var key in workflow.triggers) { + const item = workflow.triggers[key] - var node = cy.getElementById(item.id) - if (node.length > 0) { - if (inside) { - node.addClass('shuffle-hover-highlight') - } else { - node.removeClass('shuffle-hover-highlight') - } + var node = cy.getElementById(item.id) + if (node.length > 0) { + if (inside) { + node.addClass('shuffle-hover-highlight') + } else { + node.removeClass('shuffle-hover-highlight') } - } + } } + } - const handleActionHover = (inside, actionId) => { - var node = cy.getElementById(actionId) - if (node.length > 0) { - if (inside) { - node.addClass('shuffle-hover-highlight') - } else { - node.removeClass('shuffle-hover-highlight') - } + const handleActionHover = (inside, actionId) => { + var node = cy.getElementById(actionId) + if (node.length > 0) { + if (inside) { + node.addClass('shuffle-hover-highlight') + } else { + node.removeClass('shuffle-hover-highlight') } } + } - const handleMouseover = () => { - if (innerdata.type === "Execution Argument") { - handleExecArgumentHover(true) - } else if (innerdata.type === "action") { - handleActionHover(true, innerdata.id) - } - } - - const handleMouseOut = () => { - if (innerdata.type === "Execution Argument") { - handleExecArgumentHover(false) + const handleMouseover = () => { + if (innerdata.type === "Execution Argument") { + handleExecArgumentHover(true) } else if (innerdata.type === "action") { - handleActionHover(false, innerdata.id) + handleActionHover(true, innerdata.id) } + } + + const handleMouseOut = () => { + if (innerdata.type === "Execution Argument") { + handleExecArgumentHover(false) + } else if (innerdata.type === "action") { + handleActionHover(false, innerdata.id) } + } - var parsedPaths = [] - if (typeof(innerdata.example) === "object") { - parsedPaths = GetParsedPaths(innerdata.example, "") - } + var parsedPaths = [] + if (typeof(innerdata.example) === "object") { + parsedPaths = GetParsedPaths(innerdata.example, "") + } - return ( - parsedPaths.length > 0 ? - - {icon} {innerdata.name} -
- } - parentMenuOpen={!!menuPosition} - style={{backgroundColor: inputColor, color: "white", minWidth: 250,}} - onClick={() => { - handleItemClick([innerdata]) - }} - > - {parsedPaths.map((pathdata, index) => { - // FIXME: Should be recursive in here - const icon = pathdata.type === "value" ? : pathdata.type === "list" ? : - return ( - {}} - onClick={() => { - handleItemClick([innerdata, pathdata]) - }} - > - -
- {icon} {pathdata.name} -
- - - ) + return ( + parsedPaths.length > 0 ? + + {icon} {innerdata.name} +
+ } + parentMenuOpen={!!menuPosition} + style={{backgroundColor: inputColor, color: "white", minWidth: 250,}} + onClick={() => { + handleItemClick([innerdata]) + }} + > + {parsedPaths.map((pathdata, index) => { + // FIXME: Should be recursive in here + const icon = pathdata.type === "value" ? : pathdata.type === "list" ? : + return ( + {}} + onClick={() => { + handleItemClick([innerdata, pathdata]) + }} + > + +
+ {icon} {pathdata.name} +
+ + + ) - })} - - : - handleMouseover()} onMouseOut={() => {handleMouseOut()}} - onClick={() => { - handleItemClick([innerdata]) - }} - > - -
- {icon} {innerdata.name} -
- - - - ) - })} - -
+ })} + + : + handleMouseover()} onMouseOut={() => {handleMouseOut()}} + onClick={() => { + handleItemClick([innerdata]) + }} + > + +
+ {icon} {innerdata.name} +
+ + + + ) + })} + ) } @@ -3288,6 +3630,14 @@ const AngularWorkflow = (props) => { if (!data.required) { itemColor = "#ffeb3b" } + + var tmpitem = data.name.valueOf() + if (data.name.startsWith("${") && data.name.endsWith("}")) { + tmpitem = tmpitem.slice(2, data.name.length-1) + } + + tmpitem = tmpitem.charAt(0).toUpperCase()+tmpitem.substring(1) + return (
@@ -3303,7 +3653,7 @@ const AngularWorkflow = (props) => {
}
- {data.name} + {tmpitem}
{selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 ? null : @@ -3360,7 +3710,7 @@ const AngularWorkflow = (props) => { onClick={() => setShowAutocomplete(true)} fullWidth open={showAutocomplete} - style={{border: `2px solid #f85a3e`, color: "white", height: "50px", marginTop: 2,}} + style={{border: `2px solid #f85a3e`, color: "white", height: 50, marginTop: 2,}} onChange={(e) => { if (selectedActionParameters[count].value[selectedActionParameters[count].value.length-1] === ".") { e.target.value.autocomplete = e.target.value.autocomplete.slice(1, e.target.value.autocomplete.length) @@ -3475,18 +3825,63 @@ const AngularWorkflow = (props) => { }) } + const headerSize = 68 + const rightsidebarStyle = { + position: "fixed", + right: 0, + top: headerSize+1, + height: "100%", + bottom: 0, + minWidth: 350, + maxWidth: 350, + borderLeft: "1px solid rgb(91, 96, 100)", + overflow: "scroll", + overflowX: "auto", + overflowY: "auto", + zIndex: 1000, + } + const appApiView = Object.getOwnPropertyNames(selectedAction).length > 0 ?

{selectedAction.app_name}

- What are actions? - {selectedAction.errors !== null && selectedAction.errors.length > 0 ? -
- Errors: {selectedAction.errors.join("\n")} -
- : null - } +
+ { + console.log("FIND EXAMPLE RESULTS FOR ", selectedAction) + if (workflowExecutions.length > 0) { + // Look for the ID + const found = false + for (var key in workflowExecutions) { + if (workflowExecutions[key].results === undefined || workflowExecutions[key].results === null) { + continue + } + + var foundResult = workflowExecutions[key].results.find(result => result.action.id === selectedAction.id) + if (foundResult === undefined || foundResult === null) { + continue + } + + setSelectedResult(foundResult) + setCodeModalOpen(true) + break + } + } + }}> + + + + + + What are actions? + {selectedAction.errors !== null && selectedAction.errors.length > 0 ? +
+ Errors: {selectedAction.errors.join("\n")} +
+ : null + } + +
: null} {selectedAction.authentication !== undefined && selectedAction.authentication.length > 0 ? -
+
Authentication
{ + setSubworkflow(e.target.value) + setUpdate(Math.random()) + workflow.triggers[selectedTriggerIndex].parameters[0].value = e.target.value.id + setWorkflow(workflow) + }} + style={{backgroundColor: inputColor, color: "white", height: "50px"}} + > + {workflows.map((data, index) => { + if (data.id === workflow.id) { + return null + } + + return ( + + {data.name} + + ) + })} + + } + {workflow.triggers[selectedTriggerIndex].parameters[0].value.length === 0 ? null : Explore selected workflow} +
+
+
+ Execution Argument: +
+
+ { + console.log("DATA: ", e.target.value) + workflow.triggers[selectedTriggerIndex].parameters[1].value = e.target.value + setWorkflow(workflow) + }} + /> +
+
+
+ API-key: +
+
+ { + workflow.triggers[selectedTriggerIndex].parameters[2].value = e.target.value + setWorkflow(workflow) + }} + /> +
+
+
+ ) + } + + return null + } + const WebhookSidebar = () => { if (Object.getOwnPropertyNames(selectedTrigger).length > 0) { if (workflow.triggers[selectedTriggerIndex] === undefined) { @@ -4655,7 +5215,7 @@ const AngularWorkflow = (props) => {

{selectedTrigger.app_name}: {selectedTrigger.status}

- What are webhooks? + What are webhooks?
@@ -4669,7 +5229,7 @@ const AngularWorkflow = (props) => { color: "white", marginLeft: "5px", maxWidth: "95%", - height: "50px", + height: 50, fontSize: "1em", }, }} @@ -4728,23 +5288,30 @@ const AngularWorkflow = (props) => {
- Arguments + Parameters
- Webhook URI: + Webhook URI
{ - //alert.info("Saved URI to clipboard") - console.log("Copy to clipboooooard") + var copyText = document.getElementById("webhook_uri_field"); + navigator.clipboard.writeText(copyText.value) + copyText.select(); + copyText.setSelectionRange(0, 99999); /* For mobile devices */ + + /* Copy the text inside the text field */ + document.execCommand("copy"); + alert.success("Copied Webhook URL") }} InputProps={{ style:{ color: "white", - height: "50px", + height: 50, marginLeft: "5px", maxWidth: "95%", fontSize: "1em", @@ -4997,8 +5564,6 @@ const AngularWorkflow = (props) => { const UserinputSidebar = () => { if (Object.getOwnPropertyNames(selectedTrigger).length > 0 && workflow.triggers[selectedTriggerIndex] !== undefined) { - console.log(workflow.triggers[selectedTriggerIndex]) - console.log(selectedTrigger) if (workflow.triggers[selectedTriggerIndex].parameters === undefined || workflow.triggers[selectedTriggerIndex].parameters === null || workflow.triggers[selectedTriggerIndex].parameters.length === 0) { workflow.triggers[selectedTriggerIndex].parameters = [] workflow.triggers[selectedTriggerIndex].parameters[0] = {"name": "alertinfo", "value": "hello this is an alert"} @@ -5019,7 +5584,7 @@ const AngularWorkflow = (props) => {

{selectedTrigger.app_name}: {selectedTrigger.status}

- What is the user input trigger? + What is the user input trigger?
@@ -5033,7 +5598,7 @@ const AngularWorkflow = (props) => { color: "white", marginLeft: "5px", maxWidth: "95%", - height: "50px", + height: 50, fontSize: "1em", }, }} @@ -5052,7 +5617,7 @@ const AngularWorkflow = (props) => { color: "white", marginLeft: "5px", maxWidth: "95%", - height: "50px", + height: 50, fontSize: "1em", }, }} @@ -5065,7 +5630,7 @@ const AngularWorkflow = (props) => {
- Arguments + Parameters
@@ -5135,7 +5700,7 @@ const AngularWorkflow = (props) => { color: "white", marginLeft: "5px", maxWidth: "95%", - height: "50px", + height: 50, fontSize: "1em", }, }} @@ -5159,7 +5724,7 @@ const AngularWorkflow = (props) => { color: "white", marginLeft: "5px", maxWidth: "95%", - height: "50px", + height: 50, fontSize: "1em", }, }} @@ -5187,7 +5752,7 @@ const AngularWorkflow = (props) => { if (Object.getOwnPropertyNames(selectedTrigger).length > 0 && workflow.triggers[selectedTriggerIndex] !== undefined) { if (workflow.triggers[selectedTriggerIndex].parameters === undefined || workflow.triggers[selectedTriggerIndex].parameters === null || workflow.triggers[selectedTriggerIndex].parameters.length === 0) { workflow.triggers[selectedTriggerIndex].parameters = [] - workflow.triggers[selectedTriggerIndex].parameters[0] = {"name": "cron", "value": "*/2 * * * *"} + workflow.triggers[selectedTriggerIndex].parameters[0] = {"name": "cron", "value": "120"} workflow.triggers[selectedTriggerIndex].parameters[1] = {"name": "execution_argument", "value": '{"example": {"json": "is cool"}}'} setWorkflow(workflow) } @@ -5197,7 +5762,7 @@ const AngularWorkflow = (props) => {

{selectedTrigger.app_name}: {selectedTrigger.status}

- What are schedules? + What are schedules?
@@ -5211,7 +5776,7 @@ const AngularWorkflow = (props) => { color: "white", marginLeft: "5px", maxWidth: "95%", - height: "50px", + height: 50, fontSize: "1em", }, }} @@ -5274,7 +5839,7 @@ const AngularWorkflow = (props) => {
- Arguments + Parameters
@@ -5286,7 +5851,7 @@ const AngularWorkflow = (props) => { InputProps={{ style:{ color: "white", - height: "50px", + height: 50, marginLeft: "5px", maxWidth: "95%", fontSize: "1em", @@ -5625,6 +6190,12 @@ const AngularWorkflow = (props) => {
) + } else if (selectedTrigger.trigger_type === "SUBFLOW") { + return( +
+ +
+ ) } else if (selectedTrigger.trigger_type === "EMAIL") { return(
@@ -5693,30 +6264,31 @@ const AngularWorkflow = (props) => { const parsedExecutionArgument = () => { var showResult = executionData.execution_argument.trim() - showResult = showResult.split(" None").join(" \"None\"") - showResult = showResult.split("\'").join("\"") - showResult = showResult.split(" False").join(" false") - showResult = showResult.split(" True").join(" true") + const validate = validateJson(showResult) - var jsonvalid = true - try { - const tmp = String(JSON.parse(showResult)) - if (!showResult.includes("{") && !showResult.includes("[")) { - jsonvalid = false + + if (validate.valid) { + if (typeof(validate.result) === "string") { + try { + validate.result = JSON.parse(validate.result) + } catch(e) { + console.log("Error: ", e) + validate.valid = false + } } - } catch (e) { - jsonvalid = false - } - if (jsonvalid) { return ( + src={validate.result} + theme="solarized" + collapsed={true} + displayDataTypes={false} + onSelect={(select) => { + HandleJsonCopy(showResult, select, "exec") + console.log("SELECTED!: ", select) + }} + name={"Execution Argument"} + /> ) } @@ -5757,8 +6329,59 @@ const AngularWorkflow = (props) => { ) } + const HandleJsonCopy = (base, copy, base_node_name) => { + console.log("COPY: ", copy) + var newitem = JSON.parse(base) + to_be_copied = "$"+base_node_name + for (var key in copy.namespace) { + if (copy.namespace[key].includes("Results for")) { + continue + } + + if (newitem !== undefined && newitem !== null) { + newitem = newitem[copy.namespace[key]] + if (!isNaN(copy.namespace[key])) { + to_be_copied += ".#" + } else { + to_be_copied += "."+copy.namespace[key] + } + } + } + + if (newitem !== undefined && newitem !== null) { + newitem = newitem[copy.name] + if (!isNaN(copy.name)) { + to_be_copied += ".#" + } else { + to_be_copied += "."+copy.name + } + } + + //console.log(document.activeElement) + //if (document.activeElement.nodeName == 'TEXTAREA' || document.activeElement.nodeName == 'INPUT') { + // console.log("HANDLE INPUT FIELD FOR COPY!") + //} + + to_be_copied.replace(" ", "_") + const elementName = "copy_element_shuffle" + var copyText = document.getElementById(elementName); + if (copyText !== null && copyText !== undefined) { + navigator.clipboard.writeText(to_be_copied) + copyText.select(); + copyText.setSelectionRange(0, 99999); /* For mobile devices */ + + /* Copy the text inside the text field */ + document.execCommand("copy"); + alert.success("Copied "+to_be_copied) + console.log("COPYING!") + } else { + console.log("Couldn't find element ", elementName) + } + + } + const executionModal = - setExecutionModalOpen(false)} PaperProps={{style: {minWidth: 375, maxWidth: 375, backgroundColor: "#1F2023", color: "white", fontSize: 18}}}> + setExecutionModalOpen(false)} style={{resize: "both", overflow: "auto",}} PaperProps={{style: {resize: "both", overflow: "auto", minWidth: 400, maxWidth: 400, backgroundColor: "#1F2023", color: "white", fontSize: 18}}}> {executionModalView === 0 ?
@@ -5769,29 +6392,49 @@ const AngularWorkflow = (props) => { - + {workflowExecutions.length > 0 ?
- {workflowExecutions.map(data => { - const statusColor = data.status === "FINISHED" ? "green" : data.status === "ABORTED" ? "red" : "orange" + {workflowExecutions.map((data, index) => { + const statusColor = data.status === "FINISHED" ? "green" : data.status === "ABORTED" || data.status === "FAILED" ? "red" : "orange" const timeElapsed = data.completed_at-data.started_at const resultsLength = data.results !== undefined && data.results !== null ? data.results.length : 0 const timestamp = new Date(data.started_at*1000).toISOString().split('.')[0].split("T").join(" ") + var calculatedResult = data.workflow.actions.length + for (var key in data.workflow.triggers) { + const trigger = data.workflow.triggers[key] + if ((trigger.app_name === "User Input" && trigger.trigger_type === "USERINPUT") || (trigger.app_name === "Shuffle Workflow" && trigger.trigger_type === "SUBFLOW")) { + calculatedResult += 1 + } + } + return ( + {}} onMouseOut={() => {}} onClick={() => { + + if (data.result === undefined || data.result === null || data.result.length === 0) { + setExecutionRequest({ + "execution_id": data.execution_id, + "authorization": data.authorization, + }) + start() + setExecutionRunning(true) + setExecutionRequestStarted(false) + } setExecutionModalView(1) setExecutionData(data) }}>
-
+
{getExecutionSourceImage(data)}
@@ -5801,15 +6444,20 @@ const AngularWorkflow = (props) => { {data.workflow.actions !== null ?
- {resultsLength}/{data.workflow.actions.length} + {resultsLength}/{calculatedResult}
: null}
- + {lastExecution === data.execution_id ? + + : + + } + ) return })} @@ -5823,10 +6471,21 @@ const AngularWorkflow = (props) => { :
-

{setExecutionModalView(0)}}> - - See other Executions -

+ { + setExecutionRunning(false) + stop() + getWorkflowExecution(props.match.params.key) + setExecutionModalView(0) + setLastExecution(executionData.execution_id) + }}> + {}}> + + +

{ + }}> + See other Executions +

+

Executing Workflow

@@ -5884,39 +6543,52 @@ const AngularWorkflow = (props) => { return null } - - // showResult = replaceAll(showResult, " None", " \"None\"") - // Super basic check. - // // FIXME: The latter replace doens't really work if ' is used in a string var showResult = data.result.trim() - showResult = showResult.split(" None").join(" \"None\"") - showResult = showResult.split(" False").join(" false") - showResult = showResult.split(" True").join(" true") - showResult = showResult.split("\'").join("\"") - - var jsonvalid = true - try { - const tmp = String(JSON.parse(showResult)) - if (!showResult.includes("{") && !showResult.includes("[")) { - //console.log("IN HERE: ", tmp) - jsonvalid = false - } - } catch (e) { - //console.log("Error: ", e) - jsonvalid = false - } - + const validate = validateJson(showResult) + const curapp = apps.find(a => a.name === data.action.app_name && a.app_version === data.action.app_version) const imgsize = 50 const statusColor = data.status === "FINISHED" || data.status === "SUCCESS" ? "green" : data.status === "ABORTED" || data.status === "FAILURE" ? "red" : "orange" - const actionimg = curapp === null ? + + var imgSrc = curapp === undefined ? "" : curapp.large_image + if (imgSrc.length === 0) { + // Look for the node in the workflow + const action = workflow.actions.find(action => action.id === data.action.id) + if (action !== undefined && action !== null) { + imgSrc = action.large_image + } + } + + var actionimg = curapp === null ? null : - {data.action.app_name} + {data.action.app_name} + + if (triggers.length > 2) { + if (data.action.app_name === "shuffle-subflow") { + actionimg = {"Shuffle + } + + if (data.action.app_name === "User Input") { + actionimg = {"Shuffle + } + } + + if (validate.valid && typeof(validate.result) === "string") { + validate.result = JSON.parse(validate.result) + } return (
+ { + setSelectedResult(data) + setCodeModalOpen(true) + }}> + + + + {actionimg}
{data.action.label}
@@ -5924,15 +6596,26 @@ const AngularWorkflow = (props) => {
Status {data.status}
- {jsonvalid ? { + HandleJsonCopy(showResult, select, data.action.label) + console.log("SELECTED!: ", select) + }} name={"Results for "+data.action.label} /> + {data.action.app_name === "shuffle-subflow" ? + + TBD: Load subexecution result for + + : null + } + : -
+
Result  {data.result}
@@ -5944,6 +6627,182 @@ const AngularWorkflow = (props) => {
} + + // This sucks :) + const curapp = !codeModalOpen ? {} : selectedResult.action.app_name === "shuffle-subflow" ? triggers[1] : selectedResult.action.app_name === "User Input" ? triggers[2] : apps.find(a => a.name === selectedResult.action.app_name && a.app_version === selectedResult.action.app_version) + const imgsize = 50 + const statusColor = !codeModalOpen ? "red" : selectedResult.status === "FINISHED" || selectedResult.status === "SUCCESS" ? "green" : selectedResult.status === "ABORTED" || selectedResult.status === "FAILURE" ? "red" : "orange" + const validate = !codeModalOpen ? "" : validateJson(selectedResult.result.trim()) + if (validate.valid && typeof(validate.result) === "string") { + validate.result = JSON.parse(validate.result) + } + + //if (codeModalOpen && selectedResult.result.includes("file_id")) { + // console.log("SHOW RESULT WITH FILES: ", selectedResult.result) + // //const regex = "\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b" + // //const regex = /^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}/i + // const regex = /^[A-F\d]{8}-[A-F\d]{4}-4[A-F\d]{3}-[89AB][A-F\d]{3}-[A-F\d]{12}$/i + // //const found = selectedResult.result.match(regex) + // const found = "hello how are you cf80fa70-65cf-4963-b474-b459a6dead81 what".match(regex) + // const regex = /\${(\w{8}-\w{4}-\w{3}-\w{3}-\w)}/g + // const found = placeholder.match(regex) + + // console.log("FOUND: ", found) + + // //cf80fa70-65cf-4963-b474-b459a6dead81 + //} + + const codePopoutModal = !codeModalOpen ? null : + { + if (!dragging) { + setDragging(true) + } + }} + onStop={(e) => { + if (!dragging) { + return + } + + setDragging(false) + + const newoffsetX = parseInt(dragPosition.x)-parseInt(e.layerX-e.offsetX) + const newoffsetY = parseInt(dragPosition.y)-parseInt(e.layerY-e.offsetY) + if ((newoffsetX <= 40 && newoffsetX >= -40) && (newoffsetY <= 40 && newoffsetY >= -40)) { + console.log("SKIP X & Y") + return + } + + setDragPosition({ + x: e.layerX-e.offsetX, + y: e.layerY-e.offsetY, + }) + }} + position={dragPosition} + > + + + { + e.preventDefault() + for (var key in workflowExecutions) { + const execution = workflowExecutions[key] + //console.log(execution.results[0]) + const result = execution.results.find(data => data.status === "SUCCESS" && data.action.id === selectedResult.action.id) + if (result !== undefined) { + setSelectedResult(result) + setUpdate(Math.random()) + break + } + } + }}> + + + + + { + e.preventDefault() + for (var key in workflowExecutions) { + const execution = workflowExecutions[key] + //console.log(execution.results[0]) + const result = execution.results.find(data => data.action.id === selectedResult.action.id && data.status !== "SUCCESS" && data.status !== "SKIPPED" && data.status !== "WAITING") + if (result !== undefined) { + setSelectedResult(result) + setUpdate(Math.random()) + break + } + } + }}> + + + + + { + e.preventDefault() + const executionIndex = workflowExecutions.findIndex(data => data.execution_id === selectedResult.execution_id) + if (executionIndex !== -1) { + setExecutionModalOpen(true) + setExecutionModalView(1) + setExecutionData(workflowExecutions[executionIndex]) + } + }}> + + + + + { + e.preventDefault() + //console.log("CLICKING EXIT") + setCodeModalOpen(false) + }}> + + + +
{ + //event.preventDefault() + }}> +
+ {curapp === null ? null : {selectedResult.app_name}} + +
+
{selectedResult.action.label}
+
{selectedResult.action.name}
+
+
+
Status {selectedResult.status}
+ {validate.valid ? { + HandleJsonCopy(JSON.stringify(validate.result), select, selectedResult.action.label) + }} + name={"Results for "+selectedResult.action.label} + /> + : +
+ Result  + { + console.log("IN HERE TO CLICK") + to_be_copied = selectedResult.result + var copyText = document.getElementById("copy_element_shuffle"); + console.log("PRECOPY: ", to_be_copied) + if (copyText !== null && copyText !== undefined) { + console.log("COPY: ", copyText) + navigator.clipboard.writeText(to_be_copied) + copyText.select(); + copyText.setSelectionRange(0, 99999); /* For mobile devices */ + + /* Copy the text inside the text field */ + document.execCommand("copy"); + alert.success("Copied "+to_be_copied) + } else { + console.log("Failed to copy. copy_element_shuffle is undefined") + } + }}>{selectedResult.result} +
+ } +
+ + const newView = isLoggedIn ?
@@ -6248,17 +7107,26 @@ const AngularWorkflow = (props) => { console.log("FIELDS: ", newFields) newAuthOption.fields = newFields setNewAppAuth(newAuthOption) - appAuthentication.push(newAuthOption) - setAppAuthentication(appAuthentication) + //appAuthentication.push(newAuthOption) + //setAppAuthentication(appAuthentication) getAppAuthentication() setUpdate(authenticationOption.id) + /* + {selectedAction.authentication.map(data => ( + + */ + + } + + if (authenticationOption.label === null || authenticationOption.label === undefined) { + authenticationOption.label = selectedApp.name+" authentication" } return (
- What is this?
+ What is this?
These are required fields for authenticating with {selectedApp.name}
Name - what is this used for? @@ -6269,7 +7137,7 @@ const AngularWorkflow = (props) => { color: "white", marginLeft: "5px", maxWidth: "95%", - height: "50px", + height: 50, fontSize: "1em", }, }} @@ -6295,7 +7163,7 @@ const AngularWorkflow = (props) => { color: "white", marginLeft: "5px", maxWidth: "95%", - height: "50px", + height: 50, fontSize: "1em", }, }} @@ -6340,7 +7208,7 @@ const AngularWorkflow = (props) => { InputProps={{ style:{ color: "white", - height: "50px", + height: 50, fontSize: "1em", }, }} @@ -6363,6 +7231,8 @@ const AngularWorkflow = (props) => { ) } + + // This whole part is redundant. Made it part of Arguments instead. const authenticationModal = authenticationModalOpen ? { {executionVariableModal} {conditionsModal} {authenticationModal} + {codePopoutModal} +
:
diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index f47330c9..5b4cb70f 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -4,6 +4,7 @@ import {BrowserView, MobileView} from "react-device-detect"; import {Link} from 'react-router-dom'; import Paper from '@material-ui/core/Paper'; +import Typography from '@material-ui/core/Typography'; import FormControlLabel from '@material-ui/core/FormControlLabel'; import Button from '@material-ui/core/Button'; import Divider from '@material-ui/core/Divider'; @@ -18,6 +19,7 @@ import DialogActions from '@material-ui/core/DialogActions'; import TextField from '@material-ui/core/TextField'; import Tooltip from '@material-ui/core/Tooltip'; import CheckCircleIcon from '@material-ui/icons/CheckCircle'; +import AttachFileIcon from '@material-ui/icons/AttachFile'; import Breadcrumbs from '@material-ui/core/Breadcrumbs'; import AppsIcon from '@material-ui/icons/Apps'; import CircularProgress from '@material-ui/core/CircularProgress'; @@ -25,6 +27,7 @@ import CircularProgress from '@material-ui/core/CircularProgress'; import Chip from '@material-ui/core/Chip'; import ChipInput from 'material-ui-chip-input' +import YAML from 'yaml' import ErrorOutline from '@material-ui/icons/ErrorOutline'; import { useAlert } from "react-alert"; import words from "shellwords" @@ -99,7 +102,12 @@ const parseCurl = (s) => { return "" } - var args = rewrite(words.split(s)) + try { + var args = rewrite(words.split(s)) + } catch (e) { + return s + } + var out = { method: 'GET', header: {} } var state = '' @@ -187,6 +195,7 @@ const AppCreator = (props) => { const alert = useAlert() var upload = "" + const increaseAmount = 30 const actionNonBodyRequest = ["GET", "HEAD", "DELETE", "CONNECT"] const actionBodyRequest = ["POST", "PUT", "PATCH",] const authenticationOptions = ["No authentication", "API key", "Bearer auth", "Basic auth", ] @@ -218,6 +227,9 @@ const AppCreator = (props) => { const [actions, setActions] = useState([]) const [errorCode, setErrorCode] = useState("") const [appBuilding, setAppBuilding] = useState(false) + const [extraBodyFields, setExtraBodyFields] = useState([]) + const [fileUploadEnabled, setFileUploadEnabled] = useState(false) + const [actionAmount, setActionAmount] = useState(increaseAmount) //const [actions, setActions] = useState([{ // "name": "Get workflows", @@ -246,6 +258,7 @@ const AppCreator = (props) => { const [currentActionMethod, setCurrentActionMethod] = useState(actionNonBodyRequest[0]) const [currentAction, setCurrentAction] = useState({ "name": "", + "file_field": "", "description": "", "url": "", "headers": "", @@ -253,6 +266,7 @@ const AppCreator = (props) => { "queries": [], "body": "", "errors": [], + "example_response": "", "method": actionNonBodyRequest[0], }); @@ -262,7 +276,7 @@ const AppCreator = (props) => { if (firstrequest) { setFirstrequest(false) if (window.location.pathname.includes("apps/edit")) { - setIsEditing(true) + setIsEditing(true) handleEditApp() } else { checkQuery() @@ -322,15 +336,37 @@ const AppCreator = (props) => { throw new Error("NOT 200 :O") } + //console.log("DATA: ", response.text()) return response.json() }) .then((responseJson) => { + console.log("THE BODY IS HERE") setIsAppLoaded(true) if (!responseJson.success) { alert.error("Failed to verify") - } else { - const data = JSON.parse(responseJson.body) - parseIncomingOpenapiData(data) + } else{ + console.log("HMM 2") + var jsonvalid = false + var tmpvalue = "" + try { + tmpvalue = JSON.parse(responseJson.body) + jsonvalid = true + } catch (e) { + console.log("Error JSON: ", e) + } + + if (!jsonvalid) { + try { + tmpvalue = YAML.parse(responseJson.body, ) + jsonvalid = true + } catch(e) { + console.log("Error YAML: ", e) + } + } + + if (jsonvalid) { + parseIncomingOpenapiData(tmpvalue) + } } }) .catch(error => { @@ -353,16 +389,52 @@ const AppCreator = (props) => { //} } + + const handleGetRef = (parameter, data) => { + if (parameter["$ref"] === undefined) { + return parameter + } + + const paramsplit = parameter["$ref"].split("/") + if (paramsplit[0] !== "#") { + console.log("Bad param: ", paramsplit) + return parameter + } + + var newitem = data + for (var key in paramsplit) { + var tmpparam = paramsplit[key] + if (tmpparam === "#") { + continue + } + + if (newitem[tmpparam] === undefined) { + return parameter + } + + newitem = newitem[tmpparam] + } + + return newitem + + //console.log("Should get ", parameter["$ref"]) + //const subkeys = parameter["$ref"].split("/") + // setBasedata(data) + + // handleGetReference(parameter["$ref"]) + } + // Sets the data up as it should be at later points // This is the data FROM the database, not what's being saved const parseIncomingOpenapiData = (data) => { + //console.log("DATA: ", data.info) setBasedata(data) - setName(data.info.title) - setDescription(data.info.description) - document.title = "Apps - "+data.info.title - if (data.info !== null && data.info !== undefined) { + setName(data.info.title) + setDescription(data.info.description) + document.title = "Apps - "+data.info.title + if (data.info["x-logo"] !== undefined) { setFileBase64(data.info["x-logo"]) } @@ -377,11 +449,21 @@ const AppCreator = (props) => { } if (data.tags !== undefined && data.tags.length > 0) { + var newtags = [] for (var key in data.tags) { - newWorkflowTags.push(data.tags[key].name) + if (data.tags[key].name.length > 50) { + console.log("Skipping tag cus it's too long: ", data.tags[key].name.length) + continue + } + + newtags.push(data.tags[key].name) } - setNewWorkflowTags(newWorkflowTags) + if (newtags.length > 10) { + newtags = newtags.slice(0,9) + } + + setNewWorkflowTags(newtags) } // This is annoying (: @@ -410,16 +492,17 @@ const AppCreator = (props) => { for (let [path, pathvalue] of Object.entries(data.paths)) { for (let [method, methodvalue] of Object.entries(pathvalue)) { if (methodvalue === null) { - alert.info("Skipped method "+method) + alert.info("Skipped method (null)"+method) continue } if (!allowedfunctions.includes(method.toUpperCase())) { + alert.info("Skipped method (not allowed) "+method) continue } var tmpname = methodvalue.summary - if (methodvalue.operationId !== undefined && methodvalue.operationId !== null && methodvalue.operationId.length > 0) { + if (methodvalue.operationId !== undefined && methodvalue.operationId !== null && methodvalue.operationId.length > 0 && (tmpname === undefined || tmpname.length === 0)) { tmpname = methodvalue.operationId } @@ -427,16 +510,94 @@ const AppCreator = (props) => { "name": tmpname, "description": methodvalue.description, "url": path, + "file_field": "", "method": method.toUpperCase(), "headers": "", "queries": [], "paths": [], "body": "", "errors": [], + "example_response": "", + } + + if (methodvalue["requestBody"] !== undefined) { + //console.log("Handle requestbody: ", methodvalue["requestBody"]) + if (methodvalue["requestBody"]["content"] !== undefined) { + if (methodvalue["requestBody"]["content"]["application/json"] !== undefined) { + if (methodvalue["requestBody"]["content"]["application/json"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["application/json"]["schema"] !== null) { + if (methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"] !== undefined) { + var tmpobject = {} + for (let [prop, propvalue] of Object.entries(methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"])) { + tmpobject[prop] = `\$\{${prop}\}` + } + + for (var subkey in methodvalue["requestBody"]["content"]["application/json"]["schema"]["required"]) { + const tmpitem = methodvalue["requestBody"]["content"]["application/json"]["schema"]["required"][subkey] + tmpobject[tmpitem] = `\$\{${tmpitem}\}` + } + + newaction["body"] = JSON.stringify(tmpobject, null, 2) + } + } + } else if (methodvalue["requestBody"]["content"]["application/xml"] !== undefined) { + console.log("METHOD XML: ", methodvalue) + if (methodvalue["requestBody"]["content"]["application/xml"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["application/xml"]["schema"] !== null) { + if (methodvalue["requestBody"]["content"]["application/xml"]["schema"]["properties"] !== undefined) { + var tmpobject = {} + for (let [prop, propvalue] of Object.entries(methodvalue["requestBody"]["content"]["application/xml"]["schema"]["properties"])) { + tmpobject[prop] = `\$\{${prop}\}` + } + + for (var subkey in methodvalue["requestBody"]["content"]["application/xml"]["schema"]["required"]) { + const tmpitem = methodvalue["requestBody"]["content"]["application/xml"]["schema"]["required"][subkey] + tmpobject[tmpitem] = `\$\{${tmpitem}\}` + } + + //console.log("OBJ XML: ", tmpobject) + //newaction["body"] = XML.stringify(tmpobject, null, 2) + } + } + } else { + if (methodvalue["requestBody"]["content"]["example"] !== undefined) { + if (methodvalue["requestBody"]["content"]["example"]["example"] !== undefined) { + newaction["body"] = methodvalue["requestBody"]["content"]["example"]["example"] + //JSON.stringify(tmpobject, null, 2) + } + } + + console.log(methodvalue["requestBody"]["content"]) + if (methodvalue["requestBody"]["content"]["multipart/form-data"] !== undefined) { + if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"] !== null) { + if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["type"] === "object") { + const fieldname = methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["properties"]["fieldname"] + if (fieldname !== undefined) { + console.log("FIELDNAME: ", fieldname) + newaction.file_field = fieldname["value"] + } + } + } + } + } + } + } + + // HAHAHA wtf is this. + if (methodvalue.responses !== undefined && methodvalue.responses !== null) { + if (methodvalue.responses.default !== undefined) { + if (methodvalue.responses.default.content !== undefined) { + if (methodvalue.responses.default.content["text/plain"] !== undefined) { + if (methodvalue.responses.default.content["text/plain"]["schema"] !== undefined) { + if (methodvalue.responses.default.content["text/plain"]["schema"]["example"] !== undefined) { + newaction.example_response = methodvalue.responses.default.content["text/plain"]["schema"]["example"] + } + } + } + } + } } for (var key in methodvalue.parameters) { - const parameter = methodvalue.parameters[key] + const parameter = handleGetRef(methodvalue.parameters[key], data) if (parameter.in === "query") { var tmpaction = { "description": parameter.description, @@ -466,9 +627,12 @@ const AppCreator = (props) => { } } else if (parameter.in === "header") { newaction.headers += `${parameter.name}=${parameter.example}\n` + } else { + console.log("WARNING: don't know how to handle this param: ", parameter) } } + if (newaction.name === "" || newaction.name === undefined) { // Find a unique part of the string // FIXME: Looks for length between /, find the one where they differ @@ -582,6 +746,17 @@ const AppCreator = (props) => { } } + if (newActions.length > increaseAmount-1) { + setActionAmount(increaseAmount) + } else { + setActionAmount(newActions.length) + } + + if (newActions.length > 1000) { + alert.error("Cut down actions from "+newActions.length+" to 999 because of limit") + newActions = newActions.slice(0,999) + } + setActions(newActions) setIsAppLoaded(true) } @@ -659,6 +834,11 @@ const AppCreator = (props) => { } const regex = /[A-Za-z0-9 _]/g; + if (item.name === undefined) { + console.log("Skipping action ", item) + continue + } + const found = item.name.match(regex); if (found !== null) { item.name = found.join("") @@ -668,17 +848,58 @@ const AppCreator = (props) => { "responses": { "default": { "description": "default", - "schema": {} + "content": { + "text/plain": { + "schema": { + "type": "string", + "example": "", + }, + }, + }, } }, "summary": item.name, "operationId": item.name.split(" ").join("_"), "description": item.description, - "parameters": [] + "parameters": [], + "requestBody": { + "content": { + + } + }, } //console.log("ACTION: ", item) + if (item.example_response !== undefined && item.example_response.length > 0) { + // FIXME: Shallow copy of the string + var showResult = Object.assign("", item.example_response).trim() + showResult = showResult.split(" None").join(" \"None\"") + showResult = showResult.split("\'").join("\"") + showResult = showResult.split(" False").join(" false") + showResult = showResult.split(" True").join(" true") + + var jsonvalid = true + try { + const tmp = String(JSON.parse(showResult)) + if (!showResult.includes("{") && !showResult.includes("[")) { + jsonvalid = false + } + } catch (e) { + jsonvalid = false + } + + data.paths[item.url][item.method.toLowerCase()].responses["default"]["content"]["text/plain"].schema.type = "string" + if (jsonvalid) { + // FIXME: Add a JSON parser here - don't run it as a string. + data.paths[item.url][item.method.toLowerCase()].responses["default"]["content"]["text/plain"].schema.example = showResult + + } else { + data.paths[item.url][item.method.toLowerCase()].responses["default"]["content"]["text/plain"].schema.example = item.example_response + + } + } + if (item.queries.length > 0) { for (var querykey in item.queries) { const queryitem = item.queries[querykey] @@ -761,7 +982,7 @@ const AppCreator = (props) => { "type": "string", }, } - + // FIXME - add application/json if JSON example? data.paths[item.url][item.method.toLowerCase()]["requestBody"] = { "description": "Generated by Shuffler.io", @@ -773,16 +994,46 @@ const AppCreator = (props) => { }, } + /* + data.paths[item.url][item.method.toLowerCase()]["requestBody"] = { + "description": "Generated by Shuffler.io", + "required": required, + "content": { + "example": { + "example": item.body, + }, + }, + } + */ + data.paths[item.url][item.method.toLowerCase()].parameters.push(newitem) } + // https://swagger.io/docs/specification/describing-request-body/file-upload/ + if (item.file_field !== undefined && item.file_field !== null && item.file_field.length > 0) { + console.log("HANDLE FILEFIELD SAVE: ", item.file_field) + data.paths[item.url][item.method.toLowerCase()]["requestBody"]["content"]["multipart/form-data"] = { + + "schema": { + "type": "object", + "properties": { + "fieldname": { + "type": "string", + "value": item.file_field, + }, + }, + }, + } + + console.log(data.paths[item.url][item.method.toLowerCase()]["requestBody"]["content"]["multipart/form-data"]) + } + if (item.headers.length > 0) { const required = false const headersSplit = item.headers.split("\n") for (var key in headersSplit) { const header = headersSplit[key] - console.log("HEADER: ", header) var key = "" var value = "" if (header.length > 0 && header.includes("= ")) { @@ -989,6 +1240,7 @@ const AppCreator = (props) => { "name": "", "description": "", "url": "", + "file_field": "", "headers": "", "paths": [], "queries": [], @@ -1102,14 +1354,14 @@ const AppCreator = (props) => { null :
- {actions.map((data, index) => { + {actions.slice(0,actionAmount).map((data, index) => { var error = data.errors.length > 0 ? : - + @@ -1127,6 +1379,7 @@ const AppCreator = (props) => { } const url = data.url + const hasFile = data["file_field"] !== undefined && data["file_field"] !== null && data["file_field"].length > 0 return ( {error} @@ -1137,15 +1390,23 @@ const AppCreator = (props) => { setUrlPathQueries(data.queries) setUrlPath(data.url) setActionsModalOpen(true) + + if (data["body"] !== undefined && data["body"] !== null && data["body"].length > 0) { + findBodyParams(data["body"]) + } + + if (hasFile) { + setFileUploadEnabled(true) + } }}>
- {url} - {data.name} + {hasFile ? : null} {url} - {data.name} +
@@ -1177,24 +1438,51 @@ const AppCreator = (props) => { const setActionField = (field, value) => { currentAction[field] = value setCurrentAction(currentAction) + //setUrlPathQueries(currentAction.queries) } + const findBodyParams = (body) => { + const regex = /\${(\w+)}/g + const found = body.match(regex) + if (found === null) { + setExtraBodyFields([]) + } else { + setExtraBodyFields(found) + } + } + const bodyInfo = actionBodyRequest.includes(currentActionMethod) ? -
- Body - used as example in action argument +
+ Request Body: {extraBodyFields.length > 0 ? + + Variables: {extraBodyFields.join(", ")} + + : + + {`Add variables with \$\{ variable_name }`} + + } setActionField("body", e.target.value)} + onChange={e => { + setActionField("body", e.target.value) + findBodyParams(e.target.value) + }} key={currentAction} + helperText={ + + Shows an example body to the user. ${} creates variables. + + } InputProps={{ classes: { notchedOutline: classes.notchedOutline, @@ -1205,9 +1493,40 @@ const AppCreator = (props) => { }} /> +
+
: null + const exampleResponse = +
+ Example success response + setActionField("example_response", e.target.value)} + helperText={ + Helps with autocompletion and understanding of the endpoint + } + key={currentAction} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style:{ + color: "white", + }, + }} + /> +
+ const addActionToView = (errors) => { currentAction.errors = errors currentAction.queries = urlPathQueries @@ -1329,7 +1648,7 @@ const AppCreator = (props) => { const queries = values[1] if (currentAction.paths !== paths && urlPath.length > 0) { - console.log("IN PATHS SETTER: !", paths) + //console.log("IN PATHS SETTER: !", paths) setActionField("paths", paths) } @@ -1361,12 +1680,12 @@ const AppCreator = (props) => { open={actionsModalOpen} fullWidth onClose={() => { - console.log("CLOSED?") setUrlPath("") setCurrentAction({ "name": "", "description": "", "url": "", + "file_field": "", "headers": "", "paths": [], "queries": [], @@ -1377,6 +1696,7 @@ const AppCreator = (props) => { setCurrentActionMethod(apikeySelection[0]) setUrlPathQueries([]) setActionsModalOpen(false) + setFileUploadEnabled(false) }} > @@ -1453,11 +1773,13 @@ const AppCreator = (props) => { id: 'method-option', }} > - {actionNonBodyRequest.map(data => ( - + {actionNonBodyRequest.map((data, index) => { + return ( + {data} - ))} + ) + })} {actionBodyRequest.map(data => ( {data} @@ -1492,66 +1814,81 @@ const AppCreator = (props) => { }} onBlur={event => { var parsedurl = event.target.value - if (parsedurl.startsWith("curl")) { - const request = parseCurl(event.target.value) - console.log(request) - if (request.method.toUpperCase() !== currentAction.Method) { - setCurrentActionMethod(request.method.toUpperCase()) - setActionField("method", request.method.toUpperCase()) - } + if (parsedurl.startsWith("PUT ") || parsedurl.startsWith("GET ") ||parsedurl.startsWith("POST ") || parsedurl.startsWith("DELETE ") ||parsedurl.startsWith("PATCH ") || parsedurl.startsWith("CONNECT ")) { + const tmp = parsedurl.split(" ") - if (request.header !== undefined && request.header !== null) { - var headers = [] - for (let [key, value] of Object.entries(request.header)) { - if (parameterName !== undefined && key.toLowerCase() === parameterName.toLowerCase()) { - continue - } - - if (key === "Authorization" && authenticationOption === "Bearer auth") { - continue - } - - headers += key+"="+value+"\n" - } - - setActionField("headers", headers) - } - - if (request.body !== undefined && request.body !== null) { - setActionField("body", request.body) - } - - // Parse URL - if (request.url !== undefined) { - parsedurl = request.url - } - } - - if (parsedurl !== undefined) { - if (parsedurl.includes("<") && parsedurl.includes(">")) { - parsedurl = parsedurl.split("<").join("{") - parsedurl = parsedurl.split(">").join("}") - } - - if (parsedurl.startsWith("http") || parsedurl.startsWith("ftp")) { - if (parsedurl !== undefined && parsedurl.includes(parameterName)) { - // Remove <> etc. - // - - console.log("IT HAS THE PARAM NAME!") - const newurl = new URL(encodeURI(parsedurl)) - newurl.searchParams.delete(parameterName) - parsedurl = decodeURI(newurl.href) - } - - // Remove the base URL itself - if (parsedurl !== undefined && baseUrl !== undefined && baseUrl.length > 0 && parsedurl.includes(baseUrl)) { - parsedurl = parsedurl.replace(baseUrl, "") - } - - // Check URL query && headers + if (tmp.length > 1) { + parsedurl = tmp[1] setActionField("url", parsedurl) setUrlPath(parsedurl) + + setCurrentActionMethod(tmp[0].toUpperCase()) + setActionField("method", tmp[0].toUpperCase()) + } + + setUpdate(Math.random()) + } else if (parsedurl.startsWith("curl")) { + const request = parseCurl(event.target.value) + if (request !== event.target.value) { + if (request.method.toUpperCase() !== currentAction.Method) { + setCurrentActionMethod(request.method.toUpperCase()) + setActionField("method", request.method.toUpperCase()) + } + + if (request.header !== undefined && request.header !== null) { + var headers = [] + for (let [key, value] of Object.entries(request.header)) { + if (parameterName !== undefined && key.toLowerCase() === parameterName.toLowerCase()) { + continue + } + + if (key === "Authorization" && authenticationOption === "Bearer auth") { + continue + } + + headers += key+"="+value+"\n" + } + + setActionField("headers", headers) + } + + if (request.body !== undefined && request.body !== null) { + setActionField("body", request.body) + } + + // Parse URL + if (request.url !== undefined) { + parsedurl = request.url + } + } + + console.log("PARSED: ", parsedurl) + if (parsedurl !== undefined) { + if (parsedurl.includes("<") && parsedurl.includes(">")) { + parsedurl = parsedurl.split("<").join("{") + parsedurl = parsedurl.split(">").join("}") + } + + if (parsedurl.startsWith("http") || parsedurl.startsWith("ftp")) { + if (parsedurl !== undefined && parsedurl.includes(parameterName)) { + // Remove <> etc. + // + + console.log("IT HAS THE PARAM NAME!") + const newurl = new URL(encodeURI(parsedurl)) + newurl.searchParams.delete(parameterName) + parsedurl = decodeURI(newurl.href) + } + + // Remove the base URL itself + if (parsedurl !== undefined && baseUrl !== undefined && baseUrl.length > 0 && parsedurl.includes(baseUrl)) { + parsedurl = parsedurl.replace(baseUrl, "") + } + + // Check URL query && headers + setActionField("url", parsedurl) + setUrlPath(parsedurl) + } } } @@ -1563,8 +1900,38 @@ const AppCreator = (props) => { + {currentActionMethod === "POST" ? + + : null} + {fileUploadEnabled ? + setActionField("file_field", e.target.value)} + helperText={The File field to interact with} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style:{ + color: "white", + }, + }} + /> + : null}
- Headers - static for the action + Headers: static for the action { id="standard-required" defaultValue={currentAction["headers"]} multiline - rows="5" + rows="2" onChange={e => setActionField("headers", e.target.value)} helperText={Headers that are part of the request. Default: EMPTY} InputProps={{ @@ -1588,6 +1955,8 @@ const AppCreator = (props) => { }} /> {bodyInfo} + + {exampleResponse} @@ -1661,26 +2031,40 @@ const AppCreator = (props) => { const actionView =
-

Actions

+

Actions ({actions.length})

Actions are the tasks performed by an app. Read more about actions and apps here.
{loopActions} - +
+ + {actionAmount > 0 && actionAmount < actions.length ? null : + + } +
diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 69ed1dc1..15289670 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -21,6 +21,7 @@ import {Link} from 'react-router-dom'; import Breadcrumbs from '@material-ui/core/Breadcrumbs'; import ReactJson from 'react-json-view' import Chip from '@material-ui/core/Chip'; +import { useTheme } from '@material-ui/core/styles'; import CachedIcon from '@material-ui/icons/Cached'; import CloudDownloadIcon from '@material-ui/icons/CloudDownload'; @@ -46,6 +47,10 @@ const inputColor = "#383B40" export const GetParsedPaths = (inputdata, basekey) => { const splitkey = " > " var parsedValues = [] + if (inputdata === undefined || inputdata === null) { + return parsedValues + } + if (typeof(inputdata) !== "object") { return parsedValues } @@ -106,9 +111,10 @@ export const GetParsedPaths = (inputdata, basekey) => { const Apps = (props) => { - const { globalUrl, isLoggedIn, isLoaded } = props; + const { globalUrl, isLoggedIn, isLoaded, userdata } = props; //const [workflows, setWorkflows] = React.useState([]); + const theme = useTheme(); const baseRepository = "https://github.com/frikky/shuffle-apps" const alert = useAlert() const [selectedApp, setSelectedApp] = React.useState({}); @@ -134,6 +140,7 @@ const Apps = (props) => { const [field2, setField2] = React.useState("") const [cursearch, setCursearch] = React.useState("") const [sharingConfiguration, setSharingConfiguration] = React.useState("you") + const [downloadBranch, setDownloadBranch] = React.useState("master") const [isDropzone, setIsDropzone] = React.useState(false); const upload = React.useRef(null); @@ -311,10 +318,18 @@ const Apps = (props) => { boxColor = "orange" } + if (data.invalid) { + boxColor = "red" + } + + //
+ //
var imageline = data.large_image.length === 0 ? - {data.title} + {data.title} : - {data.title} + {data.title} { + //console.log("IMG LOADED!: ", event.target) + }} /> // FIXME - add label to apps, as this might be slow with A LOT of apps var newAppname = data.name @@ -336,7 +351,7 @@ const Apps = (props) => { } var description = data.description - const maxDescLen = 60 + const maxDescLen = 51 if (description.length > maxDescLen) { description = data.description.slice(0, maxDescLen)+"..." } @@ -359,8 +374,8 @@ const Apps = (props) => { } } }}> - - + + {imageline}
@@ -515,9 +530,9 @@ const Apps = (props) => { : null var imageline = selectedApp.large_image === undefined || selectedApp.large_image.length === 0 ? - {selectedApp.title} + {selectedApp.title} : - {selectedApp.title} + {selectedApp.title} const GetAppExample = () => { if (selectedAction.returns === undefined) { @@ -588,10 +603,10 @@ const Apps = (props) => {
{imageline}
-
+

{newAppname}

Version {selectedApp.app_version}

-

{description}

+

{description}

{activateButton} @@ -634,7 +649,7 @@ const Apps = (props) => { updateAppField(selectedApp.id, "sharing", !selectedApp.sharing) //setSelectedAction(event.target.value) }} - style={{width: 150, backgroundColor: inputColor, color: "white", height: 35, marginleft: 10,}} + style={{width: 150, backgroundColor: theme.palette.surfaceColor, backgroundColor: inputColor, color: "white", height: 35, marginleft: 10,}} SelectDisplayProps={{ style: { marginLeft: 10, @@ -699,7 +714,7 @@ const Apps = (props) => { {selectedAction.parameters !== undefined && selectedAction.parameters !== null ?
- Arguments + Parameters {selectedAction.parameters.map(data => { var itemColor = "#f85a3e" if (!data.required) { @@ -802,12 +817,16 @@ const Apps = (props) => { const reader = new FileReader(); - reader.addEventListener('load', (e) => { - const content = e.target.result; - setOpenApiData(content); - setIsDropzone(isDropzone); - setOpenApiModal(true) - }) + try { + reader.addEventListener('load', (e) => { + const content = e.target.result; + setOpenApiData(content); + setIsDropzone(isDropzone); + setOpenApiModal(true) + }) + } catch (e) { + console.log("Error in dropzone: ", e) + } reader.readAsText(files[0]); }; @@ -907,7 +926,7 @@ const Apps = (props) => {
{apps.length > 0 ? filteredApps.length > 0 ? -
+
{filteredApps.map(app => { return ( appPaper(app) @@ -959,6 +978,7 @@ const Apps = (props) => { const parsedData = { "url": url, + "branch": downloadBranch || 'master' } if (field1.length > 0) { @@ -988,18 +1008,23 @@ const Apps = (props) => { } setIsLoading(false) stop() + setValidation(false) return response.json() }) .then((responseJson) => { - console.log("DATA: ", responseJson) - if (responseJson.reason !== undefined) { - alert.error("Failed loading: "+responseJson.reason) - } + console.log("DATA: ", responseJson) + if (responseJson.reason !== undefined) { + alert.error("Failed loading: "+responseJson.reason) + } }) .catch(error => { console.log("ERROR: ", error.toString()) alert.error(error.toString()) + + stop() + setIsLoading(false) + setValidation(false) }) } @@ -1167,6 +1192,7 @@ const Apps = (props) => { return } + console.log("Validating response!") validateOpenApi(responseJson) }) .catch(error => { @@ -1185,10 +1211,12 @@ const Apps = (props) => { try { - return JSON.stringify(YAML.parse(apidata)) + const parsed = YAML.parse(YAML.stringify(apidata)) + //const parsed = YAML.parse(apidata)) + return YAML.stringify(parsed) } catch(error) { console.log("YAML DECODE ERROR - TRY SOMETHING ELSE?: "+error) - setOpenApiError(error.toString()) + setOpenApiError("Local error: "+ error.toString()) } return "" @@ -1197,19 +1225,23 @@ const Apps = (props) => { // Sends the data to backend, which should return a version 3 of the same API // If 200 - continue, otherwise, there's some issue somewhere const validateOpenApi = (openApidata) => { - const newApidata = escapeApiData(openApidata) + var newApidata = escapeApiData(openApidata) if (newApidata === "") { + // Used to return here + newApidata = openApidata return } + //console.log(newApidata) + setValidation(true) fetch(globalUrl+"/api/v1/validate_openapi", { - method: 'POST', + method: 'POST', headers: { 'Accept': 'application/json', }, - body: newApidata, - credentials: "include", + body: openApidata, + credentials: "include", }) .then((response) => { setValidation(false) @@ -1302,7 +1334,7 @@ const Apps = (props) => { style={{backgroundColor: inputColor}} variant="outlined" margin="normal" - defaultValue="https://github.com/frikky/shuffle-apps" + defaultValue={userdata.active_org.defaults.app_download_repo !== undefined && userdata.active_org.defaults.app_download_repo.length > 0 ? userdata.active_org.defaults.app_download_repo : "https://github.com/frikky/shuffle-apps"} InputProps={{ style:{ color: "white", @@ -1314,6 +1346,25 @@ const Apps = (props) => { placeholder="https://github.com/frikky/shuffle-apps" fullWidth /> + Branch (default value is "master"): +
+ 0 ? userdata.active_org.defaults.app_download_branch : downloadBranch} + InputProps={{ + style:{ + color: "white", + height: "50px", + fontSize: "1em", + }, + }} + onChange={e => setDownloadBranch(e.target.value)} + placeholder="master" + fullWidth + /> +
Authentication (optional - private repos etc):
diff --git a/frontend/src/views/LoginPage.jsx b/frontend/src/views/LoginPage.jsx index ef746a6d..b365336f 100644 --- a/frontend/src/views/LoginPage.jsx +++ b/frontend/src/views/LoginPage.jsx @@ -69,7 +69,7 @@ const LoginDialog = props => { }), ) .catch(error => { - setLoginInfo("Error in userdata: ", error) + setLoginInfo("Error logging in: ", error) }) } @@ -80,6 +80,7 @@ const LoginDialog = props => { const onSubmit = (e) => { e.preventDefault() + setLoginInfo("") // FIXME - add some check here ROFL // Just use this one? @@ -114,7 +115,7 @@ const LoginDialog = props => { }), ) .catch(error => { - setLoginInfo("Error in userdata: " + error) + setLoginInfo("Error logging in: " + error) }); } else { url = baseurl + '/api/v1/users/register'; @@ -130,7 +131,7 @@ const LoginDialog = props => { if (responseJson["success"] === false) { setLoginInfo(responseJson["reason"]) } else { - setLoginInfo("Successful register :)") + setLoginInfo("Successful register!") } }), ) diff --git a/frontend/src/views/SettingsPage.jsx b/frontend/src/views/SettingsPage.jsx index 0f7c572a..31dfde8d 100644 --- a/frontend/src/views/SettingsPage.jsx +++ b/frontend/src/views/SettingsPage.jsx @@ -85,7 +85,7 @@ const Settings = (props) => { const generateApikey = () => { fetch(globalUrl+"/api/v1/generateapikey", { - method: 'GET', + method: 'GET', headers: { 'Content-Type': 'application/json', 'Accept': 'application/json', @@ -99,7 +99,7 @@ const Settings = (props) => { return response.json() }) - .then((responseJson) => { + .then((responseJson) => { setUserSettings(responseJson) }) .catch(error => { diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 56ef2f86..83616e2b 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -43,8 +43,40 @@ import CloudDownloadIcon from '@material-ui/icons/CloudDownload'; const inputColor = "#383B40" const surfaceColor = "#27292D" +export const validateJson = (showResult) => { + //showResult = showResult.split(" None").join(" \"None\"") + showResult = showResult.split(" False").join(" false") + showResult = showResult.split(" True").join(" true") + + var jsonvalid = true + try { + const tmp = String(JSON.parse(showResult)) + if (!showResult.includes("{") && !showResult.includes("[")) { + jsonvalid = false + } + } catch (e) { + showResult = showResult.split("\'").join("\"") + + try { + const tmp = String(JSON.parse(showResult)) + if (!showResult.includes("{") && !showResult.includes("[")) { + jsonvalid = false + } + } catch (e) { + jsonvalid = false + } + } + + const result = jsonvalid ? JSON.parse(showResult) : showResult + //console.log("VALID: ", jsonvalid, result) + return { + "valid": jsonvalid, + "result": result, + } +} + const Workflows = (props) => { - const { globalUrl, isLoggedIn, isLoaded, removeCookie, cookies} = props; + const { globalUrl, isLoggedIn, isLoaded, removeCookie, cookies, userdata} = props; document.title = "Shuffle - Workflows" const alert = useAlert() @@ -80,7 +112,7 @@ const Workflows = (props) => { duration: 5000, startImmediate: false, callback: () => { - getWorkflowExecution(selectedWorkflow.id) + //getWorkflowExecution(selectedWorkflow.id) } }) @@ -183,7 +215,7 @@ const Workflows = (props) => { if (responseJson.length > 0){ setSelectedWorkflow(responseJson[0]) - getWorkflowExecution(responseJson[0].id) + //getWorkflowExecution(responseJson[0].id) } }) .catch(error => { @@ -202,8 +234,8 @@ const Workflows = (props) => { color: "#ffffff", width: "100%", display: "flex", - minWidth: 1366, - maxWidth: 1766, + minWidth: 1024, + maxWidth: 1024, margin: "auto", maxHeight: "90vh", } @@ -272,7 +304,7 @@ const Workflows = (props) => { setSelectedExecution(responseJson[0]) setWorkflowExecutions(responseJson) } else { - alert.info("Couldn't find executions for the workflow") + //alert.info("Couldn't find executions for the workflow") setSelectedExecution({}) setWorkflowExecutions([]) } @@ -298,7 +330,7 @@ const Workflows = (props) => { if (response.status !== 200) { console.log("Status not 200 for WORKFLOW EXECUTION :O!") } - getWorkflowExecution(workflowid) + //getWorkflowExecution(workflowid) return response.json() }) @@ -361,10 +393,24 @@ const Workflows = (props) => { data["owner"] = "" for (var key in data.triggers) { - if (data.triggers[key].status == "running") { - data.triggers[key].status = "stopped" + const trigger = data.triggers[key] + if (trigger.app_name === "Shuffle Workflow") { + if (trigger.parameters.length > 2) { + trigger.parameters[2].value = "" + } + } + + if (trigger.status == "running") { + trigger.status = "stopped" } } + + for (var key in data.actions) { + data.actions[key].authentication_id = "" + } + + //return + data["org"] = [] data["org_id"] = "" data.execution_org = {"id": ""} @@ -377,12 +423,15 @@ const Workflows = (props) => { } const copyWorkflow = (data) => { + data = JSON.parse(JSON.stringify(data)) alert.success("Copying workflow "+data.name) + console.log("data: ", data) data.id = "" data.name = data.name+"_copy" + //return fetch(globalUrl+"/api/v1/workflows", { - method: 'POST', + method: 'POST', headers: { 'Content-Type': 'application/json', 'Accept': 'application/json', @@ -397,9 +446,9 @@ const Workflows = (props) => { } return response.json() }) - .then((responseJson) => { + .then((responseJson) => { getAvailableWorkflows() - }) + }) .catch(error => { alert.error(error.toString()) }); @@ -480,7 +529,7 @@ const Workflows = (props) => {
{ if (selectedWorkflow.id !== data.id) { setSelectedWorkflow(data) - getWorkflowExecution(data.id) + //getWorkflowExecution(data.id) } }}> @@ -537,7 +586,7 @@ const Workflows = (props) => {
{ if (selectedWorkflow.id !== data.id) { setSelectedWorkflow(data) - getWorkflowExecution(data.id) + //getWorkflowExecution(data.id) } }}> @@ -684,22 +733,12 @@ const Workflows = (props) => { } var t = new Date(data.started_at*1000) - var jsonvalid = true var showResult = data.result.trim() - showResult = replaceAll(showResult, " None", " \"None\""); - try { - const tmp = String(JSON.parse(showResult)) - if (!tmp.includes("{") && !tmp.includes("[")) { - jsonvalid = false - } - } catch (e) { - jsonvalid = false - } + const validate = validateJson(showResult) - //console.log("VALID: ", jsonvalid) - if (jsonvalid) { + if (validate.valid) { showResult = { /> } else { // FIXME - have everything parsed as json, either just for frontend - // or in the backend + // or in the backend? /* const newdata = {"result": data.result} showResult = { No results yet
- const resultsLength = Object.getOwnPropertyNames(selectedExecution).length > 0 && selectedExecution.results !== null ? selectedExecution.results.length : 0 + const resultsLength = Object.getOwnPropertyNames(selectedExecution).length > 0 && selectedExecution.results !== null ? selectedExecution.results.length : 0 const ExecutionDetails = () => { var starttime = new Date(selectedExecution.started_at*1000) @@ -780,23 +819,12 @@ const Workflows = (props) => { var arg = null if (selectedExecution.execution_argument !== undefined && selectedExecution.execution_argument.length > 0) { - var jsonvalid = true - var showResult = selectedExecution.execution_argument.trim() - showResult = replaceAll(showResult, " None", " \"None\""); + const validate = validateJson(showResult) - try { - const tmp = String(JSON.parse(showResult)) - if (!tmp.includes("{") && !tmp.includes("[")) { - jsonvalid = false - } - } catch (e) { - jsonvalid = false - } - - arg = jsonvalid ? + arg = validate.valid ? { var lastresult = null if (selectedExecution.result !== undefined && selectedExecution.result.length > 0) { - var jsonvalid = true var showResult = selectedExecution.result.trim() - showResult = replaceAll(showResult, " None", " \"None\""); - - try { - const tmp = JSON.parse(showResult) - if (!tmp.includes("{") && !tmp.includes("[")) { - jsonvalid = false - } - } catch (e) { - jsonvalid = false - } - - lastresult = jsonvalid ? + const validate = validateJson(showResult) + lastresult = validate.valid ? {
:

- There are no executions for this workflow yet + Executions have been moved to the Workflow itself.
Click here to see them

) } @@ -1204,7 +1221,7 @@ const Workflows = (props) => {
-

Workflows

+

Workflows ({workflows.length})

{workflowButtons} @@ -1222,24 +1239,27 @@ const Workflows = (props) => {
-
+

Executions: {selectedWorkflow.name}

-
+ {/* +
+ */}
-
+ {/* +

Execution Timeline

@@ -1257,6 +1277,7 @@ const Workflows = (props) => {
+ */}
) } @@ -1348,7 +1369,7 @@ const Workflows = (props) => { style={{backgroundColor: inputColor}} variant="outlined" margin="normal" - value={downloadUrl} + defaultValue={userdata.active_org.defaults.workflow_download_repo !== undefined && userdata.active_org.defaults.workflow_download_repo.length > 0 ? userdata.active_org.defaults.workflow_download_repo : downloadUrl} InputProps={{ style:{ color: "white", @@ -1367,7 +1388,7 @@ const Workflows = (props) => { style={{backgroundColor: inputColor}} variant="outlined" margin="normal" - value={downloadBranch} + defaultValue={userdata.active_org.defaults.workflow_download_branch !== undefined && userdata.active_org.defaults.workflow_download_branch.length > 0 ? userdata.active_org.defaults.workflow_download_branch : downloadBranch} InputProps={{ style:{ color: "white", diff --git a/functions/extensions/aws-lambda/deploy.sh b/functions/extensions/aws-lambda/deploy.sh new file mode 100644 index 00000000..a3187a27 --- /dev/null +++ b/functions/extensions/aws-lambda/deploy.sh @@ -0,0 +1,9 @@ +GOOS=linux go build main.go +zip function.zip main + +aws lambda update-function-code \ + --function-name shuffler-forwarder \ + --runtime go1.* \ + --zip-file fileb://function.zip \ + --handler main \ + --role arn:aws:iam::123456789012:role/execution_role diff --git a/functions/extensions/cortex-responders/Shuffle/shuffle.json b/functions/extensions/cortex-responders/Shuffle/shuffle.json new file mode 100644 index 00000000..ef2610dd --- /dev/null +++ b/functions/extensions/cortex-responders/Shuffle/shuffle.json @@ -0,0 +1,35 @@ +{ + "name": "Shuffle", + "version": "1.0", + "author": "@frikkylikeme", + "url": "https://github.com/frikky/shuffle", + "license": "AGPL-V3", + "description": "Execute a workflow in Shuffle", + "dataTypeList": ["thehive:case", "thehive:alert"], + "command": "Shuffle/shuffle.py", + "baseConfig": "Shuffle", + "configurationItems": [ + { + "name": "url", + "description": "The URL to your shuffle instance", + "type": "string", + "multi": false, + "required": true, + "defaultValue": "https://shuffler.io" + }, + { + "name": "api_key", + "description": "The API key to your Shuffle user", + "type": "string", + "multi": false, + "required": true + }, + { + "name": "workflow_id", + "description": "The ID of the workflow to execute", + "type": "string", + "multi": false, + "required": true + } + ] +} diff --git a/functions/extensions/cortex-responders/Shuffle/shuffle.py b/functions/extensions/cortex-responders/Shuffle/shuffle.py new file mode 100644 index 00000000..0816ca53 --- /dev/null +++ b/functions/extensions/cortex-responders/Shuffle/shuffle.py @@ -0,0 +1,28 @@ + +#!/usr/bin/env python +# encoding: utf-8 + +from cortexutils.responder import Responder +import requests + +class Shuffle(Responder): + def __init__(self): + Responder.__init__(self) + self.api_key = self.get_param("config.api_key", "") + self.url = self.get_param("config.url", "") + self.workflow_id = self.get_param("config.workflow_id", "") + + def run(self): + Responder.run(self) + + parsed_url = "%s/api/v1/workflows/%s/execute" % (self.url, self.workflow_id) + headers = { + "Authorization": "Bearer %s" % self.api_key + } + requests.post(parsed_url, headers=headers) + + self.report({'message': 'message sent'}) + +if __name__ == '__main__': + Shuffle().run() + diff --git a/functions/extensions/wazuh/custom-shuffle b/functions/extensions/wazuh/custom-shuffle new file mode 100644 index 00000000..bd540414 --- /dev/null +++ b/functions/extensions/wazuh/custom-shuffle @@ -0,0 +1,36 @@ +#!/bin/sh +# Created by Shuffle, AS. . + +WPYTHON_BIN="framework/python/bin/python3" + +SCRIPT_PATH_NAME="$0" + +DIR_NAME="$(cd $(dirname ${SCRIPT_PATH_NAME}); pwd -P)" +SCRIPT_NAME="$(basename ${SCRIPT_PATH_NAME})" + +case ${DIR_NAME} in + */active-response/bin | */wodles*) + if [ -z "${WAZUH_PATH}" ]; then + WAZUH_PATH="$(cd ${DIR_NAME}/../..; pwd)" + fi + + PYTHON_SCRIPT="${DIR_NAME}/${SCRIPT_NAME}.py" + ;; + */bin) + if [ -z "${WAZUH_PATH}" ]; then + WAZUH_PATH="$(cd ${DIR_NAME}/..; pwd)" + fi + + PYTHON_SCRIPT="${WAZUH_PATH}/framework/scripts/${SCRIPT_NAME}.py" + ;; + */integrations) + if [ -z "${WAZUH_PATH}" ]; then + WAZUH_PATH="$(cd ${DIR_NAME}/..; pwd)" + fi + + PYTHON_SCRIPT="${DIR_NAME}/${SCRIPT_NAME}.py" + ;; +esac + + +${WAZUH_PATH}/${WPYTHON_BIN} ${PYTHON_SCRIPT} "$@" diff --git a/functions/extensions/wazuh/custom-shuffle.py b/functions/extensions/wazuh/custom-shuffle.py new file mode 100644 index 00000000..06fa4c7d --- /dev/null +++ b/functions/extensions/wazuh/custom-shuffle.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python +# Created by Shuffle, AS. . +# Based on the Slack integration using Webhooks + +import json +import sys +import time +import os + +try: + import requests + from requests.auth import HTTPBasicAuth +except Exception as e: + print("No module 'requests' found. Install: pip install requests") + sys.exit(1) + +# ADD THIS TO ossec.conf configuration: +# +# custom-shuffle +# http://:3001/api/v1/hooks/ +# 3 +# json +# + +# Global vars + +debug_enabled = False +pwd = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +json_alert = {} +now = time.strftime("%a %b %d %H:%M:%S %Z %Y") + +# Set paths +log_file = '{0}/logs/integrations.log'.format(pwd) + + +def main(args): + debug("# Starting") + + # Read args + alert_file_location = args[1] + webhook = args[3] + + debug("# Webhook") + debug(webhook) + + debug("# File location") + debug(alert_file_location) + + # Load alert. Parse JSON object. + with open(alert_file_location) as alert_file: + json_alert = json.load(alert_file) + debug("# Processing alert") + debug(json_alert) + + debug("# Generating message") + msg = generate_msg(json_alert) + if isinstance(msg, str): + if len(msg) == 0: + return + debug(msg) + + debug("# Sending message") + send_msg(msg, webhook) + + +def debug(msg): + if debug_enabled: + msg = "{0}: {1}\n".format(now, msg) + print(msg) + f = open(log_file, "a") + f.write(msg) + f.close() + +# Skips container kills to stop self-recursion +def filter_msg(alert): + # These are things that recursively happen because Shuffle starts Docker containers + # Docker integration rules: https://github.com/wazuh/wazuh-ruleset/blob/ae36745db1d3f312db0392f5925c2f2b0ec009a9/rules/0560-docker_integration_rules.xml + skip = ["87924", "87900", "87901", "87902", "87903", "87904", "86001", "86002", "86003", "87932", "80710", "87929", "87928",] + if alert["rule"]["id"] in skip: + return False + + #try: + # if "docker" in alert["rule"]["description"].lower() and " + #msg['text'] = alert.get('full_log') + #except: + # pass + #msg['title'] = alert['rule']['description'] if 'description' in alert['rule'] else "N/A" + + return True + +def generate_msg(alert): + if not filter_msg(alert): + print("Skipping rule %s" % alert["rule"]["id"]) + return "" + + level = alert['rule']['level'] + + if (level <= 4): + color = "good" + elif (level >= 5 and level <= 7): + color = "warning" + else: + color = "danger" + + msg = {} + msg['color'] = color + msg['pretext'] = "WAZUH Alert" + msg['title'] = alert['rule']['description'] if 'description' in alert['rule'] else "N/A" + msg['text'] = alert.get('full_log') + msg['rule_id'] = alert["rule"]["id"] + msg['timestamp'] = alert["timestamp"] + msg['id'] = alert['id'] + msg["all_fields"] = alert + + #msg['fields'] = [] + # msg['fields'].append({ + # "title": "Agent", + # "value": "({0}) - {1}".format( + # alert['agent']['id'], + # alert['agent']['name'] + # ), + # }) + #if 'agentless' in alert: + # msg['fields'].append({ + # "title": "Agentless Host", + # "value": alert['agentless']['host'], + # }) + + #msg['fields'].append({"title": "Location", "value": alert['location']}) + #msg['fields'].append({ + # "title": "Rule ID", + # "value": "{0} _(Level {1})_".format(alert['rule']['id'], level), + #}) + + #attach = {'attachments': [msg]} + + return json.dumps(msg) + + +def send_msg(msg, url): + headers = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'} + res = requests.post(url, data=msg, headers=headers) + debug(res) + + +if __name__ == "__main__": + try: + # Read arguments + bad_arguments = False + if len(sys.argv) >= 4: + msg = '{0} {1} {2} {3} {4}'.format( + now, + sys.argv[1], + sys.argv[2], + sys.argv[3], + sys.argv[4] if len(sys.argv) > 4 else '', + ) + debug_enabled = (len(sys.argv) > 4 and sys.argv[4] == 'debug') + else: + msg = '{0} Wrong arguments'.format(now) + bad_arguments = True + + # Logging the call + f = open(log_file, 'a') + f.write(msg + '\n') + f.close() + + if bad_arguments: + debug("# Exiting: Bad arguments.") + sys.exit(1) + + # Main function + main(sys.argv) + + except Exception as e: + debug(str(e)) + raise diff --git a/functions/extensions/wazuh/ossec.conf b/functions/extensions/wazuh/ossec.conf new file mode 100644 index 00000000..5b55f5d3 --- /dev/null +++ b/functions/extensions/wazuh/ossec.conf @@ -0,0 +1,5 @@ + + custom-shuffle + http://:3001/api/v1/hooks/webhook_ + json + diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 14499755..1aa329af 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,11 +1,11 @@ NAME=shuffle-orborus -VERSION=0.8.0 +VERSION=0.8.54 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION #docker push frikky/$NAME:$VERSION -#docker push frikky/shuffle:$NAME # docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION +docker push frikky/shuffle:$NAME docker push ghcr.io/frikky/$NAME:$VERSION diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 78984425..f4f7f50a 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -21,17 +21,22 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" + //"github.com/docker/docker/api/types/filters" dockerclient "github.com/docker/docker/client" "github.com/satori/go.uuid" //network "github.com/docker/docker/api/types/network" //natting "github.com/docker/go-connections/nat" + "github.com/mackerelio/go-osstat/cpu" + "github.com/mackerelio/go-osstat/memory" ) // Starts jobs in bulk, so this could be increased var sleepTime = 3 +var maxConcurrency = 50 // Timeout if something rashes var workerTimeoutEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_TIMEOUT") +var concurrencyEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY") var appSdkVersion = os.Getenv("SHUFFLE_APP_SDK_VERSION") var workerVersion = os.Getenv("SHUFFLE_WORKER_VERSION") @@ -47,6 +52,8 @@ var baseUrl = os.Getenv("BASE_URL") var environment = os.Getenv("ENVIRONMENT_NAME") var dockerApiVersion = os.Getenv("DOCKER_API_VERSION") var runningMode = strings.ToLower(os.Getenv("RUNNING_MODE")) +var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP")) +var workerIds = []string{} type ExecutionRequestWrapper struct { Data []ExecutionRequest `json:"data"` @@ -103,13 +110,25 @@ func getThisContainerId() { out, err := exec.Command("bash", "-c", cmd).Output() if err == nil { containerId = strings.TrimSpace(string(out)) + + // cgroup error. Hardcoding this. + // https://github.com/moby/moby/issues/7015 + //log.Printf("Checking if %s is in %s", ".scope", string(out)) + if strings.Contains(string(out), ".scope") { + containerId = "shuffle-orborus" + //docker-76c537e9a4b7c7233011f5d70e6b7f2d600b6413ac58a96519b8dca7a3f7117a.scope + } } else { - log.Printf("Failed getting container ID: %s", err) + containerId = "shuffle-orborus" + log.Printf("[WARNING] Failed getting container ID: %s", err) } } + + log.Printf("Started with containerId %s", containerId) } // Deploys the internal worker whenever something happens +// https://docs.docker.com/engine/api/sdk/examples/ func deployWorker(image string, identifier string, env []string) { // Binds is the actual "-v" volume. hostConfig := &container.HostConfig{ @@ -124,23 +143,28 @@ func deployWorker(image string, identifier string, env []string) { // form container id and use it as network source if it's not empty if containerId != "" { - log.Printf("[INFO] Found container ID %s", containerId) + //log.Printf("[INFO] Found container ID %s", containerId) hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) } else { //log.Printf("[INFO] Empty self container id, continue without NetworkMode") } + if cleanupEnv == "true" { + hostConfig.AutoRemove = true + } + config := &container.Config{ Image: image, Env: env, } - log.Printf("Identifier: %s", identifier) + //log.Printf("[INFO] Identifier: %s", identifier) cont, err := dockercli.ContainerCreate( context.Background(), config, hostConfig, nil, + nil, identifier, ) @@ -154,6 +178,7 @@ func deployWorker(image string, identifier string, env []string) { config, hostConfig, nil, + nil, identifier, ) @@ -167,7 +192,8 @@ func deployWorker(image string, identifier string, env []string) { } } - err = dockercli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{}) + containerStartOptions := types.ContainerStartOptions{} + err = dockercli.ContainerStart(context.Background(), cont.ID, containerStartOptions) if err != nil { log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err) return @@ -195,6 +221,7 @@ func deployWorker(image string, identifier string, env []string) { //} } else { log.Printf("[INFO] Container %s was created under environment %s", cont.ID, environment) + //workerIds = append(workerIds, cont.ID) } return @@ -227,11 +254,11 @@ func initializeImages() { ctx := context.Background() if appSdkVersion == "" { - appSdkVersion = "0.8.0" + appSdkVersion = "0.8.5" log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion) } if workerVersion == "" { - workerVersion = "0.8.0" + workerVersion = "0.8.54" log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion) } @@ -248,9 +275,7 @@ func initializeImages() { // check whether they are the same first images := []string{ - //fmt.Sprintf("%s/%s:app_sdk%s", baseimageregistry, baseimagename, baseimagetagsuffix), - //fmt.Sprintf("%s/%s:worker%s", baseimageregistry, baseimagename, baseimagetagsuffix), - + fmt.Sprintf("frikky/shuffle:app_sdk"), fmt.Sprintf("%s/%s/shuffle-app_sdk:%s", baseimageregistry, baseimagename, appSdkVersion), fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion), // fmt.Sprintf("docker.io/%s:app_sdk", baseimagename), @@ -275,6 +300,40 @@ func initializeImages() { } } +// Will be used for checking if there's enough to deploy based on a threshold +// E.g. having maximum CPU and maxmimum RAM +// Does this work containerized? +func getStats() { + fmt.Printf("\n") + + memory, err := memory.Get() + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + return + } + + before, err := cpu.Get() + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + return + } + time.Sleep(time.Duration(250) * time.Millisecond) + after, err := cpu.Get() + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + return + } + total := float64(after.Total - before.Total) + + fmt.Printf("[INFO] memory total: %d bytes\n", memory.Total) + fmt.Printf("[INFO] memory used: %d bytes\n", memory.Used) + fmt.Printf("[INFO] cpu used : %f%%\n", float64(after.User-before.User)/total*100) + fmt.Printf("[INFO] cpu system: %f%%\n", float64(after.System-before.System)/total*100) + fmt.Printf("[INFO] cpu idle : %f%%\n", float64(after.Idle-before.Idle)/total*100) + + fmt.Printf("\n") +} + // Initial loop etc func main() { log.Println("[INFO] Setting up execution environment") @@ -302,7 +361,19 @@ func main() { log.Printf("[INFO] Cleanup process running every %d seconds", workerTimeout) } - go zombiecheck(workerTimeout) + if concurrencyEnv != "" { + //var concurrencyEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY") + tmpInt, err := strconv.Atoi(concurrencyEnv) + if err == nil { + maxConcurrency = tmpInt + log.Printf("[INFO] Max workflow execution concurrency set to %d", maxConcurrency) + } else { + log.Printf("[WARNING] Env SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY must be a number, not %s. Defaulted to %d", workerTimeoutEnv, maxConcurrency) + } + } + + ctx := context.Background() + go zombiecheck(ctx, workerTimeout) log.Printf("[INFO] Running towards %s with Org %s", baseUrl, orgId) httpProxy := os.Getenv("HTTP_PROXY") @@ -337,6 +408,8 @@ func main() { }, } + //getStats() + if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" { client = &http.Client{} } else { @@ -367,13 +440,15 @@ func main() { hasStarted := false for { //log.Printf("Prerequest") + //go getStats() newresp, err := client.Do(req) + executionCount := getRunningWorkers(ctx, workerTimeout) //log.Printf("Postrequest") if err != nil { log.Printf("[WARNING] Failed making request: %s", err) zombiecounter += 1 if zombiecounter*sleepTime > workerTimeout { - go zombiecheck(workerTimeout) + go zombiecheck(ctx, workerTimeout) zombiecounter = 0 } time.Sleep(time.Duration(sleepTime) * time.Second) @@ -394,7 +469,7 @@ func main() { log.Printf("[ERROR] Failed reading body: %s", err) zombiecounter += 1 if zombiecounter*sleepTime > workerTimeout { - go zombiecheck(workerTimeout) + go zombiecheck(ctx, workerTimeout) zombiecounter = 0 } time.Sleep(time.Duration(sleepTime) * time.Second) @@ -408,7 +483,7 @@ func main() { sleepTime = 10 zombiecounter += 1 if zombiecounter*sleepTime > workerTimeout { - go zombiecheck(workerTimeout) + go zombiecheck(ctx, workerTimeout) zombiecounter = 0 } time.Sleep(time.Duration(sleepTime) * time.Second) @@ -423,13 +498,31 @@ func main() { if len(executionRequests.Data) == 0 { zombiecounter += 1 if zombiecounter*sleepTime > workerTimeout { - go zombiecheck(workerTimeout) + go zombiecheck(ctx, workerTimeout) zombiecounter = 0 } time.Sleep(time.Duration(sleepTime) * time.Second) continue } + // Anything below here verifies concurrency virification + if executionCount >= maxConcurrency { + if zombiecounter*sleepTime > workerTimeout { + go zombiecheck(ctx, workerTimeout) + zombiecounter = 0 + } + time.Sleep(time.Duration(sleepTime) * time.Second) + continue + } + + //log.Printf("[INFO] Got %d new requests. Executing: %d. Max: %d", len(executionRequests.Data), executionCount, maxConcurrency) + + allowed := maxConcurrency - executionCount + if len(executionRequests.Data) > allowed { + log.Printf("[WARNING] Throttle - Cutting down requests from %d to %d (MAX: %d, CUR: %d)", len(executionRequests.Data), allowed, maxConcurrency, executionCount) + executionRequests.Data = executionRequests.Data[0:allowed] + } + // New, abortable version. Should check executionid and remove everything else var toBeRemoved ExecutionRequestWrapper for _, execution := range executionRequests.Data { @@ -453,6 +546,7 @@ func main() { fmt.Sprintf("EXECUTIONID=%s", execution.ExecutionId), fmt.Sprintf("ENVIRONMENT_NAME=%s", environment), fmt.Sprintf("BASE_URL=%s", baseUrl), + fmt.Sprintf("CLEANUP=%s", cleanupEnv), } if strings.ToLower(os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) != "false" { @@ -466,7 +560,7 @@ func main() { go deployWorker(workerImage, containerName, env) - log.Printf("[INFO] %s is deployed and to be removed from queue.", execution.ExecutionId) + log.Printf("[INFO] ExecutionID %s was deployed and to be removed from queue.", execution.ExecutionId) zombiecounter += 1 toBeRemoved.Data = append(toBeRemoved.Data, execution) } @@ -528,25 +622,22 @@ func main() { } } -// FIXME - add this to remove exited workers -// Should it check what happened to the execution? idk -func zombiecheck(workerTimeout int) error { - log.Println("[INFO] Looking for old containers") - ctx := context.Background() - +// Is this ok to do with Docker? idk :) +func getRunningWorkers(ctx context.Context, workerTimeout int) int { containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{ All: true, }) + //Filters: filters.Args{ + // map[string][]string{"ancestor": {":"}}, + //}, if err != nil { - log.Printf("[ERROR] Failed creating Containerlist: %s", err) - return err + log.Printf("[ERROR] Error getting containers: %s", err) + return maxConcurrency } - containerNames := map[string]string{} - - stopContainers := []string{} - removeContainers := []string{} + currenttime := time.Now().Unix() + counter := 0 for _, container := range containers { // Skip random containers. Only handle things related to Shuffle. if !strings.Contains(container.Image, baseimagename) { @@ -568,14 +659,75 @@ func zombiecheck(workerTimeout int) error { for _, name := range container.Names { // FIXME - add name_version_uid_uid regex check as well - if strings.HasPrefix(name, "/shuffle") { + if !strings.HasPrefix(name, "/worker") { continue } - log.Printf("[INFO] NAME: %s", name) + //log.Printf("Time: %d - %d", currenttime-container.Created, int64(workerTimeout)) + if container.State == "running" && currenttime-container.Created < int64(workerTimeout) { + counter += 1 + break + } + } + } + + return counter +} + +// FIXME - add this to remove exited workers +// Should it check what happened to the execution? idk +func zombiecheck(ctx context.Context, workerTimeout int) error { + log.Println("[INFO] Looking for old containers (zombies)") + containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{ + All: true, + }) + + //log.Printf("Len: %d", len(containers)) + + if err != nil { + log.Printf("[ERROR] Failed creating Containerlist: %s", err) + return err + } + + containerNames := map[string]string{} + + stopContainers := []string{} + removeContainers := []string{} + log.Printf("[INFO] Baseimage: %s, Workertimeout: %d", baseimagename, int64(workerTimeout)) + baseString := `/bin/sh -c 'python app.py --log-level DEBUG'` + for _, container := range containers { + // Skip random containers. Only handle things related to Shuffle. + if !strings.Contains(container.Image, baseimagename) && container.Command != baseString && container.Command != "./worker" { + shuffleFound := false + for _, item := range container.Labels { + if item == "shuffle" { + shuffleFound = true + break + } + } + + // Check image name + if !shuffleFound { + log.Printf("Skipping: %s, %s", container.Labels, container.Image) + continue + } + //} else { + // log.Printf("NAME: %s", container.Image) + } else { + //log.Printf("Img: %s", container.Image) + //log.Printf("Names: %s", container.Names) + } + + for _, name := range container.Names { + // FIXME - add name_version_uid_uid regex check as well + if strings.HasPrefix(name, "/shuffle") && !strings.HasPrefix(name, "/shuffle-subflow") { + continue + } + + currenttime := time.Now().Unix() + //log.Printf("[INFO] (%s) NAME: %s. TIME: %d", container.State, name, currenttime-container.Created) // Need to check time here too because a container can be removed the same instant as its created - currenttime := time.Now().Unix() if container.State != "running" && currenttime-container.Created > int64(workerTimeout) { removeContainers = append(removeContainers, container.ID) containerNames[container.ID] = name @@ -591,9 +743,10 @@ func zombiecheck(workerTimeout int) error { } // FIXME - add killing of apps with same execution ID too + log.Printf("[INFO] Should STOP %d containers.", len(stopContainers)) for _, containername := range stopContainers { log.Printf("[INFO] Stopping and removing container %s", containerNames[containername]) - go dockercli.ContainerStop(ctx, containername, nil) + dockercli.ContainerStop(ctx, containername, nil) removeContainers = append(removeContainers, containername) } @@ -602,8 +755,9 @@ func zombiecheck(workerTimeout int) error { Force: true, } + log.Printf("[INFO] Should REMOVE %d containers.", len(removeContainers)) for _, containername := range removeContainers { - go dockercli.ContainerRemove(ctx, containername, removeOptions) + dockercli.ContainerRemove(ctx, containername, removeOptions) } return nil diff --git a/functions/onprem/worker/Dockerfile b/functions/onprem/worker/Dockerfile index c381e73b..483ab136 100644 --- a/functions/onprem/worker/Dockerfile +++ b/functions/onprem/worker/Dockerfile @@ -5,6 +5,8 @@ WORKDIR /app RUN go get -u github.com/docker/docker/api/types RUN go get -u github.com/docker/docker/api/types/container RUN go get -u github.com/docker/docker/client +RUN go get -u github.com/gorilla/mux +RUN go get -u github.com/patrickmn/go-cache COPY worker.go /app/worker.go RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker . @@ -13,7 +15,7 @@ FROM alpine:3.12 ENV SHUFFLE_BASE_IMAGE_REGISTRY=docker.io ENV SHUFFLE_BASE_IMAGE_NAME=frikky/shuffle -ENV SHUFFLE_BASE_IMAGE_TAG_SUFFIX=0.6.0 +ENV SHUFFLE_BASE_IMAGE_TAG_SUFFIX=0.8.5 RUN apk add --no-cache bash COPY --from=builder /app/ / diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 363df000..f0592678 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,12 +1,14 @@ NAME=shuffle-worker -VERSION=0.8.0 +VERSION=0.8.56 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . -docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION +docker build . -t frikky/shuffle:$NAME -t frikky/shuffle:$NAME_$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION # Push both for now.. #docker push frikky/$NAME:$VERSION -docker push frikky/shuffle:$NAME +#docker push frikky/shuffle:$NAME_$VERSION #docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION +#docker tag frikky/shuffle:0.8.51 ghcr.io/frikky/shuffle-worker:0.8.5 +docker tag frikky/shuffle:$NAME ghcr.io/frikky/shuffle-worker:0.8.52 docker push ghcr.io/frikky/$NAME:$VERSION diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index dbaea55f..1c36bdb2 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -6,9 +6,10 @@ import ( "encoding/json" "errors" "fmt" - //"io" + "io" "io/ioutil" "log" + "net" "net/http" "os" "os/exec" @@ -18,12 +19,34 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" dockerclient "github.com/docker/docker/client" + + "github.com/gorilla/mux" + "github.com/patrickmn/go-cache" ) +// This is getting out of hand :) var environment = os.Getenv("ENVIRONMENT_NAME") var baseUrl = os.Getenv("BASE_URL") +var appCallbackUrl = os.Getenv("BASE_URL") +var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP")) var baseimagename = "frikky/shuffle" +var registryName = "registry.hub.docker.com" +var fallbackName = "shuffle-orborus" var sleepTime = 2 +var requestCache *cache.Cache +var topClient *http.Client +var data string +var requestsSent = 0 + +var environments []string +var parents map[string][]string +var children map[string][]string +var visited []string +var executed []string +var nextActions []string +var containerIds []string +var extra int +var startAction string var containerId string @@ -34,6 +57,11 @@ func getThisContainerId() string { out, err := exec.Command("bash", "-c", cmd).Output() if err == nil { id = strings.TrimSpace(string(out)) + + //log.Printf("Checking if %s is in %s", ".scope", string(out)) + if strings.Contains(string(out), ".scope") { + id = fallbackName + } } return id @@ -42,12 +70,109 @@ func getThisContainerId() string { func init() { containerId = getThisContainerId() if len(containerId) == 0 { - log.Printf("[ERROR] No container ID found.") + log.Printf("[WARNING] No container ID found. Not running containerized? This should only show during testing") } else { - log.Printf("[INFO] Found container ID: %s", containerId) + log.Printf("[INFO] Found container ID for this worker: %s", containerId) } } +type Userapi struct { + Username string `datastore:"username"` + ApiKey string `datastore:"apikey"` +} + +type ExecutionInfo struct { + TotalApiUsage int64 `json:"total_api_usage" datastore:"total_api_usage"` + TotalWorkflowExecutions int64 `json:"total_workflow_executions" datastore:"total_workflow_executions"` + TotalAppExecutions int64 `json:"total_app_executions" datastore:"total_app_executions"` + TotalCloudExecutions int64 `json:"total_cloud_executions" datastore:"total_cloud_executions"` + TotalOnpremExecutions int64 `json:"total_onprem_executions" datastore:"total_onprem_executions"` + DailyApiUsage int64 `json:"daily_api_usage" datastore:"daily_api_usage"` + DailyWorkflowExecutions int64 `json:"daily_workflow_executions" datastore:"daily_workflow_executions"` + DailyAppExecutions int64 `json:"daily_app_executions" datastore:"daily_app_executions"` + DailyCloudExecutions int64 `json:"daily_cloud_executions" datastore:"daily_cloud_executions"` + DailyOnpremExecutions int64 `json:"daily_onprem_executions" datastore:"daily_onprem_executions"` +} + +type StatisticsData struct { + Timestamp int64 `json:"timestamp" datastore:"timestamp"` + Id string `json:"id" datastore:"id"` + Amount int64 `json:"amount" datastore:"amount"` +} + +type StatisticsItem struct { + Total int64 `json:"total" datastore:"total"` + Fieldname string `json:"field_name" datastore:"field_name"` + Data []StatisticsData `json:"data" datastore:"data"` +} + +// "Execution by status" +// Execution history +//type GlobalStatistics struct { +// BackendExecutions int64 `json:"backend_executions" datastore:"backend_executions"` +// WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"` +// ExecutionCount int64 `json:"execution_count" datastore:"execution_count"` +// ExecutionSuccessCount int64 `json:"execution_success_count" datastore:"execution_success_count"` +// ExecutionAbortCount int64 `json:"execution_abort_count" datastore:"execution_abort_count"` +// ExecutionFailureCount int64 `json:"execution_failure_count" datastore:"execution_failure_count"` +// ExecutionPendingCount int64 `json:"execution_pending_count" datastore:"execution_pending_count"` +// AppUsageCount int64 `json:"app_usage_count" datastore:"app_usage_count"` +// TotalAppsCount int64 `json:"total_apps_count" datastore:"total_apps_count"` +// SelfMadeAppCount int64 `json:"self_made_app_count" datastore:"self_made_app_count"` +// WebhookUsageCount int64 `json:"webhook_usage_count" datastore:"webhook_usage_count"` +// Baseline map[string]int64 `json:"baseline" datastore:"baseline"` +//} + +type ParsedOpenApi struct { + Body string `datastore:"body,noindex" json:"body"` + ID string `datastore:"id" json:"id"` + Success bool `datastore:"success,omitempty" json:"success,omitempty"` +} + +// Limits set for a user so that they can't do a shitload +type UserLimits struct { + DailyApiUsage int64 `json:"daily_api_usage" datastore:"daily_api_usage"` + DailyWorkflowExecutions int64 `json:"daily_workflow_executions" datastore:"daily_workflow_executions"` + DailyCloudExecutions int64 `json:"daily_cloud_executions" datastore:"daily_cloud_executions"` + DailyTriggers int64 `json:"daily_triggers" datastore:"daily_triggers"` + DailyMailUsage int64 `json:"daily_mail_usage" datastore:"daily_mail_usage"` + MaxTriggers int64 `json:"max_triggers" datastore:"max_triggers"` + MaxWorkflows int64 `json:"max_workflows" datastore:"max_workflows"` +} + +type retStruct struct { + Success bool `json:"success"` + SyncFeatures SyncFeatures `json:"sync_features"` + SessionKey string `json:"session_key"` + IntervalSeconds int64 `json:"interval_seconds"` + Reason string `json:"reason"` +} + +// Saves some data, not sure what to have here lol +type UserAuth struct { + Description string `json:"description" datastore:"description,noindex" yaml:"description"` + Name string `json:"name" datastore:"name" yaml:"name"` + Workflows []string `json:"workflows" datastore:"workflows"` + Username string `json:"username" datastore:"username"` + Fields []UserAuthField `json:"fields" datastore:"fields"` +} + +type UserAuthField struct { + Key string `json:"key" datastore:"key"` + Value string `json:"value" datastore:"value,noindex"` +} + +// Not environment, but execution environment +type Environment struct { + Name string `datastore:"name"` + Type string `datastore:"type"` + Registered bool `datastore:"registered"` + Default bool `datastore:"default" json:"default"` + Archived bool `datastore:"archived" json:"archived"` + Id string `datastore:"id" json:"id"` + OrgId string `datastore:"org_id" json:"org_id"` +} + type User struct { Username string `datastore:"Username" json:"username"` Password string `datastore:"password,noindex" password:"password,omitempty"` @@ -59,41 +184,293 @@ type User struct { VerificationToken string `datastore:"verification_token" json:"verification_token"` ApiKey string `datastore:"apikey" json:"apikey"` ResetReference string `datastore:"reset_reference" json:"reset_reference"` + Executions ExecutionInfo `datastore:"executions" json:"executions"` + Limits UserLimits `datastore:"limits" json:"limits"` + Authentication []UserAuth `datastore:"authentication,noindex" json:"authentication"` ResetTimeout int64 `datastore:"reset_timeout,noindex" json:"reset_timeout"` Id string `datastore:"id" json:"id"` Orgs []string `datastore:"orgs" json:"orgs"` CreationTime int64 `datastore:"creation_time" json:"creation_time"` + ActiveOrg Org `json:"active_org" datastore:"active_org"` Active bool `datastore:"active" json:"active"` } -type ExecutionRequest struct { - ExecutionId string `json:"execution_id"` - ExecutionArgument string `json:"execution_argument"` - ExecutionSource string `json:"execution_source"` - WorkflowId string `json:"workflow_id"` - Environments []string `json:"environments"` - Authorization string `json:"authorization"` - Status string `json:"status"` - Start string `json:"start"` - Type string `json:"type"` +// timeout maybe? idk +type session struct { + Username string `datastore:"Username,noindex"` + Id string `datastore:"Id,noindex"` + Session string `datastore:"session,noindex"` } +type loginStruct struct { + Username string `json:"username"` + Password string `json:"password"` +} + +type Contact struct { + Firstname string `json:"firstname"` + Lastname string `json:"lastname"` + Title string `json:"title"` + Companyname string `json:"companyname"` + Phone string `json:"phone"` + Email string `json:"email"` + Message string `json:"message"` +} + +type Translator struct { + Src struct { + Name string `json:"name" datastore:"name"` + Value string `json:"value" datastore:"value,noindex"` + Description string `json:"description" datastore:"description,noindex"` + Required string `json:"required" datastore:"required"` + Type string `json:"type" datastore:"type"` + Schema struct { + Type string `json:"type" datastore:"type"` + } `json:"schema" datastore:"schema"` + } `json:"src" datastore:"src"` + Dst struct { + Name string `json:"name" datastore:"name"` + Value string `json:"value" datastore:"value,noindex"` + Type string `json:"type" datastore:"type"` + Description string `json:"description" datastore:"description,noindex"` + Required string `json:"required" datastore:"required"` + Schema struct { + Type string `json:"type" datastore:"type"` + } `json:"schema" datastore:"schema"` + } `json:"dst" datastore:"dst"` +} + +type Appconfig struct { + Key string `json:"key" datastore:"key"` + Value string `json:"value" datastore:"value,noindex"` +} + +type ScheduleApp struct { + Foldername string `json:"foldername" datastore:"foldername,noindex"` + Name string `json:"name" datastore:"name,noindex"` + Id string `json:"id" datastore:"id,noindex"` + Description string `json:"description" datastore:"description,noindex"` + Action string `json:"action" datastore:"action,noindex"` + Config []Appconfig `json:"config,omitempty" datastore:"config,noindex"` +} + +type AppInfo struct { + SourceApp ScheduleApp `json:"sourceapp,omitempty" datastore:"sourceapp,noindex"` + DestinationApp ScheduleApp `json:"destinationapp,omitempty" datastore:"destinationapp,noindex"` +} + +// May 2020: Reused for onprem schedules - Id, Seconds, WorkflowId and argument +type ScheduleOld struct { + Id string `json:"id" datastore:"id"` + StartNode string `json:"start_node" datastore:"start_node"` + Seconds int `json:"seconds" datastore:"seconds"` + WorkflowId string `json:"workflow_id" datastore:"workflow_id", ` + Argument string `json:"argument" datastore:"argument"` + WrappedArgument string `json:"wrapped_argument" datastore:"wrapped_argument"` + AppInfo AppInfo `json:"appinfo" datastore:"appinfo,noindex"` + Finished bool `json:"finished" finished:"id"` + BaseAppLocation string `json:"base_app_location" datastore:"baseapplocation,noindex"` + Translator []Translator `json:"translator,omitempty" datastore:"translator"` + Org string `json:"org" datastore:"org"` + CreatedBy string `json:"createdby" datastore:"createdby"` + Availability string `json:"availability" datastore:"availability"` + CreationTime int64 `json:"creationtime" datastore:"creationtime,noindex"` + LastModificationtime int64 `json:"lastmodificationtime" datastore:"lastmodificationtime,noindex"` + LastRuntime int64 `json:"lastruntime" datastore:"lastruntime,noindex"` + Frequency string `json:"frequency" datastore:"frequency,noindex"` + Environment string `json:"environment" datastore:"environment"` +} + +// Returned from /GET /schedules +type Schedules struct { + Schedules []ScheduleOld `json:"schedules"` + Success bool `json:"success"` +} + +type ScheduleApps struct { + Apps []ApiYaml `json:"apps"` + Success bool `json:"success"` +} + +// The yaml that is uploaded +type ApiYaml struct { + Name string `json:"name" yaml:"name" required:"true datastore:"name"` + Foldername string `json:"foldername" yaml:"foldername" required:"true datastore:"foldername"` + Id string `json:"id" yaml:"id",required:"true, datastore:"id"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` + AppVersion string `json:"app_version" yaml:"app_version",datastore:"app_version"` + ContactInfo struct { + Name string `json:"name" datastore:"name" yaml:"name"` + Url string `json:"url" datastore:"url" yaml:"url"` + } `json:"contact_info" datastore:"contact_info" yaml:"contact_info"` + Types []string `json:"types" datastore:"types" yaml:"types"` + Input []struct { + Name string `json:"name" datastore:"name" yaml:"name"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` + InputParameters []struct { + Name string `json:"name" datastore:"name" yaml:"name"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` + Required string `json:"required" datastore:"required" yaml:"required"` + Schema struct { + Type string `json:"type" datastore:"type" yaml:"type"` + } `json:"schema" datastore:"schema" yaml:"schema"` + } `json:"inputparameters" datastore:"inputparameters" yaml:"inputparameters"` + OutputParameters []struct { + Name string `json:"name" datastore:"name" yaml:"name"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` + Required string `json:"required" datastore:"required" yaml:"required"` + Schema struct { + Type string `json:"type" datastore:"type" yaml:"type"` + } `json:"schema" datastore:"schema" yaml:"schema"` + } `json:"outputparameters" datastore:"outputparameters" yaml:"outputparameters"` + Config []struct { + Name string `json:"name" datastore:"name" yaml:"name"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` + Required string `json:"required" datastore:"required" yaml:"required"` + Schema struct { + Type string `json:"type" datastore:"type" yaml:"type"` + } `json:"schema" datastore:"schema" yaml:"schema"` + } `json:"config" datastore:"config" yaml:"config"` + } `json:"input" datastore:"input" yaml:"input"` + Output []struct { + Name string `json:"name" datastore:"name" yaml:"name"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` + Config []struct { + Name string `json:"name" datastore:"name" yaml:"name"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` + Required string `json:"required" datastore:"required" yaml:"required"` + Schema struct { + Type string `json:"type" datastore:"type" yaml:"type"` + } `json:"schema" datastore:"schema" yaml:"schema"` + } `json:"config" datastore:"config" yaml:"config"` + InputParameters []struct { + Name string `json:"name" datastore:"name" yaml:"name"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` + Required string `json:"required" datastore:"required" yaml:"required"` + Schema struct { + Type string `json:"type" datastore:"type" yaml:"type"` + } `json:"schema" datastore:"schema" yaml:"schema"` + } `json:"inputparameters" datastore:"inputparameters" yaml:"inputparameters"` + OutputParameters []struct { + Name string `json:"name" datastore:"name" yaml:"name"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` + Required string `json:"required" datastore:"required" yaml:"required"` + Schema struct { + Type string `json:"type" datastore:"type" yaml:"type"` + } `json:"schema" datastore:"schema" yaml:"schema"` + } `json:"outputparameters" datastore:"outputparameters" yaml:"outputparameters"` + } `json:"output" datastore:"output" yaml:"output"` +} + +type Hooks struct { + Hooks []Hook `json:"hooks"` + Success bool `json:"-"` +} + +type Info struct { + Url string `json:"url" datastore:"url"` + Name string `json:"name" datastore:"name"` + Description string `json:"description" datastore:"description,noindex"` +} + +// Actions to be done by webhooks etc +// Field is the actual field to use from json +type HookAction struct { + Type string `json:"type" datastore:"type"` + Name string `json:"name" datastore:"name"` + Id string `json:"id" datastore:"id"` + Field string `json:"field" datastore:"field"` +} + +type Hook struct { + Id string `json:"id" datastore:"id"` + Start string `json:"start" datastore:"start"` + Info Info `json:"info" datastore:"info"` + Actions []HookAction `json:"actions" datastore:"actions,noindex"` + Type string `json:"type" datastore:"type"` + Owner string `json:"owner" datastore:"owner"` + Status string `json:"status" datastore:"status"` + Workflows []string `json:"workflows" datastore:"workflows"` + Running bool `json:"running" datastore:"running"` + OrgId string `json:"org_id" datastore:"org_id"` + Environment string `json:"environment" datastore:"environment"` +} + +type ExecutionRequest struct { + ExecutionId string `json:"execution_id,omitempty"` + ExecutionArgument string `json:"execution_argument,omitempty"` + ExecutionSource string `json:"execution_source,omitempty"` + WorkflowId string `json:"workflow_id,omitempty"` + Environments []string `json:"environments,omitempty"` + Authorization string `json:"authorization,omitempty"` + Status string `json:"status,omitempty"` + Start string `json:"start,omitempty"` + Type string `json:"type,omitempty"` +} + +type SyncFeatures struct { + Webhook SyncData `json:"webhook" datastore:"webhook"` + Schedules SyncData `json:"schedules" datastore:"schedules"` + UserInput SyncData `json:"user_input" datastore:"user_input"` + SendMail SyncData `json:"send_mail" datastore:"send_mail"` + SendSms SyncData `json:"send_sms" datastore:"send_sms"` + Updates SyncData `json:"updates" datastore:"updates"` + Notifications SyncData `json:"notifications" datastore:"notifications"` + EmailTrigger SyncData `json:"email_trigger" datastore:"email_trigger"` + AppExecutions SyncData `json:"app_executions" datastore:"app_executions"` + WorkflowExecutions SyncData `json:"workflow_executions" datastore:"workflow_executions"` + Apps SyncData `json:"apps" datastore:"apps"` + Workflows SyncData `json:"workflows" datastore:"workflows"` + Autocomplete SyncData `json:"autocomplete" datastore:"autocomplete"` + Authentication SyncData `json:"authentication" datastore:"authentication"` + Schedule SyncData `json:"schedule" datastore:"schedule"` +} + +type SyncData struct { + Active bool `json:"active" datastore:"active"` + Type string `json:"type,omitempty" datastore:"type"` + Name string `json:"name,omitempty" datastore:"name"` + Description string `json:"description,omitempty" datastore:"description"` + Limit int64 `json:"limit,omitempty" datastore:"limit"` + StartDate int64 `json:"start_date,omitempty" datastore:"start_date"` + EndDate int64 `json:"end_date,omitempty" datastore:"end_date"` + DataCollection int64 `json:"data_collection,omitempty" datastore:"data_collection"` +} + +type SyncConfig struct { + Interval int64 `json:"interval" datastore:"interval"` + Apikey string `json:"api_key" datastore:"api_key"` +} + +// Role is just used for feedback for a user type Org struct { - Name string `json:"name"` - Org string `json:"org"` - Users []User `json:"users"` - Id string `json:"id"` + Name string `json:"name" datastore:"name"` + Description string `json:"description" datastore:"description"` + Image string `json:"image" datastore:"image,noindex"` + Id string `json:"id" datastore:"id"` + Org string `json:"org" datastore:"org"` + Users []User `json:"users" datastore:"users"` + Role string `json:"role" datastore:"role"` + Roles []string `json:"roles" datastore:"roles"` + CloudSync bool `json:"cloud_sync" datastore:"CloudSync"` + SyncConfig SyncConfig `json:"sync_config" datastore:"sync_config"` + SyncFeatures SyncFeatures `json:"sync_features" datastore:"sync_features"` + Created int64 `json:"created" datastore:"created"` + Edited int64 `json:"edited" datastore:"edited"` } type AppAuthenticationStorage struct { Active bool `json:"active" datastore:"active"` Label string `json:"label" datastore:"label"` Id string `json:"id" datastore:"id"` - App WorkflowApp `json:"app" datastore:"app"` + App WorkflowApp `json:"app" datastore:"app,noindex"` Fields []AuthenticationStore `json:"fields" datastore:"fields"` Usage []AuthenticationUsage `json:"usage" datastore:"usage"` WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"` NodeCount int64 `json:"node_count" datastore:"node_count"` + OrgId string `json:"org_id" datastore:"org_id"` + Created int64 `json:"created" datastore:"created"` + Edited int64 `json:"edited" datastore:"edited"` } type AuthenticationUsage struct { @@ -102,45 +479,61 @@ type AuthenticationUsage struct { } // An app inside Shuffle +// Source string `json:"source" datastore:"soure" yaml:"source"` - downloadlocation type WorkflowApp struct { - Name string `json:"name" yaml:"name" required:true datastore:"name"` - IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"` - ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"` - Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"` - AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"` - Generated bool `json:"generated" yaml:"generated" required:false datastore:"generated"` - Downloaded bool `json:"downloaded" yaml:"downloaded" required:false datastore:"downloaded"` - Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` - Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` - Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` - Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"` - Owner string `json:"owner" datastore:"owner" yaml:"owner"` - Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps - PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"` - Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"` - Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"` - SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` - ContactInfo struct { + Name string `json:"name" yaml:"name" required:true datastore:"name"` + IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"` + ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"` + Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"` + AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"` + SharingConfig string `json:"sharing_config" yaml:"sharing_config" datastore:"sharing_config"` + Generated bool `json:"generated" yaml:"generated" required:false datastore:"generated"` + Downloaded bool `json:"downloaded" yaml:"downloaded" required:false datastore:"downloaded"` + Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` + Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` + Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` + Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"` + Owner string `json:"owner" datastore:"owner" yaml:"owner"` + Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps + PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"` + Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"` + Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"` + SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` + LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` + ContactInfo struct { Name string `json:"name" datastore:"name" yaml:"name"` Url string `json:"url" datastore:"url" yaml:"url"` } `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false` - Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions"` + Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"` + Tags []string `json:"tags" yaml:"tags" required:false datastore:"activated"` + Categories []string `json:"categories" yaml:"categories" required:false datastore:"categories"` + Created int64 `json:"created" datastore:"created"` + Edited int64 `json:"edited" datastore:"edited"` + LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` } type WorkflowAppActionParameter struct { - Description string `json:"description" datastore:"description" yaml:"description"` - ID string `json:"id" datastore:"id" yaml:"id,omitempty"` - Name string `json:"name" datastore:"name" yaml:"name"` - Example string `json:"example" datastore:"example" yaml:"example"` - Value string `json:"value" datastore:"value" yaml:"value,omitempty"` - Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` - ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"` - Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"` - Required bool `json:"required" datastore:"required" yaml:"required"` - Configuration bool `json:"configuration" datastore:"configuration" yaml:"configuration"` - Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` + ID string `json:"id" datastore:"id" yaml:"id,omitempty"` + Name string `json:"name" datastore:"name" yaml:"name"` + Example string `json:"example" datastore:"example" yaml:"example"` + Value string `json:"value" datastore:"value,noindex" yaml:"value,omitempty"` + Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` + Options []string `json:"options" datastore:"options" yaml:"options"` + ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"` + Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"` + Required bool `json:"required" datastore:"required" yaml:"required"` + Configuration bool `json:"configuration" datastore:"configuration" yaml:"configuration"` + Tags []string `json:"tags" datastore:"tags" yaml:"tags"` + Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` + SkipMulticheck bool `json:"skip_multicheck" datastore:"skip_multicheck" yaml:"skip_multicheck"` + ValueReplace []Valuereplace `json:"value_replace" datastore:"value_replace,noindex" yaml:"value_replace,omitempty"` +} + +type Valuereplace struct { + Key string `json:"key" datastore:"key" yaml:"key"` + Value string `json:"value" datastore:"value" yaml:"value"` } type SchemaDefinition struct { @@ -148,7 +541,7 @@ type SchemaDefinition struct { } type WorkflowAppAction struct { - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id" yaml:"id,omitempty"` Name string `json:"name" datastore:"name"` Label string `json:"label" datastore:"label"` @@ -157,14 +550,15 @@ type WorkflowAppAction struct { Sharing bool `json:"sharing" datastore:"sharing"` PrivateID string `json:"private_id" datastore:"private_id"` AppID string `json:"app_id" datastore:"app_id"` - Authentication []AuthenticationStore `json:"authentication" datastore:"authentication" yaml:"authentication,omitempty"` + Tags []string `json:"tags" datastore:"tags" yaml:"tags"` + Authentication []AuthenticationStore `json:"authentication" datastore:"authentication,noindex" yaml:"authentication,omitempty"` Tested bool `json:"tested" datastore:"tested" yaml:"tested"` Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"` ExecutionVariable struct { - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id"` Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value"` + Value string `json:"value" datastore:"value,noindex"` } `json:"execution_variable" datastore:"execution_variables"` Returns struct { Description string `json:"description" datastore:"returns" yaml:"description,omitempty"` @@ -178,13 +572,15 @@ type WorkflowAppAction struct { } // FIXME: Generate a callback authentication ID? +// FIXME: Add org check .. type WorkflowExecution struct { Type string `json:"type" datastore:"type"` Status string `json:"status" datastore:"status"` Start string `json:"start" datastore:"start"` - ExecutionArgument string `json:"execution_argument" datastore:"execution_argument"` + ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` ExecutionId string `json:"execution_id" datastore:"execution_id"` ExecutionSource string `json:"execution_source" datastore:"execution_source"` + ExecutionOrg string `json:"execution_org" datastore:"execution_org"` WorkflowId string `json:"workflow_id" datastore:"workflow_id"` LastNode string `json:"last_node" datastore:"last_node"` Authorization string `json:"authorization" datastore:"authorization"` @@ -196,46 +592,46 @@ type WorkflowExecution struct { Workflow Workflow `json:"workflow" datastore:"workflow,noindex"` Results []ActionResult `json:"results" datastore:"results,noindex"` ExecutionVariables []struct { - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id"` Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value"` + Value string `json:"value" datastore:"value,noindex"` } `json:"execution_variables,omitempty" datastore:"execution_variables,omitempty"` + OrgId string `json:"org_id" datastore:"org_id"` } - -// This is for the nodes in a workflow, NOT the app action itself. type Action struct { - AppName string `json:"app_name" datastore:"app_name"` - AppVersion string `json:"app_version" datastore:"app_version"` - AppID string `json:"app_id" datastore:"app_id"` - Errors []string `json:"errors" datastore:"errors"` - ID string `json:"id" datastore:"id"` - IsValid bool `json:"is_valid" datastore:"is_valid"` - IsStartNode bool `json:"isStartNode" datastore:"isStartNode"` - Sharing bool `json:"sharing" datastore:"sharing"` - PrivateID string `json:"private_id" datastore:"private_id"` - Label string `json:"label" datastore:"label"` - SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` - Environment string `json:"environment" datastore:"environment"` - Name string `json:"name" datastore:"name"` + AppName string `json:"app_name,omitempty" datastore:"app_name"` + AppVersion string `json:"app_version,omitempty" datastore:"app_version"` + AppID string `json:"app_id,omitempty" datastore:"app_id"` + Errors []string `json:"errors,omitempty" datastore:"errors"` + ID string `json:"id,omitempty" datastore:"id"` + IsValid bool `json:"is_valid,omitempty" datastore:"is_valid"` + IsStartNode bool `json:"isStartNode,omitempty" datastore:"isStartNode"` + Sharing bool `json:"sharing,omitempty" datastore:"sharing"` + PrivateID string `json:"private_id,omitempty" datastore:"private_id"` + Label string `json:"label,omitempty" datastore:"label"` + SmallImage string `json:"small_image,omitempty" datastore:"small_image,noindex" required:false yaml:"small_image"` + LargeImage string `json:"large_image,omitempty" datastore:"large_image,noindex" yaml:"large_image" required:false` + Environment string `json:"environment,omitempty" datastore:"environment"` + Name string `json:"name,omitempty" datastore:"name"` Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` ExecutionVariable struct { - Description string `json:"description" datastore:"description"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value"` + Description string `json:"description,omitempty" datastore:"description,noindex"` + ID string `json:"id,omitempty" datastore:"id"` + Name string `json:"name,omitempty" datastore:"name"` + Value string `json:"value,omitempty" datastore:"value,noindex"` } `json:"execution_variable,omitempty" datastore:"execution_variable,omitempty"` Position struct { - X float64 `json:"x" datastore:"x"` - Y float64 `json:"y" datastore:"y"` - } `json:"position"` - Priority int `json:"priority" datastore:"priority"` - AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` - Example string `json:"example" datastore:"example"` - AuthNotRequired bool `json:"auth_not_required" datastore:"auth_not_required" yaml:"auth_not_required"` + X float64 `json:"x,omitempty" datastore:"x"` + Y float64 `json:"y,omitempty" datastore:"y"` + } `json:"position,omitempty"` + Priority int `json:"priority,omitempty" datastore:"priority"` + AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"` + Example string `json:"example,omitempty" datastore:"example"` + AuthNotRequired bool `json:"auth_not_required,omitempty" datastore:"auth_not_required" yaml:"auth_not_required"` } +// Added environment for location to execute type Trigger struct { AppName string `json:"app_name" datastore:"app_name"` Description string `json:"description" datastore:"description,noindex"` @@ -267,7 +663,7 @@ type Branch struct { SourceID string `json:"source_id" datastore:"source_id"` Label string `json:"label" datastore:"label"` HasError bool `json:"has_errors" datastore: "has_errors"` - Conditions []Condition `json:"conditions" datastore: "conditions"` + Conditions []Condition `json:"conditions" datastore: "conditions,noindex"` } // Same format for a lot of stuff @@ -280,8 +676,10 @@ type Condition struct { type Schedule struct { Name string `json:"name" datastore:"name"` Frequency string `json:"frequency" datastore:"frequency"` - ExecutionArgument string `json:"execution_argument" datastore:"execution_argument"` + ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` Id string `json:"id" datastore:"id"` + OrgId string `json:"org_id" datastore:"org_id"` + Environment string `json:"environment" datastore:"environment"` } type Workflow struct { @@ -293,33 +691,38 @@ type Workflow struct { ExitOnError bool `json:"exit_on_error" datastore:"exit_on_error"` StartFromTop bool `json:"start_from_top" datastore:"start_from_top"` } `json:"configuration,omitempty" datastore:"configuration"` + Created int64 `json:"created" datastore:"created"` + Edited int64 `json:"edited" datastore:"edited"` + LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` Errors []string `json:"errors,omitempty" datastore:"errors"` Tags []string `json:"tags,omitempty" datastore:"tags"` ID string `json:"id" datastore:"id"` IsValid bool `json:"is_valid" datastore:"is_valid"` Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` Start string `json:"start" datastore:"start"` Owner string `json:"owner" datastore:"owner"` Sharing string `json:"sharing" datastore:"sharing"` Org []Org `json:"org,omitempty" datastore:"org"` ExecutingOrg Org `json:"execution_org,omitempty" datastore:"execution_org"` + OrgId string `json:"org_id,omitempty" datastore:"org_id"` WorkflowVariables []struct { - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id"` Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value"` + Value string `json:"value" datastore:"value,noindex"` } `json:"workflow_variables" datastore:"workflow_variables"` ExecutionVariables []struct { - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id"` Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value"` + Value string `json:"value" datastore:"value,noindex"` } `json:"execution_variables,omitempty" datastore:"execution_variables"` + ExecutionEnvironment string `json:"execution_environment" datastore:"execution_environment"` } type ActionResult struct { - Action Action `json:"action" datastore:"action"` + Action Action `json:"action" datastore:"action,noindex"` ExecutionId string `json:"execution_id" datastore:"execution_id"` Authorization string `json:"authorization" datastore:"authorization"` Result string `json:"result" datastore:"result,noindex"` @@ -334,11 +737,11 @@ type Authentication struct { } type AuthenticationParams struct { - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` ID string `json:"id" datastore:"id" yaml:"id"` Name string `json:"name" datastore:"name" yaml:"name"` Example string `json:"example" datastore:"example" yaml:"example"` - Value string `json:"value,omitempty" datastore:"value" yaml:"value"` + Value string `json:"value,omitempty" datastore:"value,noindex" yaml:"value"` Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` Required bool `json:"required" datastore:"required" yaml:"required"` In string `json:"in" datastore:"in" yaml:"in"` @@ -348,44 +751,49 @@ type AuthenticationParams struct { type AuthenticationStore struct { Key string `json:"key" datastore:"key"` - Value string `json:"value" datastore:"value"` + Value string `json:"value" datastore:"value,noindex"` } type ExecutionRequestWrapper struct { Data []ExecutionRequest `json:"data"` } +type AppExecutionExample struct { + AppName string `json:"app_name" datastore:"app_name"` + AppVersion string `json:"app_version" datastore:"app_version"` + AppAction string `json:"app_action" datastore:"app_action"` + AppId string `json:"app_id" datastore:"app_id"` + ExampleId string `json:"example_id" datastore:"example_id"` + SuccessExamples []string `json:"success_examples" datastore:"success_examples,noindex"` + FailureExamples []string `json:"failure_examples" datastore:"failure_examples,noindex"` +} + // removes every container except itself (worker) func shutdown(executionId, workflowId string) { - dockercli, err := dockerclient.NewEnvClient() - if err != nil { - log.Printf("[ERROR] Unable to create docker client: %s", err) - os.Exit(3) - } + log.Printf("[INFO] Shutdown started") - containerOptions := types.ContainerListOptions{ - All: true, - } + // Might not be necessary because of cleanupEnv hostconfig autoremoval + if cleanupEnv == "true" && len(containerIds) > 0 { + /* + ctx := context.Background() + dockercli, err := dockerclient.NewEnvClient() + if err == nil { + log.Printf("[INFO] Cleaning up %d containers", len(containerIds)) + removeOptions := types.ContainerRemoveOptions{ + RemoveVolumes: true, + Force: true, + } - containers, err := dockercli.ContainerList(context.Background(), containerOptions) - if err != nil { - panic(err) - } - _ = containers - - for _, container := range containers { - for _, name := range container.Names { - if strings.Contains(name, executionId) { - // FIXME - reinstate - not here for debugging - //err = removeContainer(container.ID) - //if err != nil { - // log.Printf("Failed removing %s before shutdown.", name) - //} - - break + for _, containername := range containerIds { + log.Printf("[INFO] Should stop and and remove container %s (deprecated)", containername) + //dockercli.ContainerStop(ctx, containername, nil) + //dockercli.ContainerRemove(ctx, containername, removeOptions) + //removeContainers = append(removeContainers, containername) + } } - } - + */ + } else { + log.Printf("[INFO] NOT cleaning up containers. IDS: %d, CLEANUP env: %s", len(containerIds), cleanupEnv) } fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowId, executionId) @@ -431,7 +839,10 @@ func shutdown(executionId, workflowId string) { log.Printf("[INFO] Failed abort request: %s", err) } - log.Printf("[INFO] Finished shutdown.") + sleepDuration := 0 + log.Printf("[INFO] Finished shutdown (after %d seconds).", sleepDuration) + // Allows everything to finish in subprocesses + time.Sleep(time.Duration(sleepDuration) * time.Second) os.Exit(3) } @@ -443,6 +854,10 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] Type: "json-file", Config: map[string]string{}, }, + Resources: container.Resources{ + CPUShares: 256, + CPUPeriod: 10000, + }, } // form container id and use it as network source if it's not empty @@ -452,6 +867,11 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] log.Printf("[WARNING] Empty self container id, continue without NetworkMode") } + // Removing because log extraction should happen first + if cleanupEnv == "true" { + hostConfig.AutoRemove = true + } + config := &container.Config{ Image: image, Env: env, @@ -467,12 +887,19 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] ) if err != nil { - log.Printf("Container error: %s", err) + log.Printf("[WARNING] Container CREATE error: %s", err) return err } - cli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{}) - log.Printf("[INFO] Container %s is created", cont.ID) + err = cli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{}) + if err != nil { + log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err) + //shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + return err + } + + log.Printf("[INFO] Container %s was created for %s", cont.ID, identifier) + containerIds = append(containerIds, cont.ID) return nil } @@ -526,30 +953,688 @@ func runFilter(workflowExecution WorkflowExecution, action Action) { } -func handleExecution(client *http.Client, req *http.Request, workflowExecution WorkflowExecution) error { - // if no onprem runs (shouldn't happen, but extra check), exit - // if there are some, load the images ASAP for the app - dockercli, err := dockerclient.NewEnvClient() - if err != nil { - log.Printf("Unable to create docker client: %s", err) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) +func handleSubworkflowExecution(client *http.Client, workflowExecution WorkflowExecution, action Trigger, baseAction Action) error { + apikey := "" + workflowId := "" + executionArgument := "" + for _, parameter := range action.Parameters { + log.Printf("Parameter name: %s", parameter.Name) + if parameter.Name == "user_apikey" { + apikey = parameter.Value + } else if parameter.Name == "workflow" { + workflowId = parameter.Value + } else if parameter.Name == "data" { + executionArgument = parameter.Value + } } - onpremApps := []string{} - startAction := workflowExecution.Start + //handleSubworkflowExecution(workflowExecution, action) + status := "SUCCESS" + baseResult := `{"success": true}` + if len(apikey) == 0 || len(workflowId) == 0 { + status = "FAILURE" + baseResult = `{"success": false}` + } else { + log.Printf("Should execute workflow %s with APIKEY %s and data %s", workflowId, apikey, executionArgument) + fullUrl := fmt.Sprintf("%s/api/workflows/%s/execute", baseUrl, workflowId) + req, err := http.NewRequest( + "POST", + fullUrl, + bytes.NewBuffer([]byte(executionArgument)), + ) + + if err != nil { + log.Printf("Error building test request: %s", err) + return err + } + + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", apikey)) + newresp, err := client.Do(req) + if err != nil { + log.Printf("Error running test request: %s", err) + return err + } + + body, err := ioutil.ReadAll(newresp.Body) + if err != nil { + log.Printf("Failed reading body when waiting: %s", err) + return err + } + + log.Printf("Execution Result: %s", body) + } + + timeNow := time.Now().Unix() + //curaction := Action{ + // AppName: baseAction.AppName, + // AppVersion: baseAction.AppVersion, + // Label: baseAction.Label, + // Name: baseAction.Name, + // ID: baseAction.ID, + //} + result := ActionResult{ + Action: baseAction, + ExecutionId: workflowExecution.ExecutionId, + Authorization: workflowExecution.Authorization, + Result: baseResult, + StartedAt: timeNow, + CompletedAt: 0, + Status: status, + } + + resultData, err := json.Marshal(result) + if err != nil { + return err + } + + fullUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl) + req, err := http.NewRequest( + "POST", + fullUrl, + bytes.NewBuffer([]byte(resultData)), + ) + + if err != nil { + log.Printf("Error building test request: %s", err) + return err + } + + newresp, err := client.Do(req) + if err != nil { + log.Printf("Error running test request: %s", err) + return err + } + + body, err := ioutil.ReadAll(newresp.Body) + if err != nil { + log.Printf("Failed reading body when waiting: %s", err) + return err + } + + log.Printf("[INFO] Subworkflow Body: %s", string(body)) + + if status == "FAILURE" { + return errors.New("[ERROR] Failed to execute subworkflow") + } else { + return nil + } +} + +func removeIndex(s []string, i int) []string { + s[len(s)-1], s[i] = s[i], s[len(s)-1] + return s[:len(s)-1] +} + +func handleExecutionResult(workflowExecution WorkflowExecution) { + if len(startAction) == 0 { + startAction = workflowExecution.Start + if len(startAction) == 0 { + log.Printf("Didn't find execution start action. Setting it to workflow start action.") + startAction = workflowExecution.Workflow.Start + } + } + + //log.Printf("NEXTACTIONS: %s", nextActions) + queueNodes := []string{} + //if len(nextActions) == 0 { + // nextActions = append(nextActions, startAction) + //} + + if len(workflowExecution.Results) == 0 { + nextActions = []string{startAction} + } else { + // This is to re-check the nodes that exist and whether they should continue + appendActions := []string{} + for _, item := range workflowExecution.Results { + + // FIXME: Check whether the item should be visited or not + // Do the same check as in walkoff.go - are the parents done? + // If skipped and both parents are skipped: keep as skipped, otherwise queue + if item.Status == "SKIPPED" { + isSkipped := true + + for _, branch := range workflowExecution.Workflow.Branches { + // 1. Finds branches where the destination is our node + // 2. Finds results of those branches, and sees the status + // 3. If the status isn't skipped or failure, then it will still run this node + if branch.DestinationID == item.Action.ID { + for _, subresult := range workflowExecution.Results { + if subresult.Action.ID == branch.SourceID { + if subresult.Status != "SKIPPED" && subresult.Status != "FAILURE" { + log.Printf("\n\n\nSUBRESULT PARENT STATUS: %s\n\n\n", subresult.Status) + isSkipped = false + + break + } + } + } + } + } + + if isSkipped { + //log.Printf("Skipping %s as all parents are done", item.Action.Label) + if !arrayContains(visited, item.Action.ID) { + log.Printf("Adding visited (1): %s", item.Action.Label) + visited = append(visited, item.Action.ID) + } + } else { + log.Printf("Continuing %s as all parents are NOT done", item.Action.Label) + appendActions = append(appendActions, item.Action.ID) + } + } else { + if item.Status == "FINISHED" { + log.Printf("Adding visited (2): %s", item.Action.Label) + visited = append(visited, item.Action.ID) + } + } + + //if len(nextActions) == 0 { + //nextActions = append(nextActions, children[item.Action.ID]...) + for _, child := range children[item.Action.ID] { + if !arrayContains(nextActions, child) && !arrayContains(visited, child) && !arrayContains(visited, child) { + nextActions = append(nextActions, child) + } + } + + if len(appendActions) > 0 { + log.Printf("APPENDED NODES: %#v", appendActions) + nextActions = append(nextActions, appendActions...) + } + } + } + + //log.Printf("Nextactions: %s", nextActions) + // This is a backup in case something goes wrong in this complex hellhole. + // Max default execution time is 5 minutes for now anyway, which should take + // care if it gets stuck in a loop. + // FIXME: Force killing a worker should result in a notification somewhere + if len(nextActions) == 0 { + log.Printf("No next action. Finished? Result vs Actions: %d - %d", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)) + exit := true + for _, item := range workflowExecution.Results { + if item.Status == "EXECUTING" { + exit = false + break + } + } + + if len(environments) == 1 { + log.Printf("[INFO] Should send results to the backend because environments are %s", environments) + validateFinished(workflowExecution) + } + + if exit && len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions) { + log.Printf("Shutting down.") + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } + + // Look for the NEXT missing action + notFound := []string{} + for _, action := range workflowExecution.Workflow.Actions { + found := false + for _, result := range workflowExecution.Results { + if action.ID == result.Action.ID { + found = true + break + } + } + + if !found { + notFound = append(notFound, action.ID) + } + } + + //log.Printf("SOMETHING IS MISSING!: %#v", notFound) + for _, item := range notFound { + if arrayContains(executed, item) { + log.Printf("%s has already executed but no result!", item) + return + } + + // Visited means it's been touched in any way. + outerIndex := -1 + for index, visit := range visited { + if visit == item { + outerIndex = index + break + } + } + + if outerIndex >= 0 { + log.Printf("Removing index %s from visited") + visited = append(visited[:outerIndex], visited[outerIndex+1:]...) + } + + fixed := 0 + for _, parent := range parents[item] { + parentResult := getResult(workflowExecution, parent) + if parentResult.Status == "FINISHED" || parentResult.Status == "SUCCESS" || parentResult.Status == "SKIPPED" || parentResult.Status == "FAILURE" { + fixed += 1 + } + } + + if fixed == len(parents[item]) { + nextActions = append(nextActions, item) + } + + // If it's not executed and not in nextActions + // FIXME: Check if the item's parents are finished. If they're not, skip. + } + } + + //log.Printf("Checking nextactions: %s", nextActions) + for _, node := range nextActions { + nodeChildren := children[node] + for _, child := range nodeChildren { + if !arrayContains(queueNodes, child) { + queueNodes = append(queueNodes, child) + } + } + } + + // IF NOT VISITED && IN toExecuteOnPrem + // SKIP if it's not onprem + toRemove := []int{} + for index, nextAction := range nextActions { + action := getAction(workflowExecution, nextAction, environment) + // check visited and onprem + if arrayContains(visited, nextAction) { + //log.Printf("ALREADY VISITIED (%s): %s", action.Label, nextAction) + toRemove = append(toRemove, index) + //nextActions = removeIndex(nextActions, index) + + //validateFinished(workflowExecution) + _ = index + + continue + } + + if action.AppName == "Shuffle Workflow" { + //log.Printf("SHUFFLE WORKFLOW: %#v", action) + action.Environment = environment + action.AppName = "shuffle-subflow" + action.Name = "run_subflow" + action.AppVersion = "1.0.0" + + //appname := action.AppName + //appversion := action.AppVersion + //appname = strings.Replace(appname, ".", "-", -1) + //appversion = strings.Replace(appversion, ".", "-", -1) + // shuffle-subflow_1.0.0 + + //visited = append(visited, action.ID) + //executed = append(executed, action.ID) + + trigger := Trigger{} + for _, innertrigger := range workflowExecution.Workflow.Triggers { + if innertrigger.ID == action.ID { + trigger = innertrigger + break + } + } + + action.Parameters = []WorkflowAppActionParameter{} + for _, parameter := range trigger.Parameters { + parameter.Variant = "STATIC_VALUE" + action.Parameters = append(action.Parameters, parameter) + } + + //trigger.LargeImage = "" + //err = handleSubworkflowExecution(client, workflowExecution, trigger, action) + //if err != nil { + // log.Printf("[ERROR] Failed to execute subworkflow: %s", err) + //} else { + // log.Printf("[INFO] Executed subworkflow!") + //} + //continue + } else if action.AppName == "User Input" { + log.Printf("USER INPUT!") + + if action.ID == workflowExecution.Start { + log.Printf("Skipping because it's the startnode") + visited = append(visited, action.ID) + executed = append(executed, action.ID) + continue + } else { + log.Printf("Should stop after this iteration because it's user-input based. %#v", action) + trigger := Trigger{} + for _, innertrigger := range workflowExecution.Workflow.Triggers { + if innertrigger.ID == action.ID { + trigger = innertrigger + break + } + } + + trigger.LargeImage = "" + triggerData, err := json.Marshal(trigger) + if err != nil { + log.Printf("Failed unmarshalling action: %s", err) + triggerData = []byte("Failed unmarshalling. Cancel execution!") + } + + err = runUserInput(topClient, action, workflowExecution.Workflow.ID, workflowExecution.ExecutionId, workflowExecution.Authorization, string(triggerData)) + if err != nil { + log.Printf("Failed launching backend magic: %s", err) + os.Exit(3) + } else { + log.Printf("Launched user input node succesfully!") + os.Exit(3) + } + + break + } + } else { + //log.Printf("Handling action %#v", action) + } + + if len(toRemove) > 0 { + //toRemove = []int{} + //for index, nextAction := range nextActions { + } + + // Not really sure how this edgecase happens. + + // FIXME + // Execute, as we don't really care if env is not set? IDK + if action.Environment != environment { //&& action.Environment != "" { + //log.Printf("Action: %#v", action) + log.Printf("Bad environment for node: %s. Want %s", action.Environment, environment) + continue + } + + // check whether the parent is finished executing + //log.Printf("%s has %d parents", nextAction, len(parents[nextAction])) + + continueOuter := true + if action.IsStartNode { + continueOuter = false + } else if len(parents[nextAction]) > 0 { + // FIXME - wait for parents to finishe executing + fixed := 0 + for _, parent := range parents[nextAction] { + parentResult := getResult(workflowExecution, parent) + if parentResult.Status == "FINISHED" || parentResult.Status == "SUCCESS" || parentResult.Status == "SKIPPED" || parentResult.Status == "FAILURE" { + fixed += 1 + } + } + + if fixed == len(parents[nextAction]) { + continueOuter = false + } + } else { + continueOuter = false + } + + if continueOuter { + log.Printf("Parents of %s aren't finished: %s", nextAction, strings.Join(parents[nextAction], ", ")) + //for _, tmpaction := range parents[nextAction] { + // action := getAction(workflowExecution, tmpaction) + // _ = action + // //log.Printf("Parent: %s", action.Label) + //} + // Find the result of the nodes? + continue + } + + // get action status + actionResult := getResult(workflowExecution, nextAction) + if actionResult.Action.ID == action.ID { + log.Printf("%s already has status %s.", action.ID, actionResult.Status) + continue + } else { + log.Printf("%s:%s has no status result yet. Should execute.", action.Name, action.ID) + } + + appname := action.AppName + appversion := action.AppVersion + appname = strings.Replace(appname, ".", "-", -1) + appversion = strings.Replace(appversion, ".", "-", -1) + + image := fmt.Sprintf("%s:%s_%s", baseimagename, action.AppName, action.AppVersion) + if strings.Contains(image, " ") { + image = strings.ReplaceAll(image, " ", "-") + } + + identifier := fmt.Sprintf("%s_%s_%s_%s", appname, appversion, action.ID, workflowExecution.ExecutionId) + if strings.Contains(identifier, " ") { + identifier = strings.ReplaceAll(identifier, " ", "-") + } + + // FIXME - check whether it's running locally yet too + dockercli, err := dockerclient.NewEnvClient() + if err != nil { + log.Printf("[ERROR] Unable to create docker client (2): %s", err) + //return err + return + } + + stats, err := dockercli.ContainerInspect(context.Background(), identifier) + if err != nil || stats.ContainerJSONBase.State.Status != "running" { + // REMOVE + if err == nil { + log.Printf("Status: %s, should kill: %s", stats.ContainerJSONBase.State.Status, identifier) + err = removeContainer(identifier) + if err != nil { + log.Printf("Error killing container: %s", err) + } + } else { + //log.Printf("WHAT TO DO HERE?: %s", err) + } + } else if stats.ContainerJSONBase.State.Status == "running" { + //log.Printf(" + continue + } + + if len(action.Parameters) == 0 { + action.Parameters = []WorkflowAppActionParameter{} + } + + if len(action.Errors) == 0 { + action.Errors = []string{} + } + + // marshal action and put it in there rofl + log.Printf("Time to execute %s (%s) with app %s:%s, function %s, env %s with %d parameters.", action.ID, action.Label, action.AppName, action.AppVersion, action.Name, action.Environment, len(action.Parameters)) + + actionData, err := json.Marshal(action) + if err != nil { + log.Printf("Failed unmarshalling action: %s", err) + continue + } + + if action.AppID == "0ca8887e-b4af-4e3e-887c-87e9d3bc3d3e" { + log.Printf("\nShould run filter: %#v\n\n", action) + runFilter(workflowExecution, action) + continue + } + + executionData, err := json.Marshal(workflowExecution) + if err != nil { + log.Printf("Failed marshalling executiondata: %s", err) + executionData = []byte("") + } + + // Sending full execution so that it won't have to load in every app + // This might be an issue if they can read environments, but that's alright + // if everything is generated during execution + log.Printf("Deployed with CALLBACK_URL %s and BASE_URL %s", appCallbackUrl, baseUrl) + env := []string{ + fmt.Sprintf("ACTION=%s", string(actionData)), + fmt.Sprintf("EXECUTIONID=%s", workflowExecution.ExecutionId), + fmt.Sprintf("AUTHORIZATION=%s", workflowExecution.Authorization), + fmt.Sprintf("CALLBACK_URL=%s", baseUrl), + fmt.Sprintf("BASE_URL=%s", appCallbackUrl), + } + + // Fixes issue: + // standard_init_linux.go:185: exec user process caused "argument list too long" + // https://devblogs.microsoft.com/oldnewthing/20100203-00/?p=15083 + maxSize := 32700 - len(string(actionData)) - 2000 + if len(executionData) < maxSize { + log.Printf("[INFO] ADDING FULL_EXECUTION because size is smaller than %d", maxSize) + env = append(env, fmt.Sprintf("FULL_EXECUTION=%s", string(executionData))) + } else { + log.Printf("[WARNING] Skipping FULL_EXECUTION because size is larger than %d", maxSize) + } + + // Uses a few ways of getting / checking if an app is available + // 1. Try original + // 2. Go to lowercase + // 3. Add remote repo location + // 4. Actually download last repo + + images := []string{ + fmt.Sprintf("%s/%s:%s_%s", registryName, baseimagename, strings.ToLower(action.AppName), action.AppVersion), + image, + fmt.Sprintf("%s:%s_%s", baseimagename, strings.ToLower(action.AppName), action.AppVersion), + } + + // If cleanup is set, it should run for efficiency + pullOptions := types.ImagePullOptions{} + if cleanupEnv == "true" { + err = deployApp(dockercli, images[0], identifier, env) + if err != nil { + log.Printf("[WARNING] Failed CLEANUP execution. Downloading image remotely.") + reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) + if err != nil { + log.Printf("[ERROR] Failed getting %s. The couldn't be find locally, AND is missing.", image) + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } + + buildBuf := new(strings.Builder) + _, err = io.Copy(buildBuf, reader) + if err != nil { + log.Printf("[ERROR] Error in IO copy: %s", err) + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } else { + if strings.Contains(buildBuf.String(), "errorDetail") { + log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image) + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } + + log.Printf("[INFO] Successfully downloaded %s", image) + } + + err = deployApp(dockercli, image, identifier, env) + if err != nil { + + log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") + if strings.Contains(err.Error(), "No such image") { + //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) + log.Printf("[ERROR] Image doesn't exist. Shutting down") + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } + } + } + } else { + + err = deployApp(dockercli, image, identifier, env) + if err != nil { + // Trying to replace with lowercase to deploy again. This seems to work with Dockerhub well. + // FIXME: Should try to remotely download directly if this persists. + image = fmt.Sprintf("%s:%s_%s", baseimagename, strings.ToLower(action.AppName), action.AppVersion) + if strings.Contains(image, " ") { + image = strings.ReplaceAll(image, " ", "-") + } + + err = deployApp(dockercli, image, identifier, env) + if err != nil { + image = fmt.Sprintf("%s/%s:%s_%s", registryName, baseimagename, strings.ToLower(action.AppName), action.AppVersion) + if strings.Contains(image, " ") { + image = strings.ReplaceAll(image, " ", "-") + } + + err = deployApp(dockercli, image, identifier, env) + if err != nil { + log.Printf("[WARNING] Failed deploying image THRICE. Attempting to download the latter as last resort.") + reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) + if err != nil { + log.Printf("[ERROR] Failed getting %s. The couldn't be find locally, AND is missing.", image) + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } + + buildBuf := new(strings.Builder) + _, err = io.Copy(buildBuf, reader) + if err != nil { + log.Printf("[ERROR] Error in IO copy: %s", err) + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } else { + if strings.Contains(buildBuf.String(), "errorDetail") { + log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image) + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } + + log.Printf("[INFO] Successfully downloaded %s", image) + } + + err = deployApp(dockercli, image, identifier, env) + if err != nil { + + log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") + if strings.Contains(err.Error(), "No such image") { + //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) + log.Printf("[ERROR] Image doesn't exist. Shutting down") + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } + } + } + } + } + } + + log.Printf("Adding visited (3): %s", action.Label) + + visited = append(visited, action.ID) + executed = append(executed, action.ID) + + // If children of action.ID are NOT in executed: + // Remove them from visited. + //log.Printf("EXECUTED: %#v", executed) + } + + //log.Println(nextAction) + //log.Println(startAction, children[startAction]) + + // FIXME - new request here + // FIXME - clean up stopped (remove) containers with this execution id + + if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extra { + shutdownCheck := true + for _, result := range workflowExecution.Results { + if result.Status == "EXECUTING" { + // Cleaning up executing stuff + shutdownCheck = false + // USED TO BE CONTAINER REMOVAL + // FIXME - send POST request to kill the container + //log.Printf("Should remove (POST request) stopped containers") + //ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) + } + } + + if shutdownCheck { + log.Println("BREAKING BECAUSE RESULTS IS SAME LENGTH AS ACTIONS. SHOULD CHECK ALL RESULTS FOR WHETHER THEY'RE DONE") + validateFinished(workflowExecution) + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } + } + + time.Sleep(time.Duration(sleepTime) * time.Second) + return +} + +func executionInit(workflowExecution WorkflowExecution) error { + parents = map[string][]string{} + children = map[string][]string{} + triggersHandled := []string{} + + startAction = workflowExecution.Start if len(startAction) == 0 { log.Printf("Didn't find execution start action. Setting it to workflow start action.") startAction = workflowExecution.Workflow.Start } - log.Printf("Startaction: %s", startAction) - toExecuteOnprem := []string{} - parents := map[string][]string{} - children := map[string][]string{} + nextActions = append(nextActions, startAction) - // source = parent node, dest = child node - // parent can have more children, child can have more parents - extra := 0 for _, branch := range workflowExecution.Workflow.Branches { // Check what the parent is first. If it's trigger - skip sourceFound := false @@ -565,17 +1650,31 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W } for _, trigger := range workflowExecution.Workflow.Triggers { - if trigger.AppName != "User Input" { - continue - } + //log.Printf("Appname trigger (0): %s", trigger.AppName) + if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" { + //log.Printf("%s is a special trigger. Checking where.", trigger.AppName) - if trigger.ID == branch.SourceID { - sourceFound = true - extra += 1 - } + found := false + for _, check := range triggersHandled { + if check == trigger.ID { + found = true + break + } + } - if trigger.ID == branch.DestinationID { - destinationFound = true + if !found { + extra += 1 + } else { + triggersHandled = append(triggersHandled, trigger.ID) + } + + if trigger.ID == branch.SourceID { + log.Printf("Trigger %s is the source!", trigger.AppName) + sourceFound = true + } else if trigger.ID == branch.DestinationID { + log.Printf("Trigger %s is the destination!", trigger.AppName) + destinationFound = true + } } } @@ -593,6 +1692,8 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W } log.Printf("Actions: %d + Special Triggers: %d", len(workflowExecution.Workflow.Actions), extra) + onpremApps := []string{} + toExecuteOnprem := []string{} for _, action := range workflowExecution.Workflow.Actions { if action.Environment != environment { continue @@ -638,470 +1739,75 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W //log.Printf("Successfully downloaded and built %s", image) } + return nil +} + +func handleExecution(client *http.Client, req *http.Request, workflowExecution WorkflowExecution) error { + // if no onprem runs (shouldn't happen, but extra check), exit + // if there are some, load the images ASAP for the app + + err := executionInit(workflowExecution) + if err != nil { + log.Printf("[INFO] Workflow setup failed: %s", workflowExecution.ExecutionId, err) + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } + + log.Printf("Startaction: %s", startAction) + + // source = parent node, dest = child node + // parent can have more children, child can have more parents // Process the parents etc. How? - visited := []string{} - executed := []string{} - nextActions := []string{startAction} - firstIteration := true for { - queueNodes := []string{} + handleExecutionResult(workflowExecution) - if len(workflowExecution.Results) == 0 { - nextActions = []string{startAction} - } else if firstIteration { - firstIteration = false - } else { - // This is to re-check the nodes that exist and whether they should continue - appendActions := []string{} - for _, item := range workflowExecution.Results { + //fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowExecution.Workflow.ID, workflowExecution.ExecutionId) + fullUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl) + log.Printf("URL: %s", fullUrl) + req, err := http.NewRequest( + "POST", + fullUrl, + bytes.NewBuffer([]byte(data)), + ) - // FIXME: Check whether the item should be visited or not - // Do the same check as in walkoff.go - are the parents done? - // If skipped and both parents are skipped: keep as skipped, otherwise queue - if item.Status == "SKIPPED" { - isSkipped := true - - for _, branch := range workflowExecution.Workflow.Branches { - // 1. Finds branches where the destination is our node - // 2. Finds results of those branches, and sees the status - // 3. If the status isn't skipped or failure, then it will still run this node - if branch.DestinationID == item.Action.ID { - for _, subresult := range workflowExecution.Results { - if subresult.Action.ID == branch.SourceID { - if subresult.Status != "SKIPPED" && subresult.Status != "FAILURE" { - log.Printf("\n\n\nSUBRESULT PARENT STATUS: %s\n\n\n", subresult.Status) - isSkipped = false - - break - } - } - } - } - } - - if isSkipped { - //log.Printf("Skipping %s as all parents are done", item.Action.Label) - if !arrayContains(visited, item.Action.ID) { - log.Printf("Adding visited (1): %s", item.Action.Label) - visited = append(visited, item.Action.ID) - } - } else { - log.Printf("Continuing %s as all parents are NOT done", item.Action.Label) - appendActions = append(appendActions, item.Action.ID) - } - } else { - if item.Status == "FINISHED" { - log.Printf("Adding visited (2): %s", item.Action.Label) - visited = append(visited, item.Action.ID) - } - } - - nextActions = children[item.Action.ID] - if len(appendActions) > 0 { - log.Printf("APPENDED NODES: %#v", appendActions) - nextActions = append(nextActions, appendActions...) - } - } - } - - // This is a backup in case something goes wrong in this complex hellhole. - // Max default execution time is 5 minutes for now anyway, which should take - // care if it gets stuck in a loop. - // FIXME: Force killing a worker should result in a notification somewhere - if len(nextActions) == 0 { - log.Printf("No next action. Finished? Result vs Actions: %d - %d", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)) - exit := true - for _, item := range workflowExecution.Results { - if item.Status == "EXECUTING" { - exit = false - break - } - } - - if exit && len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions) { - log.Printf("Shutting down.") - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) - } - - // Look for the NEXT missing action - notFound := []string{} - for _, action := range workflowExecution.Workflow.Actions { - found := false - for _, result := range workflowExecution.Results { - if action.ID == result.Action.ID { - found = true - break - } - } - - if !found { - notFound = append(notFound, action.ID) - } - } - - //log.Printf("SOMETHING IS MISSING!: %#v", notFound) - for _, item := range notFound { - if arrayContains(executed, item) { - log.Printf("%s has already executed but no result!", item) - continue - } - - // Visited means it's been touched in any way. - outerIndex := -1 - for index, visit := range visited { - if visit == item { - outerIndex = index - break - } - } - - if outerIndex >= 0 { - log.Printf("Removing index %s from visited") - visited = append(visited[:outerIndex], visited[outerIndex+1:]...) - } - - fixed := 0 - for _, parent := range parents[item] { - parentResult := getResult(workflowExecution, parent) - if parentResult.Status == "FINISHED" || parentResult.Status == "SUCCESS" || parentResult.Status == "SKIPPED" || parentResult.Status == "FAILURE" { - fixed += 1 - } - } - - if fixed == len(parents[item]) { - nextActions = append(nextActions, item) - } - - // If it's not executed and not in nextActions - // FIXME: Check if the item's parents are finished. If they're not, skip. - } - } - - for _, node := range nextActions { - nodeChildren := children[node] - for _, child := range nodeChildren { - if !arrayContains(queueNodes, child) { - queueNodes = append(queueNodes, child) - } - } - } - //log.Printf("NEXT: %s", nextActions) - //log.Printf("queueNodes: %s", queueNodes) - - // IF NOT VISITED && IN toExecuteOnPrem - // SKIP if it's not onprem - for _, nextAction := range nextActions { - action := getAction(workflowExecution, nextAction, environment) - // check visited and onprem - if arrayContains(visited, nextAction) { - log.Printf("ALREADY VISITIED (%s): %s", action.Label, nextAction) - continue - } - - if action.AppName == "User Input" { - log.Printf("USER INPUT!") - - if action.ID == workflowExecution.Start { - log.Printf("Skipping because it's the startnode") - visited = append(visited, action.ID) - executed = append(executed, action.ID) - continue - } else { - log.Printf("Should stop after this iteration because it's user-input based. %#v", action) - trigger := Trigger{} - for _, innertrigger := range workflowExecution.Workflow.Triggers { - if innertrigger.ID == action.ID { - trigger = innertrigger - break - } - } - - trigger.LargeImage = "" - triggerData, err := json.Marshal(trigger) - if err != nil { - log.Printf("Failed unmarshalling action: %s", err) - triggerData = []byte("Failed unmarshalling. Cancel execution!") - } - - err = runUserInput(client, action, workflowExecution.Workflow.ID, workflowExecution.ExecutionId, workflowExecution.Authorization, string(triggerData)) - if err != nil { - log.Printf("Failed launching backend magic: %s", err) - os.Exit(3) - } else { - log.Printf("Launched user input node succesfully!") - os.Exit(3) - } - - break - } - } - - // Not really sure how this edgecase happens. - - // FIXME - // Execute, as we don't really care if env is not set? IDK - if action.Environment != environment { //&& action.Environment != "" { - log.Printf("Bad environment for node: %s. Want %s", action.Environment, environment) - continue - } - - // check whether the parent is finished executing - //log.Printf("%s has %d parents", nextAction, len(parents[nextAction])) - - continueOuter := true - if action.IsStartNode { - continueOuter = false - } else if len(parents[nextAction]) > 0 { - // FIXME - wait for parents to finishe executing - fixed := 0 - for _, parent := range parents[nextAction] { - parentResult := getResult(workflowExecution, parent) - if parentResult.Status == "FINISHED" || parentResult.Status == "SUCCESS" || parentResult.Status == "SKIPPED" || parentResult.Status == "FAILURE" { - fixed += 1 - } - } - - if fixed == len(parents[nextAction]) { - continueOuter = false - } - } else { - continueOuter = false - } - - if continueOuter { - log.Printf("Parents of %s aren't finished: %s", nextAction, strings.Join(parents[nextAction], ", ")) - //for _, tmpaction := range parents[nextAction] { - // action := getAction(workflowExecution, tmpaction) - // _ = action - // //log.Printf("Parent: %s", action.Label) - //} - // Find the result of the nodes? - continue - } - - // get action status - actionResult := getResult(workflowExecution, nextAction) - if actionResult.Action.ID == action.ID { - log.Printf("%s already has status %s.", action.ID, actionResult.Status) - continue - } else { - log.Printf("%s:%s has no status result yet. Should execute.", action.Name, action.ID) - } - - appname := action.AppName - appversion := action.AppVersion - appname = strings.Replace(appname, ".", "-", -1) - appversion = strings.Replace(appversion, ".", "-", -1) - - image := fmt.Sprintf("%s:%s_%s", baseimagename, action.AppName, action.AppVersion) - if strings.Contains(image, " ") { - image = strings.ReplaceAll(image, " ", "-") - } - - identifier := fmt.Sprintf("%s_%s_%s_%s", appname, appversion, action.ID, workflowExecution.ExecutionId) - if strings.Contains(identifier, " ") { - identifier = strings.ReplaceAll(identifier, " ", "-") - } - - // FIXME - check whether it's running locally yet too - stats, err := dockercli.ContainerInspect(context.Background(), identifier) - if err != nil || stats.ContainerJSONBase.State.Status != "running" { - // REMOVE - if err == nil { - log.Printf("Status: %s, should kill: %s", stats.ContainerJSONBase.State.Status, identifier) - err = removeContainer(identifier) - if err != nil { - log.Printf("Error killing container: %s", err) - } - } else { - //log.Printf("WHAT TO DO HERE?: %s", err) - } - } else if stats.ContainerJSONBase.State.Status == "running" { - continue - } - - if len(action.Parameters) == 0 { - action.Parameters = []WorkflowAppActionParameter{} - } - - if len(action.Errors) == 0 { - action.Errors = []string{} - } - - // marshal action and put it in there rofl - log.Printf("Time to execute %s (%s) with app %s:%s, function %s, env %s with %d parameters.", action.ID, action.Label, action.AppName, action.AppVersion, action.Name, action.Environment, len(action.Parameters)) - - actionData, err := json.Marshal(action) - if err != nil { - log.Printf("Failed unmarshalling action: %s", err) - continue - } - - if action.AppID == "0ca8887e-b4af-4e3e-887c-87e9d3bc3d3e" { - log.Printf("\nShould run filter: %#v\n\n", action) - runFilter(workflowExecution, action) - continue - } - - executionData, err := json.Marshal(workflowExecution) - if err != nil { - log.Printf("Failed marshalling executiondata: %s", err) - executionData = []byte("") - } - - // Sending full execution so that it won't have to load in every app - // This might be an issue if they can read environments, but that's alright - // if everything is generated during execution - env := []string{ - fmt.Sprintf("ACTION=%s", string(actionData)), - fmt.Sprintf("EXECUTIONID=%s", workflowExecution.ExecutionId), - fmt.Sprintf("AUTHORIZATION=%s", workflowExecution.Authorization), - fmt.Sprintf("CALLBACK_URL=%s", baseUrl), - } - - // Fixes issue: - // standard_init_linux.go:185: exec user process caused "argument list too long" - // https://devblogs.microsoft.com/oldnewthing/20100203-00/?p=15083 - maxSize := 32700 - len(string(actionData)) - 2000 - if len(executionData) < maxSize { - log.Printf("ADDING FULL_EXECUTION because size is smaller than %d", maxSize) - env = append(env, fmt.Sprintf("FULL_EXECUTION=%s", string(executionData))) - } else { - log.Printf("Skipping FULL_EXECUTION because size is larger than %d", maxSize) - } - - err = deployApp(dockercli, image, identifier, env) - if err != nil { - log.Printf("[ERROR] Failed deploying %s from image %s: %s", identifier, image, err) - if strings.Contains(err.Error(), "No such image") { - log.Printf("[ERROR] Image doesn't exist. Shutting down") - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) - } - } - - log.Printf("Adding visited (3): %s", action.Label) - - visited = append(visited, action.ID) - executed = append(executed, action.ID) - - // If children of action.ID are NOT in executed: - // Remove them from visited. - //log.Printf("EXECUTED: %#v", executed) - } - - //log.Println(nextAction) - //log.Println(startAction, children[startAction]) - - // FIXME - new request here - // FIXME - clean up stopped (remove) containers with this execution id - newresp, err := client.Do(req) + newresp, err := topClient.Do(req) if err != nil { - log.Printf("Failed making request: %s", err) + log.Printf("[ERROR] Failed making request: %s", err) time.Sleep(time.Duration(sleepTime) * time.Second) continue } body, err := ioutil.ReadAll(newresp.Body) if err != nil { - log.Printf("Failed reading body: %s", err) + log.Printf("[ERROR] Failed reading body: %s", err) time.Sleep(time.Duration(sleepTime) * time.Second) continue } if newresp.StatusCode != 200 { - log.Printf("Err: %s\nStatusCode: %d", string(body), newresp.StatusCode) + log.Printf("[ERROR] Bad statuscode: %d, %s", newresp.StatusCode, string(body)) + //shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) time.Sleep(time.Duration(sleepTime) * time.Second) continue } err = json.Unmarshal(body, &workflowExecution) if err != nil { - log.Printf("Failed workflowExecution unmarshal: %s", err) + log.Printf("[ERROR] Failed workflowExecution unmarshal: %s", err) time.Sleep(time.Duration(sleepTime) * time.Second) continue } if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" { - log.Printf("Workflow %s is finished. Exiting worker.", workflowExecution.ExecutionId) + log.Printf("[INFO] Workflow %s is finished. Exiting worker.", workflowExecution.ExecutionId) shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) } - log.Printf("Status: %s, Results: %d, actions: %d", workflowExecution.Status, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extra) + log.Printf("[INFO] Status: %s, Results: %d, actions: %d", workflowExecution.Status, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extra) if workflowExecution.Status != "EXECUTING" { - log.Printf("Exiting as worker execution has status %s!", workflowExecution.Status) + log.Printf("[WARNING] Exiting as worker execution has status %s!", workflowExecution.Status) shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) } - if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extra { - shutdownCheck := true - ctx := context.Background() - for _, result := range workflowExecution.Results { - if result.Status == "EXECUTING" { - // Cleaning up executing stuff - shutdownCheck = false - // Check status - - containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{ - All: true, - }) - if err != nil { - log.Printf("Failed listing containers: %s", err) - continue - } - - stopContainers := []string{} - removeContainers := []string{} - for _, container := range containers { - for _, name := range container.Names { - if !strings.Contains(name, result.Action.ID) { - continue - } - - if container.State != "running" { - removeContainers = append(removeContainers, container.ID) - stopContainers = append(stopContainers, container.ID) - } - } - } - - // FIXME - add killing of apps with same execution ID too - // FIXME - stahp - //for _, containername := range stopContainers { - // if err := dockercli.ContainerStop(ctx, containername, nil); err != nil { - // log.Printf("Unable to stop container: %s", err) - // } else { - // log.Printf("Stopped container %s", containername) - // } - //} - - removeOptions := types.ContainerRemoveOptions{ - RemoveVolumes: true, - Force: true, - } - - _ = removeOptions - - // FIXME - this - //for _, containername := range removeContainers { - // if err := dockercli.ContainerRemove(ctx, containername, removeOptions); err != nil { - // log.Printf("Unable to remove container: %s", err) - // } else { - // log.Printf("Removed container %s", containername) - // } - //} - - // FIXME - send POST request to kill the container - log.Printf("Should remove (POST request) stopped containers") - //ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) - } - } - - if shutdownCheck { - log.Println("BREAKING BECAUSE RESULTS IS SAME LENGTH AS ACTIONS. SHOULD CHECK ALL RESULTS FOR WHETHER THEY'RE DONE") - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) - } - } - time.Sleep(time.Duration(sleepTime) * time.Second) } return nil @@ -1142,6 +1848,7 @@ func getAction(workflowExecution WorkflowExecution, id, environment string) Acti AppName: trigger.AppName, Name: trigger.AppName, Environment: environment, + Label: trigger.Label, } log.Printf("FOUND TRIGGER: %#v!", trigger) } @@ -1191,7 +1898,7 @@ func runUserInput(client *http.Client, action Action, workflowId, workflowExecut return err } - log.Printf("[INFO] Body: %s", string(body)) + log.Printf("[INFO] User Input Body: %s", string(body)) return nil } @@ -1221,7 +1928,7 @@ func runTestExecution(client *http.Client, workflowId, apikey string) (string, s return "", "" } - log.Printf("[INFO] Body: %s", string(body)) + log.Printf("[INFO] Test Body: %s", string(body)) var workflowExecution WorkflowExecution err = json.Unmarshal(body, &workflowExecution) if err != nil { @@ -1232,6 +1939,722 @@ func runTestExecution(client *http.Client, workflowId, apikey string) (string, s return workflowExecution.Authorization, workflowExecution.ExecutionId } +func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Println("(3) Failed reading body for workflowqueue") + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + + //log.Printf("Got result: %s", string(body)) + var actionResult ActionResult + err = json.Unmarshal(body, &actionResult) + if err != nil { + log.Printf("Failed ActionResult unmarshaling: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + + // 1. Get the WorkflowExecution(ExecutionId) from the database + // 2. if ActionResult.Authentication != WorkflowExecution.Authentication -> exit + // 3. Add to and update actionResult in workflowExecution + // 4. Push to db + // IF FAIL: Set executionstatus: abort or cancel + + ctx := context.Background() + workflowExecution, err := getWorkflowExecution(ctx, actionResult.ExecutionId) + if err != nil { + log.Printf("[ERROR] Failed getting execution (workflowqueue) %s: %s", actionResult.ExecutionId, err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution ID %s because it doesn't exist."}`, actionResult.ExecutionId))) + return + } + + if workflowExecution.Authorization != actionResult.Authorization { + log.Printf("[INFO] Bad authorization key when updating node (workflowQueue) %s. Want: %s, Have: %s", actionResult.ExecutionId, workflowExecution.Authorization, actionResult.Authorization) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key"}`))) + return + } + + if workflowExecution.Status == "FINISHED" { + log.Printf("Workflowexecution is already FINISHED. No further action can be taken") + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is already finished because of %s with status %s"}`, workflowExecution.LastNode, workflowExecution.Status))) + return + } + + // Not sure what's up here + // FIXME - remove comment + if workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" { + + if workflowExecution.Workflow.Configuration.ExitOnError { + log.Printf("Workflowexecution already has status %s. No further action can be taken", workflowExecution.Status) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is aborted because of %s with result %s and status %s"}`, workflowExecution.LastNode, workflowExecution.Result, workflowExecution.Status))) + return + } else { + log.Printf("Continuing even though it's aborted.") + } + } + + //if actionResult.Status == "WAITING" && actionResult.Action.AppName == "User Input" { + // log.Printf("SHOULD WAIT A BIT AND RUN CLOUD STUFF WITH USER INPUT! WAITING!") + + // var trigger Trigger + // err = json.Unmarshal([]byte(actionResult.Result), &trigger) + // if err != nil { + // log.Printf("Failed unmarshaling actionresult for user input: %s", err) + // resp.WriteHeader(401) + // resp.Write([]byte(`{"success": false}`)) + // return + // } + + // orgId := workflowExecution.ExecutionOrg + // if len(workflowExecution.OrgId) == 0 && len(workflowExecution.Workflow.OrgId) > 0 { + // orgId = workflowExecution.Workflow.OrgId + // } + + // err := handleUserInput(trigger, orgId, workflowExecution.Workflow.ID, workflowExecution.ExecutionId) + // if err != nil { + // log.Printf("Failed userinput handler: %s", err) + // actionResult.Result = fmt.Sprintf("Cloud error: %s", err) + // workflowExecution.Results = append(workflowExecution.Results, actionResult) + // workflowExecution.Status = "ABORTED" + // err = setWorkflowExecution(ctx, *workflowExecution, true) + // if err != nil { + // log.Printf("Failed ") + // } else { + // log.Printf("Successfully set the execution to waiting.") + // } + + // resp.WriteHeader(401) + // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error: %s"}`, err))) + // } else { + // log.Printf("Successful userinput handler") + // resp.WriteHeader(200) + // resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "CLOUD IS DONE"}`))) + + // actionResult.Result = "Waiting for user feedback based on configuration" + + // workflowExecution.Results = append(workflowExecution.Results, actionResult) + // workflowExecution.Status = actionResult.Status + // err = setWorkflowExecution(ctx, *workflowExecution, true) + // if err != nil { + // log.Printf("Failed ") + // } else { + // log.Printf("Successfully set the execution to waiting.") + // } + // } + + // return + //} + + runWorkflowExecutionTransaction(ctx, 0, workflowExecution.ExecutionId, actionResult, resp) +} + +func findChildNodes(workflowExecution WorkflowExecution, nodeId string) []string { + //log.Printf("\nNODE TO FIX: %s\n\n", nodeId) + allChildren := []string{nodeId} + + // 1. Find children of this specific node + // 2. Find the children of those nodes etc. + for _, branch := range workflowExecution.Workflow.Branches { + if branch.SourceID == nodeId { + //log.Printf("Children: %s", branch.DestinationID) + allChildren = append(allChildren, branch.DestinationID) + + childNodes := findChildNodes(workflowExecution, branch.DestinationID) + for _, bottomChild := range childNodes { + found := false + for _, topChild := range allChildren { + if topChild == bottomChild { + found = true + break + } + } + + if !found { + allChildren = append(allChildren, bottomChild) + } + } + } + } + + // Remove potential duplicates + newNodes := []string{} + for _, tmpnode := range allChildren { + found := false + for _, newnode := range newNodes { + if newnode == tmpnode { + found = true + break + } + } + + if !found { + newNodes = append(newNodes, tmpnode) + } + } + + return newNodes +} + +// Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times +func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult ActionResult, resp http.ResponseWriter) { + //log.Printf("IN WORKFLOWEXECUTION SUB!") + // Should start a tx for the execution here + workflowExecution, err := getWorkflowExecution(ctx, workflowExecutionId) + if err != nil { + log.Printf("[ERROR] Failed getting execution cache: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution"}`))) + return + } + resultLength := len(workflowExecution.Results) + dbSave := false + setExecution := true + //tx, err := dbclient.NewTransaction(ctx) + //if err != nil { + // log.Printf("client.NewTransaction: %v", err) + // resp.WriteHeader(401) + // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed creating transaction"}`))) + // return + //} + + //key := datastore.NameKey("workflowexecution", workflowExecutionId, nil) + //workflowExecution := &WorkflowExecution{} + //if err := tx.Get(key, workflowExecution); err != nil { + // log.Printf("[ERROR] tx.Get bug: %v", err) + // tx.Rollback() + // resp.WriteHeader(401) + // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting the workflow key"}`))) + // return + //} + actionResult.Action = Action{ + AppName: actionResult.Action.AppName, + AppVersion: actionResult.Action.AppVersion, + Label: actionResult.Action.Label, + Name: actionResult.Action.Name, + ID: actionResult.Action.ID, + Parameters: actionResult.Action.Parameters, + } + + if actionResult.Status == "ABORTED" || actionResult.Status == "FAILURE" { + //dbSave = true + + newResults := []ActionResult{} + childNodes := []string{} + if workflowExecution.Workflow.Configuration.ExitOnError { + log.Printf("[WARNING] Actionresult is %s for node %s in %s. Should set workflowExecution and exit all running functions", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) + workflowExecution.Status = actionResult.Status + workflowExecution.LastNode = actionResult.Action.ID + // Find underlying nodes and add them + } else { + log.Printf("[WARNING] Actionresult is %s for node %s in %s. Continuing anyway because of workflow configuration.", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) + // Finds ALL childnodes to set them to SKIPPED + childNodes = findChildNodes(*workflowExecution, actionResult.Action.ID) + // Remove duplicates + //log.Printf("CHILD NODES: %d", len(childNodes)) + for _, nodeId := range childNodes { + if nodeId == actionResult.Action.ID { + continue + } + + // 1. Find the action itself + // 2. Create an actionresult + curAction := Action{ID: ""} + for _, action := range workflowExecution.Workflow.Actions { + if action.ID == nodeId { + curAction = action + break + } + } + + if len(curAction.ID) == 0 { + log.Printf("Couldn't find subnode %s", nodeId) + continue + } + + resultExists := false + for _, result := range workflowExecution.Results { + if result.Action.ID == curAction.ID { + resultExists = true + break + } + } + + if !resultExists { + // Check parents are done here. Only add it IF all parents are skipped + skipNodeAdd := false + for _, branch := range workflowExecution.Workflow.Branches { + if branch.DestinationID == nodeId { + // If the branch's source node is NOT in childNodes, it's not a skipped parent + sourceNodeFound := false + for _, item := range childNodes { + if item == branch.SourceID { + sourceNodeFound = true + break + } + } + + if !sourceNodeFound { + // FIXME: Shouldn't add skip for child nodes of these nodes. Check if this node is parent of upcoming nodes. + log.Printf("\n\n NOT setting node %s to SKIPPED", nodeId) + skipNodeAdd = true + + if !arrayContains(visited, nodeId) && !arrayContains(executed, nodeId) { + nextActions = append(nextActions, nodeId) + log.Printf("SHOULD EXECUTE NODE %s. Next actions: %s", nodeId, nextActions) + } + break + } + } + } + + if !skipNodeAdd { + newResult := ActionResult{ + Action: curAction, + ExecutionId: actionResult.ExecutionId, + Authorization: actionResult.Authorization, + Result: "Skipped because of previous node", + StartedAt: 0, + CompletedAt: 0, + Status: "SKIPPED", + } + + newResults = append(newResults, newResult) + } else { + //log.Printf("\n\nNOT adding %s as skipaction - should add to execute?", nodeId) + //var visited []string + //var executed []string + //var nextActions []string + } + } + } + } + + // Cleans up aborted, and always gives a result + lastResult := "" + // type ActionResult struct { + for _, result := range workflowExecution.Results { + if actionResult.Action.ID == result.Action.ID { + continue + } + + if result.Status == "EXECUTING" { + result.Status = actionResult.Status + result.Result = "Aborted because of error in another node (2)" + } + + if len(result.Result) > 0 { + lastResult = result.Result + } + + newResults = append(newResults, result) + } + + workflowExecution.Result = lastResult + workflowExecution.Results = newResults + } + + // FIXME rebuild to be like this or something + // workflowExecution/ExecutionId/Nodes/NodeId + // Find the appropriate action + if len(workflowExecution.Results) > 0 { + // FIXME + skip := false + found := false + outerindex := 0 + for index, item := range workflowExecution.Results { + if item.Action.ID == actionResult.Action.ID { + found = true + if item.Status == actionResult.Status { + skip = true + } + + outerindex = index + break + } + } + + if skip { + //log.Printf("Both are %s. Skipping this node", item.Status) + } else if found { + // If result exists and execution variable exists, update execution value + //log.Printf("Exec var backend: %s", workflowExecution.Results[outerindex].Action.ExecutionVariable.Name) + actionVarName := workflowExecution.Results[outerindex].Action.ExecutionVariable.Name + // Finds potential execution arguments + if len(actionVarName) > 0 { + log.Printf("EXECUTION VARIABLE LOCAL: %s", actionVarName) + for index, execvar := range workflowExecution.ExecutionVariables { + if execvar.Name == actionVarName { + // Sets the value for the variable + workflowExecution.ExecutionVariables[index].Value = actionResult.Result + break + } + } + } + + log.Printf("[INFO] Updating %s in workflow %s from %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, workflowExecution.Results[outerindex].Status, actionResult.Status) + workflowExecution.Results[outerindex] = actionResult + } else { + log.Printf("[INFO] Setting value of %s in workflow %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) + workflowExecution.Results = append(workflowExecution.Results, actionResult) + } + } else { + log.Printf("[INFO] Setting value of %s in workflow %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) + workflowExecution.Results = append(workflowExecution.Results, actionResult) + } + + // FIXME: Have a check for skippednodes and their parents + for resultIndex, result := range workflowExecution.Results { + if result.Status != "SKIPPED" { + continue + } + + // Checks if all parents are skipped or failed. Otherwise removes them from the results + for _, branch := range workflowExecution.Workflow.Branches { + if branch.DestinationID == result.Action.ID { + for _, subresult := range workflowExecution.Results { + if subresult.Action.ID == branch.SourceID { + if subresult.Status != "SKIPPED" && subresult.Status != "FAILURE" { + log.Printf("SUBRESULT PARENT STATUS: %s", subresult.Status) + log.Printf("Should remove resultIndex: %d", resultIndex) + + workflowExecution.Results = append(workflowExecution.Results[:resultIndex], workflowExecution.Results[resultIndex+1:]...) + + break + } + } + } + } + } + } + + extraInputs := 0 + for _, trigger := range workflowExecution.Workflow.Triggers { + if trigger.Name == "User Input" && trigger.AppName == "User Input" { + extraInputs += 1 + } else if trigger.Name == "Shuffle Workflow" && trigger.AppName == "Shuffle Workflow" { + extraInputs += 1 + } + } + + //log.Printf("EXTRA: %d", extraInputs) + //log.Printf("LENGTH: %d - %d", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extraInputs) + + if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extraInputs { + //log.Printf("\nIN HERE WITH RESULTS %d vs %d\n", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extraInputs) + finished := true + lastResult := "" + + // Doesn't have to be SUCCESS and FINISHED everywhere anymore. + skippedNodes := false + for _, result := range workflowExecution.Results { + if result.Status == "EXECUTING" { + finished = false + break + } + + // FIXME: Check if ALL parents are skipped or if its just one. Otherwise execute it + if result.Status == "SKIPPED" { + skippedNodes = true + + // Checks if all parents are skipped or failed. Otherwise removes them from the results + for _, branch := range workflowExecution.Workflow.Branches { + if branch.DestinationID == result.Action.ID { + for _, subresult := range workflowExecution.Results { + if subresult.Action.ID == branch.SourceID { + if subresult.Status != "SKIPPED" && subresult.Status != "FAILURE" { + //log.Printf("SUBRESULT PARENT STATUS: %s", subresult.Status) + //log.Printf("Should remove resultIndex: %d", resultIndex) + finished = false + break + } + } + } + } + + if !finished { + break + } + } + } + + lastResult = result.Result + } + + // FIXME: Handle skip nodes - change status? + _ = skippedNodes + + if finished { + dbSave = true + log.Printf("[INFO] Execution of %s finished.", workflowExecution.ExecutionId) + //log.Println("Might be finished based on length of results and everything being SUCCESS or FINISHED - VERIFY THIS. Setting status to finished.") + + workflowExecution.Result = lastResult + workflowExecution.Status = "FINISHED" + workflowExecution.CompletedAt = int64(time.Now().Unix()) + if workflowExecution.LastNode == "" { + workflowExecution.LastNode = actionResult.Action.ID + } + + } + } + + // FIXME - why isn't this how it works otherwise, wtf? + //workflow, err := getWorkflow(workflowExecution.Workflow.ID) + //newActions := []Action{} + //for _, action := range workflowExecution.Workflow.Actions { + // log.Printf("Name: %s, Env: %s", action.Name, action.Environment) + //} + + tmpJson, err := json.Marshal(workflowExecution) + if err == nil { + if len(tmpJson) >= 1048487 { + dbSave = true + log.Printf("[ERROR] Result length is too long! Need to reduce result size") + + // Result string `json:"result" datastore:"result,noindex"` + // Arbitrary reduction size + maxSize := 500000 + newResults := []ActionResult{} + for _, item := range workflowExecution.Results { + if len(item.Result) > maxSize { + item.Result = "[ERROR] Result too large to handle (https://github.com/frikky/shuffle/issues/171)" + } + + newResults = append(newResults, item) + } + + workflowExecution.Results = newResults + } + } + + // Validating that action results hasn't changed + // Handled using cachhing, so actually pretty fast + cacheKey := fmt.Sprintf("workflowexecution-%s", workflowExecution.ExecutionId) + if value, found := requestCache.Get(cacheKey); found { + parsedValue := value.(*WorkflowExecution) + if len(parsedValue.Results) > 0 && len(parsedValue.Results) != resultLength { + setExecution = false + if attempts > 5 { + //log.Printf("\n\nSkipping execution input - %d vs %d. Attempts: (%d)\n\n", len(parsedValue.Results), resultLength, attempts) + } + + attempts += 1 + if len(workflowExecution.Results) <= len(workflowExecution.Workflow.Actions) { + runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp) + return + } + } + } + + if setExecution || workflowExecution.Status == "FINISHED" || workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" { + err = setWorkflowExecution(ctx, *workflowExecution, dbSave) + if err != nil { + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution actionresult: %s"}`, err))) + return + } + } else { + log.Printf("Skipping setexec with status %s", workflowExecution.Status) + } + + //if newExecutions && len(nextActions) > 0 { + // handleExecutionResult(*workflowExecution) + //} + + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) +} + +func getWorkflowExecution(ctx context.Context, id string) (*WorkflowExecution, error) { + //log.Printf("IN GET WORKFLOW EXEC!") + cacheKey := fmt.Sprintf("workflowexecution-%s", id) + if value, found := requestCache.Get(cacheKey); found { + parsedValue := value.(*WorkflowExecution) + //log.Printf("Found execution for id %s with %d results", parsedValue.ExecutionId, len(parsedValue.Results)) + + //validateFinished(*parsedValue) + return parsedValue, nil + } + + return &WorkflowExecution{}, errors.New("No workflowexecution defined yet") +} + +func validateFinished(workflowExecution WorkflowExecution) { + log.Printf("Status: %s, Actions: %d, Extra: %d, Results: %d\n", workflowExecution.Status, len(workflowExecution.Workflow.Actions), extra, len(workflowExecution.Results)) + + //if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extra { + if (len(environments) == 1 && requestsSent == 0 && len(workflowExecution.Results) >= 1) || (len(workflowExecution.Results) >= len(workflowExecution.Workflow.Actions) && len(workflowExecution.Workflow.Actions) > 0) { + requestsSent += 1 + //log.Printf("[FINISHED] Should send full result to %s", baseUrl) + + //data = fmt.Sprintf(`{"execution_id": "%s", "authorization": "%s"}`, executionId, authorization) + data, err := json.Marshal(workflowExecution) + if err != nil { + log.Printf("[ERROR] Failed to unmarshal data for backend") + shutdown(workflowExecution.ExecutionId, "") + } + + fullUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl) + req, err := http.NewRequest( + "POST", + fullUrl, + bytes.NewBuffer([]byte(data)), + ) + + if err != nil { + log.Printf("[ERROR] Failed creating finishing request: %s", err) + shutdown(workflowExecution.ExecutionId, "") + } + + newresp, err := topClient.Do(req) + if err != nil { + log.Printf("[ERROR] Error running finishing request: %s", err) + shutdown(workflowExecution.ExecutionId, "") + } + + body, err := ioutil.ReadAll(newresp.Body) + log.Printf("BACKEND STATUS: %d", newresp.StatusCode) + if err != nil { + log.Printf("[ERROR] Failed reading body: %s", err) + } else { + log.Printf("[INFO] NEWRESP (from backend): %s", string(body)) + } + } +} + +func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Println("Failed reading body for stream result queue") + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + + var actionResult ActionResult + err = json.Unmarshal(body, &actionResult) + if err != nil { + log.Printf("Failed ActionResult unmarshaling: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + + ctx := context.Background() + workflowExecution, err := getWorkflowExecution(ctx, actionResult.ExecutionId) + if err != nil { + //log.Printf("Failed getting execution (streamresult) %s: %s", actionResult.ExecutionId, err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key or execution_id might not exist."}`))) + return + } + + // Authorization is done here + if workflowExecution.Authorization != actionResult.Authorization { + log.Printf("Bad authorization key when getting stream results %s.", actionResult.ExecutionId) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key or execution_id might not exist."}`))) + return + } + + newjson, err := json.Marshal(workflowExecution) + if err != nil { + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflow execution"}`))) + return + } + + resp.WriteHeader(200) + resp.Write(newjson) + +} + +func setWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecution, dbSave bool) error { + //log.Printf("IN SET WORKFLOW EXEC!") + //log.Printf("\n\n\nRESULT: %s\n\n\n", workflowExecution.Status) + if len(workflowExecution.ExecutionId) == 0 { + log.Printf("Workflowexeciton executionId can't be empty.") + return errors.New("ExecutionId can't be empty.") + } + + cacheKey := fmt.Sprintf("workflowexecution-%s", workflowExecution.ExecutionId) + requestCache.Set(cacheKey, &workflowExecution, cache.DefaultExpiration) + + handleExecutionResult(workflowExecution) + validateFinished(workflowExecution) + if dbSave { + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } + return nil +} + +// GetLocalIP returns the non loopback local IP of the host +func getLocalIP() string { + addrs, err := net.InterfaceAddrs() + if err != nil { + return "" + } + for _, address := range addrs { + // check the address type and if it is not a loopback the display it + if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { + if ipnet.IP.To4() != nil { + return ipnet.IP.String() + } + } + } + return "" +} + +func getAvailablePort() (net.Listener, error) { + listener, err := net.Listen("tcp", ":0") + if err != nil { + log.Printf("[WARNING] Failed to assign port by default. Defaulting to 5001") + //return ":5001" + return nil, err + } + + return listener, nil + //return fmt.Sprintf(":%d", port) +} + +func webserverSetup(workflowExecution WorkflowExecution) net.Listener { + hostname := getLocalIP() + + // FIXME: This MAY not work because of speed between first + // container being launched and port being assigned to webserver + listener, err := getAvailablePort() + if err != nil { + log.Printf("Failed to created listener: %s", err) + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } + port := listener.Addr().(*net.TCPAddr).Port + + log.Printf("\n\nStarting webserver on port %d with hostname: %s\n\n", port, hostname) + log.Printf("OLD HOSTNAME: %s", appCallbackUrl) + appCallbackUrl = fmt.Sprintf("http://%s:%d", hostname, port) + log.Printf("NEW HOSTNAME: %s", appCallbackUrl) + + return listener +} + +func runWebserver(listener net.Listener) { + r := mux.NewRouter() + r.HandleFunc("/api/v1/streams", handleWorkflowQueue).Methods("POST") + r.HandleFunc("/api/v1/streams/results", handleGetStreamResults).Methods("POST", "OPTIONS") + http.Handle("/", r) + + //log.Fatal(http.ListenAndServe(port, nil)) + log.Fatal(http.Serve(listener, nil)) +} + // Initial loop etc func main() { log.Printf("[INFO] Setting up worker environment") @@ -1283,7 +2706,7 @@ func main() { shutdown(executionId, "") } - data := fmt.Sprintf(`{"execution_id": "%s", "authorization": "%s"}`, executionId, authorization) + data = fmt.Sprintf(`{"execution_id": "%s", "authorization": "%s"}`, executionId, authorization) fullUrl := fmt.Sprintf("%s/api/v1/streams/results", baseUrl) req, err := http.NewRequest( "POST", @@ -1295,7 +2718,9 @@ func main() { log.Println("[ERROR] Failed making request builder for backend") shutdown(executionId, "") } + topClient = client + firstRequest := true for { // Because of this, it always has updated data. // Removed request requirement from app_sdk @@ -1314,7 +2739,7 @@ func main() { } if newresp.StatusCode != 200 { - log.Printf("[ERROR] %s\nStatusCode: %d", string(body), newresp.StatusCode) + log.Printf("[ERROR] %s\nStatusCode (1): %d", string(body), newresp.StatusCode) time.Sleep(time.Duration(sleepTime) * time.Second) continue } @@ -1327,6 +2752,50 @@ func main() { continue } + if firstRequest { + firstRequest = false + workflowExecution.StartedAt = int64(time.Now().Unix()) + + cacheKey := fmt.Sprintf("workflowexecution-%s", workflowExecution.ExecutionId) + requestCache = cache.New(5*time.Minute, 10*time.Minute) + requestCache.Set(cacheKey, &workflowExecution, cache.DefaultExpiration) + for _, action := range workflowExecution.Workflow.Actions { + found := false + for _, environment := range environments { + if action.Environment == environment { + found = true + break + } + } + + if !found { + environments = append(environments, action.Environment) + } + } + + log.Printf("Environments: %s. 1 = webserver, 0 or >1 = default", environments) + if len(environments) == 1 { //&& len(workflowExecution.Actions)+len(workflowExecution.Triggers) > 1 { + listener := webserverSetup(workflowExecution) + err := executionInit(workflowExecution) + if err != nil { + log.Printf("[INFO] Workflow setup failed: %s", workflowExecution.ExecutionId, err) + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } + + go func() { + time.Sleep(time.Duration(1)) + handleExecutionResult(workflowExecution) + }() + + runWebserver(listener) + //log.Printf("Before wait") + //wg := sync.WaitGroup{} + //wg.Add(1) + //wg.Wait() + } + + } + if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" { log.Printf("[INFO] Workflow %s is finished. Exiting worker.", workflowExecution.ExecutionId) shutdown(executionId, workflowExecution.Workflow.ID)