diff --git a/backend/app_sdk/Dockerfile b/backend/app_sdk/Dockerfile index c323b73e..68bd625d 100644 --- a/backend/app_sdk/Dockerfile +++ b/backend/app_sdk/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.9.1-alpine as base FROM base as builder -RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev tzdata +RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev tzdata coreutils RUN mkdir /install WORKDIR /install @@ -8,7 +8,7 @@ WORKDIR /install FROM base #--no-cache -RUN apk update && apk add --update tzdata libmagic alpine-sdk libffi libffi-dev musl-dev openssl-dev +RUN apk update && apk add --update tzdata libmagic alpine-sdk libffi libffi-dev musl-dev openssl-dev coreutils COPY --from=builder /install /usr/local COPY requirements.txt /requirements.txt diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index ce954c25..36a795dc 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -83,11 +83,11 @@ class AppBase: #self.logger.info("[INFO] URL (URL): %s" % url) try: ret = requests.post(url, headers=headers, json=action_result) - self.logger.info(f"[DEBUG] Result: {res.status_code}") - if ret.status_code != 200: - self.logger.info(f"[DEBUG] Shuffle Response: {ret.text}") + #self.logger.info(f"[DEBUG] Result: {ret.status_code}") + #if ret.status_code != 200: + # self.logger.info(f"[DEBUG] Shuffle Response: {ret.text}") - self.logger.info("[DEBUG] Stream result was sent!") + self.logger.info(f"[DEBUG] Successful request: Status= {ret.status_code} & Response= {ret.text}") except requests.exceptions.ConnectionError as e: self.logger.info(f"[DEBUG] Unexpected ConnectionError happened: {e}") return @@ -97,9 +97,11 @@ class AppBase: action_result["result"] = f"POST error: {e}" self.logger.info(f"[DEBUG] Before typeerror stream result: {e}") ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) - self.logger.info(f"[DEBUG] Result: {ret.status_code}") - if ret.status_code != 200: - self.logger.info(ret.text) + #self.logger.info(f"[DEBUG] Result: {ret.status_code}") + #if ret.status_code != 200: + # pr + + self.logger.info(f"[DEBUG] TypeError request: Status= {ret.status_code} & Response= {ret.text}") except http.client.RemoteDisconnected as e: self.logger.info(f"[DEBUG] Expected Remotedisconnect happened: {e}") return @@ -870,7 +872,7 @@ class AppBase: pass self.action = copy.deepcopy(action) - self.logger.info("Sending starting action result (EXECUTING)") + self.logger.info("[DEBUG] Sending starting action result (EXECUTING)") headers = { "Content-Type": "application/json", @@ -924,7 +926,7 @@ class AppBase: "execution_id": self.current_execution_id } - self.logger.info("Before FULLEXEC stream result") + self.logger.info("[DEBUG] Before FULLEXEC stream result") ret = requests.post( "%s/api/v1/streams/results" % (self.base_url), headers=headers, @@ -961,7 +963,7 @@ class AppBase: self.full_execution = fullexecution - self.logger.info("AFTER FULLEXEC stream result (init)") + self.logger.info("[DEBUG] AFTER FULLEXEC stream result (init)") # Gets the value at the parenthesis level you want def parse_nested_param(string, level): @@ -1019,7 +1021,6 @@ class AppBase: try: return int(data) except ValueError: - self.logger.info("[DEBUG] ValueError while casting %s to int" % data) return data if "lower" in thistype: @@ -1033,7 +1034,6 @@ class AppBase: if "split" in thistype: return data.split() if "replace" in thistype: - self.logger.info("Running replace!") splitvalues = data.split(",") if len(splitvalues) > 2: @@ -1061,47 +1061,39 @@ class AppBase: else: return f"replace({data})" if "join" in thistype: - self.logger.info(f"SHOULD JOIN: {data}") try: splitvalues = data.split(",") if "," not in data: return f"join({data})" if len(splitvalues) >= 2: - self.logger.info(f"SPLITVALUE: {splitvalues[-1]}") # 1. Take the list and parse it from string # 2. Take all the items and join them # 3. Parse them back as string and return values = ",".join(splitvalues[0:-1]) - self.logger.info(f"VALUES: {values}") tmp = json.loads(values) - self.logger.info(f"TMP: {tmp}") - #tmp = tmp[1:-1] - #self.logger.info(f"TMP2: {tmp}") try: newvalues = splitvalues[-1].join(str(item).strip() for item in tmp) except TypeError: newvalues = splitvalues[-1].join(json.dumps(item).strip() for item in tmp) - self.logger.info(f"new: {newvalues}") return newvalues else: - self.logger.info("Returning default") return f"join({data})" except (KeyError, IndexError) as e: - self.logger.info(f"ERROR in join(): {e}") + print(f"ERROR in join(): {e}") except json.decoder.JSONDecodeError as e: - self.logger.info(f"JSON ERROR in join(): {e}") + print(f"JSON ERROR in join(): {e}") if "len" in thistype or "length" in thistype or "lenght" in thistype: - self.logger.info(f"Trying to length-parse: {data}") + #self.logger.info(f"Trying to length-parse: {data}") try: tmp_len = json.loads(data, parse_float=str, parse_int=str, parse_constant=str) except (NameError, KeyError, TypeError, json.decoder.JSONDecodeError) as e: try: - self.logger.info(f"[WARNING] INITIAL Parsing bug for length in app sdk: {e}") + #self.logger.info(f"[WARNING] INITIAL Parsing bug for length in app sdk: {e}") # data = data.replace("\'", "\"") data = data.replace("True", "true") data = data.replace("False", "false") @@ -1142,7 +1134,6 @@ class AppBase: else: tmp = json.loads(parsedlist)[lastsplit[0]] - #self.logger.info(tmp) return tmp except IndexError as e: return default_error @@ -1165,9 +1156,6 @@ class AppBase: inner_value = parse_nested_param(data, maxDepth(data) - 0) outer_value = parse_nested_param(data, maxDepth(data) - 1) - #self.logger.info("[DEBUG] INNER: ", inner_value) - #self.logger.info("[DEBUG] OUTER: ", outer_value) - wrapper_group = "|".join(wrappers) parse_string = data max_depth = maxDepth(parse_string) @@ -1197,7 +1185,7 @@ class AppBase: c_parentheses = parse_nested_param(parse_string, 0)[0] match_string = re.escape(c_parentheses) custom_casting = re.findall(fr"({wrapper_group})\({match_string}", parse_string) - self.logger.info("In ELSE: %s" % custom_casting) + print("[DEBUG] In ELSE: %s" % custom_casting) # check if a wrapper was found if len(custom_casting) != 0: inner_result = parse_type(c_parentheses, custom_casting[0]) @@ -1209,7 +1197,7 @@ class AppBase: else: parse_string = inner_result - self.logger.info("PARSE STRING: %s" % parse_string) + print("PARSE STRING: %s" % parse_string) return parse_string, True # Looks for parantheses to grab special cases within a string, e.g: @@ -1308,7 +1296,6 @@ class AppBase: # $nodename.data.#min-max.info.id def recurse_json(basejson, parsersplit): match = "#([0-9a-z]+):?-?([0-9a-z]+)?#?" - #self.logger.info("Split: %s\n%s" % (parsersplit, basejson)) try: outercnt = 0 @@ -1317,9 +1304,7 @@ class AppBase: #if " " in value: # value = value.replace(" ", "_", -1) - #self.logger.info("VALUE: %s\n" % value) actualitem = re.findall(match, value, re.MULTILINE) - #self.logger.info("ACTUAL RECURSE: (%s) %s" % (value, actualitem)) if value == "#": newvalue = [] for innervalue in basejson: @@ -1339,17 +1324,16 @@ class AppBase: return newvalue, True elif len(actualitem) > 0: - #self.logger.info("[INFO] In recursion v2: ", actualitem) is_loop = True newvalue = [] firstitem = actualitem[0][0] seconditem = actualitem[0][1] - self.logger.info("[DEBUG] ACTUAL PARSED: %s" % actualitem) + print("[DEBUG] ACTUAL PARSED: %s" % actualitem) # Means it's a single item -> continue if seconditem == "": - self.logger.info("[INFO] In first - handling %s. Len: %d" % (firstitem, len(basejson))) + print("[INFO] In first - handling %s. Len: %d" % (firstitem, len(basejson))) if firstitem.lower() == "max" or firstitem.lower() == "last": firstitem = len(basejson)-1 elif firstitem.lower() == "min" or firstitem.lower() == "first": @@ -1357,14 +1341,14 @@ class AppBase: else: firstitem = int(firstitem) - self.logger.info(f"[DEBUG] Post lower checks with item {firstitem}") + print(f"[DEBUG] Post lower checks with item {firstitem}") tmpitem = basejson[int(firstitem)] try: newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:]) except IndexError: newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:]) else: - self.logger.info("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem)) + print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem)) if firstitem.lower() == "max" or firstitem.lower() == "last": firstitem = len(basejson)-1 elif firstitem.lower() == "min" or firstitem.lower() == "first": @@ -1379,7 +1363,7 @@ class AppBase: else: seconditem = int(seconditem) - self.logger.info(f"[DEBUG] Post lower checks 2: {firstitem} AND {seconditem}") + print(f"[DEBUG] Post lower checks 2: {firstitem} AND {seconditem}") newvalue = [] if int(seconditem) > len(basejson): seconditem = len(basejson) @@ -1392,63 +1376,55 @@ class AppBase: try: ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt+1:]) except IndexError: - self.logger.info("[DEBUG] INDEXERROR: ", parsersplit[outercnt]) + print("[DEBUG] INDEXERROR: ", parsersplit[outercnt]) #ret = innervalue ret, tmp_loop = recurse_json(innervalue, parsersplit[outercnt:]) - #self.logger.info("IN LIST: %s" % ret) - #exit() newvalue.append(ret) - #self.logger.info("Returning %s" % newvalue) return newvalue, is_loop else: - #self.logger.info("BEFORE NORMAL VALUE: ", basejson, value) if len(value) == 0: return basejson, False try: if isinstance(basejson, list): - self.logger.info("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) + print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) return basejson, False elif isinstance(basejson[value], str): - #self.logger.info(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value]) try: basejson = json.loads(basejson[value]) - #self.logger.info("[DEBUG] BASEJSON: %s" % basejson) except json.decoder.JSONDecodeError as e: - #self.logger.info("[DEBUG] RETURNING BECAUSE '%s' IS A NORMAL STRING (0)" % basejson[value]) return basejson[value], False else: basejson = basejson[value] except KeyError as e: - self.logger.info("[WARNING] Running secondary value check with replacement of underscore in %s: %s" % (value, e)) + print("[WARNING] Running secondary value check with replacement of underscore in %s: %s" % (value, e)) if "_" in value: value = value.replace("_", " ", -1) elif " " in value: value = value.replace(" ", "_", -1) if isinstance(basejson, list): - self.logger.info("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) + print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) return basejson, False elif isinstance(basejson[value], str): - self.logger.info(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value]) + print(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value]) try: basejson = json.loads(basejson[value]) - self.logger.info("[DEBUG] BASEJSON: %s" % basejson) + print("[DEBUG] BASEJSON: %s" % basejson) except json.decoder.JSONDecodeError as e: - self.logger.info("[DEBUG] RETURNING BECAUSE '%s' IS A NORMAL STRING (1)" % basejson[value]) + print("[DEBUG] RETURNING BECAUSE '%s' IS A NORMAL STRING (1)" % basejson[value]) return basejson[value], False else: basejson = basejson[value] - #self.logger.info("Parsed BASEJSON: %s" % basejson) outercnt += 1 except KeyError as e: - self.logger.info("[INFO] Lower keyerror: %s" % e) + print("[INFO] Lower keyerror: %s" % e) return "", False #return basejson @@ -1463,13 +1439,13 @@ class AppBase: actionname_lower = parsersplit[0][1:].lower() #Actionname: Start_node - self.logger.info(f"\n[INFO] Actionname: {actionname_lower}") + print(f"\n[INFO] Actionname: {actionname_lower}") # 1. Find the action baseresult = "" appendresult = "" - self.logger.info("[INFO] Parsersplit length: %d" % len(parsersplit)) + print("[INFO] Parsersplit length: %d" % len(parsersplit)) if (actionname_lower.startswith("exec ") or actionname_lower.startswith("webhook ") or actionname_lower.startswith("schedule ") or actionname_lower.startswith("userinput ") or actionname_lower.startswith("email_trigger ") or actionname_lower.startswith("trigger ")) and len(parsersplit) == 1: record = False for char in actionname_lower: @@ -1489,21 +1465,19 @@ class AppBase: if actionname_lower == "exec" or actionname_lower == "webhook" or actionname_lower == "schedule" or actionname_lower == "userinput" or actionname_lower == "email_trigger" or actionname_lower == "trigger": baseresult = execution_data["execution_argument"] elif actionname_lower == "shuffle_cache": - self.logger.info("SHOULD GET CACHE KEY: %s" % parsersplit) + print("[DEBUG] SHOULD GET CACHE KEY: %s" % parsersplit) if len(parsersplit) > 1: actual_key = parsersplit[1] - self.logger.info("KEY: %s" % actual_key) + print("[DEBUG] KEY: %s" % actual_key) cachedata = self.get_cache(actual_key) - self.logger.info("CACHE: %s" % cachedata) + print("CACHE: %s" % cachedata) parsersplit.pop(1) try: baseresult = json.dumps(cachedata) except json.decoder.JSONDecodeError as e: - self.logger.info("Failed json dumping: %s" % e) + print("[WARNING] Failed json dumping: %s" % e) - #returndata = str(baseresult)+str(appendresult) else: - #self.logger.info("Within execution data check. Execution data: %s", execution_data["results"]) if execution_data["results"] != None: for result in execution_data["results"]: resultlabel = result["action"]["label"].replace(" ", "_", -1).lower() @@ -1511,13 +1485,12 @@ class AppBase: baseresult = result["result"] break else: - self.logger.info("No results to get values from.") + print("[DEBUG] No results to get values from.") baseresult = "$" + parsersplit[0][1:] - self.logger.info("BEFORE VARIABLES!") + print("[DEBUG] BEFORE VARIABLES!") if len(baseresult) == 0: try: - #self.logger.info("WF Variables: %s" % execution_data["workflow"]["workflow_variables"]) for variable in execution_data["workflow"]["workflow_variables"]: variablename = variable["name"].replace(" ", "_", -1).lower() @@ -1526,47 +1499,46 @@ class AppBase: break except KeyError as e: - self.logger.info("[INFO] KeyError wf variables: %s" % e) + print("[INFO] KeyError wf variables: %s" % e) pass except TypeError as e: - self.logger.info("[INFO] TypeError wf variables: %s" % e) + print("[INFO] TypeError wf variables: %s" % e) pass - self.logger.info("BEFORE EXECUTION VAR") + print("[DEBUG] BEFORE EXECUTION VAR") if len(baseresult) == 0: try: - #self.logger.info("Execution Variables: %s" % execution_data["execution_variables"]) for variable in execution_data["execution_variables"]: variablename = variable["name"].replace(" ", "_", -1).lower() if variablename.lower() == actionname_lower: baseresult = variable["value"] break except KeyError as e: - self.logger.info("[INFO] KeyError exec variables: %s" % e) + print("[INFO] KeyError exec variables: %s" % e) pass except TypeError as e: - self.logger.info("[INFO] TypeError exec variables: %s" % e) + print("[INFO] TypeError exec variables: %s" % e) pass except KeyError as error: - self.logger.info(f"[DEBUG] KeyError in JSON: {error}") + print(f"[DEBUG] KeyError in JSON: {error}") - self.logger.info(f"[INFO] After first trycatch. Baseresult")#, baseresult) + print(f"[INFO] After first trycatch. Baseresult")#, baseresult) # 2. Find the JSON data if len(baseresult) == 0: return ""+appendresult, False - self.logger.info("[INFO] After second return") + print("[INFO] After second return") if len(parsersplit) == 1: returndata = str(baseresult)+str(appendresult) - self.logger.info("RETURNING!")#: %s" % returndata) + print("[DEBUG] RETURNING!")#: %s" % returndata) return returndata, False baseresult = baseresult.replace(" True,", " true,") baseresult = baseresult.replace(" False", " false,") - self.logger.info("[INFO] After third parser return - Formatted")#, baseresult) + print("[INFO] After third parser return - Formatted")#, baseresult) basejson = {} try: basejson = json.loads(baseresult) @@ -1575,10 +1547,10 @@ class AppBase: baseresult = baseresult.replace("\'", "\"") basejson = json.loads(baseresult) except json.decoder.JSONDecodeError as e: - self.logger.info("Parser issue with JSON: %s" % e) + print("Parser issue with JSON: %s" % e) return str(baseresult)+str(appendresult), False - self.logger.info("[INFO] After fourth parser return as JSON") + print("[INFO] After fourth parser return as JSON") data, is_loop = recurse_json(basejson, parsersplit[1:]) parseditem = data @@ -1586,31 +1558,31 @@ class AppBase: try: parseditem = json.dumps(parseditem) except json.decoder.JSONDecodeError as e: - self.logger.info("Parseditem issue: %s" % e) + print("[WARNING] Parseditem issue: %s" % e) pass - self.logger.info("DATA: (%s) %s" % (type(data), data)) + print("[DEBUG] DATA: (%s) %s" % (type(data), data)) if is_loop: - self.logger.info("DATA IS A LOOP - SHOULD WRAP") + print("[DEBUG] DATA IS A LOOP - SHOULD WRAP") if parsersplit[-1] == "#": - self.logger.info("SET DATA WRAPPER TO NORMAL!") + print("[WARNING] SET DATA WRAPPER TO NORMAL!") parseditem = "${SHUFFLE_NO_SPLITTER%s}$" % json.dumps(data) else: # Return value: ${id[12345, 45678]}$ - self.logger.info("SET DATA WRAPPER TO %s!" % parsersplit[-1]) + print("[WARNING] SET DATA WRAPPER TO %s!" % parsersplit[-1]) parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data)) - self.logger.info("Before last return with %s" % appendresult) + print("[DEBUG] Before last return with %s" % appendresult) returndata = str(parseditem)+str(appendresult) # New in 0.8.97: Don't return items without lists - self.logger.info("RETURNDATA: %s" % returndata) + #self.logger.info("RETURNDATA: %s" % returndata) #return returndata, is_loop try: return json.dumps(json.loads(returndata)), is_loop except json.decoder.JSONDecodeError as e: - self.logger.info("Error in decoder: %s" % e) + print("Error in decoder: %s" % e) return returndata, is_loop # Sending self as it's not a normal function @@ -1667,7 +1639,6 @@ class AppBase: newvalue = data for key, value in newvalue.items(): - #self.logger.info("%s: %s" % (key, value)) if isinstance(value, str) and len(value) == 0: deletekeys.append(key) continue @@ -1675,17 +1646,16 @@ class AppBase: if isinstance(value, list): try: value = json.dumps(value) - self.logger.info(value) except: - self.logger.info("[WARNING] Json parsing issue in recursed value") + print("[WARNING] Json parsing issue in recursed value") pass if value == "${%s}" % key: - self.logger.info("[WARNING] Deleting %s because key = value" % key) + print("[WARNING] Deleting %s because key = value" % key) deletekeys.append(key) continue elif "${" in value and "}" in value: - self.logger.info("[WARNING] Deleting %s because it contains ${ and }" % key) + print("[WARNING] Deleting %s because it contains ${ and }" % key) deletekeys.append(key) continue @@ -1693,9 +1663,9 @@ class AppBase: newvalue[key] = recurse_cleanup_script(value) except json.decoder.JSONDecodeError as e: - self.logger.info(f"[WARNING] Failed JSON replacement for OpenAPI keys (3) {e}") + print(f"[WARNING] Failed JSON replacement for OpenAPI keys (3) {e}") except Exception as e: - self.logger.info(f"[WARNING] Failed as an exception (1): {e}") + print(f"[WARNING] Failed as an exception (1): {e}") try: for deletekey in deletekeys: @@ -1704,11 +1674,10 @@ class AppBase: except: pass except Exception as e: - self.logger.info(f"[WARNING] Failed in deletekeys: {e}") + print(f"[WARNING] Failed in deletekeys: {e}") return data try: - #self.logger.info("Post delete: %s" % newvalue) for key, value in newvalue.items(): if isinstance(value, bool): continue @@ -1719,28 +1688,20 @@ class AppBase: value = json.loads(value) newvalue[key] = value except json.decoder.JSONDecodeError as e: - #self.logger.info("Inner overwrite issue for \"%s\": %s" % (key, e)) continue except Exception as e: - #self.logger.info("General error in newvalue items loop: %s" % e) continue try: data = json.dumps(newvalue) except json.decoder.JSONDecodeError as e: - self.logger.info("[WARNING] JsonDecodeError: %s" % e) + print("[WARNING] JsonDecodeError: %s" % e) data = newvalue except json.decoder.JSONDecodeError as e: - self.logger.info("[WARNING] Failed JSON replacement for OpenAPI keys (2) {e}") + print("[WARNING] Failed JSON replacement for OpenAPI keys (2) {e}") except Exception as e: - self.logger.info(f"[WARNING] Failed as an exception (2): {e}") - - #if isinstance(data, str): - # tmpdata = json.dumps(data) - # self.logger.info(tmpdata) - # foundvalue = re.findall(".*?(${\w+})", tmpdata, re.MULTILINE) - # self.logger.info("FOUND: %s", foundvalue) + print(f"[WARNING] Failed as an exception (2): {e}") return data @@ -1912,7 +1873,7 @@ class AppBase: return "", parameter["value"], is_loop def run_validation(sourcevalue, check, destinationvalue): - self.logger.info("Checking %s %s %s" % (sourcevalue, check, destinationvalue)) + print("[DEBUG] Checking %s %s %s" % (sourcevalue, check, destinationvalue)) if check == "=" or check.lower() == "equals": if str(sourcevalue).lower() == str(destinationvalue).lower(): @@ -1964,7 +1925,7 @@ class AppBase: continue if item.strip() in sourcevalue: - self.logger.info("[INFO] Found %s in %s" % (item, sourcevalue)) + print("[INFO] Found %s in %s" % (item, sourcevalue)) return True return False @@ -1975,7 +1936,7 @@ class AppBase: return True except AttributeError as e: - self.logger.error("[WARNING] Condition larger than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) + print("[WARNING] Condition larger than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) return False elif check.lower() == "smaller than" or check.lower() == "less than": try: @@ -1984,13 +1945,13 @@ class AppBase: return True except AttributeError as e: - self.logger.error("[WARNING] Condition smaller than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) + print("[WARNING] Condition smaller than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) return False elif check.lower() == "re" or check.lower() == "matches regex": try: found = re.search(destinationvalue, sourcevalue) except re.error as e: - self.logger.info("[WARNING] Regex error in condition: %s" % e) + print("[WARNING] Regex error in condition: %s" % e) return False if found == None: @@ -1998,7 +1959,7 @@ class AppBase: return True else: - self.logger.info("Condition: can't handle %s yet. Setting to true" % check) + print("[DEBUG] Condition: can't handle %s yet. Setting to true" % check) return False @@ -2219,7 +2180,7 @@ class AppBase: #self.logger.info(action["parameters"]) # This seems redundant now - self.logger.info("Pre parameters") + self.logger.info("[DEBUG] Pre parameters") for parameter in newparams: action["parameters"].append(parameter) @@ -2241,7 +2202,7 @@ class AppBase: # Multi_parameter has the data for each. variable minlength = 0 - self.logger.info("Pre-loading parameters") + self.logger.info("[DEBUG] Pre-loading parameters") multi_parameters = json.loads(json.dumps(params)) multiexecution = False multi_execution_lists = [] @@ -2550,7 +2511,7 @@ class AppBase: if not multiexecution: # Runs a single iteration here new_params = self.validate_unique_fields(params) - self.logger.info(f"Returned with newparams of length {len(new_params)}") + self.logger.info(f"[DEBUG] Returned with newparams of length {len(new_params)}") if isinstance(new_params, list) and len(new_params) == 1: params = new_params[0] else: @@ -2793,15 +2754,15 @@ class AppBase: if self.action_result["result"] == "": self.action_result["result"] = result - self.logger.debug(f"Executed {action['label']}-{action['id']}")#with result: {result}") + self.logger.debug(f"[DEBUG] Executed {action['label']}-{action['id']}")#with result: {result}") #self.logger.debug(f"Data: %s" % action_result) except TypeError as e: self.logger.info("TypeError issue: %s" % e) self.action_result["status"] = "FAILURE" self.action_result["result"] = "TypeError: %s" % str(e) else: - self.logger.info("Function %s doesn't exist?" % action["name"]) - self.logger.error(f"App {self.__class__.__name__}.{action['name']} is not callable") + self.logger.info("[DEBUG] Function %s doesn't exist?" % action["name"]) + self.logger.error(f"[ERROR] App {self.__class__.__name__}.{action['name']} is not callable") self.action_result["status"] = "FAILURE" self.action_result["result"] = "Function %s is not callable." % actionname @@ -2843,8 +2804,9 @@ class AppBase: app = cls(redis=None, logger=logger, console_logger=logger) if isinstance(action, str): - self.logger.info("Normal execution. Action is a string.") + print("[DEBUG] Normal execution. Action is a string.") elif isinstance(action, object): + print("[DEBUG] OBJECT execution. Action is NOT a string.") app.action = action try: diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 966bd8cc..5a0fbcb9 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -3,7 +3,7 @@ ### DEFAULT NAME=shuffle-app_sdk -VERSION=0.9.21 +VERSION=0.9.23 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 960ac02a..836ab746 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -23,7 +23,7 @@ require ( github.com/docker/go-units v0.4.0 // indirect github.com/elastic/go-elasticsearch/v7 v7.13.1 // indirect github.com/frikky/kin-openapi v0.39.0 - github.com/frikky/shuffle-shared v0.1.12 + github.com/frikky/shuffle-shared v0.1.13 github.com/fsouza/go-dockerclient v1.7.2 github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 53b58fb4..7dab4a4d 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1014,7 +1014,8 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { CreatorOrg: org.CreatorOrg, Image: org.Image, }) - // Role: "admin", + } else { + log.Printf("[WARNING] Failed to get org %s for user %s", item, userInfo.Username) } } @@ -1036,7 +1037,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { returnData, err := json.Marshal(returnValue) if err != nil { - log.Printf("[WARNING] Failed marshalling info: %s", err) + log.Printf("[WARNING] Failed marshalling info in handleinfo: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 8539dbd3..a4eb0bca 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -771,10 +771,11 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { } //log.Printf("Actionresult unmarshal: %s", string(body)) + log.Printf("[DEBUG] Got workflow result from %s of length %d", request.RemoteAddr, len(body)) err = shuffle.ValidateNewWorkerExecution(body) if err == nil { resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Success"}`))) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "success"}`))) return } else { //log.Printf("[WARNING] Handling other execution variant: %s", err) @@ -1574,7 +1575,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request newParams := []shuffle.WorkflowAppActionParameter{} if strings.ToLower(curAuth.Type) == "oauth2" { - log.Printf("\n\nShould replace auth parameters!!!\n\n") + log.Printf("[DEBUG] Should replace auth parameters (Oauth2)") for _, param := range curAuth.Fields { if param.Key == "expiration" { @@ -1588,7 +1589,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request } for _, param := range action.Parameters { - log.Printf("Param: %#v", param) + //log.Printf("Param: %#v", param) if param.Configuration { continue } diff --git a/docker-compose.yml b/docker-compose.yml index 069fedad..09f810e1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - #build: ./frontend + build: ./frontend image: ghcr.io/frikky/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - #build: ./backend + build: ./backend image: ghcr.io/frikky/shuffle-backend:nightly container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} diff --git a/frontend/src/components/Header.js b/frontend/src/components/Header.js index a25b8679..449c057b 100644 --- a/frontend/src/components/Header.js +++ b/frontend/src/components/Header.js @@ -350,7 +350,7 @@ const Header = props => { handleClose() handleClickLogout() }}> - Logout +  Logout @@ -455,7 +455,10 @@ const Header = props => { }} value={userdata.active_org.id} fullWidth - style={{marginTop: 5, backgroundColor: theme.palette.surfaceColor, marginRight: 15, color: "white", height: 50, width: 200}} + style={{zIndex: 10012, marginTop: 5, backgroundColor: theme.palette.surfaceColor, marginRight: 15, color: "white", height: 50, width: 200}} + MenuProps={{ + style: {zIndex: 10012} + }} onChange={(e) => { handleClickChangeOrg(e.target.value) }} @@ -473,7 +476,7 @@ const Header = props => { {data.name} return ( - +
diff --git a/frontend/src/components/Oauth2Auth.jsx b/frontend/src/components/Oauth2Auth.jsx index 0c680713..34ab2bfc 100644 --- a/frontend/src/components/Oauth2Auth.jsx +++ b/frontend/src/components/Oauth2Auth.jsx @@ -13,9 +13,10 @@ const AuthenticationOauth2 = (props) => { const [defaultConfigSet, setDefaultConfigSet] = React.useState(authenticationType.client_id !== undefined && authenticationType.client_id !== null && authenticationType.client_id.length > 0 && authenticationType.client_secret !== undefined && authenticationType.client_secret !== null && authenticationType.client_secret.length > 0) const [clientId, setClientId] = React.useState(defaultConfigSet ? authenticationType.client_id : "") const [clientSecret, setClientSecret] = React.useState(defaultConfigSet ? authenticationType.client_secret : "") + const [oauthUrl, setOauthUrl] = React.useState("") const [buttonClicked, setButtonClicked] = React.useState(false) - const [manuallyConfigure, setManuallyConfigure] = React.useState(false) + const [manuallyConfigure, setManuallyConfigure] = React.useState(defaultConfigSet ? false : true) const [authenticationOption, setAuthenticationOptions] = React.useState({ app: JSON.parse(JSON.stringify(selectedApp)), fields: {}, @@ -31,7 +32,7 @@ const AuthenticationOauth2 = (props) => { return null } - const handleOauth2Request = (client_id, client_secret) => { + const handleOauth2Request = (client_id, client_secret, oauth_url) => { setButtonClicked(true) //if (authenticationType.type === "oauth2" && authenticationType.redirect_uri !== undefined && authenticationType.redirect_uri !== null) { // These are test credentials @@ -40,17 +41,26 @@ const AuthenticationOauth2 = (props) => { const authentication_url = authenticationType.token_uri - const resources = "UserAuthenticationMethod.ReadWrite.All" + var resources = "" + console.log("SCOPES: ", resources) if (authenticationType.scope !== undefined && authenticationType.scope !== null) { - console.log("EDIT SCOPE!") + resources = authenticationType.scope.join(",") } - console.log(window.location) - //const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" - const redirectUri = `${window.location.protocol}//${window.location.host}/set_authentication` - const state = `workflow_id%3D${workflow.id}%26reference_action_id%3d${selectedAction.app_id}%26app_name%3d${selectedAction.app_name}%26app_id%3d${selectedAction.app_id}%26app_version%3d${selectedAction.app_version}%26authentication_url%3d${authentication_url}%26scope%3d${resources}%26client_id%3d${client_id}%26client_secret%3d${client_secret}` + resources = ["AaaServer.profile.READ"] - const url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&scope=${resources}&state=${state}` + console.log("SCOPES2: ", resources) + const redirectUri = `${window.location.protocol}//${window.location.host}/set_authentication` + var state = `workflow_id%3D${workflow.id}%26reference_action_id%3d${selectedAction.app_id}%26app_name%3d${selectedAction.app_name}%26app_id%3d${selectedAction.app_id}%26app_version%3d${selectedAction.app_version}%26authentication_url%3d${authentication_url}%26scope%3d${resources}%26client_id%3d${client_id}%26client_secret%3d${client_secret}` + if (oauth_url !== undefined && oauth_url !== null && oauth_url.length > 0) { + state += `%26oauth_url%3d${oauth_url}` + console.log("ADDING OAUTH2 URL: ", state) + } + const url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&scope=${resources}&prompt=consent&state=${state}` + + //const url = `https://accounts.zoho.com/oauth/v2/auth?response_type=code&client_id=${client_id}&scope=AaaServer.profile.Read&redirect_uri=${redirectUri}&prompt=consent` + console.log("Full URI: ", url) + console.log("Redirect Uri: ", redirectUri) // &resource=https%3A%2F%2Fgraph.microsoft.com& // FIXME: Awful, but works for prototyping @@ -64,14 +74,11 @@ const AuthenticationOauth2 = (props) => { var open = true const timer = setInterval(() => { if (newwin.closed) { + setButtonClicked(false) clearInterval(timer); //alert('"Secure Payment" window closed!'); getAppAuthentication(true, true) - setTimeout(() => { - console.log("APPAUTH: ", appAuthentication) - setAuthenticationModalOpen(false) - }, 1500) } }, 1000); //do { @@ -206,6 +213,75 @@ const AuthenticationOauth2 = (props) => { {!manuallyConfigure ? null : + {selectedApp.authentication.parameters.map((data, index) => { + //console.log(data, index) + if (data.name === "client_id" || data.name === "client_secret") { + return null + } + + if (data.name !== "url") { + return null + } + + if (oauthUrl.length === 0) { + setOauthUrl(data.value) + } + + return ( +
+ + {data.name} + + {data.schema !== undefined && data.schema !== null && data.schema.type === "bool" ? + + : + { + authenticationOption.fields[data.name] = event.target.value + console.log("Setting oauth url") + setOauthUrl(event.target.value) + //const [oauthUrl, setOauthUrl] = React.useState("") + }} + /> + } +
+ ) + })} { }
- : null} + : + null + }
) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 8c4558db..b6029cd5 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1227,6 +1227,7 @@ const AngularWorkflow = (props) => { if (selectedApp.authentication.required) { //console.log("App requires auth!!") // Setup auth here :) + var appUpdates = false const authenticationOptions = [] var findAuthId = "" if (selectedAction.authentication_id !== null && selectedAction.authentication_id !== undefined && selectedAction.authentication_id.length > 0) { @@ -1261,6 +1262,7 @@ const AngularWorkflow = (props) => { console.log("Setting auth at: ", workflow.actions[key], item.id) workflow.actions[key].selectedAuthentication = item workflow.actions[key].authentication_id = item.id + appUpdates = true //if (workflow.actions[key].selectedAuthentication === undefined || workflow.actions[key].selectedAuthentication === null || workflow.actions[key].selectedAuthentication.length === 0) { // console.log("Setting inner auth: ", workflow.actions[key]) //} @@ -1282,12 +1284,16 @@ const AngularWorkflow = (props) => { selectedAction.selectedAuthentication = {} } - setSelectedAction(selectedAction) - setWorkflow(workflow) - saveWorkflow(workflow) - //for (var key in - alert.info("Added and updated authentication!") + if (appUpdates === true) { + setAuthenticationModalOpen(false) + setSelectedAction(selectedAction) + setWorkflow(workflow) + saveWorkflow(workflow) + alert.info("Added and updated authentication!") + } else { + alert.error("Failed to find new authentication - did it work?") + } } else { alert.info("No authentication to update") } @@ -2098,6 +2104,7 @@ const AngularWorkflow = (props) => { { "type": "oauth2", "redirect_uri": curapp.authentication.redirect_uri, + "refresh_uri": curapp.authentication.refresh_uri, "token_uri": curapp.authentication.token_uri, "scope": curapp.authentication.scope, "client_id": curapp.authentication.client_id, @@ -2946,7 +2953,7 @@ const AngularWorkflow = (props) => { //console.log(event.keyCode) switch( event.keyCode ) { case 27: - console.log("ESCAPE") + //console.log("ESCAPE") if (configureWorkflowModalOpen === true) { setConfigureWorkflowModalOpen(false) } @@ -9271,7 +9278,7 @@ const AngularWorkflow = (props) => { // This whole part is redundant. Made it part of Arguments instead. - console.log(selectedApp) + //console.log(selectedApp) const authenticationModal = authenticationModalOpen ? { const actionNonBodyRequest = ["GET", "HEAD", "DELETE", "CONNECT"] const actionBodyRequest = ["POST", "PUT", "PATCH",] //const authenticationOptions = ["No authentication", "API key", "Bearer auth", "Basic auth", "JWT", "Oauth2"] - const authenticationOptions = ["No authentication", "API key", "Bearer auth", "Basic auth"]//, "Oauth2"] + const authenticationOptions = ["No authentication", "API key", "Bearer auth", "Basic auth", "Oauth2"] const apikeySelection = ["Header", "Query",] const [name, setName] = useState(""); @@ -222,7 +222,7 @@ const AppCreator = (props) => { const [parameterName, setParameterName] = useState(""); const [parameterLocation, setParameterLocation] = useState(apikeySelection.length > 0 ? apikeySelection[0] : ""); const [refreshUrl, setRefreshUrl] = useState(""); - const [oauth2Scopes, setOauth2Scopes] = useState(["google.com"]); + const [oauth2Scopes, setOauth2Scopes] = useState(["OAUTH2.SCOPE.HERE"]); const [projectCategories, setProjectCategories] = useState([]); const [selectedCategory, setSelectedCategory] = useState(""); @@ -949,6 +949,10 @@ const AppCreator = (props) => { if (value.flow.authorizationCode.refreshUrl !== undefined) { setRefreshUrl(value.flow.authorizationCode.refreshUrl) } + if (value.flow.authorizationCode.scopes !== undefined && value.flow.authorizationCode.scopes !== null && value.flow.authorizationCode.scopes.length > 0) { + setOauth2Scopes(value.flow.authorizationCode.scopes) + } + } else if (key === "ApiKeyAuth") { setAuthenticationOption("API key") @@ -1316,7 +1320,7 @@ const AppCreator = (props) => { data.paths[item.url][item.method.toLowerCase()].parameters.push(newitem) } else { - console.log("Nothing to append?") + //console.log("Nothing to append?") } // https://swagger.io/docs/specification/describing-request-body/file-upload/ @@ -1438,17 +1442,19 @@ const AppCreator = (props) => { "authorizationUrl": newparamName, "tokenUrl": parameterLocation, "refreshUrl": refreshUrl, - "scopes": [], + "scopes": oauth2Scopes === undefined || oauth2Scopes === null ? [] : oauth2Scopes, }, }, } - if (oauth2Scopes.scopes > 0) { - for (var key in oauth2Scopes) { - const scope = oauth2Scopes[key] - data.components.securitySchemes["Oauth2"]["flow"]["authorizationCode"]["scopes"].push(scope) - } - } + console.log("SCOPES: ", oauth2Scopes) + + //if (oauth2Scopes.scopes > 0) { + // for (var key in oauth2Scopes) { + // const scope = oauth2Scopes[key] + // data.components.securitySchemes["Oauth2"]["flow"]["authorizationCode"]["scopes"].push(scope) + // } + //} } if (setExtraAuth.length > 0) { @@ -1747,12 +1753,12 @@ const AppCreator = (props) => { const oauth2Auth = authenticationOption === "Oauth2" ?
Oauth2 authentication - + Base Authorization URL { }, }} /> - + Token URL { }, }} /> - + Refresh-token URL { }, }} /> + + Scopes + + { + oauth2Scopes.push(chip) + console.log(oauth2Scopes) + setOauth2Scopes(oauth2Scopes) + }} + onDelete={(chip, index) => { + oauth2Scopes.splice(index, 1) + console.log(oauth2Scopes) + setOauth2Scopes(oauth2Scopes) + }} + />
: null @@ -2429,26 +2460,46 @@ const AppCreator = (props) => { }} onBlur={event => { var parsedurl = event.target.value - console.log("URL: ", parsedurl) + //console.log("URL: ", parsedurl) + if (parsedurl.includes(" ")) { + parsedurl = parsedurl.replaceAll(" ", " ") + } + + if (parsedurl.includes(" ")) { + parsedurl = parsedurl.replaceAll(" ", " ") + } + + if (parsedurl.includes("[") && parsedurl.includes("]")) { + //console.log("REPLACE1") + parsedurl = parsedurl.replaceAll("[", "{") + parsedurl = parsedurl.replaceAll("]", "}") + } + if (parsedurl.includes("<") && parsedurl.includes(">")) { - console.log("REPLACE") + //console.log("REPLACE2") parsedurl = parsedurl.replaceAll("<", "{") parsedurl = parsedurl.replaceAll(">", "}") } + //console.log("URL2: ", parsedurl) if (parsedurl.startsWith("PUT ") || parsedurl.startsWith("GET ") ||parsedurl.startsWith("POST ") || parsedurl.startsWith("DELETE ") ||parsedurl.startsWith("PATCH ") || parsedurl.startsWith("CONNECT ")) { + const tmp = parsedurl.split(" ") if (tmp.length > 1) { - parsedurl = tmp[1] + parsedurl = tmp[1].trim() setActionField("url", parsedurl) setCurrentActionMethod(tmp[0].toUpperCase()) setActionField("method", tmp[0].toUpperCase()) } + + console.log("URL3: ", parsedurl) setUpdate(Math.random()) } else if (parsedurl.startsWith("curl")) { + console.log("URL4: ", parsedurl) + const request = parseCurl(event.target.value) if (request !== event.target.value && request.method !== undefined && request.method !== null) { if (request.method.toUpperCase() !== currentAction.Method) { diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 6f3d1801..61f8ebdb 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -480,7 +480,7 @@ const Apps = (props) => { color: "white", borderRadius: 5, backgroundColor: surfaceColor, - display: "flex", + //display: "flex", marginBottom: 10, overflow: "hidden", } @@ -522,7 +522,7 @@ const Apps = (props) => { + } diff --git a/frontend/src/views/Docs.jsx b/frontend/src/views/Docs.jsx index 7b2a7ed9..e7ac92d8 100644 --- a/frontend/src/views/Docs.jsx +++ b/frontend/src/views/Docs.jsx @@ -139,7 +139,7 @@ const Docs = (props) => { parsedline[1] = parsedline[1].replaceAll(")", "") parsedline[1] = parsedline[1].replaceAll("(", "") parsedline[1] = parsedline[1].trim() - console.log(parsedline[0], parsedline[1]) + //console.log(parsedline[0], parsedline[1]) innerTocLines.push({ "text": parsedline[0], @@ -205,12 +205,12 @@ const Docs = (props) => { // Fix location.. if (element.innerHTML.toLowerCase() === name) { - console.log(element.offsetTop) - //element.scrollIntoView({behavior: "smooth"}) - element.scrollTo({ - top: element.offsetTop+offset, - behavior: "smooth" - }) + //console.log(element.offsetTop) + element.scrollIntoView({behavior: "smooth"}) + //element.scrollTo({ + // top: element.offsetTop+offset, + // behavior: "smooth" + //}) found = true //element.scrollTo({ // top: element.offsetTop-100, @@ -222,7 +222,7 @@ const Docs = (props) => { // H# if (!found) { elements = parent.getElementsByTagName('h3') - console.log(name) + //console.log("NAMe: ", name) found = false for (key in elements) { const element = elements[key] @@ -232,11 +232,11 @@ const Docs = (props) => { // Fix location.. if (element.innerHTML.toLowerCase() === name) { - //element.scrollIntoView({behavior: "smooth"}) - element.scrollTo({ - top: element.offsetTop-offset, - behavior: "smooth" - }) + element.scrollIntoView({behavior: "smooth"}) + //element.scrollTo({ + // top: element.offsetTop-offset, + // behavior: "smooth" + //}) found = true //element.scrollTo({ // top: element.offsetTop-100, @@ -392,7 +392,7 @@ const Docs = (props) => { {itemMatching && tocLines !== null && tocLines !== undefined && tocLines.length > 0 ?
{tocLines.map((data, index) => { - console.log(data) + //console.log(data) return ( {}}> diff --git a/frontend/src/views/SetAuthentication.jsx b/frontend/src/views/SetAuthentication.jsx index 5fcd0f2b..17849a28 100644 --- a/frontend/src/views/SetAuthentication.jsx +++ b/frontend/src/views/SetAuthentication.jsx @@ -91,6 +91,10 @@ const SetAuthentication = (props) => { if (query[0] === "client_secret") { appAuthData.fields.push({"key": "client_secret", "value": query[1]}) } + + if (query[0] === "oauth_url") { + appAuthData.fields.push({"key": "oauth_url", "value": query[1]}) + } } } diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 3beae1f3..98bb5f27 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.9.14 +VERSION=0.9.23 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force