diff --git a/backend/Dockerfile b/backend/Dockerfile index d203ed78..a1887c93 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -14,6 +14,8 @@ ADD ./go-app/go.mod /app # Required files for code generation ADD ./app_sdk/app_base.py /app_sdk ADD ./app_sdk/static_baseline.py /app_sdk +ADD ./app_sdk_kali/app_base.py /app_sdk_kali +ADD ./app_sdk_kali/static_baseline.py /app_sdk_kali ADD ./app_gen /app_gen RUN go get -v @@ -28,6 +30,7 @@ FROM alpine:3.12 COPY --from=builder /app/ /app COPY --from=builder /app_sdk/ /app_sdk +COPY --from=builder /app_sdk_kali/ /app_sdk_kali COPY --from=builder /app_gen/ /app_gen COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt diff --git a/backend/app_sdk_kali/Dockerfile b/backend/app_sdk_kali/Dockerfile new file mode 100644 index 00000000..af7fd24f --- /dev/null +++ b/backend/app_sdk_kali/Dockerfile @@ -0,0 +1,19 @@ +FROM kalilinux/kali-rolling as base + +FROM base as builder + +RUN apt-get update +RUN apt-get dist-upgrade -y +RUN apt install build-essential libffi-dev musl-dev openssl python3 python3-pip -y + +RUN mkdir /install +WORKDIR /install + +COPY requirements.txt /requirements.txt +RUN pip install --prefix="/install" -r /requirements.txt + +FROM base + +COPY --from=builder /install /usr/local +COPY __init__.py /app/walkoff_app_sdk/__init__.py +COPY app_base.py /app/walkoff_app_sdk/app_base.py diff --git a/backend/app_sdk_kali/LICENSE b/backend/app_sdk_kali/LICENSE new file mode 100644 index 00000000..ce11f6f3 --- /dev/null +++ b/backend/app_sdk_kali/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Frikkylikeme + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/backend/app_sdk_kali/README.md b/backend/app_sdk_kali/README.md new file mode 100644 index 00000000..754392ff --- /dev/null +++ b/backend/app_sdk_kali/README.md @@ -0,0 +1,16 @@ +# app_sdk.py +This is the SDK used for apps to behave like they should. +To change it in the backend, upload it to Buckets/shuffler.appspot.com/generated_apps/baseline. + +# static_baseline.py +It's used for python code generation and should be under MIT. Has to be located here because it's used by the backend. + +## If you want to update apps.. PS: downloads from docker hub do overrides.. :) +1. Write your code & check if runtime works +2. Build app_base image +3. docker rm $(docker ps -aq) # Remove all stopped containers +4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...) +5. Rebuild the Docker image (click load in GUI?) + +# LICENSE +Everything in here is MIT, not AGPLv3 as indicated by the license. diff --git a/backend/app_sdk_kali/__init__.py b/backend/app_sdk_kali/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/app_sdk_kali/app_base.py b/backend/app_sdk_kali/app_base.py new file mode 100644 index 00000000..e896f67c --- /dev/null +++ b/backend/app_sdk_kali/app_base.py @@ -0,0 +1,1169 @@ +import os +import sys +import re +import time +import json +import logging +import requests +import urllib.parse + +class AppBase: + """ The base class for Python-based apps in Shuffle, handles logging and callbacks configurations""" + __version__ = None + app_name = None + + def __init__(self, redis=None, logger=None, console_logger=None):#, docker_client=None): + self.logger = logger if logger is not None else logging.getLogger("AppBaseLogger") + self.redis=redis + self.console_logger = logger if logger is not None else logging.getLogger("AppBaseLogger") + + # apikey is for the user / org + # authorization is for the specific workflow + self.url = os.getenv("CALLBACK_URL", "https://shuffler.io") + self.action = os.getenv("ACTION", "") + self.authorization = os.getenv("AUTHORIZATION", "") + self.current_execution_id = os.getenv("EXECUTIONID", "") + self.full_execution = os.getenv("FULL_EXECUTION", "") + + if isinstance(self.action, str): + self.action = json.loads(self.action) + + def send_result(self, action_result, headers, stream_path): + if action_result["status"] == "EXECUTING": + action_result["status"] = "FAILURE" + + # I wonder if this actually works + self.logger.info("Before last stream result") + try: + ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) + self.logger.info("Result: %d" % ret.status_code) + if ret.status_code != 200: + self.logger.info(ret.text) + except requests.exceptions.ConnectionError as e: + self.logger.exception(e) + return + except TypeError as e: + self.logger.exception(e) + action_result["status"] = "FAILURE" + action_result["result"] = "POST error: %s" % e + self.logger.info("Before typeerror stream result") + ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) + self.logger.info("Result: %d" % ret.status_code) + if ret.status_code != 200: + self.logger.info(ret.text) + + async def execute_action(self, action): + # FIXME - add request for the function STARTING here. Use "results stream" or something + # PAUSED, AWAITING_DATA, PENDING, COMPLETED, ABORTED, EXECUTING, SUCCESS, FAILURE + + # !!! Let this line stay - its used for some horrible codegeneration / stitching !!! # + #STARTCOPY + stream_path = "/api/v1/streams" + action_result = { + "action": action, + "authorization": self.authorization, + "execution_id": self.current_execution_id, + "result": "", + "started_at": int(time.time()), + "status": "EXECUTING" + } + self.logger.info("ACTION RESULT (start): %s", action_result) + + if len(self.action) == 0: + print("ACTION env not defined") + action_result["result"] = "Error in setup ENV: ACTION not defined" + self.send_result(action_result, headers, stream_path) + return + if len(self.authorization) == 0: + print("AUTHORIZATION env not defined") + action_result["result"] = "Error in setup ENV: AUTHORIZATION not defined" + self.send_result(action_result, headers, stream_path) + return + if len(self.current_execution_id) == 0: + print("EXECUTIONID env not defined") + action_result["result"] = "Error in setup ENV: EXECUTIONID not defined" + self.send_result(action_result, headers, stream_path) + return + + headers = { + "Content-Type": "application/json", + "Authorization": "Bearer %s" % self.authorization + } + + # Add async logger + # self.console_logger.handlers[0].stream.set_execution_id() + #self.logger.info("Before initial stream result") + try: + ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) + self.logger.info("Workflow: %d" % ret.status_code) + if ret.status_code != 200: + self.logger.info(ret.text) + except requests.exceptions.ConnectionError as e: + print("Connectionerror: %s" % e) + + action_result["result"] = "Bad setup during startup: %s" % e + self.send_result(action_result, headers, stream_path) + return + + # Verify whether there are any parameters with ACTION_RESULT required + # If found, we get the full results list from backend + fullexecution = {} + if len(self.full_execution) == 0: + print("NO EXECUTION - LOADING!") + try: + tmpdata = { + "authorization": self.authorization, + "execution_id": self.current_execution_id + } + + self.logger.info("Before FULLEXEC stream result") + ret = requests.post( + "%s/api/v1/streams/results" % (self.url), + headers=headers, + json=tmpdata + ) + + if ret.status_code == 200: + fullexecution = ret.json() + else: + self.logger.info("Error: Data: ", ret.json()) + self.logger.info("Error with status code for results. Crashing because ACTION_RESULTS or WORKFLOW_VARIABLE can't be handled. Status: %d" % ret.status_code) + action_result["result"] = "Bad result from backend: %d" % ret.status_code + self.send_result(action_result, headers, stream_path) + return + except requests.exceptions.ConnectionError as e: + self.logger.info("Connectionerror: %s" % e) + action_result["result"] = "Connection error during startup: %s" % e + self.send_result(action_result, headers, stream_path) + return + else: + try: + fullexecution = json.loads(self.full_execution) + except json.decoder.JSONDecodeError as e: + print("Json decode execution error: %s" % e) + action_result["result"] = "Json error during startup: %s" % e + self.send_result(action_result, headers, stream_path) + return + + print("") + + + self.logger.info("AFTER FULLEXEC stream result") + + # Gets the value at the parenthesis level you want + def parse_nested_param(string, level): + """ + Generate strings contained in nested (), indexing i = level + """ + if len(re.findall("\(", string)) == len(re.findall("\)", string)): + LeftRightIndex = [x for x in zip( + [Left.start()+1 for Left in re.finditer('\(', string)], + reversed([Right.start() for Right in re.finditer('\)', string)]))] + + elif len(re.findall("\(", string)) > len(re.findall("\)", string)): + return parse_nested_param(string + ')', level) + elif len(re.findall("\(", string)) < len(re.findall("\)", string)): + return parse_nested_param('(' + string, level) + + else: + return 'Failed to parse params' + + try: + return [string[LeftRightIndex[level][0]:LeftRightIndex[level][1]]] + except IndexError: + return [string[LeftRightIndex[level+1][0]:LeftRightIndex[level+1][1]]] + + # Finds the deepest level parenthesis in a string + def maxDepth(S): + current_max = 0 + max = 0 + n = len(S) + + # Traverse the input string + for i in range(n): + if S[i] == '(': + current_max += 1 + + if current_max > max: + max = current_max + elif S[i] == ')': + if current_max > 0: + current_max -= 1 + else: + return -1 + + # finally check for unbalanced string + if current_max != 0: + return -1 + + return max-1 + + # Specific type parsing + def parse_type(data, thistype): + if data == None: + return "Empty" + + if "int" in thistype or "number" in thistype: + try: + return int(data) + except ValueError: + print("ValueError while casting %s" % data) + return data + if "lower" in thistype: + return data.lower() + if "upper" in thistype: + return data.upper() + if "trim" in thistype: + return data.strip() + if "strip" in thistype: + return data.strip() + if "split" in thistype: + return data.split() + if "len" in thistype or "length" in thistype: + tmp = "" + try: + tmp = json.loads(data) + except: + pass + + if isinstance(tmp, list): + return str(len(tmp)) + + return str(len(data)) + if "parse" in thistype: + splitvalues = [] + default_error = """Error. Expected syntax: parse(["hello","test1"],0:1)""" + if "," in data: + splitvalues = data.split(",") + + for item in range(len(splitvalues)): + splitvalues[item] = splitvalues[item].strip() + else: + return default_error + + lastsplit = [] + if ":" in splitvalues[-1]: + lastsplit = splitvalues[-1].split(":") + else: + try: + lastsplit = [int(splitvalues[-1])] + except ValueError: + return default_error + + try: + parsedlist = ",".join(splitvalues[0:-1]) + if len(lastsplit) > 1: + tmp = json.loads(parsedlist)[int(lastsplit[0]):int(lastsplit[1])] + else: + tmp = json.loads(parsedlist)[lastsplit[0]] + + print(tmp) + return tmp + except IndexError as e: + return default_error + + # Parses the INNER value and recurses until everything is done + def parse_wrapper(data): + try: + if "(" not in data or ")" not in data: + return data + except TypeError: + return data + + #print("Running %s" % data) + + # Look for the INNER wrapper first, then move out + wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length"] + found = False + for wrapper in wrappers: + if wrapper not in data.lower(): + continue + + found = True + break + + if not found: + return data + + # Do stuff here. + innervalue = parse_nested_param(data, maxDepth(data)-0) + outervalue = parse_nested_param(data, maxDepth(data)-1) + print("INNER: ", innervalue) + print("OUTER: ", outervalue) + + if outervalue != innervalue: + #print("Outer: ", outervalue, " inner: ", innervalue) + for key in range(len(innervalue)): + # Replace OUTERVALUE[key] with INNERVALUE[key] in data. + print("Replace %s with %s in %s" % (outervalue[key], innervalue[key], data)) + data = data.replace(outervalue[key], innervalue[key]) + else: + for thistype in wrappers: + if thistype.lower() not in data.lower(): + continue + + parsed_value = parse_type(innervalue[0], thistype.lower()) + return parsed_value + + print("DATA: %s\n" % data) + return parse_wrapper(data) + + def parse_wrapper_start(data): + newdata = [] + newstring = "" + record = True + paranCnt = 0 + for char in data: + if char == "(": + paranCnt += 1 + + if not record: + record = True + + if record: + newstring += char + + if paranCnt == 0 and char == " ": + newdata.append(newstring) + newstring = "" + record = True + + if char == ")": + paranCnt -= 1 + + if paranCnt == 0: + record = False + + if len(newstring) > 0: + newdata.append(newstring) + + #print(newdata) + parsedlist = [] + non_string = False + for item in newdata: + ret = parse_wrapper(item) + if not isinstance(ret, str): + non_string = True + + parsedlist.append(ret) + + if len(parsedlist) > 0 and not non_string: + return " ".join(parsedlist) + elif len(parsedlist) == 1 and non_string: + return parsedlist[0] + else: + #print("Casting back to string because multi: ", parsedlist) + newlist = [] + for item in parsedlist: + try: + newlist.append(str(item)) + except ValueError: + newlist.append("parsing_error") + return " ".join(newlist) + + # Parses JSON loops and such down to the item you're looking for + def recurse_json(basejson, parsersplit): + match = "#(\d+):?-?([0-9a-z]+)?#?" + print("Split: %s\n%s" % (parsersplit, basejson)) + try: + outercnt = 0 + + # Loops over split values + for value in parsersplit: + print("VALUE: %s\n" % value) + actualitem = re.findall(match, value, re.MULTILINE) + if value == "#": + newvalue = [] + for innervalue in basejson: + # 1. Check the next item (message) + # 2. Call this function again + + try: + ret, is_loop = recurse_json(innervalue, parsersplit[outercnt+1:]) + except IndexError: + # Only in here if it's the last loop without anything in it? + ret, is_loop = recurse_json(innervalue, parsersplit[outercnt:]) + + newvalue.append(ret) + + # Magical way of returning which makes app sdk identify + # it as multi execution + return newvalue, True + elif len(actualitem) > 0: + # FIXME: This is absolutely not perfect. + print("In recursion v2: ", actualitem) + + is_loop = True + newvalue = [] + firstitem = actualitem[0][0] + seconditem = actualitem[0][1] + + # Means it's a single item -> continue + if seconditem == "": + print("In first - handling %s", seconditem) + tmpitem = basejson[int(firstitem)] + try: + newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:]) + except IndexError: + newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:]) + else: + if seconditem == "max": + seconditem = len(basejson) + if seconditem == "min": + seconditem = 0 + + newvalue = [] + for i in range(int(firstitem), int(seconditem)): + # 1. Check the next item (message) + # 2. Call this function again + print("Base: %s" % basejson[i]) + + try: + ret, is_loop = recurse_json(basejson[i], parsersplit[outercnt+1:]) + except IndexError: + print("INDEXERROR: ", parsersplit[outercnt]) + #ret = innervalue + ret, is_loop = recurse_json(innervalue, parsersplit[outercnt:]) + + print(ret) + #exit() + newvalue.append(ret) + + return newvalue, is_loop + + # FIXME: Add specific loop for other indexes + else: + #print("BEFORE NORMAL VALUE: ", basejson, value) + if len(value) == 0: + return basejson, False + + if isinstance(basejson[value], str): + print(f"LOADING STRING '%s' AS JSON" % basejson[value]) + try: + basejson = json.loads(basejson[value]) + except json.decoder.JSONDecodeError as e: + print("RETURNING BECAUSE '%s' IS A NORMAL STRING" % basejson[value]) + return basejson[value], False + else: + basejson = basejson[value] + + outercnt += 1 + + except KeyError as e: + print("Lower keyerror: %s" % e) + #return basejson + #return "KeyError: Couldn't find key: %s" % e + + return basejson, False + + # Takes a workflow execution as argument + # Returns a string if the result is single, or a list if it's a list + def get_json_value(execution_data, input_data): + parsersplit = input_data.split(".") + actionname = parsersplit[0][1:].replace(" ", "_", -1) + #Actionname: Start_node + + print(f"Actionname: {actionname}") + + # 1. Find the action + baseresult = "" + actionname_lower = actionname.lower() + try: + if actionname_lower == "exec" or actionname_lower == "webhook" or actionname_lower == "schedule" or actionname_lower == "userinput" or actionname_lower == "email_trigger" or actionname_lower == "trigger": + baseresult = execution_data["execution_argument"] + else: + for result in execution_data["results"]: + resultlabel = result["action"]["label"].replace(" ", "_", -1).lower() + if resultlabel.lower() == actionname_lower: + baseresult = result["result"] + break + + print("BEFORE VARIABLES!") + if len(baseresult) == 0: + try: + #print("WF Variables: %s" % execution_data["workflow"]["workflow_variables"]) + for variable in execution_data["workflow"]["workflow_variables"]: + variablename = variable["name"].replace(" ", "_", -1).lower() + + if variablename.lower() == actionname_lower: + baseresult = variable["value"] + break + except KeyError as e: + print("KeyError wf variables: %s" % e) + pass + except TypeError as e: + print("TypeError wf variables: %s" % e) + pass + + print("BEFORE EXECUTION VAR") + if len(baseresult) == 0: + try: + #print("Execution Variables: %s" % execution_data["execution_variables"]) + for variable in execution_data["execution_variables"]: + variablename = variable["name"].replace(" ", "_", -1).lower() + if variablename.lower() == actionname_lower: + baseresult = variable["value"] + break + except KeyError as e: + print("KeyError exec variables: %s" % e) + pass + except TypeError as e: + print("TypeError exec variables: %s" % e) + pass + + except KeyError as error: + print(f"KeyError in JSON: {error}") + + print(f"After first trycatch") + + # 2. Find the JSON data + if len(baseresult) == 0: + return "", False + + if len(parsersplit) == 1: + return baseresult, False + + baseresult = baseresult.replace("\'", "\"") + basejson = {} + try: + basejson = json.loads(baseresult) + except json.decoder.JSONDecodeError as e: + return baseresult, False + + data, is_loop = recurse_json(basejson, parsersplit[1:]) + parseditem = data + if is_loop: + print("DATA IS A LOOP - SHOULD WRAP") + if parsersplit[-1] == "#": + print("SET DATA WRAPPER TO NORMAL!") + parseditem = "${SHUFFLE_NO_SPLITTER%s}$" % json.dumps(data) + else: + # Return value: ${id[12345, 45678]}$ + print("SET DATA WRAPPER TO %s!" % parsersplit[-1]) + parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data)) + + return parseditem, is_loop + + # Parses parameters sent to it and returns whether it did it successfully with the values found + def parse_params(action, fullexecution, parameter): + # Skip if it starts with $? + jsonparsevalue = "$." + is_loop = False + + # Matches with space in the first part, but not in subsequent parts. + # JSON / yaml etc shouldn't have spaces in their fields anyway. + match = ".*?([$]{1}([a-zA-Z0-9 _-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})" + + # Regex to find all the things + if parameter["variant"] == "STATIC_VALUE": + data = parameter["value"] + actualitem = re.findall(match, data, re.MULTILINE) + #self.logger.debug(f"\n\nHandle static data with JSON: {data}\n\n") + #self.logger.info("STATIC PARSED: %s" % actualitem) + if len(actualitem) > 0: + print("ACTUAL: ", actualitem) + for replace in actualitem: + try: + to_be_replaced = replace[0] + except IndexError: + continue + + # Handles for loops etc. + value, is_loop = get_json_value(fullexecution, to_be_replaced) + + if isinstance(value, str): + parameter["value"] = parameter["value"].replace(to_be_replaced, value) + elif isinstance(value, dict): + parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) + else: + print("Unknown type %s" % type(value)) + try: + parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) + except json.decoder.JSONDecodeError as e: + parameter["value"] = parameter["value"].replace(to_be_replaced, value) + + + if parameter["variant"] == "WORKFLOW_VARIABLE": + print("Handling workflow variable") + found = False + try: + for item in fullexecution["workflow"]["workflow_variables"]: + if parameter["action_field"] == item["name"]: + found = True + parameter["value"] = item["value"] + break + except KeyError as e: + print("KeyError WF variable 1: %s" % e) + pass + except TypeError as e: + print("TypeError WF variables 1: %s" % e) + pass + + if not found: + try: + for item in fullexecution["execution_variables"]: + if parameter["action_field"] == item["name"]: + parameter["value"] = item["value"] + break + except KeyError as e: + print("KeyError WF variable 2: %s" % e) + pass + except TypeError as e: + print("TypeError WF variables 2: %s" % e) + pass + + elif parameter["variant"] == "ACTION_RESULT": + # FIXME - calculate value based on action_field and $if prominent + # FIND THE RIGHT LABEL + # GET THE LABEL'S RESULT + + tmpvalue = "" + self.logger.info("ACTION FIELD: %s" % parameter["action_field"]) + + fullname = "$" + if parameter["action_field"] == "Execution Argument": + tmpvalue = fullexecution["execution_argument"] + fullname += "exec" + else: + fullname += parameter["action_field"] + + self.logger.info("PRE Fullname: %s" % fullname) + + if parameter["value"].startswith(jsonparsevalue): + fullname += parameter["value"][1:] + #else: + # fullname = "$%s" % parameter["action_field"] + + self.logger.info("Fullname: %s" % fullname) + actualitem = re.findall(match, fullname, re.MULTILINE) + self.logger.info("ACTION PARSED: %s" % actualitem) + if len(actualitem) > 0: + for replace in actualitem: + try: + to_be_replaced = replace[0] + except IndexError: + print("Nothing to replace?: " % e) + continue + + # This will never be a loop aka multi argument + parameter["value"] = to_be_replaced + + value, is_loop = get_json_value(fullexecution, to_be_replaced) + print("Loop: %s" % is_loop) + if isinstance(value, str): + parameter["value"] = parameter["value"].replace(to_be_replaced, value) + elif isinstance(value, dict): + parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) + else: + print("Unknown type %s" % type(value)) + try: + parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) + except json.decoder.JSONDecodeError as e: + parameter["value"] = parameter["value"].replace(to_be_replaced, value) + + return "", parameter["value"], is_loop + + def run_validation(sourcevalue, check, destinationvalue): + self.logger.info("Checking %s %s %s" % (sourcevalue, check, destinationvalue)) + + if check == "=" or check.lower() == "equals": + if sourcevalue.lower() == destinationvalue.lower(): + return True + elif check == "!=" or check.lower() == "does not equal": + if sourcevalue.lower() != destinationvalue.lower(): + return True + elif check.lower() == "startswith": + if sourcevalue.lower().startswith(destinationvalue.lower()): + return True + elif check.lower() == "endswith": + if sourcevalue.lower().endswith(destinationvalue.lower()): + return True + elif check.lower() == "contains": + if destinationvalue.lower() in sourcevalue.lower(): + return True + elif check.lower() == "larger than": + try: + if sourcevalue.isdigit() and destinationvalue.isdigit(): + if int(sourcevalue) > int(destinationvalue): + return True + except AttributeError as e: + self.logger.error("Condition larger than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) + return False + elif check.lower() == "smaller than": + try: + if sourcevalue.isdigit() and destinationvalue.isdigit(): + if int(sourcevalue) < int(destinationvalue): + return True + except AttributeError as e: + self.logger.error("Condition smaller than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) + return False + else: + self.logger.info("Condition: can't handle %s yet. Setting to true" % check) + + return False + + def check_branch_conditions(action, fullexecution): + # relevantbranches = workflow.branches where destination = action + try: + if fullexecution["workflow"]["branches"] == None or len(fullexecution["workflow"]["branches"]) == 0: + return True, "" + except KeyError: + return True, "" + + relevantbranches = [] + for branch in fullexecution["workflow"]["branches"]: + if branch["destination_id"] != action["id"]: + continue + + # Remove anything without a condition + try: + if (branch["conditions"]) == 0 or branch["conditions"] == None: + continue + except KeyError: + continue + + self.logger.info("Relevant conditions: %s" % branch["conditions"]) + successful_conditions = [] + failed_conditions = [] + for condition in branch["conditions"]: + self.logger.info("Getting condition value of %s" % condition) + + # Parse all values first here + sourcevalue = condition["source"]["value"] + check, sourcevalue, is_loop = parse_params(action, fullexecution, condition["source"]) + if check: + return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check) + + + #sourcevalue = sourcevalue.encode("utf-8") + sourcevalue = parse_wrapper_start(sourcevalue) + destinationvalue = condition["destination"]["value"] + + check, destinationvalue, is_loop = parse_params(action, fullexecution, condition["destination"]) + if check: + return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check) + + #destinationvalue = destinationvalue.encode("utf-8") + destinationvalue = parse_wrapper_start(destinationvalue) + available_checks = [ + "=", + "equals", + "!=", + "does not equal", + ">", + "larger than", + "<", + "less than", + ">=", + "<=", + "startswith", + "endswith", + "contains", + "re", + "matches regex", + ] + + # FIXME - what should I do here? + if not condition["condition"]["value"] in available_checks: + self.logger.info("Skipping %s %s %s because %s is invalid." % (sourcevalue, condition["condition"]["value"], destinationvalue, condition["condition"]["value"])) + continue + + #print(destinationvalue) + # NEGATE + validation = run_validation(sourcevalue, condition["condition"]["value"], destinationvalue) + + # Configuration = negated because of WorkflowAppActionParam.. + try: + if condition["condition"]["configuration"]: + validation = not validation + except KeyError: + pass + + if not validation: + self.logger.info("Failed condition check for %s %s %s." % (sourcevalue, condition["condition"]["value"], destinationvalue)) + return False, "Failed condition: %s %s %s" % (sourcevalue, condition["condition"]["value"], destinationvalue) + + + # Make a general parser here, at least to get param["name"] = param["value"] in maparameter[string]string + #for condition in branch.conditons: + + return True, "" + + # Checks whether conditions are met, otherwise set + branchcheck, tmpresult = check_branch_conditions(action, fullexecution) + if not branchcheck: + self.logger.info("Failed one or more branch conditions.") + action_result["result"] = tmpresult + action_result["status"] = "FAILURE" + try: + ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) + self.logger.info("Result: %d" % ret.status_code) + if ret.status_code != 200: + self.logger.info(ret.text) + except requests.exceptions.ConnectionError as e: + self.logger.exception(e) + + print("\n\nRETURNING BECAUSE A BRANCH FAILED\n\n") + return + + # Replace name cus there might be issues + # Not doing lower() as there might be user-made functions + actionname = action["name"] + if " " in actionname: + actionname.replace(" ", "_", -1) + #if action.generated: + # actionname = actionname.lower() + + # Runs the actual functions + try: + func = getattr(self, actionname, None) + if func == None: + self.logger.debug("Failed executing %s because func is None." % actionname) + action_result["status"] = "FAILURE" + action_result["result"] = "Function %s doesn't exist." % actionname + elif callable(func): + try: + if len(action["parameters"]) < 1: + result = await func() + else: + # Potentially parse JSON here + # FIXME - add potential authentication as first parameter(s) here + # params[parameter["name"]] = parameter["value"] + #print(fullexecution["authentication"] + # What variables are necessary here tho hmm + + params = {} + try: + for item in action["authentication"]: + print("AUTH: ", key, value) + params[item["key"]] = item["value"] + except KeyError: + print("No authentication specified!") + pass + #action["authentication"] + + # calltimes is used to handle forloops in the app itself. + # 2 kinds of loop - one in gui with one app each, and one like this, + # which is super fast, but has a bad overview (potentially good tho) + calltimes = 1 + result = "" + + all_executions = [] + + # Multi_parameter has the data for each. variable + minlength = 0 + multi_parameters = json.loads(json.dumps(params)) + multiexecution = False + multi_execution_lists = [] + for parameter in action["parameters"]: + check, value, is_loop = parse_params(action, fullexecution, parameter) + + if check: + raise "Value check error: %s" % Exception(check) + + # Custom format for ${name[0,1,2,...]}$ + #submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" + submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" + actualitem = re.findall(submatch, value, re.MULTILINE) + try: + if action["skip_multicheck"]: + print("Skipping multicheck") + actualitem = [] + except KeyError: + pass + + print("Return value: %s" % value) + actionname = action["name"] + #print("Multicheck ", actualitem) + print("Actual item: %s" % actualitem) + if len(actualitem) > 0: + multiexecution = True + + # Loop WITHOUT JSON variables go here. + # Loop WITH variables go in else. + if len(actualitem[0]) > 2 and actualitem[0][1] == "SHUFFLE_NO_SPLITTER": + print("Pre replacement: %s" % actualitem[0][2]) + tmpitem = value + + replacement = actualitem[0][2] + if replacement.startswith("\"") and replacement.endswith("\""): + replacement = replacement[1:len(replacement)-1] + + replacement = replacement.replace("\'", "\"", -1) + print("POST replacement: %s" % replacement) + + json_replacement = replacement + try: + json_replacement = json.loads(replacement) + except json.decoder.JSONDecodeError as e: + print("JSON error singular: %s" % e) + + if len(json_replacement) > minlength: + minlength = len(json_replacement) + + tmpitem = tmpitem.replace(actualitem[0][0], replacement, 1) + params[parameter["name"]] = tmpitem + multi_execution_lists.append(json_replacement) + multi_parameters[parameter["name"]] = json_replacement + + #print("LENGTH OF ARR: %d" % len(resultarray)) + #print("RESULTARRAY: %s" % resultarray) + print("MULTI finished: %s" % replacement) + else: + + # This is here to handle for loops within variables.. kindof + # 1. Find the length of the longest array + # 2. Build an array with the base values based on parameter["value"] + # 3. Get the n'th value of the generated list from values + # 4. Execute all n answers + replacements = {} + for replace in actualitem: + try: + to_be_replaced = replace[0] + actualitem = replace[2] + except IndexError: + continue + + try: + itemlist = json.loads(actualitem) + if len(itemlist) > minlength: + minlength = len(itemlist) + except json.decoder.JSONDecodeError as e: + print("JSON Error: %s in %s" % (e, actualitem)) + + replacements[to_be_replaced] = actualitem + + # This is a result array for JUST this value.. + # What if there are more? + resultarray = [] + for i in range(0, minlength): + tmpitem = json.loads(json.dumps(parameter["value"])) + for key, value in replacements.items(): + replacement = json.dumps(json.loads(value)[i]) + if replacement.startswith("\"") and replacement.endswith("\""): + replacement = replacement[1:len(replacement)-1] + #except json.decoder.JSONDecodeError as e: + + #print("REPLACING %s with %s" % (key, replacement)) + #replacement = parse_wrapper_start(replacement) + tmpitem = tmpitem.replace(key, replacement, -1) + + resultarray.append(tmpitem) + + # With this parameter ready, add it to... a greater list of parameters. Rofl + print("LENGTH OF ARR: %d" % len(resultarray)) + print("RESULTARRAY: %s" % resultarray) + if resultarray not in multi_execution_lists: + multi_execution_lists.append(resultarray) + + multi_parameters[parameter["name"]] = resultarray + else: + # Parses things like int(value) + self.logger.info("Parsing wrapper data for %s" % value) + value = parse_wrapper_start(value) + + params[parameter["name"]] = value + multi_parameters[parameter["name"]] = value + + # Fix lists here + print("CHECKING multi execution list!") + if len(multi_execution_lists) > 0: + print("\n Multi execution list has more data: %d" % len(multi_execution_lists)) + filteredlist = [] + for listitem in multi_execution_lists: + if listitem in filteredlist: + continue + + filteredlist.append(listitem) + + #print("New list length: %d" % len(filteredlist)) + if len(filteredlist) > 1: + print("Calculating new multi-loop length with %d lists" % len(filteredlist)) + tmplength = 1 + for innerlist in filteredlist: + print("List length: %d. %d*%d" % (len(innerlist), len(innerlist), tmplength)) + tmplength = len(innerlist)*tmplength + + minlength = tmplength + + print("New multi execution length: %d\n" % tmplength) + + # FIXME - this is horrible, but works for now + #for i in range(calltimes): + if not multiexecution: + print("APP_SDK DONE: Starting NORMAL execution of function") + newres = await func(**params) + #print("NEWRES: ", newres) + if isinstance(newres, str): + result += newres + else: + try: + result += str(newres) + except ValueError: + result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) + print("Can't handle type %s value from function" % (type(newres))) + print("POST NEWRES RESULT: ", result) + else: + print("APP_SDK DONE: Starting MULTI execution with values %s of length %d" % (multi_parameters, minlength)) + # 1. Use number of executions based on the arrays being similar + # 2. Find the right value from the parsed multi_params + results = [] + json_object = False + for i in range(0, minlength): + # To be able to use the results as a list: + baseparams = json.loads(json.dumps(multi_parameters)) + # {'call': ['GoogleSafebrowsing_2_0', 'VirusTotal_GetReport_3_0']} + # 1. Check if list length is same as minlength + # 2. If NOT same length, duplicate based on length of array + # arraylength = 3 ["1", "2", "3"] + # arraylength = 4 ["1", "2", "3", "4"] + # minlength = 12 - 12/3 = 4 per item = ["1", "1", "1", "1", "2", "2", ...] + + try: + firstlist = True + for key, value in baseparams.items(): + + if isinstance(value, list): + try: + newvalue = value[i] + except IndexError: + pass + + if len(value) != minlength and len(value) > 0: + newarray = [] + print("VALUE: ", value) + additiontime = minlength/len(value) + print("Bad length for value: %d - should be %d. Additiontime: %d" % (len(value), minlength, additiontime)) + if firstlist: + print("Running normal list (FIRST)") + for subvalue in value: + for number in range(int(additiontime)): + newarray.append(subvalue) + else: + #print("Running secondary lists") + ## 1. Set up length of array + ## 2. Put values spread out + # FIXME: This works well, except if lists are same length + newarray = [""] * minlength + + cnt = 0 + for number in range(int(additiontime)): + for subvaluerange in range(len(value)): + # newlocation = number+(additiontime*subvaluerange) + # print("%d+(%d*%d) = %d. VAL: %s" % (number, additiontime, subvaluerange, newlocation, value[subvaluerange])) + # Reverse if same length? + if int(minlength/len(value)) == len(value): + tmp = int(len(value)-subvaluerange-1) + print("NEW: %d" % tmp) + newarray[cnt] = value[tmp] + else: + newarray[cnt] = value[subvaluerange] + cnt += 1 + + #print("Newarray =", newarray) + newvalue = newarray[i] + firstlist = False + + baseparams[key] = newvalue + except IndexError as e: + print("IndexError: %s" % e) + baseparams[key] = "IndexError: %s" % e + except KeyError as e: + print("KeyError: %s" % e) + baseparams[key] = "KeyError: %s" % e + + print("Running with params %s" % baseparams) + ret = await func(**baseparams) + if isinstance(ret, dict) or isinstance(ret, list): + results.append(ret) + json_object = True + else: + ret = ret.replace("\"", "\\\"", -1) + + try: + results.append(json.loads(ret)) + json_object = True + except json.decoder.JSONDecodeError as e: + #print("Json: %s" % e) + results.append(ret) + + #print("Inner ret parsed: %s" % ret) + + # Dump the result as a string of a list + #print("RESULTS: %s" % results) + if isinstance(results, list): + print("JSON OBJECT? ", json_object) + if json_object: + result = json.dumps(results) + else: + result = "[" + for item in results: + try: + json.loads(item) + result += item + except json.decoder.JSONDecodeError as e: + # Common nested issue which puts " around everything + try: + tmpitem = item.replace("\\\"", "\"", -1) + json.loads(tmpitem) + result += tmpitem + + except: + result += "\"%s\"" % item + + result += ", " + + result = result[:-2] + result += "]" + else: + print("Normal result?") + result = results + + print("RESULT: %s" % result) + action_result["status"] = "SUCCESS" + action_result["result"] = str(result) + if action_result["result"] == "": + action_result["result"] = result + + self.logger.debug(f"Executed {action['label']}-{action['id']} with result: {result}") + #self.logger.debug(f"Data: %s" % action_result) + except TypeError as e: + print("TypeError issue: %s" % e) + action_result["status"] = "FAILURE" + action_result["result"] = "TypeError: %s" % str(e) + else: + print("Function %s doesn't exist?" % action["name"]) + self.logger.error(f"App {self.__class__.__name__}.{action['name']} is not callable") + action_result["status"] = "FAILURE" + action_result["result"] = "Function %s is not callable." % actionname + + except Exception as e: + print(f"Failed to execute: {e}") + self.logger.exception(f"Failed to execute {e}-{action['id']}") + action_result["status"] = "FAILURE" + action_result["result"] = f"General exception: {e}" + + action_result["completed_at"] = int(time.time()) + + # Send the result :) + self.send_result(action_result, headers, stream_path) + return + + + #STOPCOPY + # !!! Let the above line stay - its used for some horrible codegeneration / stitching !!! # + + @classmethod + async def run(cls): + """ Connect to Redis and HTTP session, await actions """ + logging.basicConfig(format="{asctime} - {name} - {levelname}:{message}", style='{') + logger = logging.getLogger(f"{cls.__name__}") + logger.setLevel(logging.DEBUG) + print("Started execution!!") + + app = cls(redis=None, logger=logger, console_logger=logger) + + # Authorization for the app/function to control the workflow + # Function will crash if its wrong, which it probably should. + + await app.execute_action(app.action) diff --git a/backend/app_sdk_kali/build.sh b/backend/app_sdk_kali/build.sh new file mode 100644 index 00000000..78c76ced --- /dev/null +++ b/backend/app_sdk_kali/build.sh @@ -0,0 +1,13 @@ +#!/bin/bash +NAME=app_sdk_kali +VERSION=0.7.3 + +docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force +docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION + +#docker push frikky/$NAME:$VERSION +#docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION +#docker push ghcr.io/frikky/$NAME:$VERSION + +docker push frikky/shuffle:$NAME +docker push ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/app_sdk_kali/requirements.txt b/backend/app_sdk_kali/requirements.txt new file mode 100644 index 00000000..804abb1b --- /dev/null +++ b/backend/app_sdk_kali/requirements.txt @@ -0,0 +1,2 @@ +requests +urllib3 diff --git a/backend/app_sdk_kali/static_baseline.py b/backend/app_sdk_kali/static_baseline.py new file mode 100644 index 00000000..152131cd --- /dev/null +++ b/backend/app_sdk_kali/static_baseline.py @@ -0,0 +1,76 @@ +import os +import sys +import time +import logging +import requests + +# Goal here: +# * Make an app from WALKOFF able to run without app_base.py from WALKOFF +# # How: +# * Make it rely 100% on INPUT throug HTTP invocations instead of redis READS +# # But really, how? +# * Make a WORKER that reads the queue, and reuses a function + +# Here to get it global +apikey = "" +try: + apikey = os.environ["FUNCTION_APIKEY"] +except KeyError: + pass + +# Authorize the execution +def authorization(request): + # This is basically my issue, but it enforces the use of an internal API key for execution + try: + apikey = os.environ["FUNCTION_APIKEY"] + except KeyError: + return f"Internal server error", 500 + + + # Check API key from ENV authentication + authentication = request.headers.get("Authorization") + if authentication == None: return f"Unauthorized", 401 + + apikey_split = authentication.split(" ") + if apikey_split[0] != "Bearer" or len(apikey_split) != 2: + return f"Apikey error", 401 + + if apikey != apikey_split[1]: + return f"Unauthorized", 401 + + return run(request) + +class AppBase: + """ The base class for Python-based Walkoff applications, handles Redis and logging configurations. """ + __version__ = None + app_name = None + + def __init__(self, redis=None, logger=None, console_logger=None):#, docker_client=None): + self.logger = logger if logger is not None else logging.getLogger("AppBaseLogger") + self.redis=redis + self.console_logger=console_logger + self.current_execution_id = None + self.url = "https://shuffler.io" + self.apikey = apikey + + @classmethod + async def run(cls, action): + """ Connect to Redis and HTTP session, await actions """ + logging.basicConfig(format="{asctime} - {name} - {levelname}:{message}", style='{') + logger = logging.getLogger(f"{cls.__name__}") + logger.setLevel(logging.DEBUG) + + app = cls(redis=None, logger=logger, console_logger=logger) + + # Authorization for the app/function to control the workflow + # Function will crash if its wrong, which it probably should. + + await app.execute_action(action) + + async def execute_action(self, action): + # FIXME - add request for the function STARTING here. Use "results stream" or something + # PAUSED, AWAITING_DATA, PENDING, COMPLETED, ABORTED, EXECUTING, SUCCESS, FAILURE + + self.authorization = action["authorization"] + self.execution_id = action["execution_id"] + self.current_execution_id = action["execution_id"] diff --git a/docker-compose.yml b/docker-compose.yml index 6d92e85a..8457834a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,6 +52,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock environment: - SHUFFLE_APP_SDK_VERSION=0.6.0 + - SHUFFLE_APP_SDK_KALI_VERSION=0.6.0 - SHUFFLE_WORKER_VERSION=0.6.0 - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} diff --git a/functions/onprem/README.md b/functions/onprem/README.md index 51e5782d..74358762 100644 --- a/functions/onprem/README.md +++ b/functions/onprem/README.md @@ -38,3 +38,11 @@ cd app_sdk docker build . -t frikky/shuffle:app_sdk docker push frikky/shuffle:app_sdk ``` + + +Update app_sdk_kali: +``` +cd app_sdk_kali +docker build . -t frikky/shuffle:app_sdk_kali +docker push frikky/shuffle:app_sdk_kali +```