marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed issues with regex parsing

Browse Source
This commit is contained in:
frikky
2022-04-01 23:37:40 +02:00
parent fec4d63cac
commit 5b78366191
1 changed files with 28 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app_sdk/app_base.py
+28 -23
View File
@@ -275,9 +275,9 @@ class AppBase:
else: else:
self.logger.warning(f"[ERROR] Magic output not defined.") self.logger.warning(f"[ERROR] Magic output not defined.")
except KeyError as e: except KeyError as e:
self.logger.warning(f"[ERROR] Failed to run magic autoparser (send result) - keyerror: {e}") self.logger.warning(f"[DEBUG] Failed to run magic autoparser (send result) - keyerror: {e}")
except Exception as e: except Exception as e:
self.logger.warning(f"[ERROR] Failed to run magic autoparser (send result): {e}") self.logger.warning(f"[DEBUG] Failed to run magic autoparser (send result): {e}")
# Try it with some magic # Try it with some magic
@@ -383,11 +383,13 @@ class AppBase:
self.logger.info(f"[DEBUG] Expected ProtocolError happened: {e}") self.logger.info(f"[DEBUG] Expected ProtocolError happened: {e}")
# FIXME: Re-enable data flushing otherwise we'll overload it all
# Or nah?
if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true": if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true":
try: try:
pass self.log_capture_string.flush()
#self.log_capture_string.flush()
#self.log_capture_string.close() #self.log_capture_string.close()
#pass
except Exception as e: except Exception as e:
print(f"[WARNING] Failed to flush logs: {e}") print(f"[WARNING] Failed to flush logs: {e}")
pass pass
@@ -1150,10 +1152,10 @@ class AppBase:
pass pass
ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data) ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data)
self.logger.info(f"Ret CREATE: {ret.text}") #self.logger.info(f"Ret CREATE: {ret.text}")
cur_id = "" cur_id = ""
if ret.status_code == 200: if ret.status_code == 200:
self.logger.info("RET: %s" % ret.text) #self.logger.info("RET: %s" % ret.text)
ret_json = ret.json() ret_json = ret.json()
if not ret_json["success"]: if not ret_json["success"]:
self.logger.info("Not success in file upload creation.") self.logger.info("Not success in file upload creation.")
@@ -1907,10 +1909,10 @@ class AppBase:
baseresult = variable["value"] baseresult = variable["value"]
break break
except KeyError as e: except KeyError as e:
print("[INFO] KeyError exec variables: %s" % e) #print("[INFO] KeyError exec variables: %s" % e)
pass pass
except TypeError as e: except TypeError as e:
print("[INFO] TypeError exec variables: %s" % e) #print("[INFO] TypeError exec variables: %s" % e)
pass pass
except KeyError as error: except KeyError as error:
@@ -1975,7 +1977,6 @@ class AppBase:
try: try:
return json.dumps(json.loads(returndata)), is_loop return json.dumps(json.loads(returndata)), is_loop
except json.decoder.JSONDecodeError as e: except json.decoder.JSONDecodeError as e:
print("[ERROR] Error in decoder: %s" % e)
return returndata, is_loop return returndata, is_loop
# Sending self as it's not a normal function # Sending self as it's not a normal function
@@ -1984,6 +1985,7 @@ class AppBase:
errors = False errors = False
error_msg = "" error_msg = ""
try: try:
self.logger.info("In liquid")
if len(template) > 10000000: if len(template) > 10000000:
self.logger.info("[DEBUG] Skipping liquid - size too big (%d)" % len(template)) self.logger.info("[DEBUG] Skipping liquid - size too big (%d)" % len(template))
return template return template
@@ -2001,8 +2003,8 @@ class AppBase:
# return template # return template
#self.logger.info(globals()) #self.logger.info(globals())
if len(template) > 100: #if len(template) > 100:
self.logger.info("[DEBUG] Running liquid with data of length %d" % len(template)) # self.logger.info("[DEBUG] Running liquid with data of length %d" % len(template))
#self.logger.info(f"[DEBUG] Data: {template}") #self.logger.info(f"[DEBUG] Data: {template}")
run = Liquid(template, mode="wild", from_file=False, filters=shuffle_filters.filters) run = Liquid(template, mode="wild", from_file=False, filters=shuffle_filters.filters)
@@ -2052,6 +2054,7 @@ class AppBase:
error = True error = True
error_msg = e error_msg = e
self.logger.info("Done in liquid")
if error == True: if error == True:
self.action_result["status"] = "FAILURE" self.action_result["status"] = "FAILURE"
data = { data = {
@@ -2166,7 +2169,8 @@ class AppBase:
#match = ".*?([$]{1}([a-zA-Z0-9 _-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})[$/, ]?" #match = ".*?([$]{1}([a-zA-Z0-9 _-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})[$/, ]?"
#match = ".*?([$]{1}([a-zA-Z0-9 _-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})" #match = ".*?([$]{1}([a-zA-Z0-9 _-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})"
match = ".*?([$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})" # Removed space - no longer ok. Force underscore. #match = ".*?([$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})" # Removed space - no longer ok. Force underscore.
match = "([$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})" # Removed .*? to make it work with large amounts of data
# Extra replacements for certain scenarios # Extra replacements for certain scenarios
escaped_dollar = "\\$" escaped_dollar = "\\$"
@@ -2181,18 +2185,21 @@ class AppBase:
# Basic fix in case variant isn't set # Basic fix in case variant isn't set
try: try:
self.logger.info("[DEBUG] Parameter variant: %s" % parameter["variant"]) self.logger.info(f"[DEBUG] Parameter variant: {parameter['variant']} of length {len(parameter['value'])}")
except: except:
parameter["variant"] = "STATIC_VALUE" parameter["variant"] = "STATIC_VALUE"
# Regex to find all the things # Regex to find all the things
if parameter["variant"] == "STATIC_VALUE": # Should just go in here if data is ... not so big
#if parameter["variant"] == "STATIC_VALUE" and len(parameter["value"]) < 1000000:
if parameter["variant"] == "STATIC_VALUE" and len(parameter["value"]) < 5000000:
data = parameter["value"] data = parameter["value"]
actualitem = re.findall(match, data, re.MULTILINE) actualitem = re.findall(match, data, re.MULTILINE)
#self.logger.debug(f"\n\nHandle static data with JSON: {data}\n\n") #self.logger.debug(f"\n\nHandle static data with JSON: {data}\n\n")
#self.logger.info("STATIC PARSED: %s" % actualitem) #self.logger.info("STATIC PARSED: %s" % actualitem)
self.logger.info("[INFO] Done with regex matching")
if len(actualitem) > 0: if len(actualitem) > 0:
#self.logger.info("[DEACTUAL: ", actualitem) self.logger.info("[DEBUG] Matches: ", actualitem)
for replace in actualitem: for replace in actualitem:
try: try:
to_be_replaced = replace[0] to_be_replaced = replace[0]
@@ -2204,8 +2211,7 @@ class AppBase:
# Trying without string dumping. # Trying without string dumping.
value, is_loop = get_json_value(fullexecution, to_be_replaced) value, is_loop = get_json_value(fullexecution, to_be_replaced)
#self.logger.info("\n\nType of value: %s. Value: %s" % (type(value), value)) #self.logger.info(f"\n\nType of value: {type(value)}")
self.logger.info("\n\nType of value: %s" % type(value))
if isinstance(value, str): if isinstance(value, str):
parameter["value"] = parameter["value"].replace(to_be_replaced, value) parameter["value"] = parameter["value"].replace(to_be_replaced, value)
elif isinstance(value, dict) or isinstance(value, list): elif isinstance(value, dict) or isinstance(value, list):
@@ -2219,13 +2225,15 @@ class AppBase:
# parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) # parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value))
# self.logger.info("Failed parsing value as string?") # self.logger.info("Failed parsing value as string?")
else: else:
self.logger.info("Unknown type %s" % type(value)) self.logger.info("[WARNING] Unknown type %s" % type(value))
try: try:
parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value))
except json.decoder.JSONDecodeError as e: except json.decoder.JSONDecodeError as e:
parameter["value"] = parameter["value"].replace(to_be_replaced, value) parameter["value"] = parameter["value"].replace(to_be_replaced, value)
#self.logger.info("VALUE: %s" % parameter["value"]) #self.logger.info("VALUE: %s" % parameter["value"])
else:
self.logger.info(f"[ERROR] Not running static variant regex parsing (slow) on value with length {len(parameter['value'])}. Max is 5Mb~.")
if parameter["variant"] == "WORKFLOW_VARIABLE": if parameter["variant"] == "WORKFLOW_VARIABLE":
self.logger.info("[DEBUG] Handling workflow variable") self.logger.info("[DEBUG] Handling workflow variable")
@@ -2311,7 +2319,7 @@ class AppBase:
parameter["value"] = parameter["value"].replace(end_variable, "", -1) parameter["value"] = parameter["value"].replace(end_variable, "", -1)
parameter["value"] = parameter["value"].replace(escape_replacement, "$", -1) parameter["value"] = parameter["value"].replace(escape_replacement, "$", -1)
except: except:
self.logger.info("Error in datareplacement") self.logger.info(f"[ERROR] Problem in datareplacement: {e}")
# Just here in case it breaks # Just here in case it breaks
# Implemented 02.08.2021 # Implemented 02.08.2021
@@ -2321,9 +2329,6 @@ class AppBase:
except: except:
pass pass
#self.logger.info("Replaced data: %s" % parameter["value"])
#self.logger.info("POST liquid: %s" % parameter["value"])
return "", parameter["value"], is_loop return "", parameter["value"], is_loop
def run_validation(sourcevalue, check, destinationvalue): def run_validation(sourcevalue, check, destinationvalue):
@@ -3444,7 +3449,7 @@ class AppBase:
# Has to start like this due to imports in other apps # Has to start like this due to imports in other apps
# Move it outside everything? # Move it outside everything?
app = cls(redis=None, logger=logger, console_logger=logger) app = cls(redis=None, logger=logger, console_logger=logger)
logger.info(f"[DEBUG] Action: {action}") #logger.info(f"[DEBUG] Action: {action}")
if isinstance(action, str): if isinstance(action, str):
logger.info("[DEBUG] Normal execution (env var). Action is a string.") logger.info("[DEBUG] Normal execution (env var). Action is a string.")