marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed recursion bug where items that used to become empty strings became full values

Browse Source
This commit is contained in:
Frikky
2024-02-02 01:54:18 +01:00
parent 8cb9e2eb8e
commit cb9be52245
2 changed files with 186 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app_sdk/app_base.py
+66 -28
View File
@@ -2021,6 +2021,7 @@ class AppBase:
return " ".join(newlist) return " ".join(newlist)
# Parses JSON loops and such down to the item you're looking for # Parses JSON loops and such down to the item you're looking for
# Check recurse_test.py for examples and tests of this function
# $nodename.#.id # $nodename.#.id
# $nodename.data.#min-max.info.id # $nodename.data.#min-max.info.id
# $nodename.data.#1-max.info.id # $nodename.data.#1-max.info.id
@@ -2077,7 +2078,7 @@ class AppBase:
# Means it's a single item -> continue # Means it's a single item -> continue
if seconditem == "": if seconditem == "":
print("[INFO] In first - handling %s. Len: %d" % (firstitem, len(basejson))) #print("[INFO] In first - handling %s. Len: %d" % (firstitem, len(basejson)))
if str(firstitem).lower() == "max" or str(firstitem).lower() == "last" or str(firstitem).lower() == "end": if str(firstitem).lower() == "max" or str(firstitem).lower() == "last" or str(firstitem).lower() == "end":
firstitem = len(basejson)-1 firstitem = len(basejson)-1
elif str(firstitem).lower() == "min" or str(firstitem).lower() == "first": elif str(firstitem).lower() == "min" or str(firstitem).lower() == "first":
@@ -2085,14 +2086,14 @@ class AppBase:
else: else:
firstitem = int(firstitem) firstitem = int(firstitem)
print(f"[DEBUG] Post lower checks with item {firstitem}") #print(f"[DEBUG] Post lower checks with item {firstitem}")
tmpitem = basejson[int(firstitem)] tmpitem = basejson[int(firstitem)]
try: try:
newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:]) newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:])
except IndexError: except IndexError:
newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:]) newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:])
else: else:
print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem)) #print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem))
if isinstance(firstitem, str): if isinstance(firstitem, str):
if firstitem.lower() == "max" or firstitem.lower() == "last" or firstitem.lower() == "end": if firstitem.lower() == "max" or firstitem.lower() == "last" or firstitem.lower() == "end":
firstitem = len(basejson)-1 firstitem = len(basejson)-1
@@ -2113,7 +2114,7 @@ class AppBase:
else: else:
seconditem = int(seconditem) seconditem = int(seconditem)
print(f"[DEBUG] Post lower checks 2: {firstitem} AND {seconditem}") #print(f"[DEBUG] Post lower checks 2: {firstitem} AND {seconditem}")
newvalue = [] newvalue = []
if int(seconditem) > len(basejson): if int(seconditem) > len(basejson):
seconditem = len(basejson) seconditem = len(basejson)
@@ -2121,12 +2122,11 @@ class AppBase:
for i in range(int(firstitem), int(seconditem)+1): for i in range(int(firstitem), int(seconditem)+1):
# 1. Check the next item (message) # 1. Check the next item (message)
# 2. Call this function again # 2. Call this function again
#self.logger.info("Base: %s" % basejson[i])
try: try:
ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt+1:]) ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt+1:])
except IndexError: except IndexError:
print("[DEBUG] INDEXERROR: ", parsersplit[outercnt]) #print("[DEBUG] INDEXERROR (1): ", parsersplit[outercnt])
#ret = innervalue #ret = innervalue
ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt:]) ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt:])
@@ -2140,13 +2140,13 @@ class AppBase:
try: try:
if isinstance(basejson, list): if isinstance(basejson, list):
print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value)
return basejson, False return basejson, False
elif isinstance(basejson, bool): elif isinstance(basejson, bool):
print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value) #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value)
return basejson, False return basejson, False
elif isinstance(basejson, int): elif isinstance(basejson, int):
print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value) #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value)
return basejson, False return basejson, False
elif isinstance(basejson[value], str): elif isinstance(basejson[value], str):
try: try:
@@ -2154,8 +2154,16 @@ class AppBase:
basejson = json.loads(basejson[value]) basejson = json.loads(basejson[value])
else: else:
# Should we sanitize here? # Should we sanitize here?
self.logger.info("[DEBUG] VALUE TO SANITIZE?: %s" % basejson[value]) #print("[DEBUG] VALUE TO SANITIZE FOR KEY '%s'?: %s" % (value, basejson[value]))
# Check if we are on the last item?
if outercnt == len(parsersplit)-1:
#print("[DEBUG] LAST KEY")
return str(basejson[value]), False return str(basejson[value]), False
else:
#print("[DEBUG] NOT LAST KEY")
pass
except json.decoder.JSONDecodeError as e: except json.decoder.JSONDecodeError as e:
return str(basejson[value]), False return str(basejson[value]), False
else: else:
@@ -2169,54 +2177,88 @@ class AppBase:
try: try:
if isinstance(basejson, list): if isinstance(basejson, list):
print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value)
return basejson, False return basejson, False
elif isinstance(basejson, bool): elif isinstance(basejson, bool):
print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value) #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value)
return basejson, False return basejson, False
elif isinstance(basejson, int): elif isinstance(basejson, int):
print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value) #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value)
return basejson, False return basejson, False
elif isinstance(basejson[value], str): elif isinstance(basejson[value], str):
print(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value]) #print(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value])
try: try:
print("[DEBUG] BASEJSON: %s" % basejson) #print("[DEBUG] BASEJSON: %s" % basejson)
if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")): if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")):
basejson = json.loads(basejson[value]) basejson = json.loads(basejson[value])
else: else:
if outercnt == len(parsersplit)-1:
#print("LAST KEY (2)")
return str(basejson[value]), False return str(basejson[value]), False
else:
#print("NOT LAST KEY (2)")
pass
except json.decoder.JSONDecodeError as e: except json.decoder.JSONDecodeError as e:
print("[DEBUG] RETURNING BECAUSE '%s' IS A NORMAL STRING (1)" % basejson[value]) #print("[DEBUG] RETURNING BECAUSE '%s' IS A NORMAL STRING (1)" % basejson[value])
return str(basejson[value]), False return str(basejson[value]), False
else: else:
basejson = basejson[value] basejson = basejson[value]
except KeyError as e: except KeyError as e:
print("\n\n[WARNING] Running third dot notation fix that always find the correct value %s: %s" % (value, e)) # Check if previous key was handled or not
previouskey = parsersplit[outercnt-1]
#print("[DEBUG] PREVIOUS KEY: ", previouskey)
tmpval = previouskey + "." + value
#print("\n\n[WARNING] Running third dot notation fix '%s' on data %s: %s" % (value, basejson, e))
if tmpval in basejson:
return basejson[tmpval], False
try: try:
currentsplitcnt = splitcnt currentsplitcnt = splitcnt
recursed_value = value recursed_value = value
handled = False handled = False
#tmpbase = basejson
previouskey = value
while True: while True:
#print("\n\n[DEBUG] CURRENTSPLITCNT: ", currentsplitcnt)
newvalue = parsersplit[currentsplitcnt+1] newvalue = parsersplit[currentsplitcnt+1]
if newvalue == "#" or newvalue == "": if newvalue == "#" or newvalue == "":
break break
recursed_value += "." + newvalue recursed_value += "." + newvalue
#print("\n\nRECURSED: ", recursed_value)
found = False found = False
for key, value in basejson.items(): for key, value in basejson.items():
if recursed_value.lower() in key.lower(): if recursed_value.lower() in key.lower():
found = True found = True
if found == False: if found == False:
print("[INFO] DIDN'T FIND similar VALUE: ", recursed_value) #print("[INFO] DIDN'T FIND similar VALUE: ", recursed_value)
break
# Check if we are on the last key or not
return "", False
#if outercnt == len(parsersplit)-1:
# print("[DEBUG] LAST KEY (3)")
# break
#else:
# print("[DEBUG] NOT LAST KEY (3)")
# return "", False
if recursed_value in basejson: if recursed_value in basejson:
print("[INFO] FOUND RECURSED VALUE: ", recursed_value) #print("[INFO] FOUND RECURSED VALUE: ", recursed_value)
basejson = basejson[recursed_value] basejson = basejson[recursed_value]
# Whether to dig deeper or not
if isinstance(basejson, bool) or isinstance(basejson, int) or isinstance(basejson, str):
handled = False
else:
handled = True handled = True
break break
currentsplitcnt += 1 currentsplitcnt += 1
@@ -2226,9 +2268,8 @@ class AppBase:
break break
except IndexError as e: except IndexError as e:
print("[DEBUG] INDEXERROR: ", parsersplit[outercnt]) print("[DEBUG] INDEXERROR (2):", parsersplit[outercnt])
break return "", False
outercnt += 1 outercnt += 1
@@ -2237,10 +2278,7 @@ class AppBase:
return "", False return "", False
except Exception as e: except Exception as e:
print("[WARNING] Exception: %s" % e) print("[WARNING] Exception: %s" % e)
return basejson, False return "", False
#return basejson
#return "KeyError: Couldn't find key: %s" % e
return basejson, False return basejson, False
backend/app_sdk/recurse_test.py
+111 -36
View File
@@ -58,7 +58,7 @@ def recurse_json(basejson, parsersplit):
# Means it's a single item -> continue # Means it's a single item -> continue
if seconditem == "": if seconditem == "":
print("[INFO] In first - handling %s. Len: %d" % (firstitem, len(basejson))) #print("[INFO] In first - handling %s. Len: %d" % (firstitem, len(basejson)))
if str(firstitem).lower() == "max" or str(firstitem).lower() == "last" or str(firstitem).lower() == "end": if str(firstitem).lower() == "max" or str(firstitem).lower() == "last" or str(firstitem).lower() == "end":
firstitem = len(basejson)-1 firstitem = len(basejson)-1
elif str(firstitem).lower() == "min" or str(firstitem).lower() == "first": elif str(firstitem).lower() == "min" or str(firstitem).lower() == "first":
@@ -66,14 +66,14 @@ def recurse_json(basejson, parsersplit):
else: else:
firstitem = int(firstitem) firstitem = int(firstitem)
print(f"[DEBUG] Post lower checks with item {firstitem}") #print(f"[DEBUG] Post lower checks with item {firstitem}")
tmpitem = basejson[int(firstitem)] tmpitem = basejson[int(firstitem)]
try: try:
newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:]) newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:])
except IndexError: except IndexError:
newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:]) newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:])
else: else:
print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem)) #print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem))
if isinstance(firstitem, str): if isinstance(firstitem, str):
if firstitem.lower() == "max" or firstitem.lower() == "last" or firstitem.lower() == "end": if firstitem.lower() == "max" or firstitem.lower() == "last" or firstitem.lower() == "end":
firstitem = len(basejson)-1 firstitem = len(basejson)-1
@@ -94,7 +94,7 @@ def recurse_json(basejson, parsersplit):
else: else:
seconditem = int(seconditem) seconditem = int(seconditem)
print(f"[DEBUG] Post lower checks 2: {firstitem} AND {seconditem}") #print(f"[DEBUG] Post lower checks 2: {firstitem} AND {seconditem}")
newvalue = [] newvalue = []
if int(seconditem) > len(basejson): if int(seconditem) > len(basejson):
seconditem = len(basejson) seconditem = len(basejson)
@@ -106,7 +106,7 @@ def recurse_json(basejson, parsersplit):
try: try:
ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt+1:]) ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt+1:])
except IndexError: except IndexError:
print("[DEBUG] INDEXERROR: ", parsersplit[outercnt]) #print("[DEBUG] INDEXERROR (1): ", parsersplit[outercnt])
#ret = innervalue #ret = innervalue
ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt:]) ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt:])
@@ -115,20 +115,18 @@ def recurse_json(basejson, parsersplit):
return newvalue, is_loop return newvalue, is_loop
else: else:
print("IN ELSE WITH VALUE: %s" % value)
if len(value) == 0: if len(value) == 0:
return basejson, False return basejson, False
try: try:
print("PRINT:", basejson)
if isinstance(basejson, list): if isinstance(basejson, list):
print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value)
return basejson, False return basejson, False
elif isinstance(basejson, bool): elif isinstance(basejson, bool):
print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value) #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value)
return basejson, False return basejson, False
elif isinstance(basejson, int): elif isinstance(basejson, int):
print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value) #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value)
return basejson, False return basejson, False
elif isinstance(basejson[value], str): elif isinstance(basejson[value], str):
try: try:
@@ -136,8 +134,16 @@ def recurse_json(basejson, parsersplit):
basejson = json.loads(basejson[value]) basejson = json.loads(basejson[value])
else: else:
# Should we sanitize here? # Should we sanitize here?
print("[DEBUG] VALUE TO SANITIZE?: %s" % basejson[value]) #print("[DEBUG] VALUE TO SANITIZE FOR KEY '%s'?: %s" % (value, basejson[value]))
# Check if we are on the last item?
if outercnt == len(parsersplit)-1:
#print("[DEBUG] LAST KEY")
return str(basejson[value]), False return str(basejson[value]), False
else:
#print("[DEBUG] NOT LAST KEY")
pass
except json.decoder.JSONDecodeError as e: except json.decoder.JSONDecodeError as e:
return str(basejson[value]), False return str(basejson[value]), False
else: else:
@@ -151,42 +157,60 @@ def recurse_json(basejson, parsersplit):
try: try:
if isinstance(basejson, list): if isinstance(basejson, list):
print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value)
return basejson, False return basejson, False
elif isinstance(basejson, bool): elif isinstance(basejson, bool):
print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value) #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value)
return basejson, False return basejson, False
elif isinstance(basejson, int): elif isinstance(basejson, int):
print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value) #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value)
return basejson, False return basejson, False
elif isinstance(basejson[value], str): elif isinstance(basejson[value], str):
print(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value]) #print(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value])
try: try:
print("[DEBUG] BASEJSON: %s" % basejson) #print("[DEBUG] BASEJSON: %s" % basejson)
if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")): if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")):
basejson = json.loads(basejson[value]) basejson = json.loads(basejson[value])
else: else:
if outercnt == len(parsersplit)-1:
#print("LAST KEY (2)")
return str(basejson[value]), False return str(basejson[value]), False
else:
#print("NOT LAST KEY (2)")
pass
except json.decoder.JSONDecodeError as e: except json.decoder.JSONDecodeError as e:
print("[DEBUG] RETURNING BECAUSE '%s' IS A NORMAL STRING (1)" % basejson[value]) #print("[DEBUG] RETURNING BECAUSE '%s' IS A NORMAL STRING (1)" % basejson[value])
return str(basejson[value]), False return str(basejson[value]), False
else: else:
basejson = basejson[value] basejson = basejson[value]
except KeyError as e: except KeyError as e:
print("\n\n[WARNING] Running third dot notation fix %s: %s" % (value, e)) # Check if previous key was handled or not
previouskey = parsersplit[outercnt-1]
#print("[DEBUG] PREVIOUS KEY: ", previouskey)
tmpval = previouskey + "." + value
#print("\n\n[WARNING] Running third dot notation fix '%s' on data %s: %s" % (value, basejson, e))
if tmpval in basejson:
return basejson[tmpval], False
try: try:
currentsplitcnt = splitcnt currentsplitcnt = splitcnt
recursed_value = value recursed_value = value
handled = False handled = False
#tmpbase = basejson
previouskey = value
while True: while True:
#print("\n\n[DEBUG] CURRENTSPLITCNT: ", currentsplitcnt)
newvalue = parsersplit[currentsplitcnt+1] newvalue = parsersplit[currentsplitcnt+1]
if newvalue == "#" or newvalue == "": if newvalue == "#" or newvalue == "":
break break
recursed_value += "." + newvalue recursed_value += "." + newvalue
print("\n\nRECURSED: ", recursed_value) #print("\n\nRECURSED: ", recursed_value)
found = False found = False
for key, value in basejson.items(): for key, value in basejson.items():
@@ -194,13 +218,27 @@ def recurse_json(basejson, parsersplit):
found = True found = True
if found == False: if found == False:
print("[INFO] DIDN'T FIND similar VALUE: ", recursed_value) #print("[INFO] DIDN'T FIND similar VALUE: ", recursed_value)
break
# Check if we are on the last key or not
return "", False
#if outercnt == len(parsersplit)-1:
# print("[DEBUG] LAST KEY (3)")
# break
#else:
# print("[DEBUG] NOT LAST KEY (3)")
# return "", False
if recursed_value in basejson: if recursed_value in basejson:
print("[INFO] FOUND RECURSED VALUE: ", recursed_value) #print("[INFO] FOUND RECURSED VALUE: ", recursed_value)
basejson = basejson[recursed_value] basejson = basejson[recursed_value]
# Whether to dig deeper or not
if isinstance(basejson, bool) or isinstance(basejson, int) or isinstance(basejson, str):
handled = False
else:
handled = True handled = True
break break
currentsplitcnt += 1 currentsplitcnt += 1
@@ -210,8 +248,8 @@ def recurse_json(basejson, parsersplit):
break break
except IndexError as e: except IndexError as e:
print("[DEBUG] INDEXERROR: ", parsersplit[outercnt]) print("[DEBUG] INDEXERROR (2):", parsersplit[outercnt])
break return "", False
outercnt += 1 outercnt += 1
@@ -220,10 +258,7 @@ def recurse_json(basejson, parsersplit):
return "", False return "", False
except Exception as e: except Exception as e:
print("[WARNING] Exception: %s" % e) print("[WARNING] Exception: %s" % e)
return basejson, False return "", False
#return basejson
#return "KeyError: Couldn't find key: %s" % e
return basejson, False return basejson, False
@@ -231,21 +266,61 @@ print("[INFO] Starting")
#input_data = "test" #input_data = "test"
#input_data = "test2.data" #input_data = "test2.data"
#input_data = "test2.test3.data"
input_data = "test2.test5.data.hello"
parsersplit = input_data.split(".")
# Matchwith
basejson = { basejson = {
"test": "hello", "test": "hello",
"test2": { "test2": {
"data": "hello2", "test3": "hello2",
"test3.data": "hello3", "test3.data": "hello3",
"test4.data.testing": { "test4.data.testing": {
"value": "hello4" "value": "hello4"
}, },
"test5.data.hello": "wut" "test5.data.hello": "wut",
}, },
"test3": ["hello", "hello2", "hello3"],
"test4": [{
"id": "1",
}]
} }
ret, is_loop = recurse_json(basejson, parsersplit) # Inputexamples (ALL should be True)
print("\n\nOUTPUT RET (%s): %s" % (input_data, ret)) inputs = {
#"": "",
"badkey": "",
"test": "hello",
"test2.badkey": "",
"test2.test3": "hello2",
"test2.test3.data": "hello3",
"test2.test4.data.testing": "{'value': 'hello4'}", # FIXME: Doesn't work due to break vs return "", False in last exception
"test2.test4.data.testing.value": "hello4", # FIXME: Doesn't work due to break vs return "", False in last exception. Not fixed as we didn't find one of these yet.
"test2.test5.data.hello": "wut",
"test2.test5.data.badkey": "",
"test3.#1": "hello2",
"test4.#0.id": "1",
"test4.#1.id": "",
}
outputs = []
for key, value in inputs.items():
parsersplit = key.split(".")
ret, is_loop = recurse_json(basejson, parsersplit)
print("\n\nOUTPUT RET (%s): %s" % (key, ret))
outputs.append("[%s]: %s = '%s' vs '%s'" % (str(ret) == str(value), key, ret, value))
print("\n\n%s" % "\n".join(outputs))
#input_data = ""
#input_data = "badkey"
#input_data = "test"
#input_data = "test2.data"
#input_data = "test2.test3.data"
#input_data = "test2.test4.data.testing.value.as"
#input_data = "test2.test5.data.hello"