From cb9be52245a327b50a95af5e9c51e07a9b5c6033 Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 2 Feb 2024 01:54:18 +0100 Subject: [PATCH] Fixed recursion bug where items that used to become empty strings became full values --- backend/app_sdk/app_base.py | 106 +++++++++++++++------- backend/app_sdk/recurse_test.py | 153 ++++++++++++++++++++++++-------- 2 files changed, 186 insertions(+), 73 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 3937b052..9e90ec62 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -2021,6 +2021,7 @@ class AppBase: return " ".join(newlist) # Parses JSON loops and such down to the item you're looking for + # Check recurse_test.py for examples and tests of this function # $nodename.#.id # $nodename.data.#min-max.info.id # $nodename.data.#1-max.info.id @@ -2048,7 +2049,7 @@ class AppBase: for innervalue in basejson: # 1. Check the next item (message) # 2. Call this function again - + try: ret, is_loop = recurse_json(innervalue, parsersplit[outercnt+1:]) except IndexError: @@ -2077,7 +2078,7 @@ class AppBase: # Means it's a single item -> continue if seconditem == "": - print("[INFO] In first - handling %s. Len: %d" % (firstitem, len(basejson))) + #print("[INFO] In first - handling %s. Len: %d" % (firstitem, len(basejson))) if str(firstitem).lower() == "max" or str(firstitem).lower() == "last" or str(firstitem).lower() == "end": firstitem = len(basejson)-1 elif str(firstitem).lower() == "min" or str(firstitem).lower() == "first": @@ -2085,14 +2086,14 @@ class AppBase: else: firstitem = int(firstitem) - print(f"[DEBUG] Post lower checks with item {firstitem}") + #print(f"[DEBUG] Post lower checks with item {firstitem}") tmpitem = basejson[int(firstitem)] try: newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:]) except IndexError: newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:]) else: - print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem)) + #print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem)) if isinstance(firstitem, str): if firstitem.lower() == "max" or firstitem.lower() == "last" or firstitem.lower() == "end": firstitem = len(basejson)-1 @@ -2113,7 +2114,7 @@ class AppBase: else: seconditem = int(seconditem) - print(f"[DEBUG] Post lower checks 2: {firstitem} AND {seconditem}") + #print(f"[DEBUG] Post lower checks 2: {firstitem} AND {seconditem}") newvalue = [] if int(seconditem) > len(basejson): seconditem = len(basejson) @@ -2121,12 +2122,11 @@ class AppBase: for i in range(int(firstitem), int(seconditem)+1): # 1. Check the next item (message) # 2. Call this function again - #self.logger.info("Base: %s" % basejson[i]) try: ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt+1:]) except IndexError: - print("[DEBUG] INDEXERROR: ", parsersplit[outercnt]) + #print("[DEBUG] INDEXERROR (1): ", parsersplit[outercnt]) #ret = innervalue ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt:]) @@ -2137,16 +2137,16 @@ class AppBase: else: if len(value) == 0: return basejson, False - + try: if isinstance(basejson, list): - print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) + #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) return basejson, False elif isinstance(basejson, bool): - print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value) + #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value) return basejson, False elif isinstance(basejson, int): - print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value) + #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value) return basejson, False elif isinstance(basejson[value], str): try: @@ -2154,8 +2154,16 @@ class AppBase: basejson = json.loads(basejson[value]) else: # Should we sanitize here? - self.logger.info("[DEBUG] VALUE TO SANITIZE?: %s" % basejson[value]) - return str(basejson[value]), False + #print("[DEBUG] VALUE TO SANITIZE FOR KEY '%s'?: %s" % (value, basejson[value])) + + # Check if we are on the last item? + if outercnt == len(parsersplit)-1: + #print("[DEBUG] LAST KEY") + return str(basejson[value]), False + else: + #print("[DEBUG] NOT LAST KEY") + pass + except json.decoder.JSONDecodeError as e: return str(basejson[value]), False else: @@ -2169,54 +2177,88 @@ class AppBase: try: if isinstance(basejson, list): - print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) + #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) return basejson, False elif isinstance(basejson, bool): - print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value) + #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value) return basejson, False elif isinstance(basejson, int): - print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value) + #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value) return basejson, False elif isinstance(basejson[value], str): - print(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value]) + #print(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value]) try: - print("[DEBUG] BASEJSON: %s" % basejson) + #print("[DEBUG] BASEJSON: %s" % basejson) if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")): basejson = json.loads(basejson[value]) else: - return str(basejson[value]), False + + if outercnt == len(parsersplit)-1: + #print("LAST KEY (2)") + return str(basejson[value]), False + else: + #print("NOT LAST KEY (2)") + pass + except json.decoder.JSONDecodeError as e: - print("[DEBUG] RETURNING BECAUSE '%s' IS A NORMAL STRING (1)" % basejson[value]) + #print("[DEBUG] RETURNING BECAUSE '%s' IS A NORMAL STRING (1)" % basejson[value]) return str(basejson[value]), False else: basejson = basejson[value] - except KeyError as e: - print("\n\n[WARNING] Running third dot notation fix that always find the correct value %s: %s" % (value, e)) + # Check if previous key was handled or not + previouskey = parsersplit[outercnt-1] + #print("[DEBUG] PREVIOUS KEY: ", previouskey) + + tmpval = previouskey + "." + value + #print("\n\n[WARNING] Running third dot notation fix '%s' on data %s: %s" % (value, basejson, e)) + if tmpval in basejson: + return basejson[tmpval], False try: currentsplitcnt = splitcnt + recursed_value = value handled = False + + #tmpbase = basejson + previouskey = value while True: + #print("\n\n[DEBUG] CURRENTSPLITCNT: ", currentsplitcnt) newvalue = parsersplit[currentsplitcnt+1] if newvalue == "#" or newvalue == "": break recursed_value += "." + newvalue + #print("\n\nRECURSED: ", recursed_value) + found = False for key, value in basejson.items(): if recursed_value.lower() in key.lower(): found = True if found == False: - print("[INFO] DIDN'T FIND similar VALUE: ", recursed_value) - break + #print("[INFO] DIDN'T FIND similar VALUE: ", recursed_value) + + # Check if we are on the last key or not + return "", False + #if outercnt == len(parsersplit)-1: + # print("[DEBUG] LAST KEY (3)") + # break + #else: + # print("[DEBUG] NOT LAST KEY (3)") + # return "", False if recursed_value in basejson: - print("[INFO] FOUND RECURSED VALUE: ", recursed_value) + #print("[INFO] FOUND RECURSED VALUE: ", recursed_value) basejson = basejson[recursed_value] - handled = True + + # Whether to dig deeper or not + if isinstance(basejson, bool) or isinstance(basejson, int) or isinstance(basejson, str): + handled = False + else: + handled = True + break currentsplitcnt += 1 @@ -2226,21 +2268,17 @@ class AppBase: break except IndexError as e: - print("[DEBUG] INDEXERROR: ", parsersplit[outercnt]) - break - + print("[DEBUG] INDEXERROR (2):", parsersplit[outercnt]) + return "", False outercnt += 1 - + except KeyError as e: print("[INFO] Lower keyerror: %s" % e) return "", False except Exception as e: print("[WARNING] Exception: %s" % e) - return basejson, False - - #return basejson - #return "KeyError: Couldn't find key: %s" % e + return "", False return basejson, False diff --git a/backend/app_sdk/recurse_test.py b/backend/app_sdk/recurse_test.py index 9dfbd2c5..1cf2bfea 100644 --- a/backend/app_sdk/recurse_test.py +++ b/backend/app_sdk/recurse_test.py @@ -58,7 +58,7 @@ def recurse_json(basejson, parsersplit): # Means it's a single item -> continue if seconditem == "": - print("[INFO] In first - handling %s. Len: %d" % (firstitem, len(basejson))) + #print("[INFO] In first - handling %s. Len: %d" % (firstitem, len(basejson))) if str(firstitem).lower() == "max" or str(firstitem).lower() == "last" or str(firstitem).lower() == "end": firstitem = len(basejson)-1 elif str(firstitem).lower() == "min" or str(firstitem).lower() == "first": @@ -66,14 +66,14 @@ def recurse_json(basejson, parsersplit): else: firstitem = int(firstitem) - print(f"[DEBUG] Post lower checks with item {firstitem}") + #print(f"[DEBUG] Post lower checks with item {firstitem}") tmpitem = basejson[int(firstitem)] try: newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:]) except IndexError: newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:]) else: - print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem)) + #print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem)) if isinstance(firstitem, str): if firstitem.lower() == "max" or firstitem.lower() == "last" or firstitem.lower() == "end": firstitem = len(basejson)-1 @@ -94,7 +94,7 @@ def recurse_json(basejson, parsersplit): else: seconditem = int(seconditem) - print(f"[DEBUG] Post lower checks 2: {firstitem} AND {seconditem}") + #print(f"[DEBUG] Post lower checks 2: {firstitem} AND {seconditem}") newvalue = [] if int(seconditem) > len(basejson): seconditem = len(basejson) @@ -106,7 +106,7 @@ def recurse_json(basejson, parsersplit): try: ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt+1:]) except IndexError: - print("[DEBUG] INDEXERROR: ", parsersplit[outercnt]) + #print("[DEBUG] INDEXERROR (1): ", parsersplit[outercnt]) #ret = innervalue ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt:]) @@ -115,20 +115,18 @@ def recurse_json(basejson, parsersplit): return newvalue, is_loop else: - print("IN ELSE WITH VALUE: %s" % value) if len(value) == 0: return basejson, False try: - print("PRINT:", basejson) if isinstance(basejson, list): - print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) + #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) return basejson, False elif isinstance(basejson, bool): - print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value) + #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value) return basejson, False elif isinstance(basejson, int): - print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value) + #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value) return basejson, False elif isinstance(basejson[value], str): try: @@ -136,8 +134,16 @@ def recurse_json(basejson, parsersplit): basejson = json.loads(basejson[value]) else: # Should we sanitize here? - print("[DEBUG] VALUE TO SANITIZE?: %s" % basejson[value]) - return str(basejson[value]), False + #print("[DEBUG] VALUE TO SANITIZE FOR KEY '%s'?: %s" % (value, basejson[value])) + + # Check if we are on the last item? + if outercnt == len(parsersplit)-1: + #print("[DEBUG] LAST KEY") + return str(basejson[value]), False + else: + #print("[DEBUG] NOT LAST KEY") + pass + except json.decoder.JSONDecodeError as e: return str(basejson[value]), False else: @@ -151,42 +157,60 @@ def recurse_json(basejson, parsersplit): try: if isinstance(basejson, list): - print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) + #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) return basejson, False elif isinstance(basejson, bool): - print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value) + #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value) return basejson, False elif isinstance(basejson, int): - print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value) + #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value) return basejson, False elif isinstance(basejson[value], str): - print(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value]) + #print(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value]) try: - print("[DEBUG] BASEJSON: %s" % basejson) + #print("[DEBUG] BASEJSON: %s" % basejson) if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")): basejson = json.loads(basejson[value]) else: - return str(basejson[value]), False + + if outercnt == len(parsersplit)-1: + #print("LAST KEY (2)") + return str(basejson[value]), False + else: + #print("NOT LAST KEY (2)") + pass + except json.decoder.JSONDecodeError as e: - print("[DEBUG] RETURNING BECAUSE '%s' IS A NORMAL STRING (1)" % basejson[value]) + #print("[DEBUG] RETURNING BECAUSE '%s' IS A NORMAL STRING (1)" % basejson[value]) return str(basejson[value]), False else: basejson = basejson[value] except KeyError as e: - print("\n\n[WARNING] Running third dot notation fix %s: %s" % (value, e)) + # Check if previous key was handled or not + previouskey = parsersplit[outercnt-1] + #print("[DEBUG] PREVIOUS KEY: ", previouskey) + + tmpval = previouskey + "." + value + #print("\n\n[WARNING] Running third dot notation fix '%s' on data %s: %s" % (value, basejson, e)) + if tmpval in basejson: + return basejson[tmpval], False try: - currentsplitcnt = splitcnt + recursed_value = value handled = False + + #tmpbase = basejson + previouskey = value while True: + #print("\n\n[DEBUG] CURRENTSPLITCNT: ", currentsplitcnt) newvalue = parsersplit[currentsplitcnt+1] if newvalue == "#" or newvalue == "": break recursed_value += "." + newvalue - print("\n\nRECURSED: ", recursed_value) + #print("\n\nRECURSED: ", recursed_value) found = False for key, value in basejson.items(): @@ -194,13 +218,27 @@ def recurse_json(basejson, parsersplit): found = True if found == False: - print("[INFO] DIDN'T FIND similar VALUE: ", recursed_value) - break + #print("[INFO] DIDN'T FIND similar VALUE: ", recursed_value) + + # Check if we are on the last key or not + return "", False + #if outercnt == len(parsersplit)-1: + # print("[DEBUG] LAST KEY (3)") + # break + #else: + # print("[DEBUG] NOT LAST KEY (3)") + # return "", False if recursed_value in basejson: - print("[INFO] FOUND RECURSED VALUE: ", recursed_value) + #print("[INFO] FOUND RECURSED VALUE: ", recursed_value) basejson = basejson[recursed_value] - handled = True + + # Whether to dig deeper or not + if isinstance(basejson, bool) or isinstance(basejson, int) or isinstance(basejson, str): + handled = False + else: + handled = True + break currentsplitcnt += 1 @@ -210,8 +248,8 @@ def recurse_json(basejson, parsersplit): break except IndexError as e: - print("[DEBUG] INDEXERROR: ", parsersplit[outercnt]) - break + print("[DEBUG] INDEXERROR (2):", parsersplit[outercnt]) + return "", False outercnt += 1 @@ -220,10 +258,7 @@ def recurse_json(basejson, parsersplit): return "", False except Exception as e: print("[WARNING] Exception: %s" % e) - return basejson, False - - #return basejson - #return "KeyError: Couldn't find key: %s" % e + return "", False return basejson, False @@ -231,21 +266,61 @@ print("[INFO] Starting") #input_data = "test" #input_data = "test2.data" -#input_data = "test2.test3.data" -input_data = "test2.test5.data.hello" -parsersplit = input_data.split(".") + + +# Matchwith basejson = { "test": "hello", "test2": { - "data": "hello2", + "test3": "hello2", "test3.data": "hello3", "test4.data.testing": { "value": "hello4" }, - "test5.data.hello": "wut" + "test5.data.hello": "wut", }, + "test3": ["hello", "hello2", "hello3"], + "test4": [{ + "id": "1", + }] } -ret, is_loop = recurse_json(basejson, parsersplit) -print("\n\nOUTPUT RET (%s): %s" % (input_data, ret)) +# Inputexamples (ALL should be True) +inputs = { + #"": "", + "badkey": "", + "test": "hello", + "test2.badkey": "", + "test2.test3": "hello2", + "test2.test3.data": "hello3", + "test2.test4.data.testing": "{'value': 'hello4'}", # FIXME: Doesn't work due to break vs return "", False in last exception + "test2.test4.data.testing.value": "hello4", # FIXME: Doesn't work due to break vs return "", False in last exception. Not fixed as we didn't find one of these yet. + "test2.test5.data.hello": "wut", + "test2.test5.data.badkey": "", + "test3.#1": "hello2", + "test4.#0.id": "1", + "test4.#1.id": "", +} + +outputs = [] +for key, value in inputs.items(): + parsersplit = key.split(".") + ret, is_loop = recurse_json(basejson, parsersplit) + print("\n\nOUTPUT RET (%s): %s" % (key, ret)) + + outputs.append("[%s]: %s = '%s' vs '%s'" % (str(ret) == str(value), key, ret, value)) + +print("\n\n%s" % "\n".join(outputs)) + +#input_data = "" +#input_data = "badkey" +#input_data = "test" +#input_data = "test2.data" +#input_data = "test2.test3.data" +#input_data = "test2.test4.data.testing.value.as" +#input_data = "test2.test5.data.hello" + + + +