diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 9c0e165a..e896f67c 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -67,7 +67,7 @@ class AppBase: "started_at": int(time.time()), "status": "EXECUTING" } - self.logger.info("ACTION RESULT: %s", action_result) + self.logger.info("ACTION RESULT (start): %s", action_result) if len(self.action) == 0: print("ACTION env not defined") @@ -361,18 +361,115 @@ class AppBase: newlist.append("parsing_error") return " ".join(newlist) + # Parses JSON loops and such down to the item you're looking for + def recurse_json(basejson, parsersplit): + match = "#(\d+):?-?([0-9a-z]+)?#?" + print("Split: %s\n%s" % (parsersplit, basejson)) + try: + outercnt = 0 + + # Loops over split values + for value in parsersplit: + print("VALUE: %s\n" % value) + actualitem = re.findall(match, value, re.MULTILINE) + if value == "#": + newvalue = [] + for innervalue in basejson: + # 1. Check the next item (message) + # 2. Call this function again + + try: + ret, is_loop = recurse_json(innervalue, parsersplit[outercnt+1:]) + except IndexError: + # Only in here if it's the last loop without anything in it? + ret, is_loop = recurse_json(innervalue, parsersplit[outercnt:]) + + newvalue.append(ret) + + # Magical way of returning which makes app sdk identify + # it as multi execution + return newvalue, True + elif len(actualitem) > 0: + # FIXME: This is absolutely not perfect. + print("In recursion v2: ", actualitem) + + is_loop = True + newvalue = [] + firstitem = actualitem[0][0] + seconditem = actualitem[0][1] + + # Means it's a single item -> continue + if seconditem == "": + print("In first - handling %s", seconditem) + tmpitem = basejson[int(firstitem)] + try: + newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:]) + except IndexError: + newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:]) + else: + if seconditem == "max": + seconditem = len(basejson) + if seconditem == "min": + seconditem = 0 + + newvalue = [] + for i in range(int(firstitem), int(seconditem)): + # 1. Check the next item (message) + # 2. Call this function again + print("Base: %s" % basejson[i]) + + try: + ret, is_loop = recurse_json(basejson[i], parsersplit[outercnt+1:]) + except IndexError: + print("INDEXERROR: ", parsersplit[outercnt]) + #ret = innervalue + ret, is_loop = recurse_json(innervalue, parsersplit[outercnt:]) + + print(ret) + #exit() + newvalue.append(ret) + + return newvalue, is_loop + + # FIXME: Add specific loop for other indexes + else: + #print("BEFORE NORMAL VALUE: ", basejson, value) + if len(value) == 0: + return basejson, False + + if isinstance(basejson[value], str): + print(f"LOADING STRING '%s' AS JSON" % basejson[value]) + try: + basejson = json.loads(basejson[value]) + except json.decoder.JSONDecodeError as e: + print("RETURNING BECAUSE '%s' IS A NORMAL STRING" % basejson[value]) + return basejson[value], False + else: + basejson = basejson[value] + + outercnt += 1 + + except KeyError as e: + print("Lower keyerror: %s" % e) + #return basejson + #return "KeyError: Couldn't find key: %s" % e + + return basejson, False + # Takes a workflow execution as argument # Returns a string if the result is single, or a list if it's a list def get_json_value(execution_data, input_data): parsersplit = input_data.split(".") actionname = parsersplit[0][1:].replace(" ", "_", -1) + #Actionname: Start_node + print(f"Actionname: {actionname}") # 1. Find the action baseresult = "" actionname_lower = actionname.lower() try: - if actionname_lower == "exec": + if actionname_lower == "exec" or actionname_lower == "webhook" or actionname_lower == "schedule" or actionname_lower == "userinput" or actionname_lower == "email_trigger" or actionname_lower == "trigger": baseresult = execution_data["execution_argument"] else: for result in execution_data["results"]: @@ -421,76 +518,37 @@ class AppBase: # 2. Find the JSON data if len(baseresult) == 0: - return "" + return "", False if len(parsersplit) == 1: - return baseresult + return baseresult, False baseresult = baseresult.replace("\'", "\"") basejson = {} try: basejson = json.loads(baseresult) except json.decoder.JSONDecodeError as e: - return baseresult + return baseresult, False - # This whole thing should be recursive. - try: - cnt = 0 - for value in parsersplit[1:]: - cnt += 1 - - print("VALUE: %s" % value) - if value == "#": - # FIXME - not recursive - should go deeper if there are more # - print("HANDLE RECURSIVE LOOP OF %s" % basejson) - returnlist = [] - try: - for innervalue in basejson: - print("Value: %s" % innervalue[parsersplit[cnt+1]]) - returnlist.append(innervalue[parsersplit[cnt+1]]) - except IndexError as e: - print("Indexerror inner: %s" % e) - # Basically means its a normal list, not a crazy one :) - # Custom format for ${name[0,1,2,...]}$ - indexvalue = "${NO_SPLITTER%s}$" % json.dumps(basejson) - if len(returnlist) > 0: - indexvalue = "${NO_SPLITTER%s}$" % json.dumps(returnlist) - - print("INDEXVAL: ", indexvalue) - return indexvalue - except TypeError as e: - print("TypeError inner: %s" % e) - - # Example format: ${[]}$ - parseditem = "${%s%s}$" % (parsersplit[cnt+1], json.dumps(returnlist)) - print("PARSED LOOP ITEM: %s" % parseditem) - return parseditem - - else: - print("BEFORE NORMAL VALUE: ", basejson, value) - if len(value) == 0: - return basejson - - if isinstance(basejson[value], str): - print(f"LOADING STRING '%s' AS JSON" % basejson[value]) - try: - basejson = json.loads(basejson[value]) - except json.decoder.JSONDecodeError as e: - print("RETURNING BECAUSE '%s' IS A NORMAL STRING" % basejson[value]) - return basejson[value] - else: - basejson = basejson[value] - - except KeyError as e: - print("Lower keyerror: %s" % e) - return "KeyError: Couldn't find key: %s" % e + data, is_loop = recurse_json(basejson, parsersplit[1:]) + parseditem = data + if is_loop: + print("DATA IS A LOOP - SHOULD WRAP") + if parsersplit[-1] == "#": + print("SET DATA WRAPPER TO NORMAL!") + parseditem = "${SHUFFLE_NO_SPLITTER%s}$" % json.dumps(data) + else: + # Return value: ${id[12345, 45678]}$ + print("SET DATA WRAPPER TO %s!" % parsersplit[-1]) + parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data)) - return basejson + return parseditem, is_loop # Parses parameters sent to it and returns whether it did it successfully with the values found def parse_params(action, fullexecution, parameter): # Skip if it starts with $? jsonparsevalue = "$." + is_loop = False # Matches with space in the first part, but not in subsequent parts. # JSON / yaml etc shouldn't have spaces in their fields anyway. @@ -510,7 +568,9 @@ class AppBase: except IndexError: continue - value = get_json_value(fullexecution, to_be_replaced) + # Handles for loops etc. + value, is_loop = get_json_value(fullexecution, to_be_replaced) + if isinstance(value, str): parameter["value"] = parameter["value"].replace(to_be_replaced, value) elif isinstance(value, dict): @@ -588,7 +648,8 @@ class AppBase: # This will never be a loop aka multi argument parameter["value"] = to_be_replaced - value = get_json_value(fullexecution, to_be_replaced) + value, is_loop = get_json_value(fullexecution, to_be_replaced) + print("Loop: %s" % is_loop) if isinstance(value, str): parameter["value"] = parameter["value"].replace(to_be_replaced, value) elif isinstance(value, dict): @@ -600,7 +661,7 @@ class AppBase: except json.decoder.JSONDecodeError as e: parameter["value"] = parameter["value"].replace(to_be_replaced, value) - return "", parameter["value"] + return "", parameter["value"], is_loop def run_validation(sourcevalue, check, destinationvalue): self.logger.info("Checking %s %s %s" % (sourcevalue, check, destinationvalue)) @@ -653,8 +714,6 @@ class AppBase: for branch in fullexecution["workflow"]["branches"]: if branch["destination_id"] != action["id"]: continue - - self.logger.info("Relevant branch: %s" % branch) # Remove anything without a condition try: @@ -671,7 +730,7 @@ class AppBase: # Parse all values first here sourcevalue = condition["source"]["value"] - check, sourcevalue = parse_params(action, fullexecution, condition["source"]) + check, sourcevalue, is_loop = parse_params(action, fullexecution, condition["source"]) if check: return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check) @@ -680,7 +739,7 @@ class AppBase: sourcevalue = parse_wrapper_start(sourcevalue) destinationvalue = condition["destination"]["value"] - check, destinationvalue = parse_params(action, fullexecution, condition["destination"]) + check, destinationvalue, is_loop = parse_params(action, fullexecution, condition["destination"]) if check: return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check) @@ -795,8 +854,10 @@ class AppBase: minlength = 0 multi_parameters = json.loads(json.dumps(params)) multiexecution = False + multi_execution_lists = [] for parameter in action["parameters"]: - check, value = parse_params(action, fullexecution, parameter) + check, value, is_loop = parse_params(action, fullexecution, parameter) + if check: raise "Value check error: %s" % Exception(check) @@ -811,52 +872,91 @@ class AppBase: except KeyError: pass + print("Return value: %s" % value) actionname = action["name"] #print("Multicheck ", actualitem) + print("Actual item: %s" % actualitem) if len(actualitem) > 0: multiexecution = True - - # This is here to handle for loops within variables.. kindof - # 1. Find the length of the longest array - # 2. Build an array with the base values based on parameter["value"] - # 3. Get the n'th value of the generated list from values - # 4. Execute all n answers - replacements = {} - for replace in actualitem: - try: - to_be_replaced = replace[0] - actualitem = replace[2] - except IndexError: - continue + # Loop WITHOUT JSON variables go here. + # Loop WITH variables go in else. + if len(actualitem[0]) > 2 and actualitem[0][1] == "SHUFFLE_NO_SPLITTER": + print("Pre replacement: %s" % actualitem[0][2]) + tmpitem = value + + replacement = actualitem[0][2] + if replacement.startswith("\"") and replacement.endswith("\""): + replacement = replacement[1:len(replacement)-1] + + replacement = replacement.replace("\'", "\"", -1) + print("POST replacement: %s" % replacement) + + json_replacement = replacement try: - itemlist = json.loads(actualitem) - if len(itemlist) > minlength: - minlength = len(itemlist) + json_replacement = json.loads(replacement) except json.decoder.JSONDecodeError as e: - print("JSON Error: %s in %s" % (e, actualitem)) + print("JSON error singular: %s" % e) - replacements[to_be_replaced] = actualitem + if len(json_replacement) > minlength: + minlength = len(json_replacement) - # This is a result array for JUST this value.. - # What if there are more? - resultarray = [] - for i in range(0, minlength): - tmpitem = json.loads(json.dumps(parameter["value"])) - for key, value in replacements.items(): - replacement = json.dumps(json.loads(value)[i]) - if replacement.startswith("\"") and replacement.endswith("\""): - replacement = replacement[1:len(replacement)-1] - #except json.decoder.JSONDecodeError as e: + tmpitem = tmpitem.replace(actualitem[0][0], replacement, 1) + params[parameter["name"]] = tmpitem + multi_execution_lists.append(json_replacement) + multi_parameters[parameter["name"]] = json_replacement - print("REPLACING %s with %s" % (key, replacement)) - #replacement = parse_wrapper_start(replacement) - tmpitem = tmpitem.replace(key, replacement, -1) + #print("LENGTH OF ARR: %d" % len(resultarray)) + #print("RESULTARRAY: %s" % resultarray) + print("MULTI finished: %s" % replacement) + else: + + # This is here to handle for loops within variables.. kindof + # 1. Find the length of the longest array + # 2. Build an array with the base values based on parameter["value"] + # 3. Get the n'th value of the generated list from values + # 4. Execute all n answers + replacements = {} + for replace in actualitem: + try: + to_be_replaced = replace[0] + actualitem = replace[2] + except IndexError: + continue - resultarray.append(tmpitem) + try: + itemlist = json.loads(actualitem) + if len(itemlist) > minlength: + minlength = len(itemlist) + except json.decoder.JSONDecodeError as e: + print("JSON Error: %s in %s" % (e, actualitem)) - # With this parameter ready, add it to... a greater list of parameters. Rofl - multi_parameters[parameter["name"]] = resultarray + replacements[to_be_replaced] = actualitem + + # This is a result array for JUST this value.. + # What if there are more? + resultarray = [] + for i in range(0, minlength): + tmpitem = json.loads(json.dumps(parameter["value"])) + for key, value in replacements.items(): + replacement = json.dumps(json.loads(value)[i]) + if replacement.startswith("\"") and replacement.endswith("\""): + replacement = replacement[1:len(replacement)-1] + #except json.decoder.JSONDecodeError as e: + + #print("REPLACING %s with %s" % (key, replacement)) + #replacement = parse_wrapper_start(replacement) + tmpitem = tmpitem.replace(key, replacement, -1) + + resultarray.append(tmpitem) + + # With this parameter ready, add it to... a greater list of parameters. Rofl + print("LENGTH OF ARR: %d" % len(resultarray)) + print("RESULTARRAY: %s" % resultarray) + if resultarray not in multi_execution_lists: + multi_execution_lists.append(resultarray) + + multi_parameters[parameter["name"]] = resultarray else: # Parses things like int(value) self.logger.info("Parsing wrapper data for %s" % value) @@ -864,11 +964,34 @@ class AppBase: params[parameter["name"]] = value multi_parameters[parameter["name"]] = value + + # Fix lists here + print("CHECKING multi execution list!") + if len(multi_execution_lists) > 0: + print("\n Multi execution list has more data: %d" % len(multi_execution_lists)) + filteredlist = [] + for listitem in multi_execution_lists: + if listitem in filteredlist: + continue + + filteredlist.append(listitem) + + #print("New list length: %d" % len(filteredlist)) + if len(filteredlist) > 1: + print("Calculating new multi-loop length with %d lists" % len(filteredlist)) + tmplength = 1 + for innerlist in filteredlist: + print("List length: %d. %d*%d" % (len(innerlist), len(innerlist), tmplength)) + tmplength = len(innerlist)*tmplength + + minlength = tmplength + + print("New multi execution length: %d\n" % tmplength) # FIXME - this is horrible, but works for now #for i in range(calltimes): if not multiexecution: - print("APP_SDK DONE: Starting normal execution of function") + print("APP_SDK DONE: Starting NORMAL execution of function") newres = await func(**params) #print("NEWRES: ", newres) if isinstance(newres, str): @@ -881,20 +1004,67 @@ class AppBase: print("Can't handle type %s value from function" % (type(newres))) print("POST NEWRES RESULT: ", result) else: - print("APP_SDK DONE: Starting MULTI execution with", multi_parameters) - # 1. Use number of executions based on longest array + print("APP_SDK DONE: Starting MULTI execution with values %s of length %d" % (multi_parameters, minlength)) + # 1. Use number of executions based on the arrays being similar # 2. Find the right value from the parsed multi_params - results = [] json_object = False for i in range(0, minlength): # To be able to use the results as a list: baseparams = json.loads(json.dumps(multi_parameters)) - + # {'call': ['GoogleSafebrowsing_2_0', 'VirusTotal_GetReport_3_0']} + # 1. Check if list length is same as minlength + # 2. If NOT same length, duplicate based on length of array + # arraylength = 3 ["1", "2", "3"] + # arraylength = 4 ["1", "2", "3", "4"] + # minlength = 12 - 12/3 = 4 per item = ["1", "1", "1", "1", "2", "2", ...] + try: + firstlist = True for key, value in baseparams.items(): + if isinstance(value, list): - baseparams[key] = value[i] + try: + newvalue = value[i] + except IndexError: + pass + + if len(value) != minlength and len(value) > 0: + newarray = [] + print("VALUE: ", value) + additiontime = minlength/len(value) + print("Bad length for value: %d - should be %d. Additiontime: %d" % (len(value), minlength, additiontime)) + if firstlist: + print("Running normal list (FIRST)") + for subvalue in value: + for number in range(int(additiontime)): + newarray.append(subvalue) + else: + #print("Running secondary lists") + ## 1. Set up length of array + ## 2. Put values spread out + # FIXME: This works well, except if lists are same length + newarray = [""] * minlength + + cnt = 0 + for number in range(int(additiontime)): + for subvaluerange in range(len(value)): + # newlocation = number+(additiontime*subvaluerange) + # print("%d+(%d*%d) = %d. VAL: %s" % (number, additiontime, subvaluerange, newlocation, value[subvaluerange])) + # Reverse if same length? + if int(minlength/len(value)) == len(value): + tmp = int(len(value)-subvaluerange-1) + print("NEW: %d" % tmp) + newarray[cnt] = value[tmp] + else: + newarray[cnt] = value[subvaluerange] + cnt += 1 + + #print("Newarray =", newarray) + newvalue = newarray[i] + firstlist = False + + baseparams[key] = newvalue except IndexError as e: print("IndexError: %s" % e) baseparams[key] = "IndexError: %s" % e @@ -902,25 +1072,49 @@ class AppBase: print("KeyError: %s" % e) baseparams[key] = "KeyError: %s" % e - #print("Running with params %s" % baseparams) + print("Running with params %s" % baseparams) ret = await func(**baseparams) - ret = ret.replace("\"", "\\\"", -1) - print("Inner ret parsed: %s" % ret) - - try: - results.append(json.loads(ret)) - json_object = True - except json.decoder.JSONDecodeError as e: + if isinstance(ret, dict) or isinstance(ret, list): results.append(ret) + json_object = True + else: + ret = ret.replace("\"", "\\\"", -1) + + try: + results.append(json.loads(ret)) + json_object = True + except json.decoder.JSONDecodeError as e: + #print("Json: %s" % e) + results.append(ret) + + #print("Inner ret parsed: %s" % ret) # Dump the result as a string of a list - print("RESULTS: %s" % results) + #print("RESULTS: %s" % results) if isinstance(results, list): print("JSON OBJECT? ", json_object) if json_object: result = json.dumps(results) else: - result = "[\""+"\", \"".join(results)+"\"]" + result = "[" + for item in results: + try: + json.loads(item) + result += item + except json.decoder.JSONDecodeError as e: + # Common nested issue which puts " around everything + try: + tmpitem = item.replace("\\\"", "\"", -1) + json.loads(tmpitem) + result += tmpitem + + except: + result += "\"%s\"" % item + + result += ", " + + result = result[:-2] + result += "]" else: print("Normal result?") result = results @@ -932,7 +1126,7 @@ class AppBase: action_result["result"] = result self.logger.debug(f"Executed {action['label']}-{action['id']} with result: {result}") - self.logger.debug(f"Data: %s" % action_result) + #self.logger.debug(f"Data: %s" % action_result) except TypeError as e: print("TypeError issue: %s" % e) action_result["status"] = "FAILURE" @@ -947,7 +1141,7 @@ class AppBase: print(f"Failed to execute: {e}") self.logger.exception(f"Failed to execute {e}-{action['id']}") action_result["status"] = "FAILURE" - action_result["result"] = "General exception: %s" % e + action_result["result"] = f"General exception: {e}" action_result["completed_at"] = int(time.time()) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 34180390..b1ac5113 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=app_sdk -VERSION=0.6.2 +VERSION=0.7.3 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index 81aea65d..89576345 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -310,20 +310,12 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet // Specific check for SSL verification // This is critical for onprem stuff. - verifyParam := "" - verifyWrapper := "" - verifyAddin := "" - if len(swagger.Servers) == 0 { - verifyParam = ", verify=True" - verifyWrapper = `if type(ssl_verify) == str: ssl_verify = False if ssl_verify.lower() == "false" or ssl_verify == "0" else True` - verifyAddin = ", verify=ssl_verify" - } else { - if swagger.Servers[0].URL == "" { - verifyParam = ", ssl_verify=True" - verifyWrapper = `if type(ssl_verify) == str: ssl_verify = False if ssl_verify.lower() == "false" or ssl_verify == "0" else True` - verifyAddin = ", verify=ssl_verify" - } - } + //verifyParam := "" + //verifyWrapper := "" + //verifyAddin := "" + verifyParam := ", ssl_verify=False" + verifyWrapper := `if type(ssl_verify) == str: ssl_verify = False if ssl_verify.lower() == "false" or ssl_verify == "0" else True` + verifyAddin := ", verify=ssl_verify" if len(parameters) > 0 { parameterData = fmt.Sprintf(", %s", strings.Join(parameters, ", ")) @@ -404,11 +396,10 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet bodyAddin, verifyAddin, ) - - if strings.Contains(functionname, "get_returns_the_vuln") { - log.Println(data) - log.Printf("Queries: %s", queryString) - } + //if strings.Contains(functionname, "get_returns_the_vuln") { + // log.Println(data) + // log.Printf("Queries: %s", queryString) + //} //log.Printf(data) return functionname, data @@ -562,7 +553,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, }) } else if securitySchemes["BasicAuth"] != nil { api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{ - Name: "username_auth", + Name: "username_basic", Value: "", Example: "username", Description: securitySchemes["BasicAuth"].Value.Description, @@ -574,7 +565,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, }) api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{ - Name: "password_auth", + Name: "password_basic", Value: "", Example: "*****", Description: securitySchemes["BasicAuth"].Value.Description, @@ -975,31 +966,16 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} - if len(swagger.Servers) == 0 { - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } else { - if swagger.Servers[0].URL == "" { - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } - } + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) headersFound := []string{} if len(path.Connect.Parameters) > 0 { @@ -1124,31 +1100,16 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor // FIXME - remove this when authentication is properly introduced parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} - if len(swagger.Servers) == 0 { - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify the SSL certificate request", - Multiline: false, - Required: false, - Example: "False - default=True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } else { - if swagger.Servers[0].URL == "" { - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } - } + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify the SSL certificate request", + Multiline: false, + Required: false, + Example: "False - default=True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) headersFound := []string{} if len(path.Get.Parameters) > 0 { @@ -1273,31 +1234,16 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} - if len(swagger.Servers) == 0 { - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } else { - if swagger.Servers[0].URL == "" { - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } - } + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) headersFound := []string{} if len(path.Head.Parameters) > 0 { @@ -1422,31 +1368,16 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} - if len(swagger.Servers) == 0 { - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } else { - if swagger.Servers[0].URL == "" { - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } - } + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) headersFound := []string{} if len(path.Delete.Parameters) > 0 { @@ -1570,31 +1501,16 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} - if len(swagger.Servers) == 0 { - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } else { - if swagger.Servers[0].URL == "" { - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } - } + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) headersFound := []string{} if len(path.Post.Parameters) > 0 { @@ -1718,31 +1634,16 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} - if len(swagger.Servers) == 0 { - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } else { - if swagger.Servers[0].URL == "" { - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } - } + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) headersFound := []string{} if len(path.Patch.Parameters) > 0 { @@ -1866,31 +1767,16 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} - if len(swagger.Servers) == 0 { - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } else { - if swagger.Servers[0].URL == "" { - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } - } + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) headersFound := []string{} if len(path.Put.Parameters) > 0 { diff --git a/backend/go-app/main.go b/backend/go-app/main.go index bb61eaa7..4a78d089 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -150,7 +150,7 @@ type UserAuth struct { type UserAuthField struct { Key string `json:"key" datastore:"key"` - Value string `json:"value" datastore:"value"` + Value string `json:"value" datastore:"value,noindex"` } // Not environment, but execution environment @@ -181,6 +181,7 @@ type User struct { Id string `datastore:"id" json:"id"` Orgs []string `datastore:"orgs" json:"orgs"` CreationTime int64 `datastore:"creation_time" json:"creation_time"` + ActiveOrg Org `json:"active_org" datastore:"active_org"` Active bool `datastore:"active" json:"active"` } @@ -209,7 +210,7 @@ type Contact struct { type Translator struct { Src struct { Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value"` + Value string `json:"value" datastore:"value,noindex"` Description string `json:"description" datastore:"description,noindex"` Required string `json:"required" datastore:"required"` Type string `json:"type" datastore:"type"` @@ -219,7 +220,7 @@ type Translator struct { } `json:"src" datastore:"src"` Dst struct { Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value"` + Value string `json:"value" datastore:"value,noindex"` Type string `json:"type" datastore:"type"` Description string `json:"description" datastore:"description,noindex"` Required string `json:"required" datastore:"required"` @@ -231,7 +232,7 @@ type Translator struct { type Appconfig struct { Key string `json:"key" datastore:"key"` - Value string `json:"value" datastore:"value"` + Value string `json:"value" datastore:"value,noindex"` } type ScheduleApp struct { @@ -1013,13 +1014,13 @@ func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) { user, err := handleApiAuthentication(resp, request) if err != nil { resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Can't register without being admin"}`)) + resp.Write([]byte(`{"success": false, "reason": "Can't handle set env auth"}`)) return } if user.Role != "admin" { resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Can't register without being admin"}`)) + resp.Write([]byte(`{"success": false, "reason": "Can't set environment without being admin"}`)) return } @@ -1097,7 +1098,7 @@ func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": true}`)) } -func createNewUser(username, password, role, apikey string) error { +func createNewUser(username, password, role, apikey string, org Org) error { // Returns false if there is an issue // Use this for register err := checkPasswordStrength(password) @@ -1137,7 +1138,7 @@ func createNewUser(username, password, role, apikey string) error { newUser.Verified = false newUser.CreationTime = time.Now().Unix() newUser.Active = true - newUser.Orgs = []string{"default"} + newUser.Orgs = []string{org.Id} // FIXME - Remove this later if role == "admin" { @@ -1148,6 +1149,8 @@ func createNewUser(username, password, role, apikey string) error { newUser.Roles = []string{"user"} } + newUser.ActiveOrg = org + if len(apikey) > 0 { newUser.ApiKey = apikey } @@ -1182,25 +1185,25 @@ func createNewUser(username, password, role, apikey string) error { log.Printf("Error adding User %s: %s", username, err) return err } - url := fmt.Sprintf("https://shuffler.io/register/%s", verifyToken.String()) - const verifyMessage = ` -Registration URL :) - -%s - ` - addr := newUser.Username - - msg := &mail.Message{ - Sender: "Shuffle ", - To: []string{addr}, - Subject: "Verify your username - Shuffle", - Body: fmt.Sprintf(verifyMessage, url), - } - - log.Println(msg.Body) - if err := mail.Send(ctx, msg); err != nil { - log.Printf("Couldn't send email: %v", err) - } + // url := fmt.Sprintf("https://shuffler.io/register/%s", verifyToken.String()) + // const verifyMessage = ` + //Registration URL :) + // + //%s + // ` + // addr := newUser.Username + // + // msg := &mail.Message{ + // Sender: "Shuffle ", + // To: []string{addr}, + // Subject: "Verify your username - Shuffle", + // Body: fmt.Sprintf(verifyMessage, url), + // } + // + // log.Println(msg.Body) + // if err := mail.Send(ctx, msg); err != nil { + // log.Printf("Couldn't send email: %v", err) + // } err = increaseStatisticsField(ctx, "successful_register", username, 1) if err != nil { @@ -1249,7 +1252,8 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { if count == 0 { role = "admin" } - err = createNewUser(data.Username, data.Password, role, "") + + err = createNewUser(data.Username, data.Password, role, "", user.ActiveOrg) if err != nil { log.Printf("Failed registering user: %s", err) resp.WriteHeader(401) @@ -1623,6 +1627,10 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { // This is a long check to see if an inactive admin can access the site parsedAdmin := "false" + if userInfo.Role == "admin" { + parsedAdmin = "true" + } + if !userInfo.Active { if userInfo.Role == "admin" { parsedAdmin = "true" @@ -1692,16 +1700,66 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { Expires: expiration, }) + // Updating user info if there's something wrong + if (len(userInfo.ActiveOrg.Name) == 0 || len(userInfo.ActiveOrg.Id) == 0) && len(userInfo.Orgs) > 0 { + _, err := getOrg(ctx, userInfo.Orgs[0]) + if err != nil { + var orgs []Org + q := datastore.NewQuery("Organizations") + _, err = dbclient.GetAll(ctx, q, &orgs) + if err == nil { + newStringOrgs := []string{} + newOrgs := []Org{} + for _, org := range orgs { + if strings.ToLower(org.Name) == strings.ToLower(userInfo.Orgs[0]) { + newOrgs = append(newOrgs, org) + newStringOrgs = append(newStringOrgs, org.Id) + } + } + + if len(newOrgs) > 0 { + userInfo.ActiveOrg = newOrgs[0] + userInfo.Orgs = newStringOrgs + + err = setUser(ctx, &userInfo) + if err != nil { + log.Printf("Error patching User for activeOrg: %s", err) + } else { + log.Printf("Updated the users' org") + } + } + } else { + log.Printf("Failed getting orgs for user. Major issue.: %s", err) + } + + } else { + // 1. Check if the org exists by ID + // 2. if it does, overwrite user + userInfo.ActiveOrg = Org{ + Id: userInfo.Orgs[0], + } + err = setUser(ctx, &userInfo) + if err != nil { + log.Printf("Error patching User for activeOrg: %s", err) + } + } + } + + currentOrg, err := json.Marshal(userInfo.ActiveOrg) + if err != nil { + currentOrg = []byte("{}") + } + returnData := fmt.Sprintf(` { "success": true, "admin": %s, "tutorials": [], "id": "%s", - "orgs": [{"name": "Shuffle", "id": "123", "role": "admin"}], - "selected_org": {"name": "Shuffle", "id": "123", "role": "admin"}, + "orgs": [%s], + "active_org": %s, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}] - }`, parsedAdmin, userInfo.Id, userInfo.Session, expiration.Unix()) + }`, parsedAdmin, userInfo.Id, currentOrg, currentOrg, userInfo.Session, expiration.Unix()) resp.WriteHeader(200) resp.Write([]byte(returnData)) @@ -1973,7 +2031,7 @@ func handlePasswordChange(resp http.ResponseWriter, request *http.Request) { } if len(users) != 1 { - log.Printf(`Found multiple users with the same username: %s: %d`, t.Username, len(users)) + log.Printf(`Found multiple or no users with the same username: %s: %d`, t.Username, len(users)) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Found %d users with the same username: %s (%d)"}`, len(users), t.Username))) return @@ -2166,6 +2224,61 @@ func handleGetEnvironments(resp http.ResponseWriter, request *http.Request) { resp.Write(newjson) } +func handleGetOrgs(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("Api authentication failed in set new workflowhandler: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if user.Role != "admin" { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Not admin"}`)) + return + } + + ctx := context.Background() + var orgs []Org + q := datastore.NewQuery("Organizations") + _, err = dbclient.GetAll(ctx, q, &orgs) + if err != nil { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Can't get users"}`)) + return + } + + //newUsers := []User{} + //for _, item := range users { + // if len(item.Username) == 0 { + // continue + // } + + // item.Password = "" + // item.Session = "" + // item.VerificationToken = "" + + // newUsers = append(newUsers, item) + //} + + newjson, err := json.Marshal(orgs) + if err != nil { + log.Printf("Failed unmarshal: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking"}`))) + return + } + + resp.WriteHeader(200) + resp.Write(newjson) +} + func handleGetUsers(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -2281,7 +2394,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { } if len(users) != 1 { - log.Printf(`Found multiple users with the same username: %s: %d`, data.Username, len(users)) + log.Printf(`Found multiple or no users with the same username: %s: %d`, data.Username, len(users)) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Found %d users with the same username: %s"}`, len(users), data.Username))) return @@ -2383,6 +2496,28 @@ func getSession(ctx context.Context, thissession string) (*session, error) { return curUser, nil } +// ListBooks returns a list of books, ordered by title. +func getOrg(ctx context.Context, id string) (*Org, error) { + key := datastore.NameKey("Organizations", id, nil) + curOrg := &Org{} + if err := dbclient.Get(ctx, key, curOrg); err != nil { + return &Org{}, err + } + + return curOrg, nil +} + +func setOrg(ctx context.Context, data Org, id string) error { + // clear session_token and API_token for user + k := datastore.NameKey("Organizations", id, nil) + if _, err := dbclient.Put(ctx, k, &data); err != nil { + log.Println(err) + return err + } + + return nil +} + // ListBooks returns a list of books, ordered by title. func getUser(ctx context.Context, id string) (*User, error) { key := datastore.NameKey("Users", id, nil) @@ -2946,6 +3081,7 @@ func getSpecificWebhook(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() + // FIXME: Schedule = trigger? schedule, err := getSchedule(ctx, workflowId) if err != nil { log.Printf("Failed setting schedule: %s", err) @@ -4781,8 +4917,8 @@ func setTriggerAuth(ctx context.Context, trigger TriggerAuth) error { func getOutlookClient(ctx context.Context, code string, accessToken OauthToken, redirectUri string) (*http.Client, *oauth2.Token, error) { conf := &oauth2.Config{ - ClientID: "70e37005-c954-4290-b573-d4b94e484336", - ClientSecret: ".eNw/A[kQFB5zL.agvRputdEJENeJ392", + ClientID: "", + ClientSecret: "", Scopes: []string{ "Mail.Read", "User.Read", @@ -6233,6 +6369,91 @@ func runInit(ctx context.Context) { } */ + setUsers := false + orgQuery := datastore.NewQuery("Organizations") + var activeOrgs []Org + _, err = dbclient.GetAll(ctx, orgQuery, &activeOrgs) + if err != nil { + log.Printf("Error getting organizations!") + } else { + // Add all users to it + if len(activeOrgs) == 1 { + setUsers = true + } + + log.Printf("Organizations exist!") + if len(activeOrgs) == 0 { + log.Printf(`No orgs. Setting org "default"`) + orgSetupName := "default" + orgId := uuid.NewV4().String() + newOrg := Org{ + Name: orgSetupName, + Id: orgId, + Org: orgSetupName, + Users: []User{}, + Roles: []string{"admin", "user"}, + CloudSync: false, + } + + err = setOrg(ctx, newOrg, orgId) + if err != nil { + log.Printf("Failed setting organization: %s", err) + } else { + log.Printf("Successfully created the default org!") + setUsers = true + } + } else { + log.Printf("There are %d org(s).", len(activeOrgs)) + } + } + + // Adding the users to the base organization since only one exists (default) + if setUsers && len(activeOrgs) > 0 { + activeOrg := activeOrgs[0] + + q := datastore.NewQuery("Users") + var users []User + _, err = dbclient.GetAll(ctx, q, &users) + if err == nil { + setOrgBool := false + for _, user := range users { + newUser := User{ + Username: user.Username, + Id: user.Id, + ActiveOrg: Org{ + Id: activeOrg.Id, + }, + Orgs: []string{activeOrg.Id}, + Role: user.Role, + } + + found := false + for _, orgUser := range activeOrg.Users { + if user.Id == orgUser.Id { + found = true + } + } + + if !found && len(user.Username) > 0 { + log.Printf("Adding user %s to org %s", user.Username, activeOrg.Name) + activeOrg.Users = append(activeOrg.Users, newUser) + setOrgBool = true + } + } + + if setOrgBool { + err = setOrg(ctx, activeOrg, activeOrg.Id) + if err != nil { + log.Printf("Failed setting org %s: %s!", activeOrg.Name, err) + } else { + log.Printf("UPDATED org %s!", activeOrg.Name) + } + } + } + + log.Printf("Should add %d users to organization default", len(users)) + } + // Fix active users etc q := datastore.NewQuery("Users").Filter("active =", true) var activeusers []User @@ -6243,6 +6464,7 @@ func runInit(ctx context.Context) { q := datastore.NewQuery("Users") var users []User _, err := dbclient.GetAll(ctx, q, &users) + if len(activeusers) == 0 && len(users) > 0 { log.Printf("No active users found - setting ALL to active") if err == nil { @@ -6258,7 +6480,12 @@ func runInit(ctx context.Context) { } if len(user.Orgs) == 0 { - user.Orgs = []string{"default"} + defaultName := "default" + user.Orgs = []string{defaultName} + user.ActiveOrg = Org{ + Name: defaultName, + Role: "user", + } } err = setUser(ctx, &user) @@ -6281,7 +6508,11 @@ func runInit(ctx context.Context) { log.Printf("SHUFFLE_DEFAULT_USERNAME and SHUFFLE_DEFAULT_PASSWORD not defined as environments. Running without default user.") } else { apikey := os.Getenv("SHUFFLE_DEFAULT_APIKEY") - err = createNewUser(username, password, "admin", apikey) + + tmpOrg := Org{ + Name: "default", + } + err = createNewUser(username, password, "admin", apikey, tmpOrg) if err != nil { log.Printf("Failed to create default user %s: %s", username, err) } else { @@ -6449,6 +6680,164 @@ func runInit(ctx context.Context) { log.Printf("Finished INIT") } +// INFO: https://docs.google.com/drawings/d/1JJebpPeEVEbmH_qsAC6zf9Noygp7PytvesrkhE19QrY/edit +func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("Api authentication failed in verify swagger: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if user.Role != "admin" { + log.Printf("Not admin.") + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Not admin"}`)) + return + } + + body, err := ioutil.ReadAll(request.Body) + if err != nil { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`)) + return + } + + type ReturnData struct { + Apikey string `datastore:"apikey"` + Organization Org `datastore:"organization"` + } + + var tmpData ReturnData + err = json.Unmarshal(body, &tmpData) + if err != nil { + log.Printf("Failed unmarshalling test: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + ctx := context.Background() + org, err := getOrg(ctx, tmpData.Organization.Id) + if err != nil { + log.Printf("Organization doesn't exist: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // FIXME: Check if user is admin of this org + userFound := false + admin := false + for _, inneruser := range org.Users { + if inneruser.Id == user.Id { + userFound = true + log.Printf("Role: %s", inneruser.Role) + if inneruser.Role == "admin" { + admin = true + } + + break + } + } + + if !userFound { + log.Printf("User %s doesn't exist in organization %s", user.Id, org.Id) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // FIXME: Enable admin check in org for sync setup and conf. + _ = admin + //if !admin { + // log.Printf("User %s isn't admin hence can't set up sync for org %s", user.Id, org.Id) + // resp.WriteHeader(401) + // resp.Write([]byte(`{"success": false}`)) + // return + //} + + log.Printf("Apidata: %s", tmpData.Apikey) + + client := &http.Client{} + syncPath := "http://192.168.3.6:5002/api/v1/cloud/sync" + + type requestStruct struct { + ApiKey string `json:"api_key"` + } + + requestData := requestStruct{ + ApiKey: tmpData.Apikey, + } + + b, err := json.Marshal(requestData) + if err != nil { + log.Printf("Failed marshaling api key data: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed cloud sync."`, err))) + return + } + + req, err := http.NewRequest( + "POST", + syncPath, + bytes.NewBuffer(b), + ) + + newresp, err := client.Do(req) + if err != nil { + resp.WriteHeader(400) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed cloud sync: %s"`, err))) + //setBadMemcache(ctx, docPath) + return + } + + if newresp.StatusCode != 200 { + resp.WriteHeader(400) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Response code %d during sync. Expecting 200."`, newresp.StatusCode))) + return + } + + respBody, err := ioutil.ReadAll(newresp.Body) + if err != nil { + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't parse sync data"`))) + return + } + + type responseStruct struct { + Success bool `json:"success"` + Reason string `json:"reason"` + } + log.Printf("Respbody: %s", string(respBody)) + + responseData := responseStruct{} + err = json.Unmarshal(respBody, &responseData) + if err != nil { + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed handling cloud data"`))) + return + } + + if responseData.Success { + resp.WriteHeader(200) + if len(responseData.Reason) > 0 { + resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, responseData.Reason))) + } else { + resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) + } + } else { + resp.WriteHeader(400) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, responseData.Reason))) + } +} + func initHandlers() { var err error ctx := context.Background() @@ -6467,11 +6856,6 @@ func initHandlers() { r := mux.NewRouter() r.HandleFunc("/api/v1/_ah/health", healthCheckHandler) - // Sends an email if the right things are specified - r.HandleFunc("/functions/sendmail", handleSendalert).Methods("POST", "OPTIONS") - r.HandleFunc("/functions/outlook/register", handleNewOutlookRegister).Methods("GET", "OPTIONS") - r.HandleFunc("/functions/outlook/getFolders", handleGetOutlookFolders).Methods("GET", "OPTIONS") - // Make user related locations r.HandleFunc("/api/v1/users/generateapikey", handleApiGeneration).Methods("GET", "POST", "OPTIONS") r.HandleFunc("/api/v1/users/login", handleLogin).Methods("POST", "OPTIONS") @@ -6506,9 +6890,14 @@ func initHandlers() { // Queuebuilder and Workflow streams. First is to update a stream, second to get a stream // Changed from workflows/streams to streams, as appengine was messing up // This does not increase the API counter + // Used by frontend r.HandleFunc("/api/v1/streams", handleWorkflowQueue).Methods("POST") r.HandleFunc("/api/v1/streams/results", handleGetStreamResults).Methods("POST", "OPTIONS") + // Used by orborus + r.HandleFunc("/api/v1/workflows/queue", handleGetWorkflowqueue).Methods("GET") + r.HandleFunc("/api/v1/workflows/queue/confirm", handleGetWorkflowqueueConfirm).Methods("POST") + // App specific r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps/get_existing", loadSpecificApps).Methods("POST", "OPTIONS") @@ -6535,8 +6924,6 @@ func initHandlers() { /* Everything below here increases the counters*/ r.HandleFunc("/api/v1/workflows", getWorkflows).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows", setNewWorkflow).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/workflows/queue", handleGetWorkflowqueue).Methods("GET") - r.HandleFunc("/api/v1/workflows/queue/confirm", handleGetWorkflowqueueConfirm).Methods("POST") r.HandleFunc("/api/v1/workflows/schedules", handleGetSchedules).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows/download_remote", loadSpecificWorkflows).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/execute", executeWorkflow).Methods("GET", "POST", "OPTIONS") @@ -6557,7 +6944,7 @@ func initHandlers() { r.HandleFunc("/api/v1/hooks/{key}/delete", handleDeleteHook).Methods("DELETE", "OPTIONS") // Trigger hmm - r.HandleFunc("/api/v1/triggers/{key}", handleGetSpecificTrigger).Methods("GET", "OPTIONS") + //r.HandleFunc("/api/v1/triggers/{key}", handleGetSpecificTrigger).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/stats/{key}", handleGetSpecificStats).Methods("GET", "OPTIONS") @@ -6568,7 +6955,9 @@ func initHandlers() { r.HandleFunc("/api/v1/validate_openapi", validateSwagger).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/get_openapi/{key}", getOpenapi).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/execution_cleanup", cleanupExecutions).Methods("GET", "OPTIONS") + // NEW for 0.8.0 + r.HandleFunc("/api/v1/cloud/setup", handleCloudSetup).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/getorgs", handleGetOrgs).Methods("GET", "OPTIONS") http.Handle("/", r) } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index aa97385a..e20908f9 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -67,11 +67,28 @@ type ExecutionRequest struct { Type string `json:"type"` } +type SyncFeatures struct { + Apps SyncData `json:"apps" datastore:"apps"` + Workflows SyncData `json:"apps" datastore:"apps"` + Schedules SyncData `json:"apps" datastore:"apps"` + Autocomplete SyncData `json:"apps" datastore:"apps"` + Authentication SyncData `json:"apps" datastore:"apps"` +} + +type SyncData struct { + Active bool `json:"active" datastore:"active"` +} + +// Role is just used for feedback for a user type Org struct { - Name string `json:"name"` - Org string `json:"org"` - Users []User `json:"users"` - Id string `json:"id"` + Name string `json:"name" datastore:"name"` + Id string `json:"id" datastore:"id"` + Org string `json:"org" datastore:"org"` + Users []User `json:"users" datastore:"users"` + Role string `json:"role" datastore:"role"` + Roles []string `json:"roles" datastore:"roles"` + CloudSync bool `json:"cloud_sync" datastore:"CloudSync"` + SyncFeatures SyncFeatures `json:"sync_features" datastore:"sync_features"` } type AppAuthenticationStorage struct { @@ -127,7 +144,7 @@ type WorkflowAppActionParameter struct { ID string `json:"id" datastore:"id" yaml:"id,omitempty"` Name string `json:"name" datastore:"name" yaml:"name"` Example string `json:"example" datastore:"example" yaml:"example"` - Value string `json:"value" datastore:"value" yaml:"value,omitempty"` + Value string `json:"value" datastore:"value,noindex" yaml:"value,omitempty"` Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` Options []string `json:"options" datastore:"options" yaml:"options"` ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"` @@ -161,7 +178,7 @@ type WorkflowAppAction struct { Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id"` Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value"` + Value string `json:"value" datastore:"value,noindex"` } `json:"execution_variable" datastore:"execution_variables"` Returns struct { Description string `json:"description" datastore:"returns" yaml:"description,omitempty"` @@ -306,7 +323,7 @@ type Workflow struct { Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id"` Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value"` + Value string `json:"value" datastore:"value,noindex"` } `json:"workflow_variables" datastore:"workflow_variables"` ExecutionVariables []struct { Description string `json:"description" datastore:"description,noindex"` @@ -336,7 +353,7 @@ type AuthenticationParams struct { ID string `json:"id" datastore:"id" yaml:"id"` Name string `json:"name" datastore:"name" yaml:"name"` Example string `json:"example" datastore:"example" yaml:"example"` - Value string `json:"value,omitempty" datastore:"value" yaml:"value"` + Value string `json:"value,omitempty" datastore:"value,noindex" yaml:"value"` Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` Required bool `json:"required" datastore:"required" yaml:"required"` In string `json:"in" datastore:"in" yaml:"in"` @@ -346,13 +363,23 @@ type AuthenticationParams struct { type AuthenticationStore struct { Key string `json:"key" datastore:"key"` - Value string `json:"value" datastore:"value"` + Value string `json:"value" datastore:"value,noindex"` } type ExecutionRequestWrapper struct { Data []ExecutionRequest `json:"data"` } +type AppExecutionExample struct { + AppName string `json:"app_name" datastore:"app_name"` + AppVersion string `json:"app_version" datastore:"app_version"` + AppAction string `json:"app_action" datastore:"app_action"` + AppId string `json:"app_id" datastore:"app_id"` + ExampleId string `json:"example_id" datastore:"example_id"` + SuccessExamples []string `json:"success_examples" datastore:"success_examples,noindex"` + FailureExamples []string `json:"failure_examples" datastore:"failure_examples,noindex"` +} + // This might be... a bit off, but that's fine :) // This might also be stupid, as we want timelines and such // Anyway, these are super basic stupid stats. @@ -1078,6 +1105,10 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { if err != nil { log.Printf("Failed to increase success execution stats: %s", err) } + + // Handles extra statistics stuff when it's done + // Does autocomplete magic with JSON + go handleExecutionStatistics(*workflowExecution) } } @@ -1088,8 +1119,33 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { // log.Printf("Name: %s, Env: %s", action.Name, action.Environment) //} + tmpJson, err := json.Marshal(workflowExecution) + if err == nil { + if len(tmpJson) >= 1048487 { + log.Printf("[ERROR] Result length is too long! Need to reduce result size") + + // Result string `json:"result" datastore:"result,noindex"` + // Arbitrary reduction size + maxSize := 500000 + newResults := []ActionResult{} + for _, item := range workflowExecution.Results { + if len(item.Result) > maxSize { + item.Result = "[ERROR] Result too large to handle (https://github.com/frikky/shuffle/issues/171)" + } + + newResults = append(newResults, item) + } + + workflowExecution.Results = newResults + } + } + err = setWorkflowExecution(ctx, *workflowExecution) if err != nil { + //workflowExecution.Result = "Error setting workflow: result too large" + //workflowExecution.Status = "FINISHED" + //workflowExecution.CompletedAt = int64(time.Now().Unix()) + log.Printf("Error saving workflow execution actionresult setting: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution actionresult: %s"}`, err))) @@ -1100,6 +1156,90 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } +func JSONCheck(str string) bool { + var jsonStr interface{} + return json.Unmarshal([]byte(str), &jsonStr) == nil +} + +func handleExecutionStatistics(execution WorkflowExecution) { + // FIXME: CLEAN UP THE JSON THAT'S SAVED. + // https://github.com/frikky/Shuffle/issues/172 + appResults := []AppExecutionExample{} + for _, result := range execution.Results { + resultCheck := JSONCheck(result.Result) + if !resultCheck { + log.Printf("Result is NOT JSON!") + continue + } else { + log.Printf("Result IS JSON!") + + } + + appFound := false + executionIndex := 0 + for index, appExample := range appResults { + if appExample.AppId == result.Action.ID { + appFound = true + executionIndex = index + break + } + } + + if appFound { + // Append to SuccessExamples or FailureExamples + if result.Status == "ABORTED" || result.Status == "FAILURE" { + appResults[executionIndex].FailureExamples = append(appResults[executionIndex].FailureExamples, result.Result) + } else if result.Status == "FINISHED" || result.Status == "SUCCESS" { + appResults[executionIndex].SuccessExamples = append(appResults[executionIndex].SuccessExamples, result.Result) + } else { + log.Printf("[ERROR] Can't handle status %s", result.Status) + } + + // appResults = append(appResults, executionExample) + + } else { + // CREATE SuccessExamples or FailureExamples + executionExample := AppExecutionExample{ + AppName: result.Action.AppName, + AppVersion: result.Action.AppVersion, + AppAction: result.Action.Name, + AppId: result.Action.AppID, + ExampleId: fmt.Sprintf("%s_%s", execution.ExecutionId, result.Action.AppID), + } + + if result.Status == "ABORTED" || result.Status == "FAILURE" { + executionExample.FailureExamples = append(executionExample.FailureExamples, result.Result) + } else if result.Status == "FINISHED" || result.Status == "SUCCESS" { + executionExample.SuccessExamples = append(executionExample.SuccessExamples, result.Result) + } else { + log.Printf("[ERROR] Can't handle status %s", result.Status) + } + + appResults = append(appResults, executionExample) + } + } + + // ExampleId string `json:"example_id"` + // func setExampleresult(ctx context.Context, result exampleResult) error { + // log.Printf("Execution length: %d", len(appResults)) + if len(appResults) > 0 { + ctx := context.Background() + successful := 0 + for _, exampleresult := range appResults { + err := setExampleresult(ctx, exampleresult) + if err != nil { + log.Printf("Failed setting examplresult %s: %s", exampleresult.ExampleId, err) + } else { + successful += 1 + } + } + + log.Printf("Added %d exampleresults to backend", successful) + } else { + log.Printf("No examplresults necessary to be added for execution %s", execution.ExecutionId) + } +} + func getWorkflows(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -1589,7 +1729,8 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } // FIXME: Have a good way of tracking errors. ID's or similar. - if !action.IsValid { + if !action.IsValid && len(action.Errors) > 0 { + log.Printf("Node %s is invalid and needs to be remade. Errors: %s", action.Label, strings.Join(action.Errors, "\n")) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Node %s is invalid and needs to be remade."}`, action.Label))) return @@ -1602,11 +1743,43 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { workflow.Actions = newActions + newTriggers := []Trigger{} for _, trigger := range workflow.Triggers { log.Printf("Trigger %s: %s", trigger.TriggerType, trigger.Status) + // Check if it's actually running + // FIXME: Do this for other triggers too + if trigger.TriggerType == "SCHEDULE" && trigger.Status != "uninitialized" { + schedule, err := getSchedule(ctx, trigger.ID) + if err != nil { + trigger.Status = "stopped" + } else if schedule.Id == "" { + trigger.Status = "stopped" + } + } else if trigger.TriggerType == "WEBHOOK" && trigger.Status != "uninitialized" { + hook, err := getHook(ctx, trigger.ID) + if err != nil { + log.Printf("Failed getting webhook") + trigger.Status = "stopped" + } else if hook.Id == "" { + trigger.Status = "stopped" + } + } + //log.Println("TRIGGERS") allNodes = append(allNodes, trigger.ID) + newTriggers = append(newTriggers, trigger) + } + + workflow.Triggers = newTriggers + + for _, variable := range workflow.WorkflowVariables { + if len(variable.Value) == 0 { + log.Printf("Can't have an empty variable: %s", variable.Name) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Variable %s can't be empty"}`, variable.Name))) + return + } } if len(workflow.Actions) == 0 { @@ -1764,10 +1937,16 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } if !authFound { - log.Printf("App auth %s doesn't exist", action.AuthenticationId) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App auth %s doesn't exist"}`, action.AuthenticationId))) - return + log.Printf("App auth %s doesn't exist. Setting error", action.AuthenticationId) + workflow.Errors = append(workflow.Errors, fmt.Sprintf("App authentication for %s doesn't exist!", action.AppName)) + workflow.IsValid = false + + action.Errors = append(action.Errors, "App authentication doesn't exist") + action.IsValid = false + action.AuthenticationId = "" + //resp.WriteHeader(401) + //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App auth %s doesn't exist"}`, action.AuthenticationId))) + //return } } @@ -1791,70 +1970,77 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // Check to see if the whole app is valid if curapp.Name != action.AppName { - log.Printf("App %s doesn't exist.", action.AppName) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App %s doesn't exist"}`, action.AppName))) - return - } + workflow.Errors = append(workflow.Errors, fmt.Sprintf("App %s doesn't exist", action.AppName)) + action.Errors = append(action.Errors, "This app doesn't exist.") + action.IsValid = false + workflow.IsValid = false - // Check tosee if the appaction is valid - curappaction := WorkflowAppAction{} - for _, curAction := range curapp.Actions { - if action.Name == curAction.Name { - curappaction = curAction - break - } - } - - // Check to see if the action is valid - if curappaction.Name != action.Name { - log.Printf("Appaction %s doesn't exist.", action.Name) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // FIXME - check all parameters to see if they're valid - // Includes checking required fields - - newParams := []WorkflowAppActionParameter{} - for _, param := range curappaction.Parameters { - found := false - - // Handles check for parameter exists + value not empty in used fields - for _, actionParam := range action.Parameters { - if actionParam.Name == param.Name { - found = true - - if actionParam.Value == "" && actionParam.Variant == "STATIC_VALUE" && actionParam.Required == true { - log.Printf("Appaction %s with required param '%s' is empty.", action.Name, param.Name) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name))) - return - - } - - if actionParam.Variant == "" { - actionParam.Variant = "STATIC_VALUE" - } - - newParams = append(newParams, actionParam) + // Append with errors + newActions = append(newActions, action) + log.Printf("App %s doesn't exist. Adding as error.", action.AppName) + //resp.WriteHeader(401) + //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App %s doesn't exist"}`, action.AppName))) + //return + } else { + // Check tosee if the appaction is valid + curappaction := WorkflowAppAction{} + for _, curAction := range curapp.Actions { + if action.Name == curAction.Name { + curappaction = curAction break } } - // Handles check for required params - if !found && param.Required { - log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name) + // Check to see if the action is valid + if curappaction.Name != action.Name { + log.Printf("Appaction %s doesn't exist.", action.Name) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } - } + // FIXME - check all parameters to see if they're valid + // Includes checking required fields - action.Parameters = newParams - newActions = append(newActions, action) + newParams := []WorkflowAppActionParameter{} + for _, param := range curappaction.Parameters { + found := false + + // Handles check for parameter exists + value not empty in used fields + for _, actionParam := range action.Parameters { + if actionParam.Name == param.Name { + found = true + + if actionParam.Value == "" && actionParam.Variant == "STATIC_VALUE" && actionParam.Required == true { + log.Printf("Appaction %s with required param '%s' is empty.", action.Name, param.Name) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name))) + return + + } + + if actionParam.Variant == "" { + actionParam.Variant = "STATIC_VALUE" + } + + newParams = append(newParams, actionParam) + break + } + } + + // Handles check for required params + if !found && param.Required { + log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name))) + return + } + + } + + action.Parameters = newParams + newActions = append(newActions, action) + } } } @@ -1877,9 +2063,25 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { log.Printf("Failed to change total actions data: %s", err) } + type returnData struct { + Success bool `json:"success"` + Errors []string `json:"errors"` + } + + returndata := returnData{ + Success: true, + Errors: workflow.Errors, + } + log.Printf("Saved new version of workflow %s (%s)", workflow.Name, fileId) resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) + newBody, err := json.Marshal(returndata) + if err != nil { + resp.Write([]byte(`{"success": true}`)) + return + } + + resp.Write(newBody) } func getWorkflowLocal(fileId string, request *http.Request) ([]byte, error) { @@ -1923,6 +2125,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { return } + // FIXME: Check the execution if this fails. user, err := handleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in abort workflow: %s", err) @@ -2049,16 +2252,14 @@ func cleanupExecutions(resp http.ResponseWriter, request *http.Request) { return } - //if user.Role != "admin" { - // resp.WriteHeader(401) - // resp.Write([]byte(`{"success": false, "message": "Insufficient permissions"}`)) - // return - //} - - log.Printf("CLEANUP!") - log.Printf("%#v", user) + if user.Role != "admin" { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "message": "Insufficient permissions"}`)) + return + } ctx := context.Background() + // Removes three months from today timestamp := int64(time.Now().AddDate(0, -2, 0).Unix()) log.Println(timestamp) @@ -2072,8 +2273,6 @@ func cleanupExecutions(resp http.ResponseWriter, request *http.Request) { return } - log.Println(len(workflowExecutions)) - resp.WriteHeader(200) resp.Write([]byte("OK")) } @@ -2373,9 +2572,9 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf for _, authparam := range curAuth.Fields { if param.Name == authparam.Key { - log.Printf("Name: %s - value: %s", param.Name, param.Value) param.Value = authparam.Value - log.Printf("Name: %s - value: %s\n", param.Name, param.Value) + //log.Printf("Name: %s - value: %s", param.Name, param.Value) + //log.Printf("Name: %s - value: %s\n", param.Name, param.Value) break } } @@ -2634,6 +2833,8 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { err = deleteSchedule(ctx, scheduleId) if err != nil { + log.Printf("Failed deleting schedule: %s", err) + if strings.Contains(err.Error(), "Job not found") { resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) @@ -3143,6 +3344,18 @@ func getAllWorkflows(ctx context.Context) ([]Workflow, error) { return allworkflows, nil } +func setExampleresult(ctx context.Context, result AppExecutionExample) error { + key := datastore.NameKey("example_result", result.ExampleId, nil) + + // New struct, to not add body, author etc + if _, err := dbclient.Put(ctx, key, &result); err != nil { + log.Printf("Error adding workflow: %s", err) + return err + } + + return nil +} + // Hmm, so I guess this should use uuid :( // Consistency PLX func setWorkflow(ctx context.Context, workflow Workflow, id string) error { @@ -3210,6 +3423,7 @@ func deleteAppAuthentication(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": true}`)) } +// FIXME: Not suitable for cloud right now :O func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -3250,7 +3464,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { // FIXME - check whether it's in use and maybe restrict again for later? // FIXME - actually delete other than private apps too.. private := false - if app.Downloaded { + if app.Downloaded && user.Role == "admin" { log.Printf("Deleting downloaded app (authenticated users can do this)") } else if user.Id != app.Owner && user.Role != "admin" { log.Printf("Wrong user (%s) for app %s (delete)", user.Username, app.Name) @@ -3270,6 +3484,8 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { return } + // Finds workflows using the app to set errors + // FIXME: this will be WAY too big for cloud :O for _, workflow := range workflows { found := false @@ -3290,7 +3506,8 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { workflow.Actions = newActions for _, trigger := range workflow.Triggers { - log.Printf("TRIGGER: %#v", trigger) + _ = trigger + //log.Printf("TRIGGER: %#v", trigger) //err = deleteSchedule(ctx, scheduleId) //if err != nil { // if strings.Contains(err.Error(), "Job not found") { @@ -5121,8 +5338,8 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { } if len(workflowExecutions) == 0 { + resp.WriteHeader(200) resp.Write([]byte("[]")) - //resp.WriteHeader(200) return } diff --git a/docker-compose.yml b/docker-compose.yml index b90352a3..6d92e85a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,6 +60,7 @@ services: - HTTP_PROXY=${SHUFFLE_HTTP_PROXY} - HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY} - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} + - SHUFFLE_ORBORUS_EXECUTION_TIMEOUT=600 - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 1daf4734..bd6df28b 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -12177,6 +12177,11 @@ "prop-types": "^15.6.1" } }, + "material-ui-nested-menu-item": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/material-ui-nested-menu-item/-/material-ui-nested-menu-item-1.0.2.tgz", + "integrity": "sha512-LZb8xI0FrAI/A3P2vT3CB9bmSoOFWOK0dikTc1t9VvEpp1a8hZkbVUz7VhETnoLUYu3NXCkgulmXcl3zitqI9A==" + }, "material-ui-pickers": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/material-ui-pickers/-/material-ui-pickers-2.2.4.tgz", diff --git a/frontend/package.json b/frontend/package.json index 117587fc..a4c40596 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -28,6 +28,7 @@ "material-icons": "^0.3.1", "material-icons-react": "^1.0.4", "material-ui-chip-input": "^2.0.0-beta.2", + "material-ui-nested-menu-item": "^1.0.2", "md5-file": "^4.0.0", "mdbreact": "^4.21.1", "moment": "~2.20.1", diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index ea4224f0..053f0ab5 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -128,7 +128,7 @@ const App = (message, props) => { } /> } /> } /> - } /> + } /> } /> } /> } /> @@ -141,12 +141,12 @@ const App = (message, props) => { } /> } /> } /> - } /> + } /> } /> { window.location.pathname = "/docs/about" }} /> } /> } /> - { window.location.pathname = "/login" }} /> + } /> //
diff --git a/frontend/src/components/Header.js b/frontend/src/components/Header.js index 7495d06d..6902eb8f 100644 --- a/frontend/src/components/Header.js +++ b/frontend/src/components/Header.js @@ -101,7 +101,6 @@ const Header = props => { // Should be based on some path const logoCheck = !homePage ? null : null - // Handle top bar or something const loginTextBrowser = !isLoggedIn ?
@@ -184,18 +183,20 @@ const Header = props => { color="primary"> Settings - - - - - - {userdata === undefined || userdata.orgs.length <= 1 ? null : + {userdata === undefined || userdata.admin === undefined || userdata.admin === null || !userdata.admin ? null : + + + + + + } + {userdata === undefined || userdata.orgs === undefined || userdata.orgs === null || userdata.orgs.length <= 1 ? null : } + } + // Shows nested list of nodes > their JSON lists + const ActionlistWrapper = (props) => { + + const handleMenuClose = () => { + setShowAutocomplete(false) + + if (!selectedActionParameters[count].value[selectedActionParameters[count].value.length-1] === "$") { + setShowDropdown(false) + } + + setUpdate(Math.random()) + setMenuPosition(null) + } + + const handleItemClick = (values) => { + if (values === undefined || values === null || values.length === 0) { + return + } + + var toComplete = selectedActionParameters[count].value.trim().endsWith("$") ? values[0].autocomplete : "$"+values[0].autocomplete + for (var key in values) { + if (key == 0 || values[key].autocomplete.length === 0) { + continue + } + + toComplete += values[key].autocomplete + } + + selectedActionParameters[count].value += toComplete + selectedAction.parameters[count].value = selectedActionParameters[count].value + setSelectedAction(selectedAction) + setUpdate(Math.random()) + + setShowDropdown(false) + setMenuPosition(null) + } + + const iconStyle = { + marginRight: 15, + } + + return ( +
+ { + handleMenuClose() + }} + open={!!menuPosition} + style={{ + border: `2px solid #f85a3e`, + color: "white", + marginTop: 2, + }} + > + {actionlist.map(innerdata => { + const icon = innerdata.type === "action" ? : innerdata.type === "workflow_variable" || innerdata.type === "execution_variable" ? : + + const handleExecArgumentHover = (inside) => { + var exec_text_field = document.getElementById("execution_argument_input_field") + if (exec_text_field !== null) { + if (inside) { + exec_text_field.style.border = "2px solid #f85a3e" + } else { + exec_text_field.style.border = "" + } + } + + // Also doing arguments + if (workflow.triggers !== undefined && workflow.triggers !== null && workflow.triggers.length > 0) { + for (var key in workflow.triggers) { + const item = workflow.triggers[key] + + var node = cy.getElementById(item.id) + if (node.length > 0) { + if (inside) { + node.addClass('shuffle-hover-highlight') + } else { + node.removeClass('shuffle-hover-highlight') + } + } + + } + } + } + + const handleActionHover = (inside, actionId) => { + var node = cy.getElementById(actionId) + if (node.length > 0) { + if (inside) { + node.addClass('shuffle-hover-highlight') + } else { + node.removeClass('shuffle-hover-highlight') + } + } + } + + const handleMouseover = () => { + if (innerdata.type === "Execution Argument") { + handleExecArgumentHover(true) + } else if (innerdata.type === "action") { + handleActionHover(true, innerdata.id) + } + } + + const handleMouseOut = () => { + if (innerdata.type === "Execution Argument") { + handleExecArgumentHover(false) + } else if (innerdata.type === "action") { + handleActionHover(false, innerdata.id) + } + } + + var parsedPaths = [] + if (typeof(innerdata.example) === "object") { + parsedPaths = GetParsedPaths(innerdata.example, "") + } + + return ( + parsedPaths.length > 0 ? + + {icon} {innerdata.name} +
+ } + parentMenuOpen={!!menuPosition} + style={{backgroundColor: inputColor, color: "white", minWidth: 250,}} + onClick={() => { + handleItemClick([innerdata]) + }} + > + {parsedPaths.map((pathdata, index) => { + // FIXME: Should be recursive in here + const icon = pathdata.type === "value" ? : pathdata.type === "list" ? : + return ( + {}} + onClick={() => { + handleItemClick([innerdata, pathdata]) + }} + > + +
+ {icon} {pathdata.name} +
+ + + ) + + })} + + : + handleMouseover()} onMouseOut={() => {handleMouseOut()}} + onClick={() => { + handleItemClick([innerdata]) + }} + > + +
+ {icon} {innerdata.name} +
+ + + + ) + })} + +
+ ) } var itemColor = "#f85a3e" @@ -2941,7 +3207,6 @@ const AngularWorkflow = (props) => { selectedActionParameters[count].value += e.target.value.autocomplete selectedAction.parameters[count].value = selectedActionParameters[count].value - console.log("TARGET: ", selectedActionParameters) setSelectedAction(selectedAction) setUpdate(Math.random()) @@ -2968,106 +3233,7 @@ const AngularWorkflow = (props) => { : null} {showDropdown && showDropdownNumber === count && data.variant === "STATIC_VALUE" && jsonList.length === 0 ? - - Autocomplete - - + : null} @@ -3138,13 +3304,17 @@ const AngularWorkflow = (props) => { } function sortByKey(array, key) { + if (array === undefined) { + return [] + } + return array.sort(function(a, b) { var x = a[key]; var y = b[key] return ((x < y) ? -1 : ((x > y) ? 1 : 0)) }) } - const appApiView = Object.getOwnPropertyNames(selectedAction).length > 0 && Object.getOwnPropertyNames(selectedApp).length > 0 ? + const appApiView = Object.getOwnPropertyNames(selectedAction).length > 0 ?
@@ -3183,7 +3353,7 @@ const AngularWorkflow = (props) => { placeholder={selectedAction.label} onChange={selectedNameChange} /> - {selectedAction.authentication.length === 0 && requiresAuthentication ? + {selectedApp.name !== undefined && selectedAction.authentication !== undefined && selectedAction.authentication.length === 0 && requiresAuthentication ?
Authenticate {selectedApp.name}: @@ -3195,7 +3365,7 @@ const AngularWorkflow = (props) => {
: null} - {selectedAction.authentication.length > 0 ? + {selectedAction.authentication !== undefined && selectedAction.authentication.length > 0 ?
Authentication
@@ -3626,7 +3796,6 @@ const AngularWorkflow = (props) => { // Uses the target's parents, as the target should be executing the checks (I think) var parents = getParents(workflow.actions.find(a => a.id === selectedEdge["target"])) if (parents.length > 0) { - console.log(parents) data.action_field = parents[0].label } else { data.action_field = "" @@ -3790,7 +3959,6 @@ const AngularWorkflow = (props) => {
: null - var deleteButton = ((selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated) || (selectedApp.downloaded != undefined && selectedApp.downloaded == true)) && activateButton === null ? + const deleteButton = ( + (selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated) + || (selectedApp.downloaded !== undefined && selectedApp.downloaded == true) + || (!selectedApp.generated) + ) + && activateButton === null ?
{activateButton} - {props.userdata.role === "admin" || props.userdata.id === selectedApp.owner ? + {(props.userdata.role === "admin" || props.userdata.id === selectedApp.owner) || !selectedApp.generated ?
{downloadButton} {editButton} @@ -742,15 +777,17 @@ const Apps = (props) => { } const searchfield = search.toLowerCase() - const newapps = apps.filter(data => data.name.toLowerCase().includes(searchfield) || data.description.toLowerCase().includes(searchfield)) + var newapps = apps.filter(data => data.name.toLowerCase().includes(searchfield) || data.description.toLowerCase().includes(searchfield)) + var tmpapps = searchableApps.filter(data => data.name.toLowerCase().includes(searchfield) || data.description.toLowerCase().includes(searchfield)) + newapps.push(...tmpapps) - if ((newapps.length === 0 || searchBackend) && !appSearchLoading) { + setFilteredApps(newapps) + //if ((newapps.length === 0 || searchBackend) && !appSearchLoading) { - setAppSearchLoading(true) - runAppSearch(searchfield) - } else { - setFilteredApps(newapps) - } + // //setAppSearchLoading(true) + // //runAppSearch(searchfield) + //} else { + //} } const appView = isLoggedIn ? @@ -778,17 +815,8 @@ const Apps = (props) => {
-

All apps

+

Your apps ({apps.length+searchableApps.length})

- {isLoading ? : null} - Search OpenAPI
} - control={ { - handleSearchChange("") - setSearchBackend(!searchBackend)} - } />} - />