Merge pull request #173 from frikky/launch

0.7.5
This commit is contained in:
Frikky
2020-10-14 12:37:26 +02:00
committed by GitHub
18 changed files with 1898 additions and 754 deletions
+311 -117
View File
@@ -67,7 +67,7 @@ class AppBase:
"started_at": int(time.time()),
"status": "EXECUTING"
}
self.logger.info("ACTION RESULT: %s", action_result)
self.logger.info("ACTION RESULT (start): %s", action_result)
if len(self.action) == 0:
print("ACTION env not defined")
@@ -361,18 +361,115 @@ class AppBase:
newlist.append("parsing_error")
return " ".join(newlist)
# Parses JSON loops and such down to the item you're looking for
def recurse_json(basejson, parsersplit):
match = "#(\d+):?-?([0-9a-z]+)?#?"
print("Split: %s\n%s" % (parsersplit, basejson))
try:
outercnt = 0
# Loops over split values
for value in parsersplit:
print("VALUE: %s\n" % value)
actualitem = re.findall(match, value, re.MULTILINE)
if value == "#":
newvalue = []
for innervalue in basejson:
# 1. Check the next item (message)
# 2. Call this function again
try:
ret, is_loop = recurse_json(innervalue, parsersplit[outercnt+1:])
except IndexError:
# Only in here if it's the last loop without anything in it?
ret, is_loop = recurse_json(innervalue, parsersplit[outercnt:])
newvalue.append(ret)
# Magical way of returning which makes app sdk identify
# it as multi execution
return newvalue, True
elif len(actualitem) > 0:
# FIXME: This is absolutely not perfect.
print("In recursion v2: ", actualitem)
is_loop = True
newvalue = []
firstitem = actualitem[0][0]
seconditem = actualitem[0][1]
# Means it's a single item -> continue
if seconditem == "":
print("In first - handling %s", seconditem)
tmpitem = basejson[int(firstitem)]
try:
newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:])
except IndexError:
newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:])
else:
if seconditem == "max":
seconditem = len(basejson)
if seconditem == "min":
seconditem = 0
newvalue = []
for i in range(int(firstitem), int(seconditem)):
# 1. Check the next item (message)
# 2. Call this function again
print("Base: %s" % basejson[i])
try:
ret, is_loop = recurse_json(basejson[i], parsersplit[outercnt+1:])
except IndexError:
print("INDEXERROR: ", parsersplit[outercnt])
#ret = innervalue
ret, is_loop = recurse_json(innervalue, parsersplit[outercnt:])
print(ret)
#exit()
newvalue.append(ret)
return newvalue, is_loop
# FIXME: Add specific loop for other indexes
else:
#print("BEFORE NORMAL VALUE: ", basejson, value)
if len(value) == 0:
return basejson, False
if isinstance(basejson[value], str):
print(f"LOADING STRING '%s' AS JSON" % basejson[value])
try:
basejson = json.loads(basejson[value])
except json.decoder.JSONDecodeError as e:
print("RETURNING BECAUSE '%s' IS A NORMAL STRING" % basejson[value])
return basejson[value], False
else:
basejson = basejson[value]
outercnt += 1
except KeyError as e:
print("Lower keyerror: %s" % e)
#return basejson
#return "KeyError: Couldn't find key: %s" % e
return basejson, False
# Takes a workflow execution as argument
# Returns a string if the result is single, or a list if it's a list
def get_json_value(execution_data, input_data):
parsersplit = input_data.split(".")
actionname = parsersplit[0][1:].replace(" ", "_", -1)
#Actionname: Start_node
print(f"Actionname: {actionname}")
# 1. Find the action
baseresult = ""
actionname_lower = actionname.lower()
try:
if actionname_lower == "exec":
if actionname_lower == "exec" or actionname_lower == "webhook" or actionname_lower == "schedule" or actionname_lower == "userinput" or actionname_lower == "email_trigger" or actionname_lower == "trigger":
baseresult = execution_data["execution_argument"]
else:
for result in execution_data["results"]:
@@ -421,76 +518,37 @@ class AppBase:
# 2. Find the JSON data
if len(baseresult) == 0:
return ""
return "", False
if len(parsersplit) == 1:
return baseresult
return baseresult, False
baseresult = baseresult.replace("\'", "\"")
basejson = {}
try:
basejson = json.loads(baseresult)
except json.decoder.JSONDecodeError as e:
return baseresult
return baseresult, False
# This whole thing should be recursive.
try:
cnt = 0
for value in parsersplit[1:]:
cnt += 1
print("VALUE: %s" % value)
if value == "#":
# FIXME - not recursive - should go deeper if there are more #
print("HANDLE RECURSIVE LOOP OF %s" % basejson)
returnlist = []
try:
for innervalue in basejson:
print("Value: %s" % innervalue[parsersplit[cnt+1]])
returnlist.append(innervalue[parsersplit[cnt+1]])
except IndexError as e:
print("Indexerror inner: %s" % e)
# Basically means its a normal list, not a crazy one :)
# Custom format for ${name[0,1,2,...]}$
indexvalue = "${NO_SPLITTER%s}$" % json.dumps(basejson)
if len(returnlist) > 0:
indexvalue = "${NO_SPLITTER%s}$" % json.dumps(returnlist)
print("INDEXVAL: ", indexvalue)
return indexvalue
except TypeError as e:
print("TypeError inner: %s" % e)
# Example format: ${[]}$
parseditem = "${%s%s}$" % (parsersplit[cnt+1], json.dumps(returnlist))
print("PARSED LOOP ITEM: %s" % parseditem)
return parseditem
else:
print("BEFORE NORMAL VALUE: ", basejson, value)
if len(value) == 0:
return basejson
if isinstance(basejson[value], str):
print(f"LOADING STRING '%s' AS JSON" % basejson[value])
try:
basejson = json.loads(basejson[value])
except json.decoder.JSONDecodeError as e:
print("RETURNING BECAUSE '%s' IS A NORMAL STRING" % basejson[value])
return basejson[value]
else:
basejson = basejson[value]
except KeyError as e:
print("Lower keyerror: %s" % e)
return "KeyError: Couldn't find key: %s" % e
data, is_loop = recurse_json(basejson, parsersplit[1:])
parseditem = data
if is_loop:
print("DATA IS A LOOP - SHOULD WRAP")
if parsersplit[-1] == "#":
print("SET DATA WRAPPER TO NORMAL!")
parseditem = "${SHUFFLE_NO_SPLITTER%s}$" % json.dumps(data)
else:
# Return value: ${id[12345, 45678]}$
print("SET DATA WRAPPER TO %s!" % parsersplit[-1])
parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data))
return basejson
return parseditem, is_loop
# Parses parameters sent to it and returns whether it did it successfully with the values found
def parse_params(action, fullexecution, parameter):
# Skip if it starts with $?
jsonparsevalue = "$."
is_loop = False
# Matches with space in the first part, but not in subsequent parts.
# JSON / yaml etc shouldn't have spaces in their fields anyway.
@@ -510,7 +568,9 @@ class AppBase:
except IndexError:
continue
value = get_json_value(fullexecution, to_be_replaced)
# Handles for loops etc.
value, is_loop = get_json_value(fullexecution, to_be_replaced)
if isinstance(value, str):
parameter["value"] = parameter["value"].replace(to_be_replaced, value)
elif isinstance(value, dict):
@@ -588,7 +648,8 @@ class AppBase:
# This will never be a loop aka multi argument
parameter["value"] = to_be_replaced
value = get_json_value(fullexecution, to_be_replaced)
value, is_loop = get_json_value(fullexecution, to_be_replaced)
print("Loop: %s" % is_loop)
if isinstance(value, str):
parameter["value"] = parameter["value"].replace(to_be_replaced, value)
elif isinstance(value, dict):
@@ -600,7 +661,7 @@ class AppBase:
except json.decoder.JSONDecodeError as e:
parameter["value"] = parameter["value"].replace(to_be_replaced, value)
return "", parameter["value"]
return "", parameter["value"], is_loop
def run_validation(sourcevalue, check, destinationvalue):
self.logger.info("Checking %s %s %s" % (sourcevalue, check, destinationvalue))
@@ -653,8 +714,6 @@ class AppBase:
for branch in fullexecution["workflow"]["branches"]:
if branch["destination_id"] != action["id"]:
continue
self.logger.info("Relevant branch: %s" % branch)
# Remove anything without a condition
try:
@@ -671,7 +730,7 @@ class AppBase:
# Parse all values first here
sourcevalue = condition["source"]["value"]
check, sourcevalue = parse_params(action, fullexecution, condition["source"])
check, sourcevalue, is_loop = parse_params(action, fullexecution, condition["source"])
if check:
return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)
@@ -680,7 +739,7 @@ class AppBase:
sourcevalue = parse_wrapper_start(sourcevalue)
destinationvalue = condition["destination"]["value"]
check, destinationvalue = parse_params(action, fullexecution, condition["destination"])
check, destinationvalue, is_loop = parse_params(action, fullexecution, condition["destination"])
if check:
return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)
@@ -795,8 +854,10 @@ class AppBase:
minlength = 0
multi_parameters = json.loads(json.dumps(params))
multiexecution = False
multi_execution_lists = []
for parameter in action["parameters"]:
check, value = parse_params(action, fullexecution, parameter)
check, value, is_loop = parse_params(action, fullexecution, parameter)
if check:
raise "Value check error: %s" % Exception(check)
@@ -811,52 +872,91 @@ class AppBase:
except KeyError:
pass
print("Return value: %s" % value)
actionname = action["name"]
#print("Multicheck ", actualitem)
print("Actual item: %s" % actualitem)
if len(actualitem) > 0:
multiexecution = True
# This is here to handle for loops within variables.. kindof
# 1. Find the length of the longest array
# 2. Build an array with the base values based on parameter["value"]
# 3. Get the n'th value of the generated list from values
# 4. Execute all n answers
replacements = {}
for replace in actualitem:
try:
to_be_replaced = replace[0]
actualitem = replace[2]
except IndexError:
continue
# Loop WITHOUT JSON variables go here.
# Loop WITH variables go in else.
if len(actualitem[0]) > 2 and actualitem[0][1] == "SHUFFLE_NO_SPLITTER":
print("Pre replacement: %s" % actualitem[0][2])
tmpitem = value
replacement = actualitem[0][2]
if replacement.startswith("\"") and replacement.endswith("\""):
replacement = replacement[1:len(replacement)-1]
replacement = replacement.replace("\'", "\"", -1)
print("POST replacement: %s" % replacement)
json_replacement = replacement
try:
itemlist = json.loads(actualitem)
if len(itemlist) > minlength:
minlength = len(itemlist)
json_replacement = json.loads(replacement)
except json.decoder.JSONDecodeError as e:
print("JSON Error: %s in %s" % (e, actualitem))
print("JSON error singular: %s" % e)
replacements[to_be_replaced] = actualitem
if len(json_replacement) > minlength:
minlength = len(json_replacement)
# This is a result array for JUST this value..
# What if there are more?
resultarray = []
for i in range(0, minlength):
tmpitem = json.loads(json.dumps(parameter["value"]))
for key, value in replacements.items():
replacement = json.dumps(json.loads(value)[i])
if replacement.startswith("\"") and replacement.endswith("\""):
replacement = replacement[1:len(replacement)-1]
#except json.decoder.JSONDecodeError as e:
tmpitem = tmpitem.replace(actualitem[0][0], replacement, 1)
params[parameter["name"]] = tmpitem
multi_execution_lists.append(json_replacement)
multi_parameters[parameter["name"]] = json_replacement
print("REPLACING %s with %s" % (key, replacement))
#replacement = parse_wrapper_start(replacement)
tmpitem = tmpitem.replace(key, replacement, -1)
#print("LENGTH OF ARR: %d" % len(resultarray))
#print("RESULTARRAY: %s" % resultarray)
print("MULTI finished: %s" % replacement)
else:
# This is here to handle for loops within variables.. kindof
# 1. Find the length of the longest array
# 2. Build an array with the base values based on parameter["value"]
# 3. Get the n'th value of the generated list from values
# 4. Execute all n answers
replacements = {}
for replace in actualitem:
try:
to_be_replaced = replace[0]
actualitem = replace[2]
except IndexError:
continue
resultarray.append(tmpitem)
try:
itemlist = json.loads(actualitem)
if len(itemlist) > minlength:
minlength = len(itemlist)
except json.decoder.JSONDecodeError as e:
print("JSON Error: %s in %s" % (e, actualitem))
# With this parameter ready, add it to... a greater list of parameters. Rofl
multi_parameters[parameter["name"]] = resultarray
replacements[to_be_replaced] = actualitem
# This is a result array for JUST this value..
# What if there are more?
resultarray = []
for i in range(0, minlength):
tmpitem = json.loads(json.dumps(parameter["value"]))
for key, value in replacements.items():
replacement = json.dumps(json.loads(value)[i])
if replacement.startswith("\"") and replacement.endswith("\""):
replacement = replacement[1:len(replacement)-1]
#except json.decoder.JSONDecodeError as e:
#print("REPLACING %s with %s" % (key, replacement))
#replacement = parse_wrapper_start(replacement)
tmpitem = tmpitem.replace(key, replacement, -1)
resultarray.append(tmpitem)
# With this parameter ready, add it to... a greater list of parameters. Rofl
print("LENGTH OF ARR: %d" % len(resultarray))
print("RESULTARRAY: %s" % resultarray)
if resultarray not in multi_execution_lists:
multi_execution_lists.append(resultarray)
multi_parameters[parameter["name"]] = resultarray
else:
# Parses things like int(value)
self.logger.info("Parsing wrapper data for %s" % value)
@@ -864,11 +964,34 @@ class AppBase:
params[parameter["name"]] = value
multi_parameters[parameter["name"]] = value
# Fix lists here
print("CHECKING multi execution list!")
if len(multi_execution_lists) > 0:
print("\n Multi execution list has more data: %d" % len(multi_execution_lists))
filteredlist = []
for listitem in multi_execution_lists:
if listitem in filteredlist:
continue
filteredlist.append(listitem)
#print("New list length: %d" % len(filteredlist))
if len(filteredlist) > 1:
print("Calculating new multi-loop length with %d lists" % len(filteredlist))
tmplength = 1
for innerlist in filteredlist:
print("List length: %d. %d*%d" % (len(innerlist), len(innerlist), tmplength))
tmplength = len(innerlist)*tmplength
minlength = tmplength
print("New multi execution length: %d\n" % tmplength)
# FIXME - this is horrible, but works for now
#for i in range(calltimes):
if not multiexecution:
print("APP_SDK DONE: Starting normal execution of function")
print("APP_SDK DONE: Starting NORMAL execution of function")
newres = await func(**params)
#print("NEWRES: ", newres)
if isinstance(newres, str):
@@ -881,20 +1004,67 @@ class AppBase:
print("Can't handle type %s value from function" % (type(newres)))
print("POST NEWRES RESULT: ", result)
else:
print("APP_SDK DONE: Starting MULTI execution with", multi_parameters)
# 1. Use number of executions based on longest array
print("APP_SDK DONE: Starting MULTI execution with values %s of length %d" % (multi_parameters, minlength))
# 1. Use number of executions based on the arrays being similar
# 2. Find the right value from the parsed multi_params
results = []
json_object = False
for i in range(0, minlength):
# To be able to use the results as a list:
baseparams = json.loads(json.dumps(multi_parameters))
# {'call': ['GoogleSafebrowsing_2_0', 'VirusTotal_GetReport_3_0']}
# 1. Check if list length is same as minlength
# 2. If NOT same length, duplicate based on length of array
# arraylength = 3 ["1", "2", "3"]
# arraylength = 4 ["1", "2", "3", "4"]
# minlength = 12 - 12/3 = 4 per item = ["1", "1", "1", "1", "2", "2", ...]
try:
firstlist = True
for key, value in baseparams.items():
if isinstance(value, list):
baseparams[key] = value[i]
try:
newvalue = value[i]
except IndexError:
pass
if len(value) != minlength and len(value) > 0:
newarray = []
print("VALUE: ", value)
additiontime = minlength/len(value)
print("Bad length for value: %d - should be %d. Additiontime: %d" % (len(value), minlength, additiontime))
if firstlist:
print("Running normal list (FIRST)")
for subvalue in value:
for number in range(int(additiontime)):
newarray.append(subvalue)
else:
#print("Running secondary lists")
## 1. Set up length of array
## 2. Put values spread out
# FIXME: This works well, except if lists are same length
newarray = [""] * minlength
cnt = 0
for number in range(int(additiontime)):
for subvaluerange in range(len(value)):
# newlocation = number+(additiontime*subvaluerange)
# print("%d+(%d*%d) = %d. VAL: %s" % (number, additiontime, subvaluerange, newlocation, value[subvaluerange]))
# Reverse if same length?
if int(minlength/len(value)) == len(value):
tmp = int(len(value)-subvaluerange-1)
print("NEW: %d" % tmp)
newarray[cnt] = value[tmp]
else:
newarray[cnt] = value[subvaluerange]
cnt += 1
#print("Newarray =", newarray)
newvalue = newarray[i]
firstlist = False
baseparams[key] = newvalue
except IndexError as e:
print("IndexError: %s" % e)
baseparams[key] = "IndexError: %s" % e
@@ -902,25 +1072,49 @@ class AppBase:
print("KeyError: %s" % e)
baseparams[key] = "KeyError: %s" % e
#print("Running with params %s" % baseparams)
print("Running with params %s" % baseparams)
ret = await func(**baseparams)
ret = ret.replace("\"", "\\\"", -1)
print("Inner ret parsed: %s" % ret)
try:
results.append(json.loads(ret))
json_object = True
except json.decoder.JSONDecodeError as e:
if isinstance(ret, dict) or isinstance(ret, list):
results.append(ret)
json_object = True
else:
ret = ret.replace("\"", "\\\"", -1)
try:
results.append(json.loads(ret))
json_object = True
except json.decoder.JSONDecodeError as e:
#print("Json: %s" % e)
results.append(ret)
#print("Inner ret parsed: %s" % ret)
# Dump the result as a string of a list
print("RESULTS: %s" % results)
#print("RESULTS: %s" % results)
if isinstance(results, list):
print("JSON OBJECT? ", json_object)
if json_object:
result = json.dumps(results)
else:
result = "[\""+"\", \"".join(results)+"\"]"
result = "["
for item in results:
try:
json.loads(item)
result += item
except json.decoder.JSONDecodeError as e:
# Common nested issue which puts " around everything
try:
tmpitem = item.replace("\\\"", "\"", -1)
json.loads(tmpitem)
result += tmpitem
except:
result += "\"%s\"" % item
result += ", "
result = result[:-2]
result += "]"
else:
print("Normal result?")
result = results
@@ -932,7 +1126,7 @@ class AppBase:
action_result["result"] = result
self.logger.debug(f"Executed {action['label']}-{action['id']} with result: {result}")
self.logger.debug(f"Data: %s" % action_result)
#self.logger.debug(f"Data: %s" % action_result)
except TypeError as e:
print("TypeError issue: %s" % e)
action_result["status"] = "FAILURE"
@@ -947,7 +1141,7 @@ class AppBase:
print(f"Failed to execute: {e}")
self.logger.exception(f"Failed to execute {e}-{action['id']}")
action_result["status"] = "FAILURE"
action_result["result"] = "General exception: %s" % e
action_result["result"] = f"General exception: {e}"
action_result["completed_at"] = int(time.time())
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/bash
NAME=app_sdk
VERSION=0.6.2
VERSION=0.7.3
docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
+82 -196
View File
@@ -310,20 +310,12 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet
// Specific check for SSL verification
// This is critical for onprem stuff.
verifyParam := ""
verifyWrapper := ""
verifyAddin := ""
if len(swagger.Servers) == 0 {
verifyParam = ", verify=True"
verifyWrapper = `if type(ssl_verify) == str: ssl_verify = False if ssl_verify.lower() == "false" or ssl_verify == "0" else True`
verifyAddin = ", verify=ssl_verify"
} else {
if swagger.Servers[0].URL == "" {
verifyParam = ", ssl_verify=True"
verifyWrapper = `if type(ssl_verify) == str: ssl_verify = False if ssl_verify.lower() == "false" or ssl_verify == "0" else True`
verifyAddin = ", verify=ssl_verify"
}
}
//verifyParam := ""
//verifyWrapper := ""
//verifyAddin := ""
verifyParam := ", ssl_verify=False"
verifyWrapper := `if type(ssl_verify) == str: ssl_verify = False if ssl_verify.lower() == "false" or ssl_verify == "0" else True`
verifyAddin := ", verify=ssl_verify"
if len(parameters) > 0 {
parameterData = fmt.Sprintf(", %s", strings.Join(parameters, ", "))
@@ -404,11 +396,10 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet
bodyAddin,
verifyAddin,
)
if strings.Contains(functionname, "get_returns_the_vuln") {
log.Println(data)
log.Printf("Queries: %s", queryString)
}
//if strings.Contains(functionname, "get_returns_the_vuln") {
// log.Println(data)
// log.Printf("Queries: %s", queryString)
//}
//log.Printf(data)
return functionname, data
@@ -562,7 +553,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger,
})
} else if securitySchemes["BasicAuth"] != nil {
api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{
Name: "username_auth",
Name: "username_basic",
Value: "",
Example: "username",
Description: securitySchemes["BasicAuth"].Value.Description,
@@ -574,7 +565,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger,
})
api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{
Name: "password_auth",
Name: "password_basic",
Value: "",
Example: "*****",
Description: securitySchemes["BasicAuth"].Value.Description,
@@ -975,31 +966,16 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [
optionalQueries := []string{}
parameters := []string{}
optionalParameters := []WorkflowAppActionParameter{}
if len(swagger.Servers) == 0 {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
} else {
if swagger.Servers[0].URL == "" {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
}
}
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
headersFound := []string{}
if len(path.Connect.Parameters) > 0 {
@@ -1124,31 +1100,16 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
// FIXME - remove this when authentication is properly introduced
parameters := []string{}
optionalParameters := []WorkflowAppActionParameter{}
if len(swagger.Servers) == 0 {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify the SSL certificate request",
Multiline: false,
Required: false,
Example: "False - default=True",
Schema: SchemaDefinition{
Type: "string",
},
})
} else {
if swagger.Servers[0].URL == "" {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
}
}
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify the SSL certificate request",
Multiline: false,
Required: false,
Example: "False - default=True",
Schema: SchemaDefinition{
Type: "string",
},
})
headersFound := []string{}
if len(path.Get.Parameters) > 0 {
@@ -1273,31 +1234,16 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
optionalQueries := []string{}
parameters := []string{}
optionalParameters := []WorkflowAppActionParameter{}
if len(swagger.Servers) == 0 {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
} else {
if swagger.Servers[0].URL == "" {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
}
}
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
headersFound := []string{}
if len(path.Head.Parameters) > 0 {
@@ -1422,31 +1368,16 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []
optionalQueries := []string{}
parameters := []string{}
optionalParameters := []WorkflowAppActionParameter{}
if len(swagger.Servers) == 0 {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
} else {
if swagger.Servers[0].URL == "" {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
}
}
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
headersFound := []string{}
if len(path.Delete.Parameters) > 0 {
@@ -1570,31 +1501,16 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
optionalQueries := []string{}
parameters := []string{}
optionalParameters := []WorkflowAppActionParameter{}
if len(swagger.Servers) == 0 {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
} else {
if swagger.Servers[0].URL == "" {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
}
}
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
headersFound := []string{}
if len(path.Post.Parameters) > 0 {
@@ -1718,31 +1634,16 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W
optionalQueries := []string{}
parameters := []string{}
optionalParameters := []WorkflowAppActionParameter{}
if len(swagger.Servers) == 0 {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
} else {
if swagger.Servers[0].URL == "" {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
}
}
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
headersFound := []string{}
if len(path.Patch.Parameters) > 0 {
@@ -1866,31 +1767,16 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
optionalQueries := []string{}
parameters := []string{}
optionalParameters := []WorkflowAppActionParameter{}
if len(swagger.Servers) == 0 {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
} else {
if swagger.Servers[0].URL == "" {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
}
}
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
headersFound := []string{}
if len(path.Put.Parameters) > 0 {
+435 -46
View File
@@ -150,7 +150,7 @@ type UserAuth struct {
type UserAuthField struct {
Key string `json:"key" datastore:"key"`
Value string `json:"value" datastore:"value"`
Value string `json:"value" datastore:"value,noindex"`
}
// Not environment, but execution environment
@@ -181,6 +181,7 @@ type User struct {
Id string `datastore:"id" json:"id"`
Orgs []string `datastore:"orgs" json:"orgs"`
CreationTime int64 `datastore:"creation_time" json:"creation_time"`
ActiveOrg Org `json:"active_org" datastore:"active_org"`
Active bool `datastore:"active" json:"active"`
}
@@ -209,7 +210,7 @@ type Contact struct {
type Translator struct {
Src struct {
Name string `json:"name" datastore:"name"`
Value string `json:"value" datastore:"value"`
Value string `json:"value" datastore:"value,noindex"`
Description string `json:"description" datastore:"description,noindex"`
Required string `json:"required" datastore:"required"`
Type string `json:"type" datastore:"type"`
@@ -219,7 +220,7 @@ type Translator struct {
} `json:"src" datastore:"src"`
Dst struct {
Name string `json:"name" datastore:"name"`
Value string `json:"value" datastore:"value"`
Value string `json:"value" datastore:"value,noindex"`
Type string `json:"type" datastore:"type"`
Description string `json:"description" datastore:"description,noindex"`
Required string `json:"required" datastore:"required"`
@@ -231,7 +232,7 @@ type Translator struct {
type Appconfig struct {
Key string `json:"key" datastore:"key"`
Value string `json:"value" datastore:"value"`
Value string `json:"value" datastore:"value,noindex"`
}
type ScheduleApp struct {
@@ -1013,13 +1014,13 @@ func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) {
user, err := handleApiAuthentication(resp, request)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Can't register without being admin"}`))
resp.Write([]byte(`{"success": false, "reason": "Can't handle set env auth"}`))
return
}
if user.Role != "admin" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Can't register without being admin"}`))
resp.Write([]byte(`{"success": false, "reason": "Can't set environment without being admin"}`))
return
}
@@ -1097,7 +1098,7 @@ func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) {
resp.Write([]byte(`{"success": true}`))
}
func createNewUser(username, password, role, apikey string) error {
func createNewUser(username, password, role, apikey string, org Org) error {
// Returns false if there is an issue
// Use this for register
err := checkPasswordStrength(password)
@@ -1137,7 +1138,7 @@ func createNewUser(username, password, role, apikey string) error {
newUser.Verified = false
newUser.CreationTime = time.Now().Unix()
newUser.Active = true
newUser.Orgs = []string{"default"}
newUser.Orgs = []string{org.Id}
// FIXME - Remove this later
if role == "admin" {
@@ -1148,6 +1149,8 @@ func createNewUser(username, password, role, apikey string) error {
newUser.Roles = []string{"user"}
}
newUser.ActiveOrg = org
if len(apikey) > 0 {
newUser.ApiKey = apikey
}
@@ -1182,25 +1185,25 @@ func createNewUser(username, password, role, apikey string) error {
log.Printf("Error adding User %s: %s", username, err)
return err
}
url := fmt.Sprintf("https://shuffler.io/register/%s", verifyToken.String())
const verifyMessage = `
Registration URL :)
%s
`
addr := newUser.Username
msg := &mail.Message{
Sender: "Shuffle <frikky@shuffler.io>",
To: []string{addr},
Subject: "Verify your username - Shuffle",
Body: fmt.Sprintf(verifyMessage, url),
}
log.Println(msg.Body)
if err := mail.Send(ctx, msg); err != nil {
log.Printf("Couldn't send email: %v", err)
}
// url := fmt.Sprintf("https://shuffler.io/register/%s", verifyToken.String())
// const verifyMessage = `
//Registration URL :)
//
//%s
// `
// addr := newUser.Username
//
// msg := &mail.Message{
// Sender: "Shuffle <frikky@shuffler.io>",
// To: []string{addr},
// Subject: "Verify your username - Shuffle",
// Body: fmt.Sprintf(verifyMessage, url),
// }
//
// log.Println(msg.Body)
// if err := mail.Send(ctx, msg); err != nil {
// log.Printf("Couldn't send email: %v", err)
// }
err = increaseStatisticsField(ctx, "successful_register", username, 1)
if err != nil {
@@ -1249,7 +1252,8 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) {
if count == 0 {
role = "admin"
}
err = createNewUser(data.Username, data.Password, role, "")
err = createNewUser(data.Username, data.Password, role, "", user.ActiveOrg)
if err != nil {
log.Printf("Failed registering user: %s", err)
resp.WriteHeader(401)
@@ -1623,6 +1627,10 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) {
// This is a long check to see if an inactive admin can access the site
parsedAdmin := "false"
if userInfo.Role == "admin" {
parsedAdmin = "true"
}
if !userInfo.Active {
if userInfo.Role == "admin" {
parsedAdmin = "true"
@@ -1692,16 +1700,66 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) {
Expires: expiration,
})
// Updating user info if there's something wrong
if (len(userInfo.ActiveOrg.Name) == 0 || len(userInfo.ActiveOrg.Id) == 0) && len(userInfo.Orgs) > 0 {
_, err := getOrg(ctx, userInfo.Orgs[0])
if err != nil {
var orgs []Org
q := datastore.NewQuery("Organizations")
_, err = dbclient.GetAll(ctx, q, &orgs)
if err == nil {
newStringOrgs := []string{}
newOrgs := []Org{}
for _, org := range orgs {
if strings.ToLower(org.Name) == strings.ToLower(userInfo.Orgs[0]) {
newOrgs = append(newOrgs, org)
newStringOrgs = append(newStringOrgs, org.Id)
}
}
if len(newOrgs) > 0 {
userInfo.ActiveOrg = newOrgs[0]
userInfo.Orgs = newStringOrgs
err = setUser(ctx, &userInfo)
if err != nil {
log.Printf("Error patching User for activeOrg: %s", err)
} else {
log.Printf("Updated the users' org")
}
}
} else {
log.Printf("Failed getting orgs for user. Major issue.: %s", err)
}
} else {
// 1. Check if the org exists by ID
// 2. if it does, overwrite user
userInfo.ActiveOrg = Org{
Id: userInfo.Orgs[0],
}
err = setUser(ctx, &userInfo)
if err != nil {
log.Printf("Error patching User for activeOrg: %s", err)
}
}
}
currentOrg, err := json.Marshal(userInfo.ActiveOrg)
if err != nil {
currentOrg = []byte("{}")
}
returnData := fmt.Sprintf(`
{
"success": true,
"admin": %s,
"tutorials": [],
"id": "%s",
"orgs": [{"name": "Shuffle", "id": "123", "role": "admin"}],
"selected_org": {"name": "Shuffle", "id": "123", "role": "admin"},
"orgs": [%s],
"active_org": %s,
"cookies": [{"key": "session_token", "value": "%s", "expiration": %d}]
}`, parsedAdmin, userInfo.Id, userInfo.Session, expiration.Unix())
}`, parsedAdmin, userInfo.Id, currentOrg, currentOrg, userInfo.Session, expiration.Unix())
resp.WriteHeader(200)
resp.Write([]byte(returnData))
@@ -1973,7 +2031,7 @@ func handlePasswordChange(resp http.ResponseWriter, request *http.Request) {
}
if len(users) != 1 {
log.Printf(`Found multiple users with the same username: %s: %d`, t.Username, len(users))
log.Printf(`Found multiple or no users with the same username: %s: %d`, t.Username, len(users))
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Found %d users with the same username: %s (%d)"}`, len(users), t.Username)))
return
@@ -2166,6 +2224,61 @@ func handleGetEnvironments(resp http.ResponseWriter, request *http.Request) {
resp.Write(newjson)
}
func handleGetOrgs(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in set new workflowhandler: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Not admin"}`))
return
}
ctx := context.Background()
var orgs []Org
q := datastore.NewQuery("Organizations")
_, err = dbclient.GetAll(ctx, q, &orgs)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Can't get users"}`))
return
}
//newUsers := []User{}
//for _, item := range users {
// if len(item.Username) == 0 {
// continue
// }
// item.Password = ""
// item.Session = ""
// item.VerificationToken = ""
// newUsers = append(newUsers, item)
//}
newjson, err := json.Marshal(orgs)
if err != nil {
log.Printf("Failed unmarshal: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking"}`)))
return
}
resp.WriteHeader(200)
resp.Write(newjson)
}
func handleGetUsers(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
@@ -2281,7 +2394,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) {
}
if len(users) != 1 {
log.Printf(`Found multiple users with the same username: %s: %d`, data.Username, len(users))
log.Printf(`Found multiple or no users with the same username: %s: %d`, data.Username, len(users))
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Found %d users with the same username: %s"}`, len(users), data.Username)))
return
@@ -2383,6 +2496,28 @@ func getSession(ctx context.Context, thissession string) (*session, error) {
return curUser, nil
}
// ListBooks returns a list of books, ordered by title.
func getOrg(ctx context.Context, id string) (*Org, error) {
key := datastore.NameKey("Organizations", id, nil)
curOrg := &Org{}
if err := dbclient.Get(ctx, key, curOrg); err != nil {
return &Org{}, err
}
return curOrg, nil
}
func setOrg(ctx context.Context, data Org, id string) error {
// clear session_token and API_token for user
k := datastore.NameKey("Organizations", id, nil)
if _, err := dbclient.Put(ctx, k, &data); err != nil {
log.Println(err)
return err
}
return nil
}
// ListBooks returns a list of books, ordered by title.
func getUser(ctx context.Context, id string) (*User, error) {
key := datastore.NameKey("Users", id, nil)
@@ -2946,6 +3081,7 @@ func getSpecificWebhook(resp http.ResponseWriter, request *http.Request) {
}
ctx := context.Background()
// FIXME: Schedule = trigger?
schedule, err := getSchedule(ctx, workflowId)
if err != nil {
log.Printf("Failed setting schedule: %s", err)
@@ -4781,8 +4917,8 @@ func setTriggerAuth(ctx context.Context, trigger TriggerAuth) error {
func getOutlookClient(ctx context.Context, code string, accessToken OauthToken, redirectUri string) (*http.Client, *oauth2.Token, error) {
conf := &oauth2.Config{
ClientID: "70e37005-c954-4290-b573-d4b94e484336",
ClientSecret: ".eNw/A[kQFB5zL.agvRputdEJENeJ392",
ClientID: "",
ClientSecret: "",
Scopes: []string{
"Mail.Read",
"User.Read",
@@ -6233,6 +6369,91 @@ func runInit(ctx context.Context) {
}
*/
setUsers := false
orgQuery := datastore.NewQuery("Organizations")
var activeOrgs []Org
_, err = dbclient.GetAll(ctx, orgQuery, &activeOrgs)
if err != nil {
log.Printf("Error getting organizations!")
} else {
// Add all users to it
if len(activeOrgs) == 1 {
setUsers = true
}
log.Printf("Organizations exist!")
if len(activeOrgs) == 0 {
log.Printf(`No orgs. Setting org "default"`)
orgSetupName := "default"
orgId := uuid.NewV4().String()
newOrg := Org{
Name: orgSetupName,
Id: orgId,
Org: orgSetupName,
Users: []User{},
Roles: []string{"admin", "user"},
CloudSync: false,
}
err = setOrg(ctx, newOrg, orgId)
if err != nil {
log.Printf("Failed setting organization: %s", err)
} else {
log.Printf("Successfully created the default org!")
setUsers = true
}
} else {
log.Printf("There are %d org(s).", len(activeOrgs))
}
}
// Adding the users to the base organization since only one exists (default)
if setUsers && len(activeOrgs) > 0 {
activeOrg := activeOrgs[0]
q := datastore.NewQuery("Users")
var users []User
_, err = dbclient.GetAll(ctx, q, &users)
if err == nil {
setOrgBool := false
for _, user := range users {
newUser := User{
Username: user.Username,
Id: user.Id,
ActiveOrg: Org{
Id: activeOrg.Id,
},
Orgs: []string{activeOrg.Id},
Role: user.Role,
}
found := false
for _, orgUser := range activeOrg.Users {
if user.Id == orgUser.Id {
found = true
}
}
if !found && len(user.Username) > 0 {
log.Printf("Adding user %s to org %s", user.Username, activeOrg.Name)
activeOrg.Users = append(activeOrg.Users, newUser)
setOrgBool = true
}
}
if setOrgBool {
err = setOrg(ctx, activeOrg, activeOrg.Id)
if err != nil {
log.Printf("Failed setting org %s: %s!", activeOrg.Name, err)
} else {
log.Printf("UPDATED org %s!", activeOrg.Name)
}
}
}
log.Printf("Should add %d users to organization default", len(users))
}
// Fix active users etc
q := datastore.NewQuery("Users").Filter("active =", true)
var activeusers []User
@@ -6243,6 +6464,7 @@ func runInit(ctx context.Context) {
q := datastore.NewQuery("Users")
var users []User
_, err := dbclient.GetAll(ctx, q, &users)
if len(activeusers) == 0 && len(users) > 0 {
log.Printf("No active users found - setting ALL to active")
if err == nil {
@@ -6258,7 +6480,12 @@ func runInit(ctx context.Context) {
}
if len(user.Orgs) == 0 {
user.Orgs = []string{"default"}
defaultName := "default"
user.Orgs = []string{defaultName}
user.ActiveOrg = Org{
Name: defaultName,
Role: "user",
}
}
err = setUser(ctx, &user)
@@ -6281,7 +6508,11 @@ func runInit(ctx context.Context) {
log.Printf("SHUFFLE_DEFAULT_USERNAME and SHUFFLE_DEFAULT_PASSWORD not defined as environments. Running without default user.")
} else {
apikey := os.Getenv("SHUFFLE_DEFAULT_APIKEY")
err = createNewUser(username, password, "admin", apikey)
tmpOrg := Org{
Name: "default",
}
err = createNewUser(username, password, "admin", apikey, tmpOrg)
if err != nil {
log.Printf("Failed to create default user %s: %s", username, err)
} else {
@@ -6449,6 +6680,164 @@ func runInit(ctx context.Context) {
log.Printf("Finished INIT")
}
// INFO: https://docs.google.com/drawings/d/1JJebpPeEVEbmH_qsAC6zf9Noygp7PytvesrkhE19QrY/edit
func handleCloudSetup(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in verify swagger: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("Not admin.")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Not admin"}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
return
}
type ReturnData struct {
Apikey string `datastore:"apikey"`
Organization Org `datastore:"organization"`
}
var tmpData ReturnData
err = json.Unmarshal(body, &tmpData)
if err != nil {
log.Printf("Failed unmarshalling test: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
ctx := context.Background()
org, err := getOrg(ctx, tmpData.Organization.Id)
if err != nil {
log.Printf("Organization doesn't exist: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// FIXME: Check if user is admin of this org
userFound := false
admin := false
for _, inneruser := range org.Users {
if inneruser.Id == user.Id {
userFound = true
log.Printf("Role: %s", inneruser.Role)
if inneruser.Role == "admin" {
admin = true
}
break
}
}
if !userFound {
log.Printf("User %s doesn't exist in organization %s", user.Id, org.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// FIXME: Enable admin check in org for sync setup and conf.
_ = admin
//if !admin {
// log.Printf("User %s isn't admin hence can't set up sync for org %s", user.Id, org.Id)
// resp.WriteHeader(401)
// resp.Write([]byte(`{"success": false}`))
// return
//}
log.Printf("Apidata: %s", tmpData.Apikey)
client := &http.Client{}
syncPath := "http://192.168.3.6:5002/api/v1/cloud/sync"
type requestStruct struct {
ApiKey string `json:"api_key"`
}
requestData := requestStruct{
ApiKey: tmpData.Apikey,
}
b, err := json.Marshal(requestData)
if err != nil {
log.Printf("Failed marshaling api key data: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed cloud sync."`, err)))
return
}
req, err := http.NewRequest(
"POST",
syncPath,
bytes.NewBuffer(b),
)
newresp, err := client.Do(req)
if err != nil {
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed cloud sync: %s"`, err)))
//setBadMemcache(ctx, docPath)
return
}
if newresp.StatusCode != 200 {
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Response code %d during sync. Expecting 200."`, newresp.StatusCode)))
return
}
respBody, err := ioutil.ReadAll(newresp.Body)
if err != nil {
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't parse sync data"`)))
return
}
type responseStruct struct {
Success bool `json:"success"`
Reason string `json:"reason"`
}
log.Printf("Respbody: %s", string(respBody))
responseData := responseStruct{}
err = json.Unmarshal(respBody, &responseData)
if err != nil {
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed handling cloud data"`)))
return
}
if responseData.Success {
resp.WriteHeader(200)
if len(responseData.Reason) > 0 {
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, responseData.Reason)))
} else {
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
} else {
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, responseData.Reason)))
}
}
func initHandlers() {
var err error
ctx := context.Background()
@@ -6467,11 +6856,6 @@ func initHandlers() {
r := mux.NewRouter()
r.HandleFunc("/api/v1/_ah/health", healthCheckHandler)
// Sends an email if the right things are specified
r.HandleFunc("/functions/sendmail", handleSendalert).Methods("POST", "OPTIONS")
r.HandleFunc("/functions/outlook/register", handleNewOutlookRegister).Methods("GET", "OPTIONS")
r.HandleFunc("/functions/outlook/getFolders", handleGetOutlookFolders).Methods("GET", "OPTIONS")
// Make user related locations
r.HandleFunc("/api/v1/users/generateapikey", handleApiGeneration).Methods("GET", "POST", "OPTIONS")
r.HandleFunc("/api/v1/users/login", handleLogin).Methods("POST", "OPTIONS")
@@ -6506,9 +6890,14 @@ func initHandlers() {
// Queuebuilder and Workflow streams. First is to update a stream, second to get a stream
// Changed from workflows/streams to streams, as appengine was messing up
// This does not increase the API counter
// Used by frontend
r.HandleFunc("/api/v1/streams", handleWorkflowQueue).Methods("POST")
r.HandleFunc("/api/v1/streams/results", handleGetStreamResults).Methods("POST", "OPTIONS")
// Used by orborus
r.HandleFunc("/api/v1/workflows/queue", handleGetWorkflowqueue).Methods("GET")
r.HandleFunc("/api/v1/workflows/queue/confirm", handleGetWorkflowqueueConfirm).Methods("POST")
// App specific
r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/get_existing", loadSpecificApps).Methods("POST", "OPTIONS")
@@ -6535,8 +6924,6 @@ func initHandlers() {
/* Everything below here increases the counters*/
r.HandleFunc("/api/v1/workflows", getWorkflows).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/workflows", setNewWorkflow).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/queue", handleGetWorkflowqueue).Methods("GET")
r.HandleFunc("/api/v1/workflows/queue/confirm", handleGetWorkflowqueueConfirm).Methods("POST")
r.HandleFunc("/api/v1/workflows/schedules", handleGetSchedules).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/workflows/download_remote", loadSpecificWorkflows).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/execute", executeWorkflow).Methods("GET", "POST", "OPTIONS")
@@ -6557,7 +6944,7 @@ func initHandlers() {
r.HandleFunc("/api/v1/hooks/{key}/delete", handleDeleteHook).Methods("DELETE", "OPTIONS")
// Trigger hmm
r.HandleFunc("/api/v1/triggers/{key}", handleGetSpecificTrigger).Methods("GET", "OPTIONS")
//r.HandleFunc("/api/v1/triggers/{key}", handleGetSpecificTrigger).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/stats/{key}", handleGetSpecificStats).Methods("GET", "OPTIONS")
@@ -6568,7 +6955,9 @@ func initHandlers() {
r.HandleFunc("/api/v1/validate_openapi", validateSwagger).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/get_openapi/{key}", getOpenapi).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/execution_cleanup", cleanupExecutions).Methods("GET", "OPTIONS")
// NEW for 0.8.0
r.HandleFunc("/api/v1/cloud/setup", handleCloudSetup).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/getorgs", handleGetOrgs).Methods("GET", "OPTIONS")
http.Handle("/", r)
}
+300 -83
View File
@@ -67,11 +67,28 @@ type ExecutionRequest struct {
Type string `json:"type"`
}
type SyncFeatures struct {
Apps SyncData `json:"apps" datastore:"apps"`
Workflows SyncData `json:"apps" datastore:"apps"`
Schedules SyncData `json:"apps" datastore:"apps"`
Autocomplete SyncData `json:"apps" datastore:"apps"`
Authentication SyncData `json:"apps" datastore:"apps"`
}
type SyncData struct {
Active bool `json:"active" datastore:"active"`
}
// Role is just used for feedback for a user
type Org struct {
Name string `json:"name"`
Org string `json:"org"`
Users []User `json:"users"`
Id string `json:"id"`
Name string `json:"name" datastore:"name"`
Id string `json:"id" datastore:"id"`
Org string `json:"org" datastore:"org"`
Users []User `json:"users" datastore:"users"`
Role string `json:"role" datastore:"role"`
Roles []string `json:"roles" datastore:"roles"`
CloudSync bool `json:"cloud_sync" datastore:"CloudSync"`
SyncFeatures SyncFeatures `json:"sync_features" datastore:"sync_features"`
}
type AppAuthenticationStorage struct {
@@ -127,7 +144,7 @@ type WorkflowAppActionParameter struct {
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
Name string `json:"name" datastore:"name" yaml:"name"`
Example string `json:"example" datastore:"example" yaml:"example"`
Value string `json:"value" datastore:"value" yaml:"value,omitempty"`
Value string `json:"value" datastore:"value,noindex" yaml:"value,omitempty"`
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
Options []string `json:"options" datastore:"options" yaml:"options"`
ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"`
@@ -161,7 +178,7 @@ type WorkflowAppAction struct {
Description string `json:"description" datastore:"description,noindex"`
ID string `json:"id" datastore:"id"`
Name string `json:"name" datastore:"name"`
Value string `json:"value" datastore:"value"`
Value string `json:"value" datastore:"value,noindex"`
} `json:"execution_variable" datastore:"execution_variables"`
Returns struct {
Description string `json:"description" datastore:"returns" yaml:"description,omitempty"`
@@ -306,7 +323,7 @@ type Workflow struct {
Description string `json:"description" datastore:"description,noindex"`
ID string `json:"id" datastore:"id"`
Name string `json:"name" datastore:"name"`
Value string `json:"value" datastore:"value"`
Value string `json:"value" datastore:"value,noindex"`
} `json:"workflow_variables" datastore:"workflow_variables"`
ExecutionVariables []struct {
Description string `json:"description" datastore:"description,noindex"`
@@ -336,7 +353,7 @@ type AuthenticationParams struct {
ID string `json:"id" datastore:"id" yaml:"id"`
Name string `json:"name" datastore:"name" yaml:"name"`
Example string `json:"example" datastore:"example" yaml:"example"`
Value string `json:"value,omitempty" datastore:"value" yaml:"value"`
Value string `json:"value,omitempty" datastore:"value,noindex" yaml:"value"`
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
Required bool `json:"required" datastore:"required" yaml:"required"`
In string `json:"in" datastore:"in" yaml:"in"`
@@ -346,13 +363,23 @@ type AuthenticationParams struct {
type AuthenticationStore struct {
Key string `json:"key" datastore:"key"`
Value string `json:"value" datastore:"value"`
Value string `json:"value" datastore:"value,noindex"`
}
type ExecutionRequestWrapper struct {
Data []ExecutionRequest `json:"data"`
}
type AppExecutionExample struct {
AppName string `json:"app_name" datastore:"app_name"`
AppVersion string `json:"app_version" datastore:"app_version"`
AppAction string `json:"app_action" datastore:"app_action"`
AppId string `json:"app_id" datastore:"app_id"`
ExampleId string `json:"example_id" datastore:"example_id"`
SuccessExamples []string `json:"success_examples" datastore:"success_examples,noindex"`
FailureExamples []string `json:"failure_examples" datastore:"failure_examples,noindex"`
}
// This might be... a bit off, but that's fine :)
// This might also be stupid, as we want timelines and such
// Anyway, these are super basic stupid stats.
@@ -1078,6 +1105,10 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
if err != nil {
log.Printf("Failed to increase success execution stats: %s", err)
}
// Handles extra statistics stuff when it's done
// Does autocomplete magic with JSON
go handleExecutionStatistics(*workflowExecution)
}
}
@@ -1088,8 +1119,33 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
// log.Printf("Name: %s, Env: %s", action.Name, action.Environment)
//}
tmpJson, err := json.Marshal(workflowExecution)
if err == nil {
if len(tmpJson) >= 1048487 {
log.Printf("[ERROR] Result length is too long! Need to reduce result size")
// Result string `json:"result" datastore:"result,noindex"`
// Arbitrary reduction size
maxSize := 500000
newResults := []ActionResult{}
for _, item := range workflowExecution.Results {
if len(item.Result) > maxSize {
item.Result = "[ERROR] Result too large to handle (https://github.com/frikky/shuffle/issues/171)"
}
newResults = append(newResults, item)
}
workflowExecution.Results = newResults
}
}
err = setWorkflowExecution(ctx, *workflowExecution)
if err != nil {
//workflowExecution.Result = "Error setting workflow: result too large"
//workflowExecution.Status = "FINISHED"
//workflowExecution.CompletedAt = int64(time.Now().Unix())
log.Printf("Error saving workflow execution actionresult setting: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution actionresult: %s"}`, err)))
@@ -1100,6 +1156,90 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func JSONCheck(str string) bool {
var jsonStr interface{}
return json.Unmarshal([]byte(str), &jsonStr) == nil
}
func handleExecutionStatistics(execution WorkflowExecution) {
// FIXME: CLEAN UP THE JSON THAT'S SAVED.
// https://github.com/frikky/Shuffle/issues/172
appResults := []AppExecutionExample{}
for _, result := range execution.Results {
resultCheck := JSONCheck(result.Result)
if !resultCheck {
log.Printf("Result is NOT JSON!")
continue
} else {
log.Printf("Result IS JSON!")
}
appFound := false
executionIndex := 0
for index, appExample := range appResults {
if appExample.AppId == result.Action.ID {
appFound = true
executionIndex = index
break
}
}
if appFound {
// Append to SuccessExamples or FailureExamples
if result.Status == "ABORTED" || result.Status == "FAILURE" {
appResults[executionIndex].FailureExamples = append(appResults[executionIndex].FailureExamples, result.Result)
} else if result.Status == "FINISHED" || result.Status == "SUCCESS" {
appResults[executionIndex].SuccessExamples = append(appResults[executionIndex].SuccessExamples, result.Result)
} else {
log.Printf("[ERROR] Can't handle status %s", result.Status)
}
// appResults = append(appResults, executionExample)
} else {
// CREATE SuccessExamples or FailureExamples
executionExample := AppExecutionExample{
AppName: result.Action.AppName,
AppVersion: result.Action.AppVersion,
AppAction: result.Action.Name,
AppId: result.Action.AppID,
ExampleId: fmt.Sprintf("%s_%s", execution.ExecutionId, result.Action.AppID),
}
if result.Status == "ABORTED" || result.Status == "FAILURE" {
executionExample.FailureExamples = append(executionExample.FailureExamples, result.Result)
} else if result.Status == "FINISHED" || result.Status == "SUCCESS" {
executionExample.SuccessExamples = append(executionExample.SuccessExamples, result.Result)
} else {
log.Printf("[ERROR] Can't handle status %s", result.Status)
}
appResults = append(appResults, executionExample)
}
}
// ExampleId string `json:"example_id"`
// func setExampleresult(ctx context.Context, result exampleResult) error {
// log.Printf("Execution length: %d", len(appResults))
if len(appResults) > 0 {
ctx := context.Background()
successful := 0
for _, exampleresult := range appResults {
err := setExampleresult(ctx, exampleresult)
if err != nil {
log.Printf("Failed setting examplresult %s: %s", exampleresult.ExampleId, err)
} else {
successful += 1
}
}
log.Printf("Added %d exampleresults to backend", successful)
} else {
log.Printf("No examplresults necessary to be added for execution %s", execution.ExecutionId)
}
}
func getWorkflows(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
@@ -1589,7 +1729,8 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
}
// FIXME: Have a good way of tracking errors. ID's or similar.
if !action.IsValid {
if !action.IsValid && len(action.Errors) > 0 {
log.Printf("Node %s is invalid and needs to be remade. Errors: %s", action.Label, strings.Join(action.Errors, "\n"))
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Node %s is invalid and needs to be remade."}`, action.Label)))
return
@@ -1602,11 +1743,43 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
workflow.Actions = newActions
newTriggers := []Trigger{}
for _, trigger := range workflow.Triggers {
log.Printf("Trigger %s: %s", trigger.TriggerType, trigger.Status)
// Check if it's actually running
// FIXME: Do this for other triggers too
if trigger.TriggerType == "SCHEDULE" && trigger.Status != "uninitialized" {
schedule, err := getSchedule(ctx, trigger.ID)
if err != nil {
trigger.Status = "stopped"
} else if schedule.Id == "" {
trigger.Status = "stopped"
}
} else if trigger.TriggerType == "WEBHOOK" && trigger.Status != "uninitialized" {
hook, err := getHook(ctx, trigger.ID)
if err != nil {
log.Printf("Failed getting webhook")
trigger.Status = "stopped"
} else if hook.Id == "" {
trigger.Status = "stopped"
}
}
//log.Println("TRIGGERS")
allNodes = append(allNodes, trigger.ID)
newTriggers = append(newTriggers, trigger)
}
workflow.Triggers = newTriggers
for _, variable := range workflow.WorkflowVariables {
if len(variable.Value) == 0 {
log.Printf("Can't have an empty variable: %s", variable.Name)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Variable %s can't be empty"}`, variable.Name)))
return
}
}
if len(workflow.Actions) == 0 {
@@ -1764,10 +1937,16 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
}
if !authFound {
log.Printf("App auth %s doesn't exist", action.AuthenticationId)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App auth %s doesn't exist"}`, action.AuthenticationId)))
return
log.Printf("App auth %s doesn't exist. Setting error", action.AuthenticationId)
workflow.Errors = append(workflow.Errors, fmt.Sprintf("App authentication for %s doesn't exist!", action.AppName))
workflow.IsValid = false
action.Errors = append(action.Errors, "App authentication doesn't exist")
action.IsValid = false
action.AuthenticationId = ""
//resp.WriteHeader(401)
//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App auth %s doesn't exist"}`, action.AuthenticationId)))
//return
}
}
@@ -1791,70 +1970,77 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
// Check to see if the whole app is valid
if curapp.Name != action.AppName {
log.Printf("App %s doesn't exist.", action.AppName)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App %s doesn't exist"}`, action.AppName)))
return
}
workflow.Errors = append(workflow.Errors, fmt.Sprintf("App %s doesn't exist", action.AppName))
action.Errors = append(action.Errors, "This app doesn't exist.")
action.IsValid = false
workflow.IsValid = false
// Check tosee if the appaction is valid
curappaction := WorkflowAppAction{}
for _, curAction := range curapp.Actions {
if action.Name == curAction.Name {
curappaction = curAction
break
}
}
// Check to see if the action is valid
if curappaction.Name != action.Name {
log.Printf("Appaction %s doesn't exist.", action.Name)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// FIXME - check all parameters to see if they're valid
// Includes checking required fields
newParams := []WorkflowAppActionParameter{}
for _, param := range curappaction.Parameters {
found := false
// Handles check for parameter exists + value not empty in used fields
for _, actionParam := range action.Parameters {
if actionParam.Name == param.Name {
found = true
if actionParam.Value == "" && actionParam.Variant == "STATIC_VALUE" && actionParam.Required == true {
log.Printf("Appaction %s with required param '%s' is empty.", action.Name, param.Name)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name)))
return
}
if actionParam.Variant == "" {
actionParam.Variant = "STATIC_VALUE"
}
newParams = append(newParams, actionParam)
// Append with errors
newActions = append(newActions, action)
log.Printf("App %s doesn't exist. Adding as error.", action.AppName)
//resp.WriteHeader(401)
//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App %s doesn't exist"}`, action.AppName)))
//return
} else {
// Check tosee if the appaction is valid
curappaction := WorkflowAppAction{}
for _, curAction := range curapp.Actions {
if action.Name == curAction.Name {
curappaction = curAction
break
}
}
// Handles check for required params
if !found && param.Required {
log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name)
// Check to see if the action is valid
if curappaction.Name != action.Name {
log.Printf("Appaction %s doesn't exist.", action.Name)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
// FIXME - check all parameters to see if they're valid
// Includes checking required fields
action.Parameters = newParams
newActions = append(newActions, action)
newParams := []WorkflowAppActionParameter{}
for _, param := range curappaction.Parameters {
found := false
// Handles check for parameter exists + value not empty in used fields
for _, actionParam := range action.Parameters {
if actionParam.Name == param.Name {
found = true
if actionParam.Value == "" && actionParam.Variant == "STATIC_VALUE" && actionParam.Required == true {
log.Printf("Appaction %s with required param '%s' is empty.", action.Name, param.Name)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name)))
return
}
if actionParam.Variant == "" {
actionParam.Variant = "STATIC_VALUE"
}
newParams = append(newParams, actionParam)
break
}
}
// Handles check for required params
if !found && param.Required {
log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name)))
return
}
}
action.Parameters = newParams
newActions = append(newActions, action)
}
}
}
@@ -1877,9 +2063,25 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
log.Printf("Failed to change total actions data: %s", err)
}
type returnData struct {
Success bool `json:"success"`
Errors []string `json:"errors"`
}
returndata := returnData{
Success: true,
Errors: workflow.Errors,
}
log.Printf("Saved new version of workflow %s (%s)", workflow.Name, fileId)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
newBody, err := json.Marshal(returndata)
if err != nil {
resp.Write([]byte(`{"success": true}`))
return
}
resp.Write(newBody)
}
func getWorkflowLocal(fileId string, request *http.Request) ([]byte, error) {
@@ -1923,6 +2125,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) {
return
}
// FIXME: Check the execution if this fails.
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in abort workflow: %s", err)
@@ -2049,16 +2252,14 @@ func cleanupExecutions(resp http.ResponseWriter, request *http.Request) {
return
}
//if user.Role != "admin" {
// resp.WriteHeader(401)
// resp.Write([]byte(`{"success": false, "message": "Insufficient permissions"}`))
// return
//}
log.Printf("CLEANUP!")
log.Printf("%#v", user)
if user.Role != "admin" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "message": "Insufficient permissions"}`))
return
}
ctx := context.Background()
// Removes three months from today
timestamp := int64(time.Now().AddDate(0, -2, 0).Unix())
log.Println(timestamp)
@@ -2072,8 +2273,6 @@ func cleanupExecutions(resp http.ResponseWriter, request *http.Request) {
return
}
log.Println(len(workflowExecutions))
resp.WriteHeader(200)
resp.Write([]byte("OK"))
}
@@ -2373,9 +2572,9 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
for _, authparam := range curAuth.Fields {
if param.Name == authparam.Key {
log.Printf("Name: %s - value: %s", param.Name, param.Value)
param.Value = authparam.Value
log.Printf("Name: %s - value: %s\n", param.Name, param.Value)
//log.Printf("Name: %s - value: %s", param.Name, param.Value)
//log.Printf("Name: %s - value: %s\n", param.Name, param.Value)
break
}
}
@@ -2634,6 +2833,8 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) {
err = deleteSchedule(ctx, scheduleId)
if err != nil {
log.Printf("Failed deleting schedule: %s", err)
if strings.Contains(err.Error(), "Job not found") {
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
@@ -3143,6 +3344,18 @@ func getAllWorkflows(ctx context.Context) ([]Workflow, error) {
return allworkflows, nil
}
func setExampleresult(ctx context.Context, result AppExecutionExample) error {
key := datastore.NameKey("example_result", result.ExampleId, nil)
// New struct, to not add body, author etc
if _, err := dbclient.Put(ctx, key, &result); err != nil {
log.Printf("Error adding workflow: %s", err)
return err
}
return nil
}
// Hmm, so I guess this should use uuid :(
// Consistency PLX
func setWorkflow(ctx context.Context, workflow Workflow, id string) error {
@@ -3210,6 +3423,7 @@ func deleteAppAuthentication(resp http.ResponseWriter, request *http.Request) {
resp.Write([]byte(`{"success": true}`))
}
// FIXME: Not suitable for cloud right now :O
func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
@@ -3250,7 +3464,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) {
// FIXME - check whether it's in use and maybe restrict again for later?
// FIXME - actually delete other than private apps too..
private := false
if app.Downloaded {
if app.Downloaded && user.Role == "admin" {
log.Printf("Deleting downloaded app (authenticated users can do this)")
} else if user.Id != app.Owner && user.Role != "admin" {
log.Printf("Wrong user (%s) for app %s (delete)", user.Username, app.Name)
@@ -3270,6 +3484,8 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) {
return
}
// Finds workflows using the app to set errors
// FIXME: this will be WAY too big for cloud :O
for _, workflow := range workflows {
found := false
@@ -3290,7 +3506,8 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) {
workflow.Actions = newActions
for _, trigger := range workflow.Triggers {
log.Printf("TRIGGER: %#v", trigger)
_ = trigger
//log.Printf("TRIGGER: %#v", trigger)
//err = deleteSchedule(ctx, scheduleId)
//if err != nil {
// if strings.Contains(err.Error(), "Job not found") {
@@ -5121,8 +5338,8 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) {
}
if len(workflowExecutions) == 0 {
resp.WriteHeader(200)
resp.Write([]byte("[]"))
//resp.WriteHeader(200)
return
}
+1
View File
@@ -60,6 +60,7 @@ services:
- HTTP_PROXY=${SHUFFLE_HTTP_PROXY}
- HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY}
- SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY}
- SHUFFLE_ORBORUS_EXECUTION_TIMEOUT=600
- SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME}
- SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY}
- SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX}
+5
View File
@@ -12177,6 +12177,11 @@
"prop-types": "^15.6.1"
}
},
"material-ui-nested-menu-item": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/material-ui-nested-menu-item/-/material-ui-nested-menu-item-1.0.2.tgz",
"integrity": "sha512-LZb8xI0FrAI/A3P2vT3CB9bmSoOFWOK0dikTc1t9VvEpp1a8hZkbVUz7VhETnoLUYu3NXCkgulmXcl3zitqI9A=="
},
"material-ui-pickers": {
"version": "2.2.4",
"resolved": "https://registry.npmjs.org/material-ui-pickers/-/material-ui-pickers-2.2.4.tgz",
+1
View File
@@ -28,6 +28,7 @@
"material-icons": "^0.3.1",
"material-icons-react": "^1.0.4",
"material-ui-chip-input": "^2.0.0-beta.2",
"material-ui-nested-menu-item": "^1.0.2",
"md5-file": "^4.0.0",
"mdbreact": "^4.21.1",
"moment": "~2.20.1",
+3 -3
View File
@@ -128,7 +128,7 @@ const App = (message, props) => {
<Route exact path="/oauth2" render={props => <Oauth2 isLoaded={isLoaded} globalUrl={globalUrl} {...props} />} />
<Route exact path="/contact" render={props => <Contact isLoaded={isLoaded} globalUrl={globalUrl} {...props} />} />
<Route exact path="/login" render={props => <LoginPage isLoggedIn={isLoggedIn} setIsLoggedIn={setIsLoggedIn} register={true} isLoaded={isLoaded} globalUrl={globalUrl} setCookie={setCookie} cookies={cookies} {...props} />} />
<Route exact path="/admin" render={props => <Admin isLoggedIn={isLoggedIn} setIsLoggedIn={setIsLoggedIn} register={true} isLoaded={isLoaded} globalUrl={globalUrl} setCookie={setCookie} cookies={cookies} {...props} />} />
<Route exact path="/admin" render={props => <Admin userdata={userdata} isLoggedIn={isLoggedIn} setIsLoggedIn={setIsLoggedIn} register={true} isLoaded={isLoaded} globalUrl={globalUrl} setCookie={setCookie} cookies={cookies} {...props} />} />
<Route exact path="/admin/:key" render={props => <Admin isLoggedIn={isLoggedIn} setIsLoggedIn={setIsLoggedIn} register={true} isLoaded={isLoaded} globalUrl={globalUrl} setCookie={setCookie} cookies={cookies} {...props} />} />
<Route exact path="/settings" render={props => <SettingsPage isLoaded={isLoaded} userdata={userdata} globalUrl={globalUrl} {...props} />} />
<Route exact path="/AdminSetup" render={props => <AdminSetup isLoaded={isLoaded} userdata={userdata} globalUrl={globalUrl} {...props} />} />
@@ -141,12 +141,12 @@ const App = (message, props) => {
<Route exact path="/apps/edit/:appid" render={props => <AppCreator isLoaded={isLoaded} isLoggedIn={isLoggedIn} globalUrl={globalUrl} {...props} />} />
<Route exact path="/schedules/:key" render={props => <EditSchedule globalUrl={globalUrl} {...props} />} />
<Route exact path="/workflows" render={props => <Workflows isLoaded={isLoaded} isLoggedIn={isLoggedIn} globalUrl={globalUrl} cookies={cookies} {...props} />} />
<Route exact path="/workflows/:key" render={props => <AngularWorkflow globalUrl={globalUrl} isLoaded={isLoaded} isLoggedIn={isLoggedIn} {...props} />} />
<Route exact path="/workflows/:key" render={props => <AngularWorkflow userdata={userdata} globalUrl={globalUrl} isLoaded={isLoaded} isLoggedIn={isLoggedIn} {...props} />} />
<Route exact path="/docs/:key" render={props => <Docs isLoaded={isLoaded} globalUrl={globalUrl} {...props} />} />
<Route exact path="/docs" render={props => { window.location.pathname = "/docs/about" }} />
<Route exact path="/introduction" render={props => <Introduction isLoaded={isLoaded} globalUrl={globalUrl} {...props} />} />
<Route exact path="/introduction/:key" render={props => <Introduction isLoaded={isLoaded} globalUrl={globalUrl} {...props} />} />
<Route exact path="/" render={props => { window.location.pathname = "/login" }} />
<Route exact path="/" render={props => <LoginPage isLoggedIn={isLoggedIn} setIsLoggedIn={setIsLoggedIn} register={true} isLoaded={isLoaded} globalUrl={globalUrl} setCookie={setCookie} cookies={cookies} {...props} />} />
</div>
// <div style={{backgroundColor: "rgba(21, 32, 43, 1)", color: "#fffff", minHeight: "100vh"}}>
+14 -13
View File
@@ -101,7 +101,6 @@ const Header = props => {
// Should be based on some path
const logoCheck = !homePage ? null : null
// Handle top bar or something
const loginTextBrowser = !isLoggedIn ?
<div style={{display: "flex"}}>
@@ -184,18 +183,20 @@ const Header = props => {
color="primary"> Settings</Button>
</Link>
</ListItem>
<ListItem>
<Link to="/admin" style={hrefStyle}>
<Button
style={{}}
variant="contained"
color="primary"
>
Admin
</Button>
</Link>
</ListItem>
{userdata === undefined || userdata.orgs.length <= 1 ? null :
{userdata === undefined || userdata.admin === undefined || userdata.admin === null || !userdata.admin ? null :
<ListItem>
<Link to="/admin" style={hrefStyle}>
<Button
style={{}}
variant="contained"
color="primary"
>
Admin
</Button>
</Link>
</ListItem>
}
{userdata === undefined || userdata.orgs === undefined || userdata.orgs === null || userdata.orgs.length <= 1 ? null :
<ListItem>
<Select
SelectDisplayProps={{
+183 -19
View File
@@ -36,9 +36,16 @@ const Admin = (props) => {
const [firstRequest, setFirstRequest] = React.useState(true);
const [modalUser, setModalUser] = React.useState({});
const [modalOpen, setModalOpen] = React.useState(false);
const [cloudSyncModalOpen, setCloudSyncModalOpen] = React.useState(false);
const [cloudSyncApikey, setCloudSyncApikey] = React.useState("");
const [loading, setLoading] = React.useState(false);
const [selectedOrganization, setSelectedOrganization] = React.useState({});
const [loginInfo, setLoginInfo] = React.useState("");
const [curTab, setCurTab] = React.useState(0);
const [users, setUsers] = React.useState([]);
const [organizations, setOrganizations] = React.useState([]);
const [environments, setEnvironments] = React.useState([]);
const [authentication, setAuthentication] = React.useState([]);
const [schedules, setSchedules] = React.useState([])
@@ -175,6 +182,41 @@ const Admin = (props) => {
});
}
const enableCloudSync = (apikey, organization) => {
const data = {
apikey: apikey,
organization: organization,
}
const url = globalUrl + '/api/v1/cloud/setup';
fetch(url, {
mode: 'cors',
method: 'POST',
body: JSON.stringify(data),
credentials: 'include',
crossDomain: true,
withCredentials: true,
headers: {
'Content-Type': 'application/json; charset=utf-8',
},
})
.then(response =>
response.json().then(responseJson => {
setLoading(false)
console.log(responseJson)
if (responseJson["success"] === false) {
alert.error("Failed setting up cloud sync")
} else {
alert.success("Set up cloud sync!")
}
}),
)
.catch(error => {
setLoading(false)
alert.error("Err: " + error.toString())
});
}
const onPasswordChange = () => {
const data = { "username": selectedUser.username, "newpassword": newPassword }
const url = globalUrl + '/api/v1/users/passwordchange';
@@ -471,6 +513,31 @@ const Admin = (props) => {
});
}
const getOrgs = () => {
fetch(globalUrl + "/api/v1/getorgs", {
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
},
credentials: "include",
})
.then((response) => {
if (response.status !== 200) {
console.log("Status not 200 for apps :O!")
return
}
return response.json()
})
.then((responseJson) => {
setOrganizations(responseJson)
})
.catch(error => {
alert.error(error.toString())
});
}
const getUsers = () => {
fetch(globalUrl + "/api/v1/getusers", {
method: 'GET',
@@ -482,7 +549,7 @@ const Admin = (props) => {
})
.then((response) => {
if (response.status !== 200) {
console.log("Status not 200 for apps :O!")
window.location.pathname = "/workflows"
return
}
@@ -713,8 +780,69 @@ const Admin = (props) => {
</DialogContent>
</Dialog>
const cloudSyncModal =
<Dialog
open={cloudSyncModalOpen}
onClose={() => { setCloudSyncModalOpen(false) }}
PaperProps={{
style: {
backgroundColor: theme.palette.surfaceColor,
color: "white",
minWidth: "800px",
minHeight: "320px",
},
}}
>
<DialogTitle><span style={{ color: "white" }}>
Enable cloud features
</span></DialogTitle>
<DialogContent>
What does <a href="https://shuffler.io/docs/hybrid#cloud_sync" target="_blank" style={{textDecoration: "none", color: "#f85a3e"}}>cloud sync</a> do?
<div style={{marginTop: 5}}/>
Cloud Apikey
<div style={{display: "flex", marginBottom: 20, }}>
<TextField
color="primary"
style={{backgroundColor: theme.palette.inputColor, marginRight: 10, }}
InputProps={{
style: {
height: "50px",
color: "white",
fontSize: "1em",
},
}}
required
fullWidth={true}
autoComplete="cloud apikey"
id="apikey_field"
margin="normal"
placeholder="Cloud Apikey"
variant="outlined"
onChange={(event) => {
setCloudSyncApikey(event.target.value)
}}
/>
<Button disabled={cloudSyncApikey.length === 0 || loading} variant="contained" style={{ marginLeft: 15, height: 60, margin: "auto", borderRadius: "0px" }} onClick={() => {
setLoading(true)
enableCloudSync(
cloudSyncApikey,
selectedOrganization,
)
}} color="primary">
Test sync
</Button>
</div>
* New triggers (userinput, hotmail realtime)<div/>
* Execute in the cloud rather than onprem<div/>
* Apps can be built in the cloud<div/>
* Easily share apps and workflows<div/>
* Access to powerful cloud search
</DialogContent>
</Dialog>
const modalView =
<Dialog modal
<Dialog
open={modalOpen}
onClose={() => { setModalOpen(false) }}
PaperProps={{
@@ -1229,7 +1357,10 @@ const Admin = (props) => {
style={{}}
variant="contained"
color="primary"
onClick={() => setModalOpen(true)}
disabled
onClick={() => {
setModalOpen(true)
}}
>
Add organization
</Button>
@@ -1241,28 +1372,58 @@ const Admin = (props) => {
style={{minWidth: 150, maxWidth: 150}}
/>
<ListItemText
primary="Orborus running (TBD)"
primary="id"
style={{minWidth: 200, maxWidth: 200}}
/>
<ListItemText
primary="Actions"
style={{minWidth: 150, maxWidth: 150}}
/>
</ListItem>
<ListItem>
<ListItemText
primary="Enabled"
style={{minWidth: 150, maxWidth: 150}}
/>
<ListItemText
primary="false"
style={{minWidth: 200, maxWidth: 200}}
/>
<ListItemText
primary=<Switch checked={false} onChange={() => {console.log("INVERT")}} />
primary="Your role"
style={{minWidth: 150, maxWidth: 150}}
/>
<ListItemText
primary="Selected"
style={{minWidth: 150, maxWidth: 150}}
/>
<ListItemText
primary="Cloud Sync"
style={{minWidth: 150, maxWidth: 150}}
/>
</ListItem>
{organizations !== undefined && organizations !== null && organizations.length > 0 ?
<span>
{organizations.map((data, index) => {
const isSelected = props.userdata.active_org.id === undefined ? "False" : props.userdata.active_org.id === data.id ? "True" : "False"
return (
<ListItem key={index}>
<ListItemText
primary={data.name}
style={{minWidth: 150, maxWidth: 150}}
/>
<ListItemText
primary={data.id}
style={{minWidth: 200, maxWidth: 200}}
/>
<ListItemText
primary={data.role}
style={{minWidth: 150, maxWidth: 150}}
/>
<ListItemText
primary={isSelected}
style={{minWidth: 150, maxWidth: 150}}
/>
<ListItemText
primary=<Switch checked={data.cloud_sync} onChange={() => {
setCloudSyncModalOpen(true)
setSelectedOrganization(data)
console.log("INVERT CLOUD SYNC")
}} />
style={{minWidth: 150, maxWidth: 150}}
/>
</ListItem>
)
})}
</span>
: null}
</List>
</div>
: null
@@ -1316,6 +1477,8 @@ const Admin = (props) => {
getEnvironments()
} else if (newValue === 3) {
getSchedules()
} else if (newValue === 5) {
getOrgs()
}
if (newValue === 6) {
@@ -1360,6 +1523,7 @@ const Admin = (props) => {
return (
<div>
{modalView}
{cloudSyncModal}
{editUserModal}
{editAuthenticationModal}
{data}
File diff suppressed because one or more lines are too long
+59 -21
View File
@@ -376,16 +376,6 @@ const AppCreator = (props) => {
}
}
if (data.servers !== undefined && data.servers.length > 0) {
var firstUrl = data.servers[0].url
if (firstUrl.endsWith("/")) {
setBaseUrl(firstUrl.slice(0, firstUrl.length-1))
} else {
setBaseUrl(firstUrl)
}
}
if (data.tags !== undefined && data.tags.length > 0) {
for (var key in data.tags) {
newWorkflowTags.push(data.tags[key].name)
@@ -425,12 +415,16 @@ const AppCreator = (props) => {
}
if (!allowedfunctions.includes(method.toUpperCase())) {
console.log(method, path)
continue
}
var tmpname = methodvalue.summary
if (methodvalue.operationId !== undefined && methodvalue.operationId !== null && methodvalue.operationId.length > 0) {
tmpname = methodvalue.operationId
}
var newaction = {
"name": methodvalue.summary,
"name": tmpname,
"description": methodvalue.description,
"url": path,
"method": method.toUpperCase(),
@@ -532,6 +526,30 @@ const AppCreator = (props) => {
}
newActions.push(newaction)
}
if (data.servers !== undefined && data.servers.length > 0) {
var firstUrl = data.servers[0].url
if (firstUrl.includes("{") && firstUrl.includes("}") && data.servers[0].variables !== undefined) {
const regex = /{\w+}/g
const found = firstUrl.match(regex)
if (found !== null) {
for (var key in found) {
const item = found[key].slice(1, found[key].length-1)
const foundVar = data.servers[0].variables[item]
if (foundVar["default"] !== undefined) {
firstUrl = firstUrl.replace(found[key], foundVar["default"])
}
}
}
}
if (firstUrl.endsWith("/")) {
setBaseUrl(firstUrl.slice(0, firstUrl.length-1))
} else {
setBaseUrl(firstUrl)
}
}
}
@@ -655,6 +673,7 @@ const AppCreator = (props) => {
}
},
"summary": item.name,
"operationId": item.name.split(" ").join("_"),
"description": item.description,
"parameters": []
}
@@ -1060,19 +1079,29 @@ const AppCreator = (props) => {
:
<div>
{actions.map((data, index) => {
var error = <Tooltip color="secondary" title={data.errors.join("\n")} placement="bottom">
var error = data.errors.length > 0 ?
<Tooltip color="primary" title={data.errors.join("\n")} placement="bottom">
<ErrorOutline />
</Tooltip>
:
<Tooltip color="secondary" title={data.errors.join("\n")} placement="bottom">
<CheckCircleIcon />
</Tooltip>
// "ERROR: "+data.errors.join("\n")
if (data.errors.length > 0) {
error =
<Tooltip color="primary" title={data.errors.join("\n")} placement="bottom">
<ErrorOutline />
</Tooltip>
var bgColor = "#61afee"
if (data.method === "POST") {
bgColor = "#49cc90"
} else if (data.method === "PUT") {
bgColor = "#fca130"
} else if (data.method === "PATCH") {
bgColor = "#50e3c2"
} else if (data.method === "DELETE") {
bgColor = "#f93e3e"
} else if (data.method === "HEAD") {
bgColor = "#9012fe"
}
const url = data.url
return (
<Paper style={actionListStyle}>
@@ -1085,7 +1114,16 @@ const AppCreator = (props) => {
setUrlPath(data.url)
setActionsModalOpen(true)
}}>
{data.method} - {url} - {data.name}
<div style={{display: "flex"}}>
<Chip
style={{backgroundColor: bgColor, color: "white", borderRadius: 5, minWidth: 80, marginRight: 10, marginTop: 2, cursor: "pointer", fontSize: 14,}}
label={data.method}
variant="contained"
/>
<span style={{fontSize: 16, marginTop: "auto", marginBottom: "auto",}}>
{url} - {data.name}
</span>
</div>
</div>
</Tooltip>
{/*
+71 -34
View File
@@ -43,13 +43,31 @@ const inputColor = "#383B40"
// Parses JSON data into keys that can be used everywhere :)
export const GetParsedPaths = (inputdata, basekey) => {
const splitkey = " => "
const splitkey = " > "
var parsedValues = []
for (const [key, value] of Object.entries(inputdata)) {
if (typeof(inputdata) !== "object") {
return parsedValues
}
for (const [key, value] of Object.entries(inputdata)) {
// Check if loop or JSON
const extra = basekey.length > 0 ? splitkey : ""
const basekeyname = `${basekey.slice(1,basekey.length).split(".").join(splitkey)}${extra}${key}`
const basekeyname = `${basekey.slice(1, basekey.length).split(".").join(splitkey)}${extra}${key}`
// Handle direct loop!
if (!isNaN(key) && basekey === "") {
console.log("Handling direct loop.")
parsedValues.push({"type": "object", "name": "Node", "autocomplete": `${basekey}`})
parsedValues.push({"type": "list", "name": `${splitkey}list`, "autocomplete": `${basekey}.#`})
const returnValues = GetParsedPaths(value, `${basekey}.#`)
for (var subkey in returnValues) {
parsedValues.push(returnValues[subkey])
}
return parsedValues
}
//console.log("KEY: ", key, "VALUE: ", value, "BASEKEY: ", basekeyname)
if (typeof(value) === 'object') {
if (Array.isArray(value)) {
// Check if each item is object
@@ -101,6 +119,7 @@ const Apps = (props) => {
const [appSearchLoading, setAppSearchLoading] = React.useState(false)
const [selectedAction, setSelectedAction] = React.useState({})
const [searchBackend, setSearchBackend] = React.useState(false)
const [searchableApps, setSearchableApps] = React.useState([])
const [openApi, setOpenApi] = React.useState("")
const [openApiData, setOpenApiData] = React.useState("")
@@ -112,6 +131,7 @@ const Apps = (props) => {
const [openApiError, setOpenApiError] = React.useState("")
const [field1, setField1] = React.useState("")
const [field2, setField2] = React.useState("")
const [cursearch, setCursearch] = React.useState("")
const [sharingConfiguration, setSharingConfiguration] = React.useState("you")
const { start, stop } = useInterval({
@@ -165,6 +185,8 @@ const Apps = (props) => {
maxHeight: 130,
minWidth: "100%",
maxWidth: "100%",
marginBottom: 5,
borderRadius: 5,
color: "white",
backgroundColor: surfaceColor,
cursor: "pointer",
@@ -200,6 +222,8 @@ const Apps = (props) => {
setSelectedAction({})
}
}
runAppSearch("")
})
.catch(error => {
alert.error(error.toString())
@@ -277,6 +301,10 @@ const Apps = (props) => {
boxColor = "green"
}
if (!data.activated && data.generated) {
boxColor = "orange"
}
var imageline = data.large_image.length === 0 ?
<img alt={data.title} style={{width: 100, height: 100}} />
:
@@ -307,6 +335,7 @@ const Apps = (props) => {
description = data.description.slice(0, maxDescLen)+"..."
}
const version = data.app_version
return (
<Paper square key={data.id} style={paperAppStyle} onClick={() => {
if (selectedApp.id !== data.id) {
@@ -319,7 +348,6 @@ const Apps = (props) => {
setSelectedAction({})
}
console.log("Sharing: ", data.sharing_config)
if (data.sharing) {
setSharingConfiguration("everyone")
}
@@ -375,9 +403,10 @@ const Apps = (props) => {
const dividerColor = "rgb(225, 228, 232)"
const uploadViewPaperStyle = {
minWidth: "100%",
minWidth: 662.5,
maxWidth: 662.5,
color: "white",
borderRadius: 5,
backgroundColor: surfaceColor,
display: "flex",
marginBottom: 10,
@@ -430,7 +459,7 @@ const Apps = (props) => {
</Tooltip>
</Link> : null
var activateButton = selectedApp.generated && !selectedApp.activated ?
const activateButton = selectedApp.generated && !selectedApp.activated ?
<div>
<Link to={activateUrl} style={{textDecoration: "none"}}>
<Button
@@ -458,7 +487,12 @@ const Apps = (props) => {
</div>
: null
var deleteButton = ((selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated) || (selectedApp.downloaded != undefined && selectedApp.downloaded == true)) && activateButton === null ?
const deleteButton = (
(selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated)
|| (selectedApp.downloaded !== undefined && selectedApp.downloaded == true)
|| (!selectedApp.generated)
)
&& activateButton === null ?
<Tooltip title={"Delete app"}>
<Button
variant="outlined"
@@ -507,10 +541,10 @@ const Apps = (props) => {
return (
<div>
{paths.map(data => {
{paths.map((data, index) => {
const circleSize = 10
return (
<MenuItem style={{backgroundColor: inputColor, color: "white"}} value={data} onClick={() => console.log(data.autocomplete)}>
<MenuItem key={index} style={{backgroundColor: inputColor, color: "white"}} value={data} onClick={() => console.log(data.autocomplete)}>
{data.name}
</MenuItem>
)
@@ -549,12 +583,13 @@ const Apps = (props) => {
{imageline}
</div>
<div style={{maxWidth: "75%", overflow: "hidden"}}>
<h2>{newAppname}</h2>
<p>{description}</p>
<h2 style={{marginTop: 20, marginBottom: 0, }}>{newAppname}</h2>
<p style={{marginTop: 5, marginBottom: 0,}}>Version {selectedApp.app_version}</p>
<p style={{marginTop: 5, marginBottom: 0}}>{description}</p>
</div>
</div>
{activateButton}
{props.userdata.role === "admin" || props.userdata.id === selectedApp.owner ?
{(props.userdata.role === "admin" || props.userdata.id === selectedApp.owner) || !selectedApp.generated ?
<div>
{downloadButton}
{editButton}
@@ -742,15 +777,17 @@ const Apps = (props) => {
}
const searchfield = search.toLowerCase()
const newapps = apps.filter(data => data.name.toLowerCase().includes(searchfield) || data.description.toLowerCase().includes(searchfield))
var newapps = apps.filter(data => data.name.toLowerCase().includes(searchfield) || data.description.toLowerCase().includes(searchfield))
var tmpapps = searchableApps.filter(data => data.name.toLowerCase().includes(searchfield) || data.description.toLowerCase().includes(searchfield))
newapps.push(...tmpapps)
if ((newapps.length === 0 || searchBackend) && !appSearchLoading) {
setFilteredApps(newapps)
//if ((newapps.length === 0 || searchBackend) && !appSearchLoading) {
setAppSearchLoading(true)
runAppSearch(searchfield)
} else {
setFilteredApps(newapps)
}
// //setAppSearchLoading(true)
// //runAppSearch(searchfield)
//} else {
//}
}
const appView = isLoggedIn ?
@@ -778,17 +815,8 @@ const Apps = (props) => {
<div style={{flex: 1, marginLeft: 10, marginRight: 10}}>
<div style={{display: "flex"}}>
<div style={{flex: 1}}>
<h2>All apps</h2>
<h2>Your apps ({apps.length+searchableApps.length})</h2>
</div>
{isLoading ? <CircularProgress style={{marginTop: 13, marginRight: 15}} /> : null}
<FormControlLabel
style={{color: "white", marginBottom: "0px", marginTop: "10px"}}
label={<div style={{color: "white"}}>Search OpenAPI</div>}
control={<Switch checked={searchBackend} onChange={() => {
handleSearchChange("")
setSearchBackend(!searchBackend)}
} />}
/>
<Tooltip title={"Reload apps locally"} style={{marginTop: "28px", width: "100%"}} aria-label={"Upload"}>
<Button
variant="outlined"
@@ -833,6 +861,7 @@ const Apps = (props) => {
placeholder={"Search apps"}
onChange={(event) => {
handleSearchChange(event.target.value)
setCursearch(event.target.value)
}}
/>
<div style={{marginTop: 15}}>
@@ -844,11 +873,18 @@ const Apps = (props) => {
appPaper(app)
)
})}
{cursearch.length > 0 ? null :
searchableApps.map(app => {
return (
appPaper(app)
)
})
}
</div>
:
<Paper square style={uploadViewPaperStyle}>
<h4>
Try a broader search term, e.g. "http", "alert", "ticket" etc.
<h4 style={{margin: 10, }}>
Try a broader search term, e.g. http, alert, ticket etc.
</h4>
<div/>
@@ -859,7 +895,7 @@ const Apps = (props) => {
</Paper>
:
<Paper square style={uploadViewPaperStyle}>
<h4>
<h4 style={{margin: 10}}>
No apps have been created, uploaded or downloaded yet. Click "Load existing apps" above to get the baseline. This may take a while as its building docker images.
</h4>
</Paper>
@@ -939,6 +975,7 @@ const Apps = (props) => {
setIsLoading(false)
if (response.status === 200) {
alert.success("Hotloaded apps!")
getApps()
}
return response.json()
@@ -1045,10 +1082,10 @@ const Apps = (props) => {
return response.json()
})
.then((responseJson) => {
//console.log(responseJson)
if (responseJson.success) {
if (responseJson.reason !== undefined && responseJson.reason.length > 0) {
setFilteredApps(responseJson.reason)
setSearchableApps(responseJson.reason)
//setFilteredApps(responseJson.reason)
}
}
})
+1 -2
View File
@@ -123,8 +123,7 @@ const Settings = (props) => {
return response.json()
})
.then((responseJson) => {
console.log(responseJson)
.then((responseJson) => {
setUserSettings(responseJson)
})
.catch(error => {
+24 -19
View File
@@ -543,22 +543,24 @@ const Workflows = (props) => {
</div>
</Grid>
</Grid>
<Grid container style={{maxWidth: 35, marginRight: 10,}}>
<Tooltip title={`Actions: ${data.actions.length}`} placement="right">
<AppsIcon style={{width: imgSize, height: imgSize}} />
</Tooltip>
{data.actions !== undefined && data.actions !== null ?
<Grid container style={{maxWidth: 35, marginRight: 10,}}>
<Tooltip title={`Actions: ${data.actions.length}`} placement="right">
<AppsIcon style={{width: imgSize, height: imgSize}} />
</Tooltip>
{webhooks > 0 ?
<Tooltip title={`Webhooks: ${webhooks}`} placement="right">
<img alt={data.title} style={{width: imgSize, height: imgSize, marginTop: 5}} src={webhookImg} />
</Tooltip>
: null}
{schedules > 0 ?
<Tooltip title={`Schedules: ${schedules}`} placement="right">
<img alt={data.title} style={{width: imgSize, height: imgSize, marginTop: 5}} src={scheduleImg} />
</Tooltip>
: null}
</Grid>
{webhooks > 0 ?
<Tooltip title={`Webhooks: ${webhooks}`} placement="right">
<img alt={data.title} style={{width: imgSize, height: imgSize, marginTop: 5}} src={webhookImg} />
</Tooltip>
: null}
{schedules > 0 ?
<Tooltip title={`Schedules: ${schedules}`} placement="right">
<img alt={data.title} style={{width: imgSize, height: imgSize, marginTop: 5}} src={scheduleImg} />
</Tooltip>
: null}
</Grid>
: null}
</Paper>
)
}
@@ -587,7 +589,7 @@ const Workflows = (props) => {
}
return (
<Paper key={data.name} square style={paperAppStyle} onClick={() => {
<Paper key={data.execution_id} square style={paperAppStyle} onClick={() => {
setSelectedExecution(data)
}}>
<div style={{marginLeft: "10px", marginTop: "5px", marginBottom: "5px", width: boxWidth, backgroundColor: boxColor}} />
@@ -682,7 +684,7 @@ const Workflows = (props) => {
}
return (
<Paper key={data.name} square style={resultPaperAppStyle} onClick={() => {}}>
<Paper key={data.execution_id} square style={resultPaperAppStyle} onClick={() => {}}>
<div style={{marginLeft: "10px", marginTop: "5px", marginBottom: "5px", marginRight: "5px", width: boxWidth, backgroundColor: boxColor}}>
</div>
<Grid container style={{margin: "10px 10px 10px 10px", flex: "1"}}>
@@ -715,9 +717,11 @@ const Workflows = (props) => {
const resultsHandler = Object.getOwnPropertyNames(selectedExecution).length > 0 && selectedExecution.results !== null ?
<div>
{selectedExecution.results.sort((a, b) => a.started_at - b.started_at).map(data => {
{selectedExecution.results.sort((a, b) => a.started_at - b.started_at).map((data, index) => {
return (
resultsPaper(data)
<div key={index}>
{resultsPaper(data)}
</div>
)
})}
</div>
@@ -912,6 +916,7 @@ const Workflows = (props) => {
const importFiles = (event) => {
console.log("Importing!")
const file = event.target.value
if (event.target.files.length > 0) {
for (var key in event.target.files) {
+22 -8
View File
@@ -15,6 +15,7 @@ import (
"net/http"
"os"
"os/exec"
"strconv"
"strings"
"time"
@@ -29,7 +30,7 @@ import (
var sleepTime = 3
// Timeout if something rashes
var workerTimeout = 300
var workerTimeoutEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_TIMEOUT")
var appSdkVersion = os.Getenv("SHUFFLE_APP_SDK_VERSION")
var workerVersion = os.Getenv("SHUFFLE_WORKER_VERSION")
@@ -93,7 +94,7 @@ func getThisContainerId() {
default:
fCol = "3" // for backward-compatibility with production
log.Printf("[WARNING] RUNNING_MODE not set, so I can't figure out the current container ID! Defaulting to Docker (not Kubernetes).")
log.Printf("[WARNING] RUNNING_MODE not set - defaulting to Docker (NOT Kubernetes).")
}
if fCol != "" {
@@ -250,7 +251,6 @@ func initializeImages() {
// Initial loop etc
func main() {
go zombiecheck()
log.Println("[INFO] Setting up execution environment")
//FIXME
@@ -264,6 +264,20 @@ func main() {
os.Exit(3)
}
workerTimeout := 600
if workerTimeoutEnv != "" {
tmpInt, err := strconv.Atoi(workerTimeoutEnv)
if err == nil {
workerTimeout = tmpInt
} else {
log.Printf("[WARNING] Env SHUFFLE_ORBORUS_EXECUTION_TIMEOUT must be a number, not %s", workerTimeoutEnv)
}
log.Printf("[INFO] Cleanup process running every %d seconds", workerTimeout)
}
go zombiecheck(workerTimeout)
log.Printf("[INFO] Running towards %s with Org %s", baseUrl, orgId)
httpProxy := os.Getenv("HTTP_PROXY")
httpsProxy := os.Getenv("HTTPS_PROXY")
@@ -330,7 +344,7 @@ func main() {
log.Printf("[WARNING] Failed making request: %s", err)
zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout {
go zombiecheck()
go zombiecheck(workerTimeout)
zombiecounter = 0
}
time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -351,7 +365,7 @@ func main() {
log.Printf("[ERROR] Failed reading body: %s", err)
zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout {
go zombiecheck()
go zombiecheck(workerTimeout)
zombiecounter = 0
}
time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -365,7 +379,7 @@ func main() {
sleepTime = 10
zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout {
go zombiecheck()
go zombiecheck(workerTimeout)
zombiecounter = 0
}
time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -380,7 +394,7 @@ func main() {
if len(executionRequests.Data) == 0 {
zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout {
go zombiecheck()
go zombiecheck(workerTimeout)
zombiecounter = 0
}
time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -487,7 +501,7 @@ func main() {
// FIXME - add this to remove exited workers
// Should it check what happened to the execution? idk
func zombiecheck() error {
func zombiecheck(workerTimeout int) error {
log.Println("[INFO] Looking for old containers")
ctx := context.Background()
+9 -1
View File
@@ -694,7 +694,15 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W
// FIXME: Force killing a worker should result in a notification somewhere
if len(nextActions) == 0 {
log.Printf("No next action. Finished? Result vs Actions: %d - %d", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions))
if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions) {
//exit := true
//for _, item := range workflowExecution.Results {
// if item == "EXECUTING" {
// exit = false
// break
// }
//}
if exit && len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions) {
shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID)
}