marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

#60: Created API endpoint to hotload apps from directory

Browse Source
This commit is contained in:
frikky
2020-06-15 12:02:52 +02:00
parent d4291cd268 4357f889b3
commit 054da3771d
21 changed files with 1587 additions and 577 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app_sdk/app_base.py
+2 -7
View File
@@ -73,12 +73,9 @@ class AppBase:
except requests.exceptions.ConnectionError as e:
print("Connectionerror: %s" % e)
return
#self.logger.info("AFTER initial stream result")
self.logger.info("THIS IS THE NEW UPDATE")
# Verify whether there are any parameters with ACTION_RESULT required
# If found, we get the full results list from backend
fullexecution = {}
try:
tmpdata = {
@@ -195,10 +192,9 @@ class AppBase:
# Regex to find all the things
if parameter["variant"] == "STATIC_VALUE":
data = parameter["value"]
self.logger.debug(f"\n\nHandle static data with JSON: {data}\n\n")
actualitem = re.findall(match, data, re.MULTILINE)
self.logger.info("STATIC PARSED: %s" % actualitem)
#self.logger.debug(f"\n\nHandle static data with JSON: {data}\n\n")
#self.logger.info("STATIC PARSED: %s" % actualitem)
if len(actualitem) > 0:
for replace in actualitem:
try:
@@ -487,7 +483,6 @@ class AppBase:
# With this parameter ready, add it to... a greater list of parameters. Rofl
multi_parameters[parameter["name"]] = resultarray
else:
print("Hello, in here?: %s" % value)
params[parameter["name"]] = value
multi_parameters[parameter["name"]] = value
backend/go-app/codegen.go
+407 -103
View File
@@ -242,7 +242,9 @@ func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) {
return appPath, nil
}
func makePythoncode(swagger *openapi3.Swagger, name, url, method string, parameters, optionalQueries []string) (string, string) {
// This function generates the python code that's being used.
// This is really meta when you program it. Handling parameters is hard here.
func makePythoncode(swagger *openapi3.Swagger, name, url, method string, parameters, optionalQueries, headers []string) (string, string) {
method = strings.ToLower(method)
queryString := ""
queryData := ""
@@ -259,9 +261,6 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet
}
}
// How to add authentication?
// I think it should be like:
// async def(self, auth, baseurl, data):
// api.Authentication.Parameters[0].Value = "BearerAuth"
authenticationParameter := ""
authenticationSetup := ""
@@ -270,14 +269,15 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet
if swagger.Components.SecuritySchemes != nil {
if swagger.Components.SecuritySchemes["BearerAuth"] != nil {
authenticationParameter = ", apikey"
authenticationSetup = "headers[\"Authorization\"] = f\"Bearer {apikey}\""
authenticationSetup = "if apikey != \" \": headers[\"Authorization\"] = f\"Bearer {apikey}\""
} else if swagger.Components.SecuritySchemes["BasicAuth"] != nil {
authenticationParameter = ", username, password"
authenticationAddin = ", auth=(username, password)"
} else if swagger.Components.SecuritySchemes["ApiKeyAuth"] != nil {
authenticationParameter = ", apikey"
if swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.In == "header" {
authenticationSetup = fmt.Sprintf("headers[\"%s\"] = apikey", swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name)
// This is a way to bypass apikeys by passing " "
authenticationSetup = fmt.Sprintf(`if apikey != " ": headers["%s"] = apikey`, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name)
} else if swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.In == "query" {
// This might suck lol
key := "?"
@@ -285,7 +285,7 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet
key = "&"
}
authenticationSetup = fmt.Sprintf("url+=f\"%s%s={apikey}\"", key, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name)
authenticationSetup = fmt.Sprintf("if apikey != \" \": url+=f\"%s%s={apikey}\"", key, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name)
}
}
}
@@ -301,6 +301,23 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet
urlInline = "{url}"
}
// Specific check for SSL verification
// This is critical for onprem stuff.
verifyParam := ""
verifyWrapper := ""
verifyAddin := ""
if len(swagger.Servers) == 0 {
verifyParam = ", verify=True"
verifyWrapper = `if type(ssl_verify) == str: ssl_verify = False if ssl_verify.lower() == "false" or ssl_verify == "0" else True`
verifyAddin = ", verify=ssl_verify"
} else {
if swagger.Servers[0].URL == "" {
verifyParam = ", ssl_verify=True"
verifyWrapper = `if type(ssl_verify) == str: ssl_verify = False if ssl_verify.lower() == "false" or ssl_verify == "0" else True`
verifyAddin = ", verify=ssl_verify"
}
}
if len(parameters) > 0 {
parameterData = fmt.Sprintf(", %s", strings.Join(parameters, ", "))
}
@@ -333,15 +350,39 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet
}
}
preparedHeaders := "headers={}"
if len(headers) > 0 {
preparedHeaders = "headers={"
for count, header := range headers {
headerSplit := strings.Split(header, "=")
added := false
if len(headerSplit) == 2 {
if strings.Contains(preparedHeaders, headerSplit[0]) {
continue
}
preparedHeaders += fmt.Sprintf(`"%s": "%s"`, headerSplit[0], headerSplit[1])
added = true
}
if count != len(headers)-1 && added {
preparedHeaders += ","
}
}
preparedHeaders += "}"
}
// Extra param for url if it's changeable
// Extra param for authentication scheme(s)
data := fmt.Sprintf(` async def %s(self%s%s%s%s%s):
headers={}
data := fmt.Sprintf(` async def %s(self%s%s%s%s%s%s):
%s
url=f"%s%s"
%s
%s
%s
%s
return requests.%s(url, headers=headers%s%s).text
return requests.%s(url, headers=headers%s%s%s).text
`,
functionname,
authenticationParameter,
@@ -349,14 +390,18 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet
parameterData,
queryString,
bodyParameter,
verifyParam,
preparedHeaders,
urlInline,
url,
verifyWrapper,
authenticationSetup,
queryData,
bodyFormatter,
method,
authenticationAddin,
bodyAddin,
verifyAddin,
)
//log.Println(data)
@@ -412,7 +457,6 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger,
//log.Printf("%s", j)
api.SmallImage = string(j)
api.LargeImage = string(j)
log.Printf("Set images!")
}
}
@@ -444,6 +488,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger,
Description: "The apikey to use",
Multiline: false,
Required: true,
Example: "The API key to use. Space = skip",
Schema: SchemaDefinition{
Type: "string",
},
@@ -460,6 +505,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger,
Description: "The apikey to use",
Multiline: false,
Required: true,
Example: "The API key to use. Space = skip",
Schema: SchemaDefinition{
Type: "string",
},
@@ -475,6 +521,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger,
Description: "The username to use",
Multiline: false,
Required: true,
Example: "The username to use",
Schema: SchemaDefinition{
Type: "string",
},
@@ -484,6 +531,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger,
Description: "The password to use",
Multiline: false,
Required: true,
Example: "The password to use",
Schema: SchemaDefinition{
Type: "string",
},
@@ -508,7 +556,11 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger,
// Could just as well be go at this point lol
pythonFunctions := []string{}
for actualPath, path := range swagger.Paths {
// FIXME: Add everything from here:
actualPath = strings.Replace(actualPath, " ", "_", -1)
actualPath = strings.Replace(actualPath, ".", "", -1)
actualPath = strings.Replace(actualPath, ".", "", -1)
actualPath = strings.Replace(actualPath, "\\", "", -1)
// https://godoc.org/github.com/getkin/kin-openapi/openapi3#PathItem
if path.Get != nil {
action, curCode := handleGet(swagger, api, extraParameters, path, actualPath)
@@ -759,13 +811,52 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [
optionalQueries := []string{}
parameters := []string{}
optionalParameters := []WorkflowAppActionParameter{}
if len(swagger.Servers) == 0 {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
} else {
if swagger.Servers[0].URL == "" {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
}
}
headersFound := []string{}
if len(path.Connect.Parameters) > 0 {
for _, param := range path.Connect.Parameters {
if param.Value.Schema == nil || param.Value.In == "header" {
for counter, param := range path.Connect.Parameters {
if param.Value.Schema == nil {
continue
} else if param.Value.In == "header" {
headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example))
continue
}
parsedName := param.Value.Name
parsedName = strings.ReplaceAll(parsedName, " ", "_")
parsedName = strings.ReplaceAll(parsedName, ",", "_")
parsedName = strings.ReplaceAll(parsedName, ".", "_")
parsedName = strings.ReplaceAll(parsedName, "|", "_")
param.Value.Name = parsedName
path.Connect.Parameters[counter].Value.Name = parsedName
curParam := WorkflowAppActionParameter{
Name: param.Value.Name,
Name: parsedName,
Description: param.Value.Description,
Multiline: false,
Required: param.Value.Required,
@@ -792,15 +883,8 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [
}
}
if param.Value.Required {
action.Parameters = append(action.Parameters, curParam)
} else {
optionalParameters = append(optionalParameters, curParam)
}
if param.Value.In == "path" {
//log.Printf("PATH!: %s", param.Value.Name)
parameters = append(parameters, param.Value.Name)
parameters = append(parameters, curParam.Name)
//baseUrl = fmt.Sprintf("%s%s", baseUrl)
} else if param.Value.In == "query" {
//log.Printf("QUERY!: %s", param.Value.Name)
@@ -823,6 +907,12 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [
firstQuery = false
}
if param.Value.Required {
action.Parameters = append(action.Parameters, curParam)
} else {
optionalParameters = append(optionalParameters, curParam)
}
}
}
@@ -832,7 +922,7 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "connect", parameters, optionalQueries)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "connect", parameters, optionalQueries, headersFound)
if len(functionname) > 0 {
action.Name = functionname
@@ -857,8 +947,6 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
action.Returns.Schema.Type = "string"
baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath)
//log.Println(path.Parameters)
// Parameters: []WorkflowAppActionParameter{},
// FIXME - add data for POST stuff
firstQuery := true
@@ -866,16 +954,53 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
// FIXME - remove this when authentication is properly introduced
parameters := []string{}
optionalParameters := []WorkflowAppActionParameter{}
if len(swagger.Servers) == 0 {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify the SSL certificate request",
Multiline: false,
Required: false,
Example: "False - default=True",
Schema: SchemaDefinition{
Type: "string",
},
})
} else {
if swagger.Servers[0].URL == "" {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
}
}
headersFound := []string{}
if len(path.Get.Parameters) > 0 {
for _, param := range path.Get.Parameters {
if param.Value.Schema == nil || param.Value.In == "header" {
for counter, param := range path.Get.Parameters {
if param.Value.Schema == nil {
continue
} else if param.Value.In == "header" {
headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example))
continue
}
parsedName := param.Value.Name
parsedName = strings.ReplaceAll(parsedName, " ", "_")
parsedName = strings.ReplaceAll(parsedName, ",", "_")
parsedName = strings.ReplaceAll(parsedName, ".", "_")
parsedName = strings.ReplaceAll(parsedName, "|", "_")
param.Value.Name = parsedName
path.Get.Parameters[counter].Value.Name = parsedName
curParam := WorkflowAppActionParameter{
Name: param.Value.Name,
Name: parsedName,
Description: param.Value.Description,
Multiline: false,
Required: param.Value.Required,
@@ -902,15 +1027,8 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
}
}
if param.Value.Required {
action.Parameters = append(action.Parameters, curParam)
} else {
optionalParameters = append(optionalParameters, curParam)
}
if param.Value.In == "path" {
log.Printf("PATH!: %s", param.Value.Name)
parameters = append(parameters, param.Value.Name)
parameters = append(parameters, curParam.Name)
//baseUrl = fmt.Sprintf("%s%s", baseUrl)
} else if param.Value.In == "query" {
//log.Printf("QUERY!: %s", param.Value.Name)
@@ -933,6 +1051,11 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
}
firstQuery = false
}
if param.Value.Required {
action.Parameters = append(action.Parameters, curParam)
} else {
optionalParameters = append(optionalParameters, curParam)
}
}
}
@@ -943,7 +1066,7 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "get", parameters, optionalQueries)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "get", parameters, optionalQueries, headersFound)
if len(functionname) > 0 {
action.Name = functionname
@@ -976,13 +1099,52 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
optionalQueries := []string{}
parameters := []string{}
optionalParameters := []WorkflowAppActionParameter{}
if len(swagger.Servers) == 0 {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
} else {
if swagger.Servers[0].URL == "" {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
}
}
headersFound := []string{}
if len(path.Head.Parameters) > 0 {
for _, param := range path.Head.Parameters {
if param.Value.Schema == nil || param.Value.In == "header" {
for counter, param := range path.Head.Parameters {
if param.Value.Schema == nil {
continue
} else if param.Value.In == "header" {
headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example))
continue
}
parsedName := param.Value.Name
parsedName = strings.ReplaceAll(parsedName, " ", "_")
parsedName = strings.ReplaceAll(parsedName, ",", "_")
parsedName = strings.ReplaceAll(parsedName, ".", "_")
parsedName = strings.ReplaceAll(parsedName, "|", "_")
param.Value.Name = parsedName
path.Head.Parameters[counter].Value.Name = parsedName
curParam := WorkflowAppActionParameter{
Name: param.Value.Name,
Name: parsedName,
Description: param.Value.Description,
Multiline: false,
Required: param.Value.Required,
@@ -1009,15 +1171,8 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
}
}
if param.Value.Required {
action.Parameters = append(action.Parameters, curParam)
} else {
optionalParameters = append(optionalParameters, curParam)
}
if param.Value.In == "path" {
//log.Printf("PATH!: %s", param.Value.Name)
parameters = append(parameters, param.Value.Name)
parameters = append(parameters, curParam.Name)
//baseUrl = fmt.Sprintf("%s%s", baseUrl)
} else if param.Value.In == "query" {
//log.Printf("QUERY!: %s", param.Value.Name)
@@ -1040,6 +1195,12 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
firstQuery = false
}
if param.Value.Required {
action.Parameters = append(action.Parameters, curParam)
} else {
optionalParameters = append(optionalParameters, curParam)
}
}
}
@@ -1049,7 +1210,7 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "head", parameters, optionalQueries)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "head", parameters, optionalQueries, headersFound)
if len(functionname) > 0 {
action.Name = functionname
@@ -1082,13 +1243,52 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []
optionalQueries := []string{}
parameters := []string{}
optionalParameters := []WorkflowAppActionParameter{}
if len(swagger.Servers) == 0 {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
} else {
if swagger.Servers[0].URL == "" {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
}
}
headersFound := []string{}
if len(path.Delete.Parameters) > 0 {
for _, param := range path.Delete.Parameters {
if param.Value.Schema == nil || param.Value.In == "header" {
for counter, param := range path.Delete.Parameters {
if param.Value.Schema == nil {
continue
} else if param.Value.In == "header" {
headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example))
continue
}
parsedName := param.Value.Name
parsedName = strings.ReplaceAll(parsedName, " ", "_")
parsedName = strings.ReplaceAll(parsedName, ",", "_")
parsedName = strings.ReplaceAll(parsedName, ".", "_")
parsedName = strings.ReplaceAll(parsedName, "|", "_")
param.Value.Name = parsedName
path.Delete.Parameters[counter].Value.Name = parsedName
curParam := WorkflowAppActionParameter{
Name: param.Value.Name,
Name: parsedName,
Description: param.Value.Description,
Multiline: false,
Required: param.Value.Required,
@@ -1115,15 +1315,8 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []
}
}
if param.Value.Required {
action.Parameters = append(action.Parameters, curParam)
} else {
optionalParameters = append(optionalParameters, curParam)
}
if param.Value.In == "path" {
//log.Printf("PATH!: %s", param.Value.Name)
parameters = append(parameters, param.Value.Name)
parameters = append(parameters, curParam.Name)
//baseUrl = fmt.Sprintf("%s%s", baseUrl)
} else if param.Value.In == "query" {
//log.Printf("QUERY!: %s", param.Value.Name)
@@ -1146,6 +1339,12 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []
firstQuery = false
}
if param.Value.Required {
action.Parameters = append(action.Parameters, curParam)
} else {
optionalParameters = append(optionalParameters, curParam)
}
}
}
@@ -1155,7 +1354,7 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound)
if len(functionname) > 0 {
action.Name = functionname
@@ -1178,10 +1377,6 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
Parameters: extraParameters,
}
if path.Post.RequestBody != nil {
log.Printf("RequestBody: %#v", path.Post.RequestBody)
}
action.Returns.Schema.Type = "string"
baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath)
@@ -1191,15 +1386,52 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
optionalQueries := []string{}
parameters := []string{}
optionalParameters := []WorkflowAppActionParameter{}
if len(swagger.Servers) == 0 {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
} else {
if swagger.Servers[0].URL == "" {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
}
}
headersFound := []string{}
if len(path.Post.Parameters) > 0 {
for _, param := range path.Post.Parameters {
if param.Value.Schema == nil || param.Value.In == "header" {
for counter, param := range path.Post.Parameters {
if param.Value.Schema == nil {
continue
} else if param.Value.In == "header" {
headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example))
continue
}
parsedName := param.Value.Name
parsedName = strings.ReplaceAll(parsedName, " ", "_")
parsedName = strings.ReplaceAll(parsedName, ",", "_")
parsedName = strings.ReplaceAll(parsedName, ".", "_")
parsedName = strings.ReplaceAll(parsedName, "|", "_")
param.Value.Name = parsedName
path.Post.Parameters[counter].Value.Name = parsedName
curParam := WorkflowAppActionParameter{
Name: param.Value.Name,
Name: parsedName,
Description: param.Value.Description,
Multiline: false,
Required: param.Value.Required,
@@ -1212,7 +1444,7 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
if param.Value.Example != nil {
curParam.Example = param.Value.Example.(string)
if param.Value.Name == "body" {
if parsedName == "body" {
curParam.Value = param.Value.Example.(string)
}
}
@@ -1226,15 +1458,8 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
}
}
if param.Value.Required {
action.Parameters = append(action.Parameters, curParam)
} else {
optionalParameters = append(optionalParameters, curParam)
}
if param.Value.In == "path" {
//log.Printf("PATH!: %s", param.Value.Name)
parameters = append(parameters, param.Value.Name)
parameters = append(parameters, curParam.Name)
//baseUrl = fmt.Sprintf("%s%s", baseUrl)
} else if param.Value.In == "query" {
//log.Printf("QUERY!: %s", param.Value.Name)
@@ -1257,6 +1482,11 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
firstQuery = false
}
if param.Value.Required {
action.Parameters = append(action.Parameters, curParam)
} else {
optionalParameters = append(optionalParameters, curParam)
}
}
}
@@ -1266,7 +1496,7 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "post", parameters, optionalQueries)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "post", parameters, optionalQueries, headersFound)
if len(functionname) > 0 {
action.Name = functionname
@@ -1299,13 +1529,52 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W
optionalQueries := []string{}
parameters := []string{}
optionalParameters := []WorkflowAppActionParameter{}
if len(swagger.Servers) == 0 {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
} else {
if swagger.Servers[0].URL == "" {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
}
}
headersFound := []string{}
if len(path.Patch.Parameters) > 0 {
for _, param := range path.Patch.Parameters {
if param.Value.Schema == nil || param.Value.In == "header" {
for counter, param := range path.Patch.Parameters {
if param.Value.Schema == nil {
continue
} else if param.Value.In == "header" {
headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example))
continue
}
parsedName := param.Value.Name
parsedName = strings.ReplaceAll(parsedName, " ", "_")
parsedName = strings.ReplaceAll(parsedName, ",", "_")
parsedName = strings.ReplaceAll(parsedName, ".", "_")
parsedName = strings.ReplaceAll(parsedName, "|", "_")
param.Value.Name = parsedName
path.Patch.Parameters[counter].Value.Name = parsedName
curParam := WorkflowAppActionParameter{
Name: param.Value.Name,
Name: parsedName,
Description: param.Value.Description,
Multiline: false,
Required: param.Value.Required,
@@ -1332,15 +1601,8 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W
}
}
if param.Value.Required {
action.Parameters = append(action.Parameters, curParam)
} else {
optionalParameters = append(optionalParameters, curParam)
}
if param.Value.In == "path" {
//log.Printf("PATH!: %s", param.Value.Name)
parameters = append(parameters, param.Value.Name)
parameters = append(parameters, curParam.Name)
//baseUrl = fmt.Sprintf("%s%s", baseUrl)
} else if param.Value.In == "query" {
//log.Printf("QUERY!: %s", param.Value.Name)
@@ -1363,6 +1625,11 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W
firstQuery = false
}
if param.Value.Required {
action.Parameters = append(action.Parameters, curParam)
} else {
optionalParameters = append(optionalParameters, curParam)
}
}
}
@@ -1372,7 +1639,7 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "patch", parameters, optionalQueries)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound)
if len(functionname) > 0 {
action.Name = functionname
@@ -1405,14 +1672,52 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
optionalQueries := []string{}
parameters := []string{}
optionalParameters := []WorkflowAppActionParameter{}
if len(swagger.Servers) == 0 {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
} else {
if swagger.Servers[0].URL == "" {
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Schema: SchemaDefinition{
Type: "string",
},
})
}
}
headersFound := []string{}
if len(path.Put.Parameters) > 0 {
for _, param := range path.Put.Parameters {
if param.Value.Schema == nil || param.Value.In == "header" {
for counter, param := range path.Put.Parameters {
if param.Value.Schema == nil {
continue
} else if param.Value.In == "header" {
headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example))
continue
}
parsedName := param.Value.Name
parsedName = strings.ReplaceAll(parsedName, " ", "_")
parsedName = strings.ReplaceAll(parsedName, ",", "_")
parsedName = strings.ReplaceAll(parsedName, ".", "_")
parsedName = strings.ReplaceAll(parsedName, "|", "_")
param.Value.Name = parsedName
path.Put.Parameters[counter].Value.Name = parsedName
curParam := WorkflowAppActionParameter{
Name: param.Value.Name,
Name: parsedName,
Description: param.Value.Description,
Multiline: false,
Required: param.Value.Required,
@@ -1439,14 +1744,7 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
}
}
if param.Value.Required {
action.Parameters = append(action.Parameters, curParam)
} else {
optionalParameters = append(optionalParameters, curParam)
}
if param.Value.In == "path" {
//log.Printf("PATH!: %s", param.Value.Name)
parameters = append(parameters, param.Value.Name)
//baseUrl = fmt.Sprintf("%s%s", baseUrl)
} else if param.Value.In == "query" {
@@ -1470,6 +1768,12 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
firstQuery = false
}
if param.Value.Required {
action.Parameters = append(action.Parameters, curParam)
} else {
optionalParameters = append(optionalParameters, curParam)
}
}
}
@@ -1479,7 +1783,7 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "put", parameters, optionalQueries)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound)
if len(functionname) > 0 {
action.Name = functionname
backend/go-app/main.go
+159 -6
View File
@@ -9,6 +9,7 @@ import (
"encoding/hex"
"encoding/json"
"errors"
"path/filepath"
"fmt"
"io"
@@ -36,7 +37,9 @@ import (
"github.com/google/go-github/v28/github"
"golang.org/x/oauth2"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-billy/v5/osfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/storage/memory"
@@ -238,7 +241,7 @@ type AppInfo struct {
type ScheduleOld struct {
Id string `json:"id" datastore:"id"`
Seconds int `json:"seconds" datastore:"seconds"`
WorkflowId string `json:"workflow_id datastore:"workflow_id", `
WorkflowId string `json:"workflow_id" datastore:"workflow_id", `
Argument string `json:"argument" datastore:"argument"`
AppInfo AppInfo `json:"appinfo" datastore:"appinfo,noindex"`
Finished bool `json:"finished" finished:"id"`
@@ -1803,6 +1806,49 @@ func getUserCount() (int, error) {
return count, nil
}
func handleGetSchedules(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in set new workflowhandler: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Admin required"}`))
return
}
ctx := context.Background()
schedules, err := getAllSchedules(ctx)
if err != nil {
log.Printf("Failed getting schedules: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Couldn't get schedules"}`))
return
}
newjson, err := json.Marshal(schedules)
if err != nil {
log.Printf("Failed unmarshal: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking environments"}`)))
return
}
//log.Printf("Existing environments: %s", string(newjson))
resp.WriteHeader(200)
resp.Write(newjson)
}
func handleGetEnvironments(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
@@ -1897,10 +1943,12 @@ func handleGetUsers(resp http.ResponseWriter, request *http.Request) {
}
func checkAdminLogin(resp http.ResponseWriter, request *http.Request) {
log.Printf("HELLO?")
cors := handleCors(resp, request)
if cors {
return
}
log.Printf("HELLO2?")
count, err := getUserCount()
if err != nil {
@@ -2635,6 +2683,21 @@ func handleDeleteSchedule(resp http.ResponseWriter, request *http.Request) {
return
}
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in set new workflowhandler: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// FIXME: IAM - Get workflow and check owner
if user.Role != "admin" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Admin required"}`))
return
}
location := strings.Split(request.URL.String(), "/")
var workflowId string
@@ -2655,7 +2718,7 @@ func handleDeleteSchedule(resp http.ResponseWriter, request *http.Request) {
}
ctx := context.Background()
err := DeleteKey(ctx, "schedules", workflowId)
err = DeleteKey(ctx, "schedules", workflowId)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "message": "Can't delete"}`))
@@ -4648,7 +4711,7 @@ func handleDeleteOutlookSub(resp http.ResponseWriter, request *http.Request) {
ctx := context.Background()
workflow, err := getWorkflow(ctx, workflowId)
if err != nil {
log.Printf("Failed getting the workflow locally: %s", err)
log.Printf("Failed getting the workflow locally (delete outlook): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
@@ -4782,7 +4845,7 @@ func createOutlookSub(resp http.ResponseWriter, request *http.Request) {
ctx := context.Background()
workflow, err := getWorkflow(ctx, workflowId)
if err != nil {
log.Printf("Failed getting the workflow locally: %s", err)
log.Printf("Failed getting the workflow locally (outlook sub): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
@@ -5729,6 +5792,86 @@ func healthCheckHandler(resp http.ResponseWriter, request *http.Request) {
fmt.Fprint(resp, "OK")
}
// Creates osfs from folderpath with a basepath as directory base
func createFs(basepath, pathname string) (billy.Filesystem, error) {
fs := osfs.New("")
err := filepath.Walk(pathname,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if strings.Contains(path, ".git") {
return nil
}
fullpath := fmt.Sprintf("%s%s", basepath, path)
switch mode := info.Mode(); {
case mode.IsDir():
err = fs.MkdirAll(fullpath, 0644)
if err != nil {
log.Printf("Failed making folder: %s", err)
}
case mode.IsRegular():
srcData, err := ioutil.ReadFile(path)
if err != nil {
log.Printf("Src error: %s", err)
return err
}
//if strings.Contains(path, "yaml") {
// log.Printf("PATH: %s -> %s", path, fullpath)
// //log.Printf("DATA: %s", string(srcData))
//}
dst, err := fs.Create(fullpath)
if err != nil {
log.Printf("Dst error: %s", err)
return err
}
_, err = dst.Write(srcData)
if err != nil {
log.Printf("Dst write error: %s", err)
return err
}
}
return nil
})
return fs, err
}
// Hotloads new apps from a folder
// FIXME: Not finished
func handleAppHotload(location string) error {
basepath := "base"
fs, err := createFs(basepath, location)
if err != nil {
log.Printf("Failed making files and stuff: %s", err)
return err
} else {
log.Printf("Hotloading from %s finished", location)
}
dir, err := fs.ReadDir(basepath)
if err != nil {
log.Printf("Failed reading folder: %s", err)
return err
}
err = iterateAppGithubFolders(fs, dir, "", "")
if err != nil {
log.Printf("Err: %s", err)
return err
}
return nil
}
// Handles configuration items during Shuffle startup
func runInit(ctx context.Context) {
// Setting stats for backend starts (failure count as well)
err := increaseStatisticsField(ctx, "backend_executions", "", 1)
@@ -5765,7 +5908,7 @@ func runInit(ctx context.Context) {
_, _, err := handleExecution(schedule.WorkflowId, Workflow{}, request)
if err != nil {
log.Printf("Failed to execute: %s", err)
log.Printf("Failed to execute %s: %s", schedule.WorkflowId, err)
}
}
@@ -5822,10 +5965,16 @@ func runInit(ctx context.Context) {
// FIXME: Get all the apps?
iterateAppGithubFolders(fs, dir, "", "")
// Hotloads locally
location := os.Getenv("APP_HOTLOAD_FOLDER")
if len(location) != 0 {
handleAppHotload(location)
}
}
log.Printf("Downloading OpenAPI data for search - EXTRA APPS")
apis := "https://github.com/frikky/OpenAPI-security-definitions"
apis := "https://github.com/frikky/security-openapis"
// THis gets memory problems hahah
//apis := "https://github.com/APIs-guru/openapi-directory"
@@ -5896,7 +6045,9 @@ func init() {
r.HandleFunc("/api/v1/streams/results", handleGetStreamResults).Methods("POST", "OPTIONS")
// App specific
r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/get_existing", loadSpecificApps).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/download_remote", loadSpecificApps).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/validate", validateAppInput).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appId}", deleteWorkflowApp).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appId}/config", getWorkflowAppConfig).Methods("GET", "OPTIONS")
@@ -5914,6 +6065,8 @@ func init() {
/* Everything below here increases the counters*/
r.HandleFunc("/api/v1/workflows", getWorkflows).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/workflows", setNewWorkflow).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/schedules", handleGetSchedules).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/workflows/download_remote", loadSpecificWorkflows).Methods("POST", "OPTIONS")
//r.HandleFunc("/api/v1/workflows/{key}/execute_fs", executeWorkflowFS)
r.HandleFunc("/api/v1/workflows/{key}/execute", executeWorkflow).Methods("GET", "POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/schedule", scheduleWorkflow).Methods("POST", "OPTIONS")
backend/go-app/walkoff.go
+347 -94
View File
File diff suppressed because one or more lines are too long