diff --git a/.env b/.env index a07a0644..7f864f6b 100644 --- a/.env +++ b/.env @@ -13,4 +13,4 @@ BACKEND_PORT=5001 FRONTEND_PORT=3001 FRONTEND_PORT_HTTPS=3443 OUTER_HOSTNAME=shuffle-backend -DB_LOCATION=/etc/shuffle +DB_LOCATION=./shuffle-database diff --git a/README.md b/README.md index 9d8c916d..d87c6087 100644 --- a/README.md +++ b/README.md @@ -3,21 +3,30 @@ **It's in BETA** - [Get in touch](https://shuffler.io/contact), send a mail to [frikky@shuffler.io](mailto:frikky@shuffler.io) or poke me on twitter [@frikkylikeme](https://twitter.com/frikkylikeme) -![Example Shuffle webhook integration](shuffle_webhook.png) +![Example Shuffle webhook integration](https://github.com/frikky/Shuffle/blob/master/frontend/src/assets/img/shuffle_webhook.png) ## Try it * Self-hosted: Check out the [installation guide](https://github.com/frikky/shuffle/blob/master/install-guide.md). -* Cloud: Register at https://shuffler.io/register and get cooking (old, missing a lot of features) +* Cloud: Register at https://shuffler.io/register and get cooking (missing a lot of features) + +## Getting started +* Self-hosted: Check out the [installation guide](https://github.com/frikky/shuffle/blob/master/install-guide.md) and [getting started](https://shuffler.io/docs/getting_started) +* Cloud: Register at https://shuffler.io/register and get cooking (there are some differences!) + +## Blogposts +* [1. Introducing Shuffle](https://medium.com/security-operation-capybara/introducing-shuffle-an-open-source-soar-platform-part-1-58a529de7d12) +* [2. Getting started with Shuffle](https://medium.com/security-operation-capybara/getting-started-with-shuffle-an-open-source-soar-platform-part-2-1d7c67a64244) +* [3. Integrating Shuffle with Virustotal and TheHive](https://medium.com/@Frikkylikeme/integrating-shuffle-with-virustotal-and-thehive-open-source-soar-part-3-8e2e0d3396a9) + +## Documentation +[Documentation](https://shuffler.io/docs) can be found on https://shuffler.io/docs and is written in https://github.com/frikky/shuffle-docs. ## Related repositories * Apps: https://github.com/frikky/shuffle-apps -* Workflows: https://github.com/frikky/shuffle-workflows (empty) -* Security OpenAPI apps: https://github.com/frikky/OpenAPI-security-definitions +* Workflows: https://github.com/frikky/shuffle-workflows +* Security OpenAPI apps: https://github.com/frikky/security-openapis * Documentation: https://github.com/frikky/shuffle-docs -## Documentation -Documentation can be found on https://shuffler.io/docs. - ## Features * Simple workflow automation editor * Premade apps for a number of security tools diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 33fa9314..8bf5736b 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -73,12 +73,9 @@ class AppBase: except requests.exceptions.ConnectionError as e: print("Connectionerror: %s" % e) return - #self.logger.info("AFTER initial stream result") - self.logger.info("THIS IS THE NEW UPDATE") # Verify whether there are any parameters with ACTION_RESULT required # If found, we get the full results list from backend - fullexecution = {} try: tmpdata = { @@ -195,10 +192,9 @@ class AppBase: # Regex to find all the things if parameter["variant"] == "STATIC_VALUE": data = parameter["value"] - self.logger.debug(f"\n\nHandle static data with JSON: {data}\n\n") - actualitem = re.findall(match, data, re.MULTILINE) - self.logger.info("STATIC PARSED: %s" % actualitem) + #self.logger.debug(f"\n\nHandle static data with JSON: {data}\n\n") + #self.logger.info("STATIC PARSED: %s" % actualitem) if len(actualitem) > 0: for replace in actualitem: try: @@ -487,7 +483,6 @@ class AppBase: # With this parameter ready, add it to... a greater list of parameters. Rofl multi_parameters[parameter["name"]] = resultarray else: - print("Hello, in here?: %s" % value) params[parameter["name"]] = value multi_parameters[parameter["name"]] = value diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index e995afea..7a728aa9 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -242,7 +242,9 @@ func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) { return appPath, nil } -func makePythoncode(swagger *openapi3.Swagger, name, url, method string, parameters, optionalQueries []string) (string, string) { +// This function generates the python code that's being used. +// This is really meta when you program it. Handling parameters is hard here. +func makePythoncode(swagger *openapi3.Swagger, name, url, method string, parameters, optionalQueries, headers []string) (string, string) { method = strings.ToLower(method) queryString := "" queryData := "" @@ -259,9 +261,6 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet } } - // How to add authentication? - // I think it should be like: - // async def(self, auth, baseurl, data): // api.Authentication.Parameters[0].Value = "BearerAuth" authenticationParameter := "" authenticationSetup := "" @@ -270,14 +269,15 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet if swagger.Components.SecuritySchemes != nil { if swagger.Components.SecuritySchemes["BearerAuth"] != nil { authenticationParameter = ", apikey" - authenticationSetup = "headers[\"Authorization\"] = f\"Bearer {apikey}\"" + authenticationSetup = "if apikey != \" \": headers[\"Authorization\"] = f\"Bearer {apikey}\"" } else if swagger.Components.SecuritySchemes["BasicAuth"] != nil { authenticationParameter = ", username, password" authenticationAddin = ", auth=(username, password)" } else if swagger.Components.SecuritySchemes["ApiKeyAuth"] != nil { authenticationParameter = ", apikey" if swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.In == "header" { - authenticationSetup = fmt.Sprintf("headers[\"%s\"] = apikey", swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name) + // This is a way to bypass apikeys by passing " " + authenticationSetup = fmt.Sprintf(`if apikey != " ": headers["%s"] = apikey`, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name) } else if swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.In == "query" { // This might suck lol key := "?" @@ -285,7 +285,7 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet key = "&" } - authenticationSetup = fmt.Sprintf("url+=f\"%s%s={apikey}\"", key, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name) + authenticationSetup = fmt.Sprintf("if apikey != \" \": url+=f\"%s%s={apikey}\"", key, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name) } } } @@ -301,6 +301,23 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet urlInline = "{url}" } + // Specific check for SSL verification + // This is critical for onprem stuff. + verifyParam := "" + verifyWrapper := "" + verifyAddin := "" + if len(swagger.Servers) == 0 { + verifyParam = ", verify=True" + verifyWrapper = `if type(ssl_verify) == str: ssl_verify = False if ssl_verify.lower() == "false" or ssl_verify == "0" else True` + verifyAddin = ", verify=ssl_verify" + } else { + if swagger.Servers[0].URL == "" { + verifyParam = ", ssl_verify=True" + verifyWrapper = `if type(ssl_verify) == str: ssl_verify = False if ssl_verify.lower() == "false" or ssl_verify == "0" else True` + verifyAddin = ", verify=ssl_verify" + } + } + if len(parameters) > 0 { parameterData = fmt.Sprintf(", %s", strings.Join(parameters, ", ")) } @@ -333,15 +350,39 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet } } + preparedHeaders := "headers={}" + if len(headers) > 0 { + preparedHeaders = "headers={" + for count, header := range headers { + headerSplit := strings.Split(header, "=") + added := false + if len(headerSplit) == 2 { + if strings.Contains(preparedHeaders, headerSplit[0]) { + continue + } + + preparedHeaders += fmt.Sprintf(`"%s": "%s"`, headerSplit[0], headerSplit[1]) + added = true + } + + if count != len(headers)-1 && added { + preparedHeaders += "," + } + } + + preparedHeaders += "}" + } + // Extra param for url if it's changeable // Extra param for authentication scheme(s) - data := fmt.Sprintf(` async def %s(self%s%s%s%s%s): - headers={} + data := fmt.Sprintf(` async def %s(self%s%s%s%s%s%s): + %s url=f"%s%s" %s + %s %s %s - return requests.%s(url, headers=headers%s%s).text + return requests.%s(url, headers=headers%s%s%s).text `, functionname, authenticationParameter, @@ -349,14 +390,18 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet parameterData, queryString, bodyParameter, + verifyParam, + preparedHeaders, urlInline, url, + verifyWrapper, authenticationSetup, queryData, bodyFormatter, method, authenticationAddin, bodyAddin, + verifyAddin, ) //log.Println(data) @@ -412,7 +457,6 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, //log.Printf("%s", j) api.SmallImage = string(j) api.LargeImage = string(j) - log.Printf("Set images!") } } @@ -444,6 +488,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, Description: "The apikey to use", Multiline: false, Required: true, + Example: "The API key to use. Space = skip", Schema: SchemaDefinition{ Type: "string", }, @@ -460,6 +505,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, Description: "The apikey to use", Multiline: false, Required: true, + Example: "The API key to use. Space = skip", Schema: SchemaDefinition{ Type: "string", }, @@ -475,6 +521,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, Description: "The username to use", Multiline: false, Required: true, + Example: "The username to use", Schema: SchemaDefinition{ Type: "string", }, @@ -484,6 +531,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, Description: "The password to use", Multiline: false, Required: true, + Example: "The password to use", Schema: SchemaDefinition{ Type: "string", }, @@ -508,7 +556,11 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, // Could just as well be go at this point lol pythonFunctions := []string{} for actualPath, path := range swagger.Paths { - // FIXME: Add everything from here: + actualPath = strings.Replace(actualPath, " ", "_", -1) + actualPath = strings.Replace(actualPath, ".", "", -1) + actualPath = strings.Replace(actualPath, ".", "", -1) + actualPath = strings.Replace(actualPath, "\\", "", -1) + // https://godoc.org/github.com/getkin/kin-openapi/openapi3#PathItem if path.Get != nil { action, curCode := handleGet(swagger, api, extraParameters, path, actualPath) @@ -759,13 +811,52 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} + if len(swagger.Servers) == 0 { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } else { + if swagger.Servers[0].URL == "" { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } + } + + headersFound := []string{} if len(path.Connect.Parameters) > 0 { - for _, param := range path.Connect.Parameters { - if param.Value.Schema == nil || param.Value.In == "header" { + for counter, param := range path.Connect.Parameters { + if param.Value.Schema == nil { + continue + } else if param.Value.In == "header" { + headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) continue } + + parsedName := param.Value.Name + parsedName = strings.ReplaceAll(parsedName, " ", "_") + parsedName = strings.ReplaceAll(parsedName, ",", "_") + parsedName = strings.ReplaceAll(parsedName, ".", "_") + parsedName = strings.ReplaceAll(parsedName, "|", "_") + param.Value.Name = parsedName + path.Connect.Parameters[counter].Value.Name = parsedName + curParam := WorkflowAppActionParameter{ - Name: param.Value.Name, + Name: parsedName, Description: param.Value.Description, Multiline: false, Required: param.Value.Required, @@ -792,15 +883,8 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ } } - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - if param.Value.In == "path" { - //log.Printf("PATH!: %s", param.Value.Name) - parameters = append(parameters, param.Value.Name) + parameters = append(parameters, curParam.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { //log.Printf("QUERY!: %s", param.Value.Name) @@ -823,6 +907,12 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ firstQuery = false } + if param.Value.Required { + action.Parameters = append(action.Parameters, curParam) + } else { + optionalParameters = append(optionalParameters, curParam) + } + } } @@ -832,7 +922,7 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "connect", parameters, optionalQueries) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "connect", parameters, optionalQueries, headersFound) if len(functionname) > 0 { action.Name = functionname @@ -857,8 +947,6 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor action.Returns.Schema.Type = "string" baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - //log.Println(path.Parameters) - // Parameters: []WorkflowAppActionParameter{}, // FIXME - add data for POST stuff firstQuery := true @@ -866,16 +954,53 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor // FIXME - remove this when authentication is properly introduced parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} + if len(swagger.Servers) == 0 { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify the SSL certificate request", + Multiline: false, + Required: false, + Example: "False - default=True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } else { + if swagger.Servers[0].URL == "" { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } + } + + headersFound := []string{} if len(path.Get.Parameters) > 0 { - for _, param := range path.Get.Parameters { - if param.Value.Schema == nil || param.Value.In == "header" { + for counter, param := range path.Get.Parameters { + if param.Value.Schema == nil { + continue + } else if param.Value.In == "header" { + headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) continue } + parsedName := param.Value.Name + parsedName = strings.ReplaceAll(parsedName, " ", "_") + parsedName = strings.ReplaceAll(parsedName, ",", "_") + parsedName = strings.ReplaceAll(parsedName, ".", "_") + parsedName = strings.ReplaceAll(parsedName, "|", "_") + param.Value.Name = parsedName + path.Get.Parameters[counter].Value.Name = parsedName + curParam := WorkflowAppActionParameter{ - Name: param.Value.Name, + Name: parsedName, Description: param.Value.Description, Multiline: false, Required: param.Value.Required, @@ -902,15 +1027,8 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor } } - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - if param.Value.In == "path" { - log.Printf("PATH!: %s", param.Value.Name) - parameters = append(parameters, param.Value.Name) + parameters = append(parameters, curParam.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { //log.Printf("QUERY!: %s", param.Value.Name) @@ -933,6 +1051,11 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor } firstQuery = false } + if param.Value.Required { + action.Parameters = append(action.Parameters, curParam) + } else { + optionalParameters = append(optionalParameters, curParam) + } } } @@ -943,7 +1066,7 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "get", parameters, optionalQueries) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "get", parameters, optionalQueries, headersFound) if len(functionname) > 0 { action.Name = functionname @@ -976,13 +1099,52 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} + if len(swagger.Servers) == 0 { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } else { + if swagger.Servers[0].URL == "" { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } + } + + headersFound := []string{} if len(path.Head.Parameters) > 0 { - for _, param := range path.Head.Parameters { - if param.Value.Schema == nil || param.Value.In == "header" { + for counter, param := range path.Head.Parameters { + if param.Value.Schema == nil { + continue + } else if param.Value.In == "header" { + headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) continue } + + parsedName := param.Value.Name + parsedName = strings.ReplaceAll(parsedName, " ", "_") + parsedName = strings.ReplaceAll(parsedName, ",", "_") + parsedName = strings.ReplaceAll(parsedName, ".", "_") + parsedName = strings.ReplaceAll(parsedName, "|", "_") + param.Value.Name = parsedName + path.Head.Parameters[counter].Value.Name = parsedName + curParam := WorkflowAppActionParameter{ - Name: param.Value.Name, + Name: parsedName, Description: param.Value.Description, Multiline: false, Required: param.Value.Required, @@ -1009,15 +1171,8 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo } } - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - if param.Value.In == "path" { - //log.Printf("PATH!: %s", param.Value.Name) - parameters = append(parameters, param.Value.Name) + parameters = append(parameters, curParam.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { //log.Printf("QUERY!: %s", param.Value.Name) @@ -1040,6 +1195,12 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo firstQuery = false } + if param.Value.Required { + action.Parameters = append(action.Parameters, curParam) + } else { + optionalParameters = append(optionalParameters, curParam) + } + } } @@ -1049,7 +1210,7 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "head", parameters, optionalQueries) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "head", parameters, optionalQueries, headersFound) if len(functionname) > 0 { action.Name = functionname @@ -1082,13 +1243,52 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} + if len(swagger.Servers) == 0 { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } else { + if swagger.Servers[0].URL == "" { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } + } + + headersFound := []string{} if len(path.Delete.Parameters) > 0 { - for _, param := range path.Delete.Parameters { - if param.Value.Schema == nil || param.Value.In == "header" { + for counter, param := range path.Delete.Parameters { + if param.Value.Schema == nil { + continue + } else if param.Value.In == "header" { + headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) continue } + + parsedName := param.Value.Name + parsedName = strings.ReplaceAll(parsedName, " ", "_") + parsedName = strings.ReplaceAll(parsedName, ",", "_") + parsedName = strings.ReplaceAll(parsedName, ".", "_") + parsedName = strings.ReplaceAll(parsedName, "|", "_") + param.Value.Name = parsedName + path.Delete.Parameters[counter].Value.Name = parsedName + curParam := WorkflowAppActionParameter{ - Name: param.Value.Name, + Name: parsedName, Description: param.Value.Description, Multiline: false, Required: param.Value.Required, @@ -1115,15 +1315,8 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] } } - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - if param.Value.In == "path" { - //log.Printf("PATH!: %s", param.Value.Name) - parameters = append(parameters, param.Value.Name) + parameters = append(parameters, curParam.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { //log.Printf("QUERY!: %s", param.Value.Name) @@ -1146,6 +1339,12 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] firstQuery = false } + if param.Value.Required { + action.Parameters = append(action.Parameters, curParam) + } else { + optionalParameters = append(optionalParameters, curParam) + } + } } @@ -1155,7 +1354,7 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound) if len(functionname) > 0 { action.Name = functionname @@ -1178,10 +1377,6 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo Parameters: extraParameters, } - if path.Post.RequestBody != nil { - log.Printf("RequestBody: %#v", path.Post.RequestBody) - } - action.Returns.Schema.Type = "string" baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) @@ -1191,15 +1386,52 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} + if len(swagger.Servers) == 0 { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } else { + if swagger.Servers[0].URL == "" { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } + } + headersFound := []string{} if len(path.Post.Parameters) > 0 { - for _, param := range path.Post.Parameters { - if param.Value.Schema == nil || param.Value.In == "header" { + for counter, param := range path.Post.Parameters { + if param.Value.Schema == nil { + continue + } else if param.Value.In == "header" { + headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) continue } + parsedName := param.Value.Name + parsedName = strings.ReplaceAll(parsedName, " ", "_") + parsedName = strings.ReplaceAll(parsedName, ",", "_") + parsedName = strings.ReplaceAll(parsedName, ".", "_") + parsedName = strings.ReplaceAll(parsedName, "|", "_") + param.Value.Name = parsedName + path.Post.Parameters[counter].Value.Name = parsedName + curParam := WorkflowAppActionParameter{ - Name: param.Value.Name, + Name: parsedName, Description: param.Value.Description, Multiline: false, Required: param.Value.Required, @@ -1212,7 +1444,7 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo if param.Value.Example != nil { curParam.Example = param.Value.Example.(string) - if param.Value.Name == "body" { + if parsedName == "body" { curParam.Value = param.Value.Example.(string) } } @@ -1226,15 +1458,8 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo } } - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - if param.Value.In == "path" { - //log.Printf("PATH!: %s", param.Value.Name) - parameters = append(parameters, param.Value.Name) + parameters = append(parameters, curParam.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { //log.Printf("QUERY!: %s", param.Value.Name) @@ -1257,6 +1482,11 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo firstQuery = false } + if param.Value.Required { + action.Parameters = append(action.Parameters, curParam) + } else { + optionalParameters = append(optionalParameters, curParam) + } } } @@ -1266,7 +1496,7 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "post", parameters, optionalQueries) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "post", parameters, optionalQueries, headersFound) if len(functionname) > 0 { action.Name = functionname @@ -1299,13 +1529,52 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} + if len(swagger.Servers) == 0 { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } else { + if swagger.Servers[0].URL == "" { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } + } + + headersFound := []string{} if len(path.Patch.Parameters) > 0 { - for _, param := range path.Patch.Parameters { - if param.Value.Schema == nil || param.Value.In == "header" { + for counter, param := range path.Patch.Parameters { + if param.Value.Schema == nil { + continue + } else if param.Value.In == "header" { + headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) continue } + + parsedName := param.Value.Name + parsedName = strings.ReplaceAll(parsedName, " ", "_") + parsedName = strings.ReplaceAll(parsedName, ",", "_") + parsedName = strings.ReplaceAll(parsedName, ".", "_") + parsedName = strings.ReplaceAll(parsedName, "|", "_") + param.Value.Name = parsedName + path.Patch.Parameters[counter].Value.Name = parsedName + curParam := WorkflowAppActionParameter{ - Name: param.Value.Name, + Name: parsedName, Description: param.Value.Description, Multiline: false, Required: param.Value.Required, @@ -1332,15 +1601,8 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W } } - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - if param.Value.In == "path" { - //log.Printf("PATH!: %s", param.Value.Name) - parameters = append(parameters, param.Value.Name) + parameters = append(parameters, curParam.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { //log.Printf("QUERY!: %s", param.Value.Name) @@ -1363,6 +1625,11 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W firstQuery = false } + if param.Value.Required { + action.Parameters = append(action.Parameters, curParam) + } else { + optionalParameters = append(optionalParameters, curParam) + } } } @@ -1372,7 +1639,7 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "patch", parameters, optionalQueries) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound) if len(functionname) > 0 { action.Name = functionname @@ -1405,14 +1672,52 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} + if len(swagger.Servers) == 0 { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } else { + if swagger.Servers[0].URL == "" { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } + } + headersFound := []string{} if len(path.Put.Parameters) > 0 { - for _, param := range path.Put.Parameters { - if param.Value.Schema == nil || param.Value.In == "header" { + for counter, param := range path.Put.Parameters { + if param.Value.Schema == nil { + continue + } else if param.Value.In == "header" { + headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) continue } + + parsedName := param.Value.Name + parsedName = strings.ReplaceAll(parsedName, " ", "_") + parsedName = strings.ReplaceAll(parsedName, ",", "_") + parsedName = strings.ReplaceAll(parsedName, ".", "_") + parsedName = strings.ReplaceAll(parsedName, "|", "_") + param.Value.Name = parsedName + path.Put.Parameters[counter].Value.Name = parsedName + curParam := WorkflowAppActionParameter{ - Name: param.Value.Name, + Name: parsedName, Description: param.Value.Description, Multiline: false, Required: param.Value.Required, @@ -1439,14 +1744,7 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor } } - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - if param.Value.In == "path" { - //log.Printf("PATH!: %s", param.Value.Name) parameters = append(parameters, param.Value.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { @@ -1470,6 +1768,12 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor firstQuery = false } + if param.Value.Required { + action.Parameters = append(action.Parameters, curParam) + } else { + optionalParameters = append(optionalParameters, curParam) + } + } } @@ -1479,7 +1783,7 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "put", parameters, optionalQueries) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound) if len(functionname) > 0 { action.Name = functionname diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 0bfeeda9..4e18c4e1 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -9,6 +9,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "path/filepath" "fmt" "io" @@ -36,7 +37,9 @@ import ( "github.com/google/go-github/v28/github" "golang.org/x/oauth2" + "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" + "github.com/go-git/go-billy/v5/osfs" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/storage/memory" @@ -238,7 +241,7 @@ type AppInfo struct { type ScheduleOld struct { Id string `json:"id" datastore:"id"` Seconds int `json:"seconds" datastore:"seconds"` - WorkflowId string `json:"workflow_id datastore:"workflow_id", ` + WorkflowId string `json:"workflow_id" datastore:"workflow_id", ` Argument string `json:"argument" datastore:"argument"` AppInfo AppInfo `json:"appinfo" datastore:"appinfo,noindex"` Finished bool `json:"finished" finished:"id"` @@ -1803,6 +1806,49 @@ func getUserCount() (int, error) { return count, nil } +func handleGetSchedules(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("Api authentication failed in set new workflowhandler: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if user.Role != "admin" { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Admin required"}`)) + return + } + + ctx := context.Background() + schedules, err := getAllSchedules(ctx) + if err != nil { + log.Printf("Failed getting schedules: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Couldn't get schedules"}`)) + return + } + + newjson, err := json.Marshal(schedules) + if err != nil { + log.Printf("Failed unmarshal: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking environments"}`))) + return + } + + //log.Printf("Existing environments: %s", string(newjson)) + + resp.WriteHeader(200) + resp.Write(newjson) +} + func handleGetEnvironments(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -1897,10 +1943,12 @@ func handleGetUsers(resp http.ResponseWriter, request *http.Request) { } func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { + log.Printf("HELLO?") cors := handleCors(resp, request) if cors { return } + log.Printf("HELLO2?") count, err := getUserCount() if err != nil { @@ -2635,6 +2683,21 @@ func handleDeleteSchedule(resp http.ResponseWriter, request *http.Request) { return } + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("Api authentication failed in set new workflowhandler: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // FIXME: IAM - Get workflow and check owner + if user.Role != "admin" { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Admin required"}`)) + return + } + location := strings.Split(request.URL.String(), "/") var workflowId string @@ -2655,7 +2718,7 @@ func handleDeleteSchedule(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - err := DeleteKey(ctx, "schedules", workflowId) + err = DeleteKey(ctx, "schedules", workflowId) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "Can't delete"}`)) @@ -4648,7 +4711,7 @@ func handleDeleteOutlookSub(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := getWorkflow(ctx, workflowId) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (delete outlook): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -4782,7 +4845,7 @@ func createOutlookSub(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := getWorkflow(ctx, workflowId) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (outlook sub): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -5729,6 +5792,86 @@ func healthCheckHandler(resp http.ResponseWriter, request *http.Request) { fmt.Fprint(resp, "OK") } +// Creates osfs from folderpath with a basepath as directory base +func createFs(basepath, pathname string) (billy.Filesystem, error) { + fs := osfs.New("") + + err := filepath.Walk(pathname, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if strings.Contains(path, ".git") { + return nil + } + + fullpath := fmt.Sprintf("%s%s", basepath, path) + switch mode := info.Mode(); { + case mode.IsDir(): + err = fs.MkdirAll(fullpath, 0644) + if err != nil { + log.Printf("Failed making folder: %s", err) + } + case mode.IsRegular(): + srcData, err := ioutil.ReadFile(path) + if err != nil { + log.Printf("Src error: %s", err) + return err + } + + //if strings.Contains(path, "yaml") { + // log.Printf("PATH: %s -> %s", path, fullpath) + // //log.Printf("DATA: %s", string(srcData)) + //} + + dst, err := fs.Create(fullpath) + if err != nil { + log.Printf("Dst error: %s", err) + return err + } + + _, err = dst.Write(srcData) + if err != nil { + log.Printf("Dst write error: %s", err) + return err + } + } + + return nil + }) + + return fs, err +} + +// Hotloads new apps from a folder +// FIXME: Not finished +func handleAppHotload(location string) error { + basepath := "base" + fs, err := createFs(basepath, location) + if err != nil { + log.Printf("Failed making files and stuff: %s", err) + return err + } else { + log.Printf("Hotloading from %s finished", location) + } + + dir, err := fs.ReadDir(basepath) + if err != nil { + log.Printf("Failed reading folder: %s", err) + return err + } + + err = iterateAppGithubFolders(fs, dir, "", "") + if err != nil { + log.Printf("Err: %s", err) + return err + } + + return nil +} + +// Handles configuration items during Shuffle startup func runInit(ctx context.Context) { // Setting stats for backend starts (failure count as well) err := increaseStatisticsField(ctx, "backend_executions", "", 1) @@ -5765,7 +5908,7 @@ func runInit(ctx context.Context) { _, _, err := handleExecution(schedule.WorkflowId, Workflow{}, request) if err != nil { - log.Printf("Failed to execute: %s", err) + log.Printf("Failed to execute %s: %s", schedule.WorkflowId, err) } } @@ -5822,10 +5965,16 @@ func runInit(ctx context.Context) { // FIXME: Get all the apps? iterateAppGithubFolders(fs, dir, "", "") + + // Hotloads locally + location := os.Getenv("APP_HOTLOAD_FOLDER") + if len(location) != 0 { + handleAppHotload(location) + } } log.Printf("Downloading OpenAPI data for search - EXTRA APPS") - apis := "https://github.com/frikky/OpenAPI-security-definitions" + apis := "https://github.com/frikky/security-openapis" // THis gets memory problems hahah //apis := "https://github.com/APIs-guru/openapi-directory" @@ -5896,7 +6045,9 @@ func init() { r.HandleFunc("/api/v1/streams/results", handleGetStreamResults).Methods("POST", "OPTIONS") // App specific + r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps/get_existing", loadSpecificApps).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/apps/download_remote", loadSpecificApps).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/validate", validateAppInput).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}", deleteWorkflowApp).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}/config", getWorkflowAppConfig).Methods("GET", "OPTIONS") @@ -5914,6 +6065,8 @@ func init() { /* Everything below here increases the counters*/ r.HandleFunc("/api/v1/workflows", getWorkflows).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows", setNewWorkflow).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/workflows/schedules", handleGetSchedules).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/workflows/download_remote", loadSpecificWorkflows).Methods("POST", "OPTIONS") //r.HandleFunc("/api/v1/workflows/{key}/execute_fs", executeWorkflowFS) r.HandleFunc("/api/v1/workflows/{key}/execute", executeWorkflow).Methods("GET", "POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/schedule", scheduleWorkflow).Methods("POST", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index b9fde0ba..a7420d53 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -428,7 +428,7 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, frequency _, _, err := handleExecution(workflowId, Workflow{}, request) if err != nil { - log.Printf("Failed to execute: %s", err) + log.Printf("Failed to execute %s: %s", workflowId, err) } } @@ -966,18 +966,10 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { workflow.Sharing = "private" ctx := context.Background() - err = setWorkflow(ctx, workflow, workflow.ID) - if err != nil { - log.Printf("Failed setting workflow: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - log.Printf("Saved new workflow %s with name %s", workflow.ID, workflow.Name) err = increaseStatisticsField(ctx, "total_workflows", workflow.ID, 1) if err != nil { - log.Printf("Failed to increase total workflows: %s", err) + log.Printf("Failed to increase total workflows stats: %s", err) } if len(workflow.Actions) == 0 { @@ -1003,6 +995,37 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { newActions = append(newActions, action) } + // Initialized without functions = adding a hello world node. + if len(newActions) == 0 { + log.Printf("APPENDING NEW APP FOR NEW WORKFLOW") + //nodeId := "40447f30-fa44-4a4f-a133-4ee710368737" + //workflow.Start = nodeId + //newActions = append(newActions, Action{ + // Label: "Start node", + // Name: "hello_world", + // AppName: "testing", + // Environment: "Shuffle", + // Parameters: []WorkflowAppActionParameter{}, + // Position: struct { + // X float64 "json:\"x\" datastore:\"x\"" + // Y float64 "json:\"y\" datastore:\"y\"" + // }{X: 449.5, Y: 446}, + // Priority: 0, + // AppVersion: "1.0.0", + // AppID: "c567fc10-9c15-403e-b72c-6550e9e76bc8", + // Errors: []string{}, + // ID: nodeId, + // IsValid: true, + // IsStartNode: true, + // Sharing: true, + // PrivateID: "", + // SmallImage: "", + // LargeImage: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABkCAIAAAD/gAIDAAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAABmJLR0QA/wD/AP+gvaeTAAAAB3RJTUUH4wgeDy4zYzmH5gAADkRJREFUeNrtXV1QG9cV3nt3V2AwkvgRrRE4nTFxMFKATGyLh1gONG8Rxn2qIQ8GOSYdfpy4rknATacPBRx7ak+d2K0dkDWTYvutMRB3nKllYZLaQD2B8GcPJBNwpIxBIPSDEbt39/ZhG+qglZDEjwT4e9Q9e38+3Xv2nHOPjgDGmHiGwADDPYG1hGdkBYFnZAWBZ2QFASrcEyAIgsAYcxyHEGIYhmVZnucJgoAQ0jQtkUgoiiJJEgAQ7mmGiSyEkMPhsFqtIyMjVqvVYvneYrFMTNgYhkHo/2RRFC2RSBSKJKVSqVSmpqRsSU9P37IlRS6XU1QYZr6qQ7pcrpGRke7urp6env7+fpvN5nQ6WBYBQEAICYLw3j6CZcPzPMYETVMymSwhIVGtfjErKys3V7NtW7pUKl21+YNVsLM8Hs/Q0JDJZOrouPPgwZDT6eR5niRJCGGwhwtjzPM8x3EQgrg4WUZGhlarzc/P37EjMzo6em2TNTU1ZTbfvn79emdnp9PpIAhiGbWPoOkIgpBKZbt37y4sLHz11bzExMS1R5bNZmtra7t69crg4ABCiKKoldPQGGOEEEmSKpW6qKhYp9MlJSWtDbLcbveNGzcMhqa+vq8xxqupiRFCAAC1+kW9Xv/667rNmzdHLlk8z//nP93nz583m80sy4TlhUUQBEKIpmmtdm9FReWuXbtIkow4sux2u9F4uampcXJykqYpggizWcSybHx8gl5/SK/XJyQkLEufy0NWX19fQ0N9e7sZACAYAZEA4Q2g1e6tqanNzs5eeodLJQshdOPGjfr6P42OjtI0HW5+RMCybFra1traWp2uYImaYUlkeTyexsbGc+f+MjPjXkbVsOzgOC42Nraq6sibbx7etGlTyP2ETpbL5T59+gOj0cDzOHKOni/wPA8hLCkpPX68Oi4uLrROQiTL6XTW1dU1N38CAIgEFzcQYIwxJoqL3zhx4vcyWShOUig7wu1219XVNTf/HcI1wxRBEAAACMGVK3+vq/uTy+UKoYegyfJ4PKdOfdDc/Ing+YabgeAXDOGVK82nT5/yeDxBPxuUNM/zjY0fG40GAMBaZEoAAITRePnSpYtCLChwBEdWa2vruXPneB6vodPnDQAAz/MfffTh9evXg3swcAXf19d3+PCbjx6NRbKVEDg4jlMqUy9d+jgnJyfARwLdWXb7VEND/ejod+uDKYIgSJJ89Gjs5MmGqampAB8JiCyMsdFobG83R6aNHjJomr5zp91gaApQeQVEVldXV1NT45rWU75AkqTB0NTZeS8Q4cXJcrvdFy6cn5ycjHwzPQQAAOx2+4ULF9xu96LCi6//xo3PzGYzTUfEpdlKgKbp9nZzS0vLopKLkGWz2QwGA8sya9eqCgQIIaPx8sTEuH+xRchqa2vr6/s6XDHPVQNFUQMD/a2trf7F/JE1OTl59eqVDZKThDG+du2azWbzI+OPLLPZPDg4sO63lQCKogYHB0wmkx8Zn2R5PJ6Wlk8RQuFexeqB47iWluuzs7O+BHySNTQ01NnZtUG2lQCKorq7uwcHB30J+CTLZDI5nY51aYj6AgDA6XSYTLd8CYiT5XK5OjruhHvyYQAAoKOjw+l0iraKkzUyMvLgwdC68ZkDB0mSDx8+HB4eFm0VJ6u7u8vpdG6oMyhAOIn37om7iiJkIYR6enqCjSKuG2CMe3t7WJb1bhIhy+Fw9Pf3b8AzKIAkyYGBgenpae8mEbKsVqvNZluXMYZAACG026csFotIk/dHIyMjG81oeBoAAIfD8c0333g3idicVqsVISSRSIIdBgvXmF5ji2aKBig5D57nMcZ+Ek9E+wykZ29wHGe1WgMiy2L5PliaCILgeS4uTuad2DkzMzM7++Snc8VSqVQiiVogyTCMyyVi4HAcR9P0z3++RSKRzM3NjY8/5jhuAWUY402bNsXGxhGEyHfAMHO+TCdfsFpFSFhIFkLIYrEEq7AQQhkZO+rq6lJSlDzPz6cdkyR55syfr127+nTwHmPi2LHjr7322tMvXIqiPv/88z/+8Q/eTKlU6oqKSo1GExMT43a7zWbz2bNnrFbL068glmX37/9VdfW7QpYpxgRBYGFD2e32+vq6L7/8IvBXFoRwbGxMyO5chKyJCVuAnQrgef7557d/8MGpl19+WRhJWKSQI7l5c+yC04ExoVAotm7dKqQeCx+SJOmdcsbzfGpq6pkzZ9VqtcfjmZmZUSgUxcXFiYmJ77zzttvtmt+wGOOYmNi0tDSOQwQBBF54nnc4HB9+eO7u3X8H+3K326dZll2ELIZhGIYJ6oQDACiKbGy89Ne/sjt37iovr3A4HGfPnhFuGIeGhnx5452dnZcu/Q0AgDGGED5+/Nj7m9Nq96rVarvdXl19/P79+zpdwdGjRzUaTWZm5t27X1LU/zYsTdNffNFRVvYmx3GpqWlHj/5WLpdfvHjxn//8rKenJ9iXFQBgbo5hGGZBftLCZbAsixAbRMcEAQB48ODB4OAAw7Acx5eXV8zNzd2+fWtoaIgkSZKkfB3q6enpnp7e+XHt9kkISa/OIUEQEokkKyvLarW2trbcv98VHR09PDz8tDCEcHT0u2+/HeE47oUXMsrLK4Qv4+7du6ElZCHEetulC8nieT4E250kSZIkhUSV//VLUTQt8b/58/Lybt78nPjRDvzNb8oW+FgURZlM/+rs7NRoNFVVR/T6Q6Ojo7du/au5+RObbXLBhoUQQiiBkKMo8sdPQMghJoQ4Qf39ZIjQ+lodQAitVutbb5WdPHnyq6++Qgjt2LGjquqI0fjJCy9keC9mpbGQeAjhqtnut2/frq2tEbYSy7IOh2PB0BzH/fKXr+XkvDQ2NvrGG8XJycn5+fnl5RUZGRmvv65b0bgIRZHenS8ki6bpea250pDJZFlZWQAQGBMQwvHx8a+/7v0pWfxzz/3i2LFjLpeLpumbN2+aTKZf//pAUlJSVFTQNnOQZNHeuQoLyZJIJBKJJOQbHUHhBPjyyc3N3bVr14+Toz799B9HjlT9dMbkZ5+1FRQUaDSaurr6t99+Jzo6WqFQ2Gw2k8m0cicAYxwVJfH2YRaSRVGUQhH6D1+ePHkyNjY2NTXJsj5vOgAgbDabxWJ5WulQFOXt6EMIx8cf/+53xyorq/bs2SOXyxFCnZ2dFy/+rbu7y9cZZFlksVgYhnny5EnIC4mPl3vvLJH8rNraGoOhKTTfUCKRyGRyjPH0tN23AsaCY/T00ACAuTmPqFPCcRxFUcnJP4uKiuI4bmpq0uVy+XnNkSQpl8cL/jDDzIUQEWAYpqSkpKHh1IJHRYZUKlOD7X1+wQzDjI8/Jn60430JOp0Oh2Pa+3HRhZEkiTH+4QerQC6E0L9BwHGccBEfciY1xkRKSqr3oyKjpqRsCdk8CXB+ISwjKA21RHVGUaRSqRTp1vujbdvSZTLZBrm19wbGWCaTbdu2zbtJhKyUlJSEhMQNG4PneT4+PiHQnSWXy9Vq9erbxxECjuNUKpVcLvduEiGLoqisrGwIN25YOTs7RzR7VlwRajSauLiNqLYwxlKpNDdXI9oqTlZ6enpGRhg81bCD47jt27enpz8v2ipOllQq1Wq14Z55GIAx1mr3ymQy0Vaf9kheXr5UurFOIsZYKpXl5eX7EvBJVmZm5u7duzdUMhtCaOfOnZmZKl8CPsmKjo7et69wQ13iQ0gWFu6PifEZhvbnFuTlvapSqTfI5kIIZWZm5uXl+ZHxR1ZiYtKBA0Ub5B4fAFBUVKRQKPzILOJw6nQFavWL635zIYRUKrVOV+BfbBGyFIqk0tLSdZ+GS5LUwYMlycnJ/sUWD2XodAV7974qmty1PsCyrFar3bdv36KSi5O1efPm8vKK+PiEdWlzYYzj4xMqKioDKfYQUJBMo9Ho9fp16f1wHFdSUpqbmxuIcEBkQQj1+kNa7d51dhhZln3llT2HDh0KMLIaaPg1ISHhvfdq0tK2rpv9JaSQ1NTUBl75LohYdU5OTk1NbWxs7DoIovI8HxMT+957NS+99FLgTwUX2C8sLKyqOgIhXNPKXki3rKys3L9/f1APBkcWhPDw4bKSktK1zBWBMT54sKSs7K1gPd+gr4yio6OPH68uLn5jjR5GnucPHCiurn43hLytUEzzuLi4Eyd+TxDElSvNABBrxXkU0pkPHCh+//33Q6ubu5RiY67Tp08ZjZeFOl7hpmIR8DwPADh4sKS6+t2QKwwvqYzd7Ozsxx9f+uijD2dmZiI58sVxXExMbGVlZVnZW+EpYycAIdTW1lZfX/fo0VhkFl9hWVawp/bt2xfOAonz6O3tbWiov3OnPUIKtwsQSm++8sqe2toTgZcy8oNlK+o6NTVlMDQZDE12uz0StphQ1LW0tFSvP7Rc1amXs1wwx3FdXV0XLpxvbzd7/zxh1YAQIklKq9VWVlbu3q2JxHLB83C73a2trZcvGwYG+sNSiFqlUpeUlOp0umWvq79SJc4nJiba2lqvXr06ODggpO6tdIlzCMnMzMyioqKCggKFInnp3a4eWQImJydNplstLS1dXV3CbxiXvXi+cDO6c+fOwsL9+fn5K1QJfjXIEjA7Ozs4OGgy3ero6Hj48KHT6cAYL/FvGQAAUql0+/btWu3evLx8lUq1FAMqgsiah9PpHB4evnfvXm9v78BAv90+5XA4EOIC/MMPiiJlMll8fIJKpc7Ozs7NzU1PT/eVl7DmyZoHy7LT09NWq2Vk5Bur1WK1WsbGxuz26bk5BiEWIY4gCIoiKYqOipLEx8vT0tJSUlKVSmV6+raUFKVcLg+LdRIeshZA0D4sywp/UiQEY0mSFP6kiKbpCLF1I4KstYJIjxZEFJ6RFQSekRUEnpEVBP4LiQWypqHC6doAAAAldEVYdGRhdGU6Y3JlYXRlADIwMTktMDgtMzBUMTU6NDc6MjQtMDQ6MDCzXTa0AAAAJXRFWHRkYXRlOm1vZGlmeQAyMDE5LTA4LTMwVDE1OjQ2OjUxLTA0OjAwdT/DiAAAAABJRU5ErkJggg==", + //}) + } else { + log.Printf("Has actions already?") + } + workflow.Actions = newActions workflow.IsValid = true @@ -1014,6 +1037,14 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { return } + err = setWorkflow(ctx, workflow, workflow.ID) + if err != nil { + log.Printf("Failed setting workflow: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + //memcacheName := fmt.Sprintf("%s_workflows", user.Username) //memcache.Delete(ctx, memcacheName) @@ -1058,7 +1089,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := getWorkflow(ctx, fileId) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (delete workflow): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -1128,7 +1159,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { return } - log.Println("Start") + //log.Println("Start") user, userErr := handleApiAuthentication(resp, request) if userErr != nil { log.Printf("Api authentication failed in edit workflow: %s", userErr) @@ -1137,7 +1168,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { return } - log.Println("PostUser") + //log.Println("PostUser") location := strings.Split(request.URL.String(), "/") var fileId string @@ -1164,7 +1195,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { tmpworkflow, err := getWorkflow(ctx, fileId) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (save workflow): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -1211,7 +1242,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // FIXME - this shouldn't be necessary with proper API checks newActions := []Action{} allNodes := []string{} - log.Println("Pre") + //log.Println("Pre") for _, action := range workflow.Actions { allNodes = append(allNodes, action.ID) @@ -1237,7 +1268,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { workflow.Actions = newActions for _, trigger := range workflow.Triggers { - log.Println("TRIGGERS") + //log.Println("TRIGGERS") allNodes = append(allNodes, trigger.ID) } @@ -1260,7 +1291,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { foundNodes := []string{} for _, node := range allNodes { for _, branch := range workflow.Branches { - log.Println("branch") + //log.Println("branch") //log.Println(node) //log.Println(branch.DestinationID) if node == branch.DestinationID || node == branch.SourceID { @@ -1462,7 +1493,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { log.Printf("Failed to change total actions data: %s", err) } - log.Printf("Saved new version of workflow %s", fileId) + log.Printf("Saved new version of workflow %s (%s)", workflow.Name, fileId) resp.WriteHeader(200) resp.Write([]byte(`{"success": true}`)) } @@ -1668,7 +1699,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if workflow.ID == "" || workflow.ID != id { tmpworkflow, err := getWorkflow(ctx, id) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (execution cleanup): %s", err) return WorkflowExecution{}, "Failed getting workflow", err } @@ -1736,6 +1767,18 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if len(execution.Start) == 36 { log.Printf("SHOULD START ON NODE %s", execution.Start) workflow.Start = execution.Start + + found := false + for _, action := range workflow.Actions { + if action.ID == workflow.Start { + found = true + } + } + + if !found { + log.Printf("ACTION %s WAS NOT FOUND!", workflow.Start) + return WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", workflow.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", workflow.Start)) + } } if len(execution.ExecutionId) == 36 { @@ -1993,7 +2036,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := getWorkflow(ctx, fileId) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (execute workflow): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -2021,6 +2064,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { } func stopSchedule(resp http.ResponseWriter, request *http.Request) { + log.Printf("Delete?") cors := handleCors(resp, request) if cors { return @@ -2064,7 +2108,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := getWorkflow(ctx, fileId) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (stop schedule): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -2079,19 +2123,6 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { return } - if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} - } - if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} - } - if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} - } - if len(workflow.Errors) == 0 { - workflow.Errors = []string{} - } - err = deleteSchedule(ctx, scheduleId) if err != nil { if strings.Contains(err.Error(), "Job not found") { @@ -2153,7 +2184,7 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := getWorkflow(ctx, fileId) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (stop schedule GCP): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -2275,7 +2306,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := getWorkflow(ctx, fileId) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (schedule workflow): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -3276,6 +3307,154 @@ func deployWebhookFunction(ctx context.Context, name, localization, applocation return nil } +func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + // Just need to be logged in + // FIXME - should have some permissions? + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("Api authentication failed in load apps: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if user.Role != "admin" { + log.Printf("Wrong user (%s) when downloading from github", user.Username) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Printf("Error with body read: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // Field1 & 2 can be a lot of things.. + type tmpStruct struct { + URL string `json:"url"` + Field1 string `json:"field_1"` + Field2 string `json:"field_2"` + } + //log.Printf("Body: %s", string(body)) + + var tmpBody tmpStruct + err = json.Unmarshal(body, &tmpBody) + if err != nil { + log.Printf("Error with unmarshal tmpBody: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fs := memfs.New() + + if strings.Contains(tmpBody.URL, "github") || strings.Contains(tmpBody.URL, "gitlab") || strings.Contains(tmpBody.URL, "bitbucket") { + cloneOptions := &git.CloneOptions{ + URL: tmpBody.URL, + } + + // FIXME: Better auth. + if len(tmpBody.Field1) > 0 && len(tmpBody.Field2) > 0 { + cloneOptions.Auth = &http2.BasicAuth{ + + Username: tmpBody.Field1, + Password: tmpBody.Field2, + } + } + + storer := memory.NewStorage() + r, err := git.Clone(storer, fs, cloneOptions) + if err != nil { + log.Printf("Failed loading repo into memory: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + + dir, err := fs.ReadDir("/") + if err != nil { + log.Printf("FAiled reading folder: %s", err) + } + _ = r + + log.Printf("Starting workflow folder iteration") + iterateWorkflowGithubFolders(fs, dir, "", "") + + } else if strings.Contains(tmpBody.URL, "s3") { + //https://docs.aws.amazon.com/sdk-for-go/api/service/s3/ + + //sess := session.Must(session.NewSession()) + //downloader := s3manager.NewDownloader(sess) + + //// Write the contents of S3 Object to the file + //storer := memory.NewStorage() + //n, err := downloader.Download(storer, &s3.GetObjectInput{ + // Bucket: aws.String(myBucket), + // Key: aws.String(myString), + //}) + //if err != nil { + // return fmt.Errorf("failed to download file, %v", err) + //} + //fmt.Printf("file downloaded, %d bytes\n", n) + } else { + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s is unsupported. Try e.g. github"}`, tmpBody.URL))) + return + } + + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) +} + +func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + // Just need to be logged in + // FIXME - should have some permissions? + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("Api authentication failed in app hotload: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if user.Role != "admin" { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Must be admin to hotload apps"}`)) + return + } + + location := os.Getenv("APP_HOTLOAD_FOLDER") + if len(location) == 0 { + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "APP_HOTLOAD_FOLDER not specified in .env"}`, err))) + return + } + + err = handleAppHotload(location) + if err != nil { + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed loading apps: %s"}`, err))) + return + } + + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) +} + func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -3286,7 +3465,7 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { // FIXME - should have some permissions? _, err := handleApiAuthentication(resp, request) if err != nil { - log.Printf("Api authentication failed in load apps: %s", err) + log.Printf("Api authentication failed in load specific apps: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -3391,16 +3570,19 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, switch mode := file.Mode(); { case mode.IsDir(): tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name()) + //log.Printf("TMPEXTRA: %s", tmpExtra) dir, err := fs.ReadDir(tmpExtra) if err != nil { - log.Printf("Failed to read dir: %s", err) - break + log.Printf("Failed reading dir in openapi: %s", err) + continue } // Go routine? Hmm, this can be super quick I guess err = iterateOpenApiGithub(fs, dir, tmpExtra, "") if err != nil { - break + log.Printf("Failed recursion in openapi: %s", err) + continue + //break } case mode.IsRegular(): // Check the file @@ -3501,10 +3683,70 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, return nil } +// Onlyname is used to +func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error { + var err error + + for _, file := range dir { + if len(onlyname) > 0 && file.Name() != onlyname { + continue + } + + // Folder? + switch mode := file.Mode(); { + case mode.IsDir(): + tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name()) + dir, err := fs.ReadDir(tmpExtra) + if err != nil { + log.Printf("Failed to read dir: %s", err) + break + } + + // Go routine? Hmm, this can be super quick I guess + err = iterateWorkflowGithubFolders(fs, dir, tmpExtra, "") + if err != nil { + break + } + case mode.IsRegular(): + // Check the file + filename := file.Name() + if strings.HasSuffix(filename, ".json") { + path := fmt.Sprintf("%s%s", extra, file.Name()) + fileReader, err := fs.Open(path) + if err != nil { + log.Printf("Error reading file: %s", err) + continue + } + + readFile, err := ioutil.ReadAll(fileReader) + if err != nil { + log.Printf("Error reading file: %s", err) + continue + } + + var workflow Workflow + err = json.Unmarshal(readFile, &workflow) + if err != nil { + continue + } + + ctx := context.Background() + err = setWorkflow(ctx, workflow, workflow.ID) + if err != nil { + log.Printf("Failed setting (download) workflow: %s", err) + continue + } + log.Printf("Uploaded workflow %s!", filename) + } + } + } + + return err +} + // Onlyname is used to func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error { var err error - runUpload := false for _, file := range dir { if len(onlyname) > 0 && file.Name() != onlyname { continue @@ -3529,85 +3771,87 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin // Check the file filename := file.Name() if filename == "Dockerfile" { + + // Quick Dockerfile check + dockerdata, err := ioutil.ReadFile(fmt.Sprintf("%sDockerfile", extra)) + if err != nil { + continue + } + + if len(dockerdata) == 0 { + continue + } + log.Printf("Handle Dockerfile in location %s", extra) - extraSplit := strings.Split(extra, "/") - tags := []string{} - if len(extraSplit) > 1 { - tags = []string{ - fmt.Sprintf("%s:%s_%s", baseDockerName, strings.ReplaceAll(extraSplit[0], " ", "-"), extraSplit[1]), - // Version = folder of last part of extra - // Name = first folder of extra - } - } else { - // Skip - runUpload = false - log.Printf("Skipping folder %s because the extra variable is empty~", extra) - break - //return nil - } - - /// Only upload if successful and no errors - err := buildImageMemory(fs, tags, extra) - if err != nil { - log.Printf("Failed image build memory: %s", err) - runUpload = false - } else { - runUpload = true - } - } - } - } - - // Done sequentailly to prevent bad uploads - if runUpload && err == nil { - for _, file := range dir { - if file.Name() == "api.yaml" || file.Name() == "api.yaml" { - log.Printf("Run update of %sapi.yaml in backend if it doesn't exist!!", extra) - fullPath := fmt.Sprintf("%s%s", extra, file.Name()) - + // Try api.yaml and api.yml + fullPath := fmt.Sprintf("%s%s", extra, "api.yaml") fileReader, err := fs.Open(fullPath) if err != nil { - return err + fullPath = fmt.Sprintf("%s%s", extra, "api.yml") + fileReader, err = fs.Open(fullPath) + if err != nil { + log.Printf("Failed finding api.yaml/yml: %s", err) + continue + } } readFile, err := ioutil.ReadAll(fileReader) if err != nil { - log.Printf("Filereader error: %s", err) - return err + log.Printf("Failed reading %s: %s", fullPath, err) + continue + } + + if len(readFile) == 0 { + log.Printf("Failed reading %s - length is 0.", fullPath) + continue } var workflowapp WorkflowApp err = gyaml.Unmarshal(readFile, &workflowapp) if err != nil { - log.Printf("Failed api.yaml unmarshal: %s", err) - return err + log.Printf("Failed unmarshaling %s: %s", fullPath, err) + continue } - log.Printf("APIName: %s", workflowapp.Name) - extraSplit := strings.Split(extra, "/") - appName := fmt.Sprintf("%s_%s", strings.ReplaceAll(extraSplit[0], " ", "-"), extraSplit[1]) + newName := workflowapp.Name + newName = strings.ReplaceAll(newName, " ", "-") + + tags := []string{ + fmt.Sprintf("%s:%s_%s", baseDockerName, newName, workflowapp.AppVersion), + } ctx := context.Background() allapps, err := getAllWorkflowApps(ctx) if err != nil { log.Printf("Failed getting apps to verify: %s", err) - return err + continue + //return err } - log.Printf("APPS: %d", len(allapps)) - + // Make an option to override existing apps? + removeApps := []string{} for _, app := range allapps { if app.Name == workflowapp.Name && app.AppVersion == workflowapp.AppVersion { - log.Printf("App upload for %s:%s already exists.", app.Name, app.AppVersion) - return errors.New(fmt.Sprintf("App %s already exists. ", appName)) + //log.Printf("App upload for %s:%s already exists.", app.Name, app.AppVersion) + log.Printf("Overriding app %s:%s as it exists.", app.Name, app.AppVersion) + removeApps = append(removeApps, app.ID) } } err = checkWorkflowApp(workflowapp) if err != nil { log.Printf("%s for app %s:%s", err, workflowapp.Name, workflowapp.AppVersion) - return err + continue + } + + if len(removeApps) > 0 { + for _, item := range removeApps { + err = DeleteKey(ctx, "workflowapp", item) + if err != nil { + log.Printf("Failed deleting %s", item) + } + } } //if workflowapp.Environment == "" { @@ -3623,7 +3867,8 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin err = setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err != nil { log.Printf("Failed setting workflowapp: %s", err) - return err + continue + //return err } err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1) @@ -3637,8 +3882,16 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion) - //memcache.Delete(ctx, "all_apps") - //os.Exit(3) + + /// Only upload if successful and no errors + err = buildImageMemory(fs, tags, extra) + if err != nil { + log.Printf("Failed image build memory: %s", err) + } else { + if len(tags) > 0 { + log.Printf("Successfully built image %s", tags[0]) + } + } } } } @@ -3773,7 +4026,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := getWorkflow(ctx, fileId) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (get executions): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return diff --git a/docker-compose.yml b/docker-compose.yml index 0fbc7c41..77ee3a1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,20 +13,8 @@ services: restart: unless-stopped depends_on: - backend - database: - #build: ./backend/database - image: frikky/shuffle:database - container_name: shuffle-database - hostname: shuffle-database - ports: - - "8000:8000" - networks: - - shuffle - restart: unless-stopped - volumes: - - ${DB_LOCATION}:/etc/shuffle backend: - #build: ./backend + build: ./backend image: frikky/shuffle:backend container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} @@ -59,6 +47,18 @@ services: - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} - DOCKER_API_VERSION=1.40 restart: unless-stopped + database: + #build: ./backend/database + image: frikky/shuffle:database + container_name: shuffle-database + hostname: shuffle-database + ports: + - "8000:8000" + networks: + - shuffle + restart: unless-stopped + volumes: + - ${DB_LOCATION}:/etc/shuffle networks: shuffle: driver: bridge diff --git a/frontend/src/Admin.js b/frontend/src/Admin.js index a0a1fa74..c3700307 100644 --- a/frontend/src/Admin.js +++ b/frontend/src/Admin.js @@ -8,7 +8,8 @@ import ListItem from '@material-ui/core/ListItem'; import Button from '@material-ui/core/Button'; import Tabs from '@material-ui/core/Tabs'; import Tab from '@material-ui/core/Tab'; - +import ListItemText from '@material-ui/core/ListItemText'; +import ListItemSecondaryAction from '@material-ui/core/ListItemSecondaryAction'; import { useAlert } from "react-alert"; @@ -29,9 +30,39 @@ const Admin = (props) => { const [curTab, setCurTab] = React.useState(0); const [users, setUsers] = React.useState([]); const [environments, setEnvironments] = React.useState([]); + const [schedules, setSchedules] = React.useState([]) const alert = useAlert() + const deleteSchedule = (data) => { + // FIXME - add some check here ROFL + console.log("INPUT: ", data) + + // Just use this one? + const url = globalUrl+'/api/v1/workflows/'+data["workflow_id datastore:"]+"/schedule/"+data.id + console.log("URL: ", url) + fetch(url, { + method: 'DELETE', + credentials: "include", + headers: { + 'Content-Type': 'application/json', + }, + }) + .then(response => + response.json().then(responseJson => { + if (responseJson["success"] === false) { + alert.error("Failed stopping schedule") + } else { + getSchedules() + alert.success("Successfully stopped schedule!") + } + }), + ) + .catch(error => { + console.log("Error in userdata: ", error) + }); + } + const submitUser = (data) => { // FIXME - add some check here ROFL console.log("INPUT: ", data) @@ -132,6 +163,31 @@ const Admin = (props) => { }); } + const getSchedules = () => { + fetch(globalUrl+"/api/v1/workflows/schedules", { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for apps :O!") + return + } + + return response.json() + }) + .then((responseJson) => { + setSchedules(responseJson) + }) + .catch(error => { + alert.error(error.toString()) + }); + } + const getEnvironments = () => { fetch(globalUrl+"/api/v1/getenvironments", { method: 'GET', @@ -329,6 +385,54 @@ const Admin = (props) => { : null + const schedulesView = curTab === 2 ? +
+

+ Schedules +

+ + + + + + + + {schedules === undefined || schedules === null ? null : schedules.map(schedule => { + return ( + + + + + + + + ) + })} + +
+ : null + const environmentView = curTab === 1 ?

@@ -359,6 +463,8 @@ const Admin = (props) => { const setConfig = (event, newValue) => { if (newValue === 1) { getEnvironments() + } else if (newValue === 2) { + getSchedules() } setModalUser({}) @@ -377,10 +483,12 @@ const Admin = (props) => { > +
{usersView} {environmentView} + {schedulesView}
diff --git a/frontend/src/AngularWorkflow.js b/frontend/src/AngularWorkflow.js index c1701c69..c74ddf19 100644 --- a/frontend/src/AngularWorkflow.js +++ b/frontend/src/AngularWorkflow.js @@ -10,6 +10,8 @@ import Drawer from '@material-ui/core/Drawer'; import Button from '@material-ui/core/Button'; import Paper from '@material-ui/core/Paper'; import Grid from '@material-ui/core/Grid'; +import Tabs from '@material-ui/core/Tabs'; +import Tab from '@material-ui/core/Tab'; import ButtonBase from '@material-ui/core/ButtonBase'; import Tooltip from '@material-ui/core/Tooltip'; import Select from '@material-ui/core/Select'; @@ -60,8 +62,6 @@ import cxtmenu from 'cytoscape-cxtmenu'; import { w3cwebsocket as W3CWebSocket } from "websocket"; import { useAlert } from "react-alert"; -const hoverColor = "#f85a3e" -const hoverOutColor = "#e8eaf6" const surfaceColor = "#27292D" const inputColor = "#383B40" @@ -100,7 +100,7 @@ const AngularWorkflow = (props) => { const [cy, setCy] = React.useState() const [appSearch, setAppSearch] = React.useState("") - const [currentView, setCurrentView] = React.useState("apps") + const [currentView, setCurrentView] = React.useState(0) const [triggerAuthentication, setTriggerAuthentication] = React.useState({}) const [triggerFolders, setTriggerFolders] = React.useState([]) @@ -164,9 +164,6 @@ const AngularWorkflow = (props) => { const [lastSaved, setLastSaved] = React.useState(true) - const [AppsHoverColor, setAppsHoverColor] = useState(hoverOutColor); - const [VariablesHoverColor, setVariablesHoverColor] = useState(hoverOutColor); - const [HookHoverColor, setHookHoverColor] = useState(hoverOutColor); const [appAdded, setAppAdded] = useState(false) const [update, setUpdate] = useState(""); const [workflowExecutions, setWorkflowExecutions] = React.useState([]); @@ -262,7 +259,7 @@ const AngularWorkflow = (props) => { const abortExecution = () => { setExecutionRunning(false) - alert.success("Aborting execution") + alert.info("Aborting execution") fetch(globalUrl+"/api/v1/workflows/"+props.match.params.key+"/executions/"+executionRequest.execution_id+"/abort", { method: 'GET', headers: { @@ -274,7 +271,9 @@ const AngularWorkflow = (props) => { .then((response) => { if (response.status !== 200) { console.log("Status not 200 for WORKFLOW EXECUTION :O!") - } + } else { + alert.success("Execution aborted") + } return response.json() }) @@ -547,68 +546,6 @@ const AngularWorkflow = (props) => { return true } - const executeWorkflowWebsocket = () => { - if (!lastSaved) { - //alert.error("You might have forgotten to save before executing.") - console.log("FIXME: Might have forgotten to save before executing.") - } - - var returncheck = monitorUpdates() - if (!returncheck) { - alert.error("No startnode set.") - return - } - - setVisited([]) - setExecutionRunning(true) - setExecutionRequest({}) - - var curelements = cy.elements() - for (var i = 0; i < curelements.length; i++) { - curelements[i].addClass("not-executing-highlight") - } - - if (executionText.length > 0) { - alert.success("Starting execution with argument "+executionText) - } else { - alert.success("Starting execution") - } - - const data = {"execution_argument": executionText} - fetch(globalUrl+"/api/v1/workflows/"+props.match.params.key+"/execute_fs", { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - }, - credentials: "include", - body: JSON.stringify(data), - }) - .then((response) => { - if (response.status !== 200) { - console.log("Status not 200 for WORKFLOW EXECUTION :O!") - } - - return response.json() - }) - .then((responseJson) => { - if (!responseJson.success) { - alert.error("Failed to start: "+responseJson.reason) - stop() - return - } - - setExecutionRequest({ - "execution_id": responseJson.execution_id, - "authorization": responseJson.authorization, - }) - setExecutingNodes([workflow.start]) - }) - .catch(error => { - alert.error(error.toString()) - }); - } - const executeWorkflow = () => { if (!lastSaved) { //alert.error("You might have forgotten to save before executing.") @@ -639,14 +576,14 @@ const AngularWorkflow = (props) => { const data = {"execution_argument": executionText, "start": workflow.start} fetch(globalUrl+"/api/v1/workflows/"+props.match.params.key+"/execute", { - method: 'POST', + method: 'POST', headers: { 'Content-Type': 'application/json', 'Accept': 'application/json', }, credentials: "include", body: JSON.stringify(data), - }) + }) .then((response) => { if (response.status !== 200) { console.log("Status not 200 for WORKFLOW EXECUTION :O!") @@ -657,7 +594,13 @@ const AngularWorkflow = (props) => { .then((responseJson) => { if (!responseJson.success) { alert.error("Failed to start: "+responseJson.reason) + setExecutionRunning(false) + setExecutionRequestStarted(false) stop() + + for (var i = 0; i < curelements.length; i++) { + curelements[i].removeClass("not-executing-highlight") + } return } else { setExecutionRunning(true) @@ -1197,7 +1140,6 @@ const AngularWorkflow = (props) => { } const onNodeHover = (event) => { - //event.target.style("border-width", "5px") event.target.animate({ style: { "border-width": "5px", @@ -1211,24 +1153,22 @@ const AngularWorkflow = (props) => { } const onEdgeHoverOut = (event) => { - //event.target.removeStyle() + event.target.removeStyle() } // This is here to have a proper transition for lines const onEdgeHover = (event) => { - - //console.log(event.target.data()) - //const sourcecolor = cy.getElementById(event.target.data("source")).style("border-color") - //const targetcolor = cy.getElementById(event.target.data("target")).style("border-color") - //event.target.animate({ - // style: { - // "line-fill": "linear-gradient", - // 'target-arrow-color': targetcolor, - // "line-gradient-stop-colors": [sourcecolor, targetcolor], - // "line-gradient-stop-positions": [0, 1], - // }, - // duration: 0, - //}) + const sourcecolor = cy.getElementById(event.target.data("source")).style("border-color") + const targetcolor = cy.getElementById(event.target.data("target")).style("border-color") + event.target.animate({ + style: { + "line-fill": "linear-gradient", + 'target-arrow-color': targetcolor, + "line-gradient-stop-colors": [sourcecolor, targetcolor], + "line-gradient-stop-positions": [0, 1], + }, + duration: 0, + }) } @@ -1429,16 +1369,17 @@ const AngularWorkflow = (props) => { } const appViewStyle = { - marginLeft: "5px", - marginRight: "5px", + marginLeft: 5, + marginRight: 5, display: "flex", flexDirection: "column", + height: "100%", } const scrollStyle = { - marginTop: "10px", + marginTop: 10, overflow: "scroll", - height: "66vh", + height: "100%", overflowX: "auto", overflowY: "auto", } @@ -1455,37 +1396,6 @@ const AngularWorkflow = (props) => { display: "flex", } - // All this is stupid lmao - const handleHookHover = () => { - setHookHoverColor(hoverColor) - setAppsHoverColor(hoverOutColor) - setVariablesHoverColor(hoverOutColor) - } - - const handleHookHoverOut = () => { - setHookHoverColor(hoverOutColor) - } - - const handleAppsHover = () => { - setAppsHoverColor(hoverColor) - setHookHoverColor(hoverOutColor) - setVariablesHoverColor(hoverOutColor) - } - - const handleAppsHoverOut = () => { - setAppsHoverColor(hoverOutColor) - } - - const handleVariablesHover = () => { - setVariablesHoverColor(hoverColor) - setHookHoverColor(hoverOutColor) - setAppsHoverColor(hoverOutColor) - } - - const handleVariablesHoverOut = () => { - setVariablesHoverColor(hoverOutColor) - } - const paperVariableStyle = { minHeight: "50px", maxHeight: "50px", @@ -1560,6 +1470,7 @@ const AngularWorkflow = (props) => { aria-controls="long-menu" aria-haspopup="true" onClick={menuClick} + style={{color: "white"}} > @@ -1605,57 +1516,77 @@ const AngularWorkflow = (props) => {

) } + + const curTab = 0 + const handleSetTab = (event, newValue) => { + setCurrentView(newValue) + } const HandleLeftView = () => { // Defaults to apps. var thisview = - if (currentView === "triggers") { + if (currentView === 1) { thisview = - } else if (currentView === "variables") { + } else if (currentView === 2) { thisview = } + const tabStyle = { + maxWidth: leftBarSize/3, + minWidth: leftBarSize/3, + flex: 1, + textTransform: "none", + } + + const iconStyle = { + marginTop: 3, + marginRight: 5, + } + return(
- -
+
{thisview}
-
- -
-
{setCurrentView("apps")}}> - - - - + + + - Apps - + -
-
{setCurrentView("triggers")}}> - - - - + + Apps + + + } style={tabStyle} /> + - Triggers - + -
-
{setCurrentView("variables")}}> - - - - + + Triggers + + + } style={tabStyle} /> + - Variables - + -
-
-
+ + Variables + + + }style={tabStyle} /> +
) } @@ -1965,7 +1896,7 @@ const AngularWorkflow = (props) => { node.data.type = "ACTION" node.isStartNode = action["id"] === workflow.start - return node; + return node }) const tmpelements = [].concat(actions) @@ -2001,14 +1932,16 @@ const AngularWorkflow = (props) => { } } - const handleDragStop = (e) => { + const handleDragStop = (e, app) => { newNodeId = "" + console.log("STOP!: ", e) + console.log("APP!: ", app) } const appScrollStyle = { overflow: "scroll", - maxHeight: bodyHeight-appBarSize-150, - minHeight: bodyHeight-appBarSize-150, + maxHeight: bodyHeight-appBarSize-55, + minHeight: bodyHeight-appBarSize-55, overflowY: "auto", overflowX: "hidden", } @@ -2063,7 +1996,8 @@ const AngularWorkflow = (props) => { return( {handleAppDrag(e, app)}} - onStop={(e) => {handleDragStop(e)}} + onStop={(e) => {handleDragStop(e, app)}} + key={app.id} dragging={false} position={{ x: 0, @@ -2347,6 +2281,7 @@ const AngularWorkflow = (props) => { rows="5" color="primary" defaultValue={data.value} + type={placeholder.includes("***") ? "password" : "text"} placeholder={placeholder} onChange={(event) => { changeActionParameter(event, count) @@ -2427,7 +2362,7 @@ const AngularWorkflow = (props) => { } else if (data.variant === "WORKFLOW_VARIABLE") { varcolor = "#f85a3e" if (workflow.workflow_variables === null || workflow.workflow_variables === undefined || workflow.workflow_variables.length === 0) { - setCurrentView("variables") + setCurrentView(2) datafield =
@@ -2477,7 +2412,7 @@ const AngularWorkflow = (props) => {
- {data.name}: + {data.name}
{ @@ -2563,6 +2498,13 @@ const AngularWorkflow = (props) => { //setStartNode(selectedAction.id) } + function sortByKey(array, key) { + return array.sort(function(a, b) { + var x = a[key]; var y = b[key] + return ((x < y) ? -1 : ((x > y) ? 1 : 0)) + }) + } + const appApiView = Object.getOwnPropertyNames(selectedAction).length > 0 && Object.getOwnPropertyNames(selectedApp).length > 0 ?
@@ -2644,7 +2586,7 @@ const AngularWorkflow = (props) => { : null*/}
-
+
Actions
+ {selectedAction.description !== undefined && selectedAction.description.length > 0 ? +
+ {selectedAction.description} +
: null}
@@ -2703,11 +2649,11 @@ const AngularWorkflow = (props) => { } const setTriggerFolderWrapperMulti = event => { - const { options } = event.target; - const value = []; + const { options } = event.target + const value = [] for (let i = 0, l = options.length; i < l; i += 1) { if (options[i].selected) { - value.push(options[i].value); + value.push(options[i].value) } } @@ -2783,7 +2729,7 @@ const AngularWorkflow = (props) => { if (splitItems.includes(value)) { for( var i = 0; i < splitItems.length; i++){ if (splitItems[i] === value) { - splitItems.splice(i, 1); + splitItems.splice(i, 1) } } @@ -2793,7 +2739,7 @@ const AngularWorkflow = (props) => { for( var i = 0; i < splitItems.length; i++){ if (splitItems[i] === "") { - splitItems.splice(i, 1); + splitItems.splice(i, 1) } } @@ -2824,7 +2770,7 @@ const AngularWorkflow = (props) => { } const AppConditionHandler = (props) => { - const { tmpdata, type } = props; + const { tmpdata, type } = props if (tmpdata === undefined) { return tmpdata @@ -2933,7 +2879,7 @@ const AngularWorkflow = (props) => { } else if (data.variant === "WORKFLOW_VARIABLE") { varcolor = "#f85a3e" if (workflow.workflow_variables === null || workflow.workflow_variables === undefined || workflow.workflow_variables.length === 0) { - setCurrentView("variables") + setCurrentView(2) datafield =
@@ -3192,8 +3138,8 @@ const AngularWorkflow = (props) => { const EdgeSidebar = () => { const ConditionHandler = (condition, index) => { - const [open, setOpen] = React.useState(false); - const [anchorEl, setAnchorEl] = React.useState(null); + const [open, setOpen] = React.useState(false) + const [anchorEl, setAnchorEl] = React.useState(null) const deleteCondition = (conditionIndex) => { console.log(selectedEdge) @@ -3223,7 +3169,7 @@ const AngularWorkflow = (props) => { const menuClick = (event) => { console.log("MENU CLICK") setOpen(!open) - setAnchorEl(event.currentTarget); + setAnchorEl(event.currentTarget) } return ( @@ -3363,7 +3309,7 @@ const AngularWorkflow = (props) => { }) .catch(error => { console.log(error.toString()) - }); + }) } const getTriggerAuth = () => { @@ -3384,7 +3330,7 @@ const AngularWorkflow = (props) => { }) .catch(error => { console.log(error.toString()) - }); + }) } // Getting the triggers and the folders if they exist @@ -3426,8 +3372,8 @@ const AngularWorkflow = (props) => { }) .catch(error => { console.log(error.toString()) - }); - }, 2500); + }) + }, 2500) console.log(data) saveWorkflow(workflow) @@ -3801,7 +3747,7 @@ const AngularWorkflow = (props) => { }) .catch(error => { alert.error(error.toString()) - }); + }) } const startMailSub = (trigger, triggerindex) => { @@ -3857,7 +3803,7 @@ const AngularWorkflow = (props) => { }) .catch(error => { alert.error(error.toString()) - }); + }) } const newWebhook = (trigger) => { @@ -3903,7 +3849,7 @@ const AngularWorkflow = (props) => { }) .catch(error => { console.log(error.toString()) - }); + }) } const deleteWebhook = (trigger, triggerindex) => { @@ -3942,7 +3888,7 @@ const AngularWorkflow = (props) => { }) .catch(error => { alert.error(error.toString()) - }); + }) } const UserinputSidebar = () => { @@ -4356,7 +4302,7 @@ const AngularWorkflow = (props) => { } if (Object.getOwnPropertyNames(selectedAction).length > 0 && Object.getOwnPropertyNames(selectedApp).length > 0) { - //console.time('ACTIONSTART'); + //console.time('ACTIONSTART') return(
{appApiView} @@ -4479,11 +4425,13 @@ const AngularWorkflow = (props) => {
{timestamp}
- -
- {resultsLength}/{data.workflow.actions.length} -
- + {data.workflow.actions !== null ? + +
+ {resultsLength}/{data.workflow.actions.length} +
+ + : null}
@@ -4545,7 +4493,7 @@ const AngularWorkflow = (props) => { executionData.results.map(data => { var showResult = data.result.trim() showResult.split(" None").join(" \"None\"") - //showResult = replaceAll(showResult, " None", " \"None\""); + //showResult = replaceAll(showResult, " None", " \"None\"") var jsonvalid = true try { JSON.parse(showResult) @@ -4569,7 +4517,7 @@ const AngularWorkflow = (props) => { {jsonvalid ? @@ -4600,6 +4548,9 @@ const AngularWorkflow = (props) => { boxSelectionEnabled={true} autounselectify={false} cy={(incy) => { + // FIXME: There's something specific loading when + // you do the first hover of a node. Why is this different? + console.log("CY: ", incy) setCy(incy) }} /> @@ -4608,7 +4559,6 @@ const AngularWorkflow = (props) => { -
:
@@ -4872,4 +4822,4 @@ const AngularWorkflow = (props) => { ) } -export default AngularWorkflow; +export default AngularWorkflow diff --git a/frontend/src/App.js b/frontend/src/App.js index e40d7660..af034001 100644 --- a/frontend/src/App.js +++ b/frontend/src/App.js @@ -147,13 +147,13 @@ const App = (message, props) => { // This is a mess hahahah return ( - - - - {includedData} - - - + + + + {includedData} + + + ); }; diff --git a/frontend/src/AppCreator.js b/frontend/src/AppCreator.js index 7925246a..bd04dc60 100644 --- a/frontend/src/AppCreator.js +++ b/frontend/src/AppCreator.js @@ -4,11 +4,13 @@ import {BrowserView, MobileView} from "react-device-detect"; import {Link} from 'react-router-dom'; import Paper from '@material-ui/core/Paper'; +import FormControlLabel from '@material-ui/core/FormControlLabel'; import Button from '@material-ui/core/Button'; import Divider from '@material-ui/core/Divider'; import Select from '@material-ui/core/Select'; import MenuItem from '@material-ui/core/MenuItem'; import FormControl from '@material-ui/core/FormControl'; +import Switch from '@material-ui/core/Switch'; import Dialog from '@material-ui/core/Dialog'; import DialogTitle from '@material-ui/core/DialogTitle'; import DialogContent from '@material-ui/core/DialogContent'; @@ -65,7 +67,7 @@ const useStyles = makeStyles({ const rewrite = (args) => { return args.reduce(function(args, a){ - if (0 == a.indexOf('-X')) { + if (0 === a.indexOf('-X')) { args.push('-X') args.push(a.slice(2)) } else { @@ -103,35 +105,35 @@ const parseCurl = (s) => { out.url = arg break; - case arg == '-A' || arg == '--user-agent': + case arg === '-A' || arg === '--user-agent': state = 'user-agent' break; - case arg == '-H' || arg == '--header': + case arg === '-H' || arg === '--header': state = 'header' break; - case arg == '-d' || arg == '--data' || arg == '--data-ascii': + case arg === '-d' || arg === '--data' || arg === '--data-ascii': state = 'data' break; - case arg == '-u' || arg == '--user': + case arg === '-u' || arg === '--user': state = 'user' break; - case arg == '-I' || arg == '--head': + case arg === '-I' || arg === '--head': out.method = 'HEAD' break; - case arg == '-X' || arg == '--request': + case arg === '-X' || arg === '--request': state = 'method' break; - case arg == '-b' || arg =='--cookie': + case arg === '-b' || arg === '--cookie': state = 'cookie' break; - case arg == '--compressed': + case arg === '--compressed': out.header['Accept-Encoding'] = out.header['Accept-Encoding'] || 'deflate, gzip' break; @@ -147,7 +149,7 @@ const parseCurl = (s) => { state = '' break; case 'data': - if (out.method == 'GET' || out.method == 'HEAD') out.method = 'POST' + if (out.method === 'GET' || out.method === 'HEAD') out.method = 'POST' out.header['Content-Type'] = out.header['Content-Type'] || 'application/x-www-form-urlencoded' out.body = out.body ? out.body + '&' + arg @@ -196,6 +198,7 @@ const AppCreator = (props) => { const [updater, setUpdater] = useState("tmp") const [baseUrl, setBaseUrl] = useState(""); const [actionsModalOpen, setActionsModalOpen] = useState(false); + const [authenticationRequired, setAuthenticationRequired] = useState(false); const [authenticationOption, setAuthenticationOption] = useState(authenticationOptions[0]); const [parameterName, setParameterName] = useState(""); const [parameterLocation, setParameterLocation] = useState(apikeySelection.length > 0 ? apikeySelection[0] : ""); @@ -372,30 +375,6 @@ const AppCreator = (props) => { securitySchemes = data.components.securitySchemes } - // FIXME: Have multiple authentication options? - if (securitySchemes !== undefined) { - for (const [key, value] of Object.entries(securitySchemes)) { - if (value.scheme === "bearer") { - setAuthenticationOption("Bearer auth") - break - } else if (value.type === "apiKey") { - setAuthenticationOption("API key") - - value.in = value.in.charAt(0).toUpperCase() + value.in.slice(1); - setParameterLocation(value.in) - if (!apikeySelection.includes(value.in)) { - console.log("APIKEY SELECT: ", apikeySelection) - alert.error("Might be error in setting up API key authentication") - } - - setParameterName(value.name) - break - } else if (value.scheme === "basic") { - setAuthenticationOption("Basic auth") - break - } - } - } console.log(data) @@ -451,9 +430,39 @@ const AppCreator = (props) => { } } + + // FIXME: Have multiple authentication options? + if (securitySchemes !== undefined) { + for (const [key, value] of Object.entries(securitySchemes)) { + if (value.scheme === "bearer") { + setAuthenticationOption("Bearer auth") + setAuthenticationRequired(true) + break + } else if (value.type === "apiKey") { + setAuthenticationOption("API key") + + value.in = value.in.charAt(0).toUpperCase() + value.in.slice(1); + setParameterLocation(value.in) + if (!apikeySelection.includes(value.in)) { + console.log("APIKEY SELECT: ", apikeySelection) + alert.error("Might be error in setting up API key authentication") + } + + console.log("PARAM NAME: ", value.name) + setParameterName(value.name) + setAuthenticationRequired(true) + break + } else if (value.scheme === "basic") { + setAuthenticationOption("Basic auth") + setAuthenticationRequired(true) + break + } + } + } + setActions(newActions) } - + // Saving the app that's been configured. const submitApp = () => { alert.info("Uploading and building app " + name) @@ -605,12 +614,25 @@ const AppCreator = (props) => { const headersSplit = item.headers.split("\n") for (var key in headersSplit) { const header = headersSplit[key] + console.log("HEADER: ", header) var key = "" var value = "" - if (header.length > 0 && header.includes("=")) { + if (header.length > 0 && header.includes("= ")) { + const headersplit = header.split("= ") + key = headersplit[0] + value = headersplit[1] + } else if (header.length > 0 && header.includes("=")) { const headersplit = header.split("=") key = headersplit[0] value = headersplit[1] + } else if (header.length > 0 && header.includes(": ")) { + const headersplit = header.split(": ") + key = headersplit[0] + value = headersplit[1] + } else if (header.length > 0 && header.includes(":")) { + const headersplit = header.split(":") + key = headersplit[0] + value = headersplit[1] } else { continue } @@ -701,7 +723,6 @@ const AppCreator = (props) => { Users will be required to submit their API as the header "Authorization: Bearer APIKEY" -
: null @@ -714,7 +735,6 @@ const AppCreator = (props) => { Users will be required to submit a valid username and password before using the API -
: null @@ -810,7 +830,7 @@ const AppCreator = (props) => { id="standard-required" margin="normal" variant="outlined" - defaultValue={parameterName} + value={parameterName} helperText={
Can't be empty. Can't contain any of the following characters: !#$%&'^+-._~|]+$
} onChange={e => setParameterName(e.target.value)} InputProps={{ @@ -847,7 +867,6 @@ const AppCreator = (props) => { )} )} -
: null @@ -926,11 +945,13 @@ const AppCreator = (props) => { {data.method} - {url} - {data.name}
+ {/*
{testAction(index)}}> Test
+ */}
{deleteAction(index)}}> Delete @@ -999,7 +1020,7 @@ const AppCreator = (props) => { // Url verification if (currentAction.url.length === 0) { errormessage.push("URL path can't be empty.") - } else if (!currentAction.url.startsWith("/")) { + } else if (!currentAction.url.startsWith("/") && baseUrl.length > 0) { errormessage.push("URL must start with /") } @@ -1261,6 +1282,14 @@ const AppCreator = (props) => { if (request.header !== undefined && request.header !== null) { var headers = [] for (let [key, value] of Object.entries(request.header)) { + if (parameterName !== undefined && key.toLowerCase() === parameterName.toLowerCase()) { + continue + } + + if (key === "Authorization" && authenticationOption === "Bearer auth") { + continue + } + headers += key+"="+value+"\n" } @@ -1272,33 +1301,37 @@ const AppCreator = (props) => { } // Parse URL - parsedurl = request.url + if (request.url !== undefined) { + parsedurl = request.url + } } - if (parsedurl.includes("<") && parsedurl.includes(">")) { - parsedurl = parsedurl.split("<").join("{") - parsedurl = parsedurl.split(">").join("}") - } - - if (parsedurl.startsWith("http") || parsedurl.startsWith("ftp")) { - if (parsedurl !== undefined && parsedurl.includes(parameterName)) { - // Remove <> etc. - // - - console.log("IT HAS THE PARAM NAME!") - const newurl = new URL(encodeURI(parsedurl)) - newurl.searchParams.delete(parameterName) - parsedurl = decodeURI(newurl.href) + if (parsedurl !== undefined) { + if (parsedurl.includes("<") && parsedurl.includes(">")) { + parsedurl = parsedurl.split("<").join("{") + parsedurl = parsedurl.split(">").join("}") } - // Remove the base URL itself - if (parsedurl !== undefined && baseUrl !== undefined && baseUrl.length > 0 && parsedurl.includes(baseUrl)) { - parsedurl = parsedurl.replace(baseUrl, "") - } + if (parsedurl.startsWith("http") || parsedurl.startsWith("ftp")) { + if (parsedurl !== undefined && parsedurl.includes(parameterName)) { + // Remove <> etc. + // + + console.log("IT HAS THE PARAM NAME!") + const newurl = new URL(encodeURI(parsedurl)) + newurl.searchParams.delete(parameterName) + parsedurl = decodeURI(newurl.href) + } - // Check URL query && headers - setActionField("url", parsedurl) - setUrlPath(parsedurl) + // Remove the base URL itself + if (parsedurl !== undefined && baseUrl !== undefined && baseUrl.length > 0 && parsedurl.includes(baseUrl)) { + parsedurl = parsedurl.replace(baseUrl, "") + } + + // Check URL query && headers + setActionField("url", parsedurl) + setUrlPath(parsedurl) + } } //console.log("URL: ", request.url) @@ -1550,11 +1583,17 @@ const AppCreator = (props) => { }} /> -
Authentication
+
Authentication +
upload = ref} onChange={importFiles} />
@@ -962,10 +983,160 @@ const Workflows = (props) => {
+ + const importWorkflowsFromUrl = (url) => { + console.log("IMPORT WORKFLOWS FROM ", downloadUrl) + + const parsedData = { + "url": url, + } + + if (field1.length > 0) { + parsedData["field_1"] = field1 + } + + if (field2.length > 0) { + parsedData["field_2"] = field2 + } + + alert.success("Getting specific workflows from your URL.") + var cors = "cors" + fetch(globalUrl+"/api/v1/workflows/download_remote", { + method: "POST", + mode: "cors", + headers: { + 'Accept': 'application/json', + }, + body: JSON.stringify(parsedData), + credentials: "include", + }) + .then((response) => { + if (response.status === 200) { + response.text().then(function (text) { + console.log("RETURN: ", text) + alert.success("Loaded existing apps!") + }) + } + + return response.json() + }) + .then((responseJson) => { + console.log("DATA: ", responseJson) + if (responseJson.reason !== undefined) { + alert.error("Failed loading: "+responseJson.reason) + } else { + alert.error("Failed loading") + } + }) + .catch(error => { + alert.error(error.toString()) + }) + } + + const handleGithubValidation = () => { + importWorkflowsFromUrl(downloadUrl) + setLoadWorkflowsModalOpen(false) + } + + const workflowDownloadModalOpen = loadWorkflowsModalOpen ? + { + }} + PaperProps={{ + style: { + backgroundColor: surfaceColor, + color: "white", + minWidth: "800px", + minHeight: "320px", + }, + }} + > + +
+ Load workflows from github repo +
+ + + +
+
+ + + Repository (supported: github, gitlab, bitbucket) + setDownloadUrl(e.target.value)} + placeholder="https://github.com/frikky/shuffle-apps" + fullWidth + /> + + Authentication (optional - private repos etc): +
+ setField1(e.target.value)} + type="username" + placeholder="Username / APIkey (optional)" + fullWidth + /> + setField2(e.target.value)} + type="password" + placeholder="Password (optional)" + fullWidth + /> +
+ + + + + + + : null + const loadedCheck = isLoaded && isLoggedIn && workflowDone ?
{workflowView} {modalView} + {workflowDownloadModalOpen}
:
diff --git a/frontend/src/assets/img/logo.png b/frontend/src/assets/img/logo.png new file mode 100644 index 00000000..c6826517 Binary files /dev/null and b/frontend/src/assets/img/logo.png differ diff --git a/frontend/src/assets/img/logo.svg b/frontend/src/assets/img/logo.svg new file mode 100644 index 00000000..3c516967 --- /dev/null +++ b/frontend/src/assets/img/logo.svg @@ -0,0 +1,26 @@ + + + + + + + + + + + + + background + + + + Layer 1 + + + + + + + + + \ No newline at end of file diff --git a/frontend/src/assets/img/logo.webp b/frontend/src/assets/img/logo.webp new file mode 100644 index 00000000..d1dadedd Binary files /dev/null and b/frontend/src/assets/img/logo.webp differ diff --git a/shuffle_adminaccount.png b/frontend/src/assets/img/shuffle_adminaccount.png similarity index 100% rename from shuffle_adminaccount.png rename to frontend/src/assets/img/shuffle_adminaccount.png diff --git a/shuffle_webhook.png b/frontend/src/assets/img/shuffle_webhook.png similarity index 100% rename from shuffle_webhook.png rename to frontend/src/assets/img/shuffle_webhook.png diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index cbd49b83..4c65fbd2 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -106,9 +106,9 @@ func deployWorker(cli *dockerclient.Client, image string, identifier string, env err = cli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{}) if err != nil { - log.Printf("Failed to start container: %s", err) + log.Printf("Failed to start container in environment %s: %s", environment, err) } else { - log.Printf("Container %s is created", cont.ID) + log.Printf("Container %s was created under environment %s", cont.ID, environment) } return nil } @@ -401,7 +401,7 @@ func main() { //log.Println(string(body)) //log.Println(resultResp) if len(toBeRemoved.Data) == len(executionRequests.Data) { - log.Println("Should remove ALL!") + //log.Println("Should remove ALL!") } else { log.Printf("NOT IMPLEMENTED: Should remove %d workflows from backend because they're executed!", len(toBeRemoved.Data)) } diff --git a/setup.sh b/setup.sh deleted file mode 100644 index 6ceb1699..00000000 --- a/setup.sh +++ /dev/null @@ -1,33 +0,0 @@ -# Build script to make it work as an open source platform - -# 1. Grab builtin functions - Done in backend as a button click -# 2. Upload to database & build docker images -# 3. Run docker-compose: frontend, backend, database & orborus -# 4. Set up docker swarm for apps? - -# Basic overview of how it works: -# -# Backend: o -# | -# Orborus: o -# / \ -# Workers: o o -# / / \ -# Apps: o o o - -# 1. Grab builtin functions -# Where should I have these? Maybe OpenAPI github repo and just preload? - -echo "Building frontend" -cd frontend -npm run build -rm -rf ../backend/go-app/build -cp -r build/ ../backend/go-app/build - -echo "Setting up backend" -cd ../backend/go-app -go build -go test - -#gcloud app deploy $GOPATH/src/github.com/frikky/shuffle/app.yaml -echo "This script is not finished. Please follow the docker installation guide here: https://github.com/frikky/shuffle/blob/master/install-guide.md"