diff --git a/README.md b/README.md index 04fefd6e..7262a756 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,16 @@ **It's in BETA** - [Get in touch](https://shuffler.io/contact), send a mail to [frikky@shuffler.io](mailto:frikky@shuffler.io) or poke me on twitter [@frikkylikeme](https://twitter.com/frikkylikeme) -![Example Shuffle webhook integration](shuffle_webhook.png) +![Example Shuffle webhook integration](https://github.com/frikky/Shuffle/blob/master/frontend/src/assets/img/shuffle_webhook.png) -## Try it -* Self-hosted: Check out the [installation guide](https://github.com/frikky/shuffle/blob/master/install-guide.md). +## Getting started +* Self-hosted: Check out the [installation guide](https://github.com/frikky/shuffle/blob/master/install-guide.md) and [getting started](https://shuffler.io/docs/getting_started) * Cloud: Register at https://shuffler.io/register and get cooking (there are some differences!) +## Blogposts +[1. Introducing Shuffle](https://medium.com/security-operation-capybara/introducing-shuffle-an-open-source-soar-platform-part-1-58a529de7d12) +[2. Getting started with Shuffle](https://medium.com/security-operation-capybara/getting-started-with-shuffle-an-open-source-soar-platform-part-2-1d7c67a64244) + ## Related repositories * Apps: https://github.com/frikky/shuffle-apps * Workflows: https://github.com/frikky/shuffle-workflows (empty) diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index e995afea..48711b1a 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -242,6 +242,7 @@ func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) { return appPath, nil } +// This function generates the python code that's being used. func makePythoncode(swagger *openapi3.Swagger, name, url, method string, parameters, optionalQueries []string) (string, string) { method = strings.ToLower(method) queryString := "" @@ -259,9 +260,6 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet } } - // How to add authentication? - // I think it should be like: - // async def(self, auth, baseurl, data): // api.Authentication.Parameters[0].Value = "BearerAuth" authenticationParameter := "" authenticationSetup := "" @@ -270,14 +268,15 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet if swagger.Components.SecuritySchemes != nil { if swagger.Components.SecuritySchemes["BearerAuth"] != nil { authenticationParameter = ", apikey" - authenticationSetup = "headers[\"Authorization\"] = f\"Bearer {apikey}\"" + authenticationSetup = "if apikey != \" \": headers[\"Authorization\"] = f\"Bearer {apikey}\"" } else if swagger.Components.SecuritySchemes["BasicAuth"] != nil { authenticationParameter = ", username, password" authenticationAddin = ", auth=(username, password)" } else if swagger.Components.SecuritySchemes["ApiKeyAuth"] != nil { authenticationParameter = ", apikey" if swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.In == "header" { - authenticationSetup = fmt.Sprintf("headers[\"%s\"] = apikey", swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name) + // This is a way to bypass apikeys by passing " " + authenticationSetup = fmt.Sprintf(`if apikey != " ": headers["%s"] = apikey`, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name) } else if swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.In == "query" { // This might suck lol key := "?" @@ -285,7 +284,7 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet key = "&" } - authenticationSetup = fmt.Sprintf("url+=f\"%s%s={apikey}\"", key, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name) + authenticationSetup = fmt.Sprintf("if apikey != \" \": url+=f\"%s%s={apikey}\"", key, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name) } } } @@ -301,6 +300,23 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet urlInline = "{url}" } + // Specific check for SSL verification + // This is critical for onprem stuff. + verifyParam := "" + verifyWrapper := "" + verifyAddin := "" + if len(swagger.Servers) == 0 { + verifyParam = ", verify=True" + verifyWrapper = `if type(ssl_verify) == str: ssl_verify = False if ssl_verify.lower() == "false" or ssl_verify == "0" else True` + verifyAddin = ", verify=ssl_verify" + } else { + if swagger.Servers[0].URL == "" { + verifyParam = ", ssl_verify=True" + verifyWrapper = `if type(ssl_verify) == str: ssl_verify = False if ssl_verify.lower() == "false" or ssl_verify == "0" else True` + verifyAddin = ", verify=ssl_verify" + } + } + if len(parameters) > 0 { parameterData = fmt.Sprintf(", %s", strings.Join(parameters, ", ")) } @@ -335,13 +351,14 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet // Extra param for url if it's changeable // Extra param for authentication scheme(s) - data := fmt.Sprintf(` async def %s(self%s%s%s%s%s): + data := fmt.Sprintf(` async def %s(self%s%s%s%s%s%s): headers={} url=f"%s%s" %s + %s %s %s - return requests.%s(url, headers=headers%s%s).text + return requests.%s(url, headers=headers%s%s%s).text `, functionname, authenticationParameter, @@ -349,16 +366,21 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet parameterData, queryString, bodyParameter, + verifyParam, urlInline, url, + verifyWrapper, authenticationSetup, queryData, bodyFormatter, method, authenticationAddin, bodyAddin, + verifyAddin, ) + log.Printf("CODE: %s", data) + //log.Println(data) //log.Println(functionname) return functionname, data @@ -412,7 +434,6 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, //log.Printf("%s", j) api.SmallImage = string(j) api.LargeImage = string(j) - log.Printf("Set images!") } } @@ -444,6 +465,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, Description: "The apikey to use", Multiline: false, Required: true, + Example: "The API key to use. Space = skip", Schema: SchemaDefinition{ Type: "string", }, @@ -460,6 +482,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, Description: "The apikey to use", Multiline: false, Required: true, + Example: "The API key to use. Space = skip", Schema: SchemaDefinition{ Type: "string", }, @@ -475,6 +498,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, Description: "The username to use", Multiline: false, Required: true, + Example: "The username to use", Schema: SchemaDefinition{ Type: "string", }, @@ -484,6 +508,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, Description: "The password to use", Multiline: false, Required: true, + Example: "The password to use", Schema: SchemaDefinition{ Type: "string", }, @@ -759,6 +784,32 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} + if len(swagger.Servers) == 0 { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } else { + if swagger.Servers[0].URL == "" { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } + } + if len(path.Connect.Parameters) > 0 { for _, param := range path.Connect.Parameters { if param.Value.Schema == nil || param.Value.In == "header" { @@ -799,7 +850,6 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ } if param.Value.In == "path" { - //log.Printf("PATH!: %s", param.Value.Name) parameters = append(parameters, param.Value.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { @@ -866,8 +916,33 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor // FIXME - remove this when authentication is properly introduced parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} + if len(swagger.Servers) == 0 { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify the SSL certificate request", + Multiline: false, + Required: false, + Example: "False - default=True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } else { + if swagger.Servers[0].URL == "" { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } + } + if len(path.Get.Parameters) > 0 { for _, param := range path.Get.Parameters { if param.Value.Schema == nil || param.Value.In == "header" { @@ -909,7 +984,6 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor } if param.Value.In == "path" { - log.Printf("PATH!: %s", param.Value.Name) parameters = append(parameters, param.Value.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { @@ -976,6 +1050,31 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} + if len(swagger.Servers) == 0 { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } else { + if swagger.Servers[0].URL == "" { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } + } if len(path.Head.Parameters) > 0 { for _, param := range path.Head.Parameters { if param.Value.Schema == nil || param.Value.In == "header" { @@ -1016,7 +1115,6 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo } if param.Value.In == "path" { - //log.Printf("PATH!: %s", param.Value.Name) parameters = append(parameters, param.Value.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { @@ -1082,6 +1180,31 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} + if len(swagger.Servers) == 0 { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } else { + if swagger.Servers[0].URL == "" { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } + } if len(path.Delete.Parameters) > 0 { for _, param := range path.Delete.Parameters { if param.Value.Schema == nil || param.Value.In == "header" { @@ -1122,7 +1245,6 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] } if param.Value.In == "path" { - //log.Printf("PATH!: %s", param.Value.Name) parameters = append(parameters, param.Value.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { @@ -1178,9 +1300,9 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo Parameters: extraParameters, } - if path.Post.RequestBody != nil { - log.Printf("RequestBody: %#v", path.Post.RequestBody) - } + //if path.Post.RequestBody != nil { + // log.Printf("RequestBody: %#v", path.Post.RequestBody) + //} action.Returns.Schema.Type = "string" baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) @@ -1191,6 +1313,31 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} + if len(swagger.Servers) == 0 { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } else { + if swagger.Servers[0].URL == "" { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } + } if len(path.Post.Parameters) > 0 { for _, param := range path.Post.Parameters { @@ -1233,7 +1380,6 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo } if param.Value.In == "path" { - //log.Printf("PATH!: %s", param.Value.Name) parameters = append(parameters, param.Value.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { @@ -1299,6 +1445,31 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} + if len(swagger.Servers) == 0 { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } else { + if swagger.Servers[0].URL == "" { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } + } if len(path.Patch.Parameters) > 0 { for _, param := range path.Patch.Parameters { if param.Value.Schema == nil || param.Value.In == "header" { @@ -1339,7 +1510,6 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W } if param.Value.In == "path" { - //log.Printf("PATH!: %s", param.Value.Name) parameters = append(parameters, param.Value.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { @@ -1405,6 +1575,31 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} + if len(swagger.Servers) == 0 { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } else { + if swagger.Servers[0].URL == "" { + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } + } if len(path.Put.Parameters) > 0 { for _, param := range path.Put.Parameters { @@ -1446,7 +1641,6 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor } if param.Value.In == "path" { - //log.Printf("PATH!: %s", param.Value.Name) parameters = append(parameters, param.Value.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 0bfeeda9..29f3cf9a 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -238,7 +238,7 @@ type AppInfo struct { type ScheduleOld struct { Id string `json:"id" datastore:"id"` Seconds int `json:"seconds" datastore:"seconds"` - WorkflowId string `json:"workflow_id datastore:"workflow_id", ` + WorkflowId string `json:"workflow_id" datastore:"workflow_id", ` Argument string `json:"argument" datastore:"argument"` AppInfo AppInfo `json:"appinfo" datastore:"appinfo,noindex"` Finished bool `json:"finished" finished:"id"` @@ -1803,6 +1803,49 @@ func getUserCount() (int, error) { return count, nil } +func handleGetSchedules(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("Api authentication failed in set new workflowhandler: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if user.Role != "admin" { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Admin required"}`)) + return + } + + ctx := context.Background() + schedules, err := getAllSchedules(ctx) + if err != nil { + log.Printf("Failed getting schedules: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Couldn't get schedules"}`)) + return + } + + newjson, err := json.Marshal(schedules) + if err != nil { + log.Printf("Failed unmarshal: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking environments"}`))) + return + } + + //log.Printf("Existing environments: %s", string(newjson)) + + resp.WriteHeader(200) + resp.Write(newjson) +} + func handleGetEnvironments(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -2635,6 +2678,21 @@ func handleDeleteSchedule(resp http.ResponseWriter, request *http.Request) { return } + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("Api authentication failed in set new workflowhandler: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // FIXME: IAM - Get workflow and check owner + if user.Role != "admin" { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Admin required"}`)) + return + } + location := strings.Split(request.URL.String(), "/") var workflowId string @@ -2655,7 +2713,7 @@ func handleDeleteSchedule(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - err := DeleteKey(ctx, "schedules", workflowId) + err = DeleteKey(ctx, "schedules", workflowId) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "Can't delete"}`)) @@ -4648,7 +4706,7 @@ func handleDeleteOutlookSub(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := getWorkflow(ctx, workflowId) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (delete outlook): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -4782,7 +4840,7 @@ func createOutlookSub(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := getWorkflow(ctx, workflowId) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (outlook sub): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -5765,7 +5823,7 @@ func runInit(ctx context.Context) { _, _, err := handleExecution(schedule.WorkflowId, Workflow{}, request) if err != nil { - log.Printf("Failed to execute: %s", err) + log.Printf("Failed to execute %s: %s", schedule.WorkflowId, err) } } @@ -5825,7 +5883,7 @@ func runInit(ctx context.Context) { } log.Printf("Downloading OpenAPI data for search - EXTRA APPS") - apis := "https://github.com/frikky/OpenAPI-security-definitions" + apis := "https://github.com/frikky/security-openapis" // THis gets memory problems hahah //apis := "https://github.com/APIs-guru/openapi-directory" @@ -5897,6 +5955,7 @@ func init() { // App specific r.HandleFunc("/api/v1/apps/get_existing", loadSpecificApps).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/apps/download_remote", loadSpecificApps).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/validate", validateAppInput).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}", deleteWorkflowApp).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}/config", getWorkflowAppConfig).Methods("GET", "OPTIONS") @@ -5914,6 +5973,8 @@ func init() { /* Everything below here increases the counters*/ r.HandleFunc("/api/v1/workflows", getWorkflows).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows", setNewWorkflow).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/workflows/schedules", handleGetSchedules).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/workflows/download_remote", loadSpecificWorkflows).Methods("POST", "OPTIONS") //r.HandleFunc("/api/v1/workflows/{key}/execute_fs", executeWorkflowFS) r.HandleFunc("/api/v1/workflows/{key}/execute", executeWorkflow).Methods("GET", "POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/schedule", scheduleWorkflow).Methods("POST", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index b9fde0ba..2f3b6391 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -428,7 +428,7 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, frequency _, _, err := handleExecution(workflowId, Workflow{}, request) if err != nil { - log.Printf("Failed to execute: %s", err) + log.Printf("Failed to execute %s: %s", workflowId, err) } } @@ -966,18 +966,10 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { workflow.Sharing = "private" ctx := context.Background() - err = setWorkflow(ctx, workflow, workflow.ID) - if err != nil { - log.Printf("Failed setting workflow: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - log.Printf("Saved new workflow %s with name %s", workflow.ID, workflow.Name) err = increaseStatisticsField(ctx, "total_workflows", workflow.ID, 1) if err != nil { - log.Printf("Failed to increase total workflows: %s", err) + log.Printf("Failed to increase total workflows stats: %s", err) } if len(workflow.Actions) == 0 { @@ -1003,6 +995,37 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { newActions = append(newActions, action) } + // Initialized without functions = adding a hello world node. + if len(newActions) == 0 { + log.Printf("APPENDING NEW APP FOR NEW WORKFLOW") + //nodeId := "40447f30-fa44-4a4f-a133-4ee710368737" + //workflow.Start = nodeId + //newActions = append(newActions, Action{ + // Label: "Start node", + // Name: "hello_world", + // AppName: "testing", + // Environment: "Shuffle", + // Parameters: []WorkflowAppActionParameter{}, + // Position: struct { + // X float64 "json:\"x\" datastore:\"x\"" + // Y float64 "json:\"y\" datastore:\"y\"" + // }{X: 449.5, Y: 446}, + // Priority: 0, + // AppVersion: "1.0.0", + // AppID: "c567fc10-9c15-403e-b72c-6550e9e76bc8", + // Errors: []string{}, + // ID: nodeId, + // IsValid: true, + // IsStartNode: true, + // Sharing: true, + // PrivateID: "", + // SmallImage: "", + // LargeImage: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABkCAIAAAD/gAIDAAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAABmJLR0QA/wD/AP+gvaeTAAAAB3RJTUUH4wgeDy4zYzmH5gAADkRJREFUeNrtXV1QG9cV3nt3V2AwkvgRrRE4nTFxMFKATGyLh1gONG8Rxn2qIQ8GOSYdfpy4rknATacPBRx7ak+d2K0dkDWTYvutMRB3nKllYZLaQD2B8GcPJBNwpIxBIPSDEbt39/ZhG+qglZDEjwT4e9Q9e38+3Xv2nHOPjgDGmHiGwADDPYG1hGdkBYFnZAWBZ2QFASrcEyAIgsAYcxyHEGIYhmVZnucJgoAQ0jQtkUgoiiJJEgAQ7mmGiSyEkMPhsFqtIyMjVqvVYvneYrFMTNgYhkHo/2RRFC2RSBSKJKVSqVSmpqRsSU9P37IlRS6XU1QYZr6qQ7pcrpGRke7urp6env7+fpvN5nQ6WBYBQEAICYLw3j6CZcPzPMYETVMymSwhIVGtfjErKys3V7NtW7pUKl21+YNVsLM8Hs/Q0JDJZOrouPPgwZDT6eR5niRJCGGwhwtjzPM8x3EQgrg4WUZGhlarzc/P37EjMzo6em2TNTU1ZTbfvn79emdnp9PpIAhiGbWPoOkIgpBKZbt37y4sLHz11bzExMS1R5bNZmtra7t69crg4ABCiKKoldPQGGOEEEmSKpW6qKhYp9MlJSWtDbLcbveNGzcMhqa+vq8xxqupiRFCAAC1+kW9Xv/667rNmzdHLlk8z//nP93nz583m80sy4TlhUUQBEKIpmmtdm9FReWuXbtIkow4sux2u9F4uampcXJykqYpggizWcSybHx8gl5/SK/XJyQkLEufy0NWX19fQ0N9e7sZACAYAZEA4Q2g1e6tqanNzs5eeodLJQshdOPGjfr6P42OjtI0HW5+RMCybFra1traWp2uYImaYUlkeTyexsbGc+f+MjPjXkbVsOzgOC42Nraq6sibbx7etGlTyP2ETpbL5T59+gOj0cDzOHKOni/wPA8hLCkpPX68Oi4uLrROQiTL6XTW1dU1N38CAIgEFzcQYIwxJoqL3zhx4vcyWShOUig7wu1219XVNTf/HcI1wxRBEAAACMGVK3+vq/uTy+UKoYegyfJ4PKdOfdDc/Ing+YabgeAXDOGVK82nT5/yeDxBPxuUNM/zjY0fG40GAMBaZEoAAITRePnSpYtCLChwBEdWa2vruXPneB6vodPnDQAAz/MfffTh9evXg3swcAXf19d3+PCbjx6NRbKVEDg4jlMqUy9d+jgnJyfARwLdWXb7VEND/ejod+uDKYIgSJJ89Gjs5MmGqampAB8JiCyMsdFobG83R6aNHjJomr5zp91gaApQeQVEVldXV1NT45rWU75AkqTB0NTZeS8Q4cXJcrvdFy6cn5ycjHwzPQQAAOx2+4ULF9xu96LCi6//xo3PzGYzTUfEpdlKgKbp9nZzS0vLopKLkGWz2QwGA8sya9eqCgQIIaPx8sTEuH+xRchqa2vr6/s6XDHPVQNFUQMD/a2trf7F/JE1OTl59eqVDZKThDG+du2azWbzI+OPLLPZPDg4sO63lQCKogYHB0wmkx8Zn2R5PJ6Wlk8RQuFexeqB47iWluuzs7O+BHySNTQ01NnZtUG2lQCKorq7uwcHB30J+CTLZDI5nY51aYj6AgDA6XSYTLd8CYiT5XK5OjruhHvyYQAAoKOjw+l0iraKkzUyMvLgwdC68ZkDB0mSDx8+HB4eFm0VJ6u7u8vpdG6oMyhAOIn37om7iiJkIYR6enqCjSKuG2CMe3t7WJb1bhIhy+Fw9Pf3b8AzKIAkyYGBgenpae8mEbKsVqvNZluXMYZAACG026csFotIk/dHIyMjG81oeBoAAIfD8c0333g3idicVqsVISSRSIIdBgvXmF5ji2aKBig5D57nMcZ+Ek9E+wykZ29wHGe1WgMiy2L5PliaCILgeS4uTuad2DkzMzM7++Snc8VSqVQiiVogyTCMyyVi4HAcR9P0z3++RSKRzM3NjY8/5jhuAWUY402bNsXGxhGEyHfAMHO+TCdfsFpFSFhIFkLIYrEEq7AQQhkZO+rq6lJSlDzPz6cdkyR55syfr127+nTwHmPi2LHjr7322tMvXIqiPv/88z/+8Q/eTKlU6oqKSo1GExMT43a7zWbz2bNnrFbL068glmX37/9VdfW7QpYpxgRBYGFD2e32+vq6L7/8IvBXFoRwbGxMyO5chKyJCVuAnQrgef7557d/8MGpl19+WRhJWKSQI7l5c+yC04ExoVAotm7dKqQeCx+SJOmdcsbzfGpq6pkzZ9VqtcfjmZmZUSgUxcXFiYmJ77zzttvtmt+wGOOYmNi0tDSOQwQBBF54nnc4HB9+eO7u3X8H+3K326dZll2ELIZhGIYJ6oQDACiKbGy89Ne/sjt37iovr3A4HGfPnhFuGIeGhnx5452dnZcu/Q0AgDGGED5+/Nj7m9Nq96rVarvdXl19/P79+zpdwdGjRzUaTWZm5t27X1LU/zYsTdNffNFRVvYmx3GpqWlHj/5WLpdfvHjxn//8rKenJ9iXFQBgbo5hGGZBftLCZbAsixAbRMcEAQB48ODB4OAAw7Acx5eXV8zNzd2+fWtoaIgkSZKkfB3q6enpnp7e+XHt9kkISa/OIUEQEokkKyvLarW2trbcv98VHR09PDz8tDCEcHT0u2+/HeE47oUXMsrLK4Qv4+7du6ElZCHEetulC8nieT4E250kSZIkhUSV//VLUTQt8b/58/Lybt78nPjRDvzNb8oW+FgURZlM/+rs7NRoNFVVR/T6Q6Ojo7du/au5+RObbXLBhoUQQiiBkKMo8sdPQMghJoQ4Qf39ZIjQ+lodQAitVutbb5WdPHnyq6++Qgjt2LGjquqI0fjJCy9keC9mpbGQeAjhqtnut2/frq2tEbYSy7IOh2PB0BzH/fKXr+XkvDQ2NvrGG8XJycn5+fnl5RUZGRmvv65b0bgIRZHenS8ki6bpea250pDJZFlZWQAQGBMQwvHx8a+/7v0pWfxzz/3i2LFjLpeLpumbN2+aTKZf//pAUlJSVFTQNnOQZNHeuQoLyZJIJBKJJOQbHUHhBPjyyc3N3bVr14+Toz799B9HjlT9dMbkZ5+1FRQUaDSaurr6t99+Jzo6WqFQ2Gw2k8m0cicAYxwVJfH2YRaSRVGUQhH6D1+ePHkyNjY2NTXJsj5vOgAgbDabxWJ5WulQFOXt6EMIx8cf/+53xyorq/bs2SOXyxFCnZ2dFy/+rbu7y9cZZFlksVgYhnny5EnIC4mPl3vvLJH8rNraGoOhKTTfUCKRyGRyjPH0tN23AsaCY/T00ACAuTmPqFPCcRxFUcnJP4uKiuI4bmpq0uVy+XnNkSQpl8cL/jDDzIUQEWAYpqSkpKHh1IJHRYZUKlOD7X1+wQzDjI8/Jn60430JOp0Oh2Pa+3HRhZEkiTH+4QerQC6E0L9BwHGccBEfciY1xkRKSqr3oyKjpqRsCdk8CXB+ISwjKA21RHVGUaRSqRTp1vujbdvSZTLZBrm19wbGWCaTbdu2zbtJhKyUlJSEhMQNG4PneT4+PiHQnSWXy9Vq9erbxxECjuNUKpVcLvduEiGLoqisrGwIN25YOTs7RzR7VlwRajSauLiNqLYwxlKpNDdXI9oqTlZ6enpGRhg81bCD47jt27enpz8v2ipOllQq1Wq14Z55GIAx1mr3ymQy0Vaf9kheXr5UurFOIsZYKpXl5eX7EvBJVmZm5u7duzdUMhtCaOfOnZmZKl8CPsmKjo7et69wQ13iQ0gWFu6PifEZhvbnFuTlvapSqTfI5kIIZWZm5uXl+ZHxR1ZiYtKBA0Ub5B4fAFBUVKRQKPzILOJw6nQFavWL635zIYRUKrVOV+BfbBGyFIqk0tLSdZ+GS5LUwYMlycnJ/sUWD2XodAV7974qmty1PsCyrFar3bdv36KSi5O1efPm8vKK+PiEdWlzYYzj4xMqKioDKfYQUJBMo9Ho9fp16f1wHFdSUpqbmxuIcEBkQQj1+kNa7d51dhhZln3llT2HDh0KMLIaaPg1ISHhvfdq0tK2rpv9JaSQ1NTUBl75LohYdU5OTk1NbWxs7DoIovI8HxMT+957NS+99FLgTwUX2C8sLKyqOgIhXNPKXki3rKys3L9/f1APBkcWhPDw4bKSktK1zBWBMT54sKSs7K1gPd+gr4yio6OPH68uLn5jjR5GnucPHCiurn43hLytUEzzuLi4Eyd+TxDElSvNABBrxXkU0pkPHCh+//33Q6ubu5RiY67Tp08ZjZeFOl7hpmIR8DwPADh4sKS6+t2QKwwvqYzd7Ozsxx9f+uijD2dmZiI58sVxXExMbGVlZVnZW+EpYycAIdTW1lZfX/fo0VhkFl9hWVawp/bt2xfOAonz6O3tbWiov3OnPUIKtwsQSm++8sqe2toTgZcy8oNlK+o6NTVlMDQZDE12uz0StphQ1LW0tFSvP7Rc1amXs1wwx3FdXV0XLpxvbzd7/zxh1YAQIklKq9VWVlbu3q2JxHLB83C73a2trZcvGwYG+sNSiFqlUpeUlOp0umWvq79SJc4nJiba2lqvXr06ODggpO6tdIlzCMnMzMyioqKCggKFInnp3a4eWQImJydNplstLS1dXV3CbxiXvXi+cDO6c+fOwsL9+fn5K1QJfjXIEjA7Ozs4OGgy3ero6Hj48KHT6cAYL/FvGQAAUql0+/btWu3evLx8lUq1FAMqgsiah9PpHB4evnfvXm9v78BAv90+5XA4EOIC/MMPiiJlMll8fIJKpc7Ozs7NzU1PT/eVl7DmyZoHy7LT09NWq2Vk5Bur1WK1WsbGxuz26bk5BiEWIY4gCIoiKYqOipLEx8vT0tJSUlKVSmV6+raUFKVcLg+LdRIeshZA0D4sywp/UiQEY0mSFP6kiKbpCLF1I4KstYJIjxZEFJ6RFQSekRUEnpEVBP4LiQWypqHC6doAAAAldEVYdGRhdGU6Y3JlYXRlADIwMTktMDgtMzBUMTU6NDc6MjQtMDQ6MDCzXTa0AAAAJXRFWHRkYXRlOm1vZGlmeQAyMDE5LTA4LTMwVDE1OjQ2OjUxLTA0OjAwdT/DiAAAAABJRU5ErkJggg==", + //}) + } else { + log.Printf("Has actions already?") + } + workflow.Actions = newActions workflow.IsValid = true @@ -1014,6 +1037,14 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { return } + err = setWorkflow(ctx, workflow, workflow.ID) + if err != nil { + log.Printf("Failed setting workflow: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + //memcacheName := fmt.Sprintf("%s_workflows", user.Username) //memcache.Delete(ctx, memcacheName) @@ -1058,7 +1089,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := getWorkflow(ctx, fileId) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (delete workflow): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -1128,7 +1159,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { return } - log.Println("Start") + //log.Println("Start") user, userErr := handleApiAuthentication(resp, request) if userErr != nil { log.Printf("Api authentication failed in edit workflow: %s", userErr) @@ -1137,7 +1168,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { return } - log.Println("PostUser") + //log.Println("PostUser") location := strings.Split(request.URL.String(), "/") var fileId string @@ -1164,7 +1195,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { tmpworkflow, err := getWorkflow(ctx, fileId) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (save workflow): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -1211,7 +1242,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // FIXME - this shouldn't be necessary with proper API checks newActions := []Action{} allNodes := []string{} - log.Println("Pre") + //log.Println("Pre") for _, action := range workflow.Actions { allNodes = append(allNodes, action.ID) @@ -1237,7 +1268,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { workflow.Actions = newActions for _, trigger := range workflow.Triggers { - log.Println("TRIGGERS") + //log.Println("TRIGGERS") allNodes = append(allNodes, trigger.ID) } @@ -1260,7 +1291,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { foundNodes := []string{} for _, node := range allNodes { for _, branch := range workflow.Branches { - log.Println("branch") + //log.Println("branch") //log.Println(node) //log.Println(branch.DestinationID) if node == branch.DestinationID || node == branch.SourceID { @@ -1462,7 +1493,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { log.Printf("Failed to change total actions data: %s", err) } - log.Printf("Saved new version of workflow %s", fileId) + log.Printf("Saved new version of workflow %s (%s)", workflow.Name, fileId) resp.WriteHeader(200) resp.Write([]byte(`{"success": true}`)) } @@ -1668,7 +1699,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if workflow.ID == "" || workflow.ID != id { tmpworkflow, err := getWorkflow(ctx, id) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (execution cleanup): %s", err) return WorkflowExecution{}, "Failed getting workflow", err } @@ -1736,6 +1767,18 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if len(execution.Start) == 36 { log.Printf("SHOULD START ON NODE %s", execution.Start) workflow.Start = execution.Start + + found := false + for _, action := range workflow.Actions { + if action.ID == workflow.Start { + found = true + } + } + + if !found { + log.Printf("ACTION %s WAS NOT FOUND!", workflow.Start) + return WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", workflow.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", workflow.Start)) + } } if len(execution.ExecutionId) == 36 { @@ -1993,7 +2036,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := getWorkflow(ctx, fileId) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (execute workflow): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -2021,6 +2064,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { } func stopSchedule(resp http.ResponseWriter, request *http.Request) { + log.Printf("Delete?") cors := handleCors(resp, request) if cors { return @@ -2064,7 +2108,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := getWorkflow(ctx, fileId) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (stop schedule): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -2079,19 +2123,6 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { return } - if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} - } - if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} - } - if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} - } - if len(workflow.Errors) == 0 { - workflow.Errors = []string{} - } - err = deleteSchedule(ctx, scheduleId) if err != nil { if strings.Contains(err.Error(), "Job not found") { @@ -2153,7 +2184,7 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := getWorkflow(ctx, fileId) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (stop schedule GCP): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -2275,7 +2306,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := getWorkflow(ctx, fileId) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (schedule workflow): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -3276,6 +3307,114 @@ func deployWebhookFunction(ctx context.Context, name, localization, applocation return nil } +func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + // Just need to be logged in + // FIXME - should have some permissions? + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("Api authentication failed in load apps: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if user.Role != "admin" { + log.Printf("Wrong user (%s) when downloading from github", user.Username) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Printf("Error with body read: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // Field1 & 2 can be a lot of things.. + type tmpStruct struct { + URL string `json:"url"` + Field1 string `json:"field_1"` + Field2 string `json:"field_2"` + } + //log.Printf("Body: %s", string(body)) + + var tmpBody tmpStruct + err = json.Unmarshal(body, &tmpBody) + if err != nil { + log.Printf("Error with unmarshal tmpBody: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fs := memfs.New() + + if strings.Contains(tmpBody.URL, "github") || strings.Contains(tmpBody.URL, "gitlab") || strings.Contains(tmpBody.URL, "bitbucket") { + cloneOptions := &git.CloneOptions{ + URL: tmpBody.URL, + } + + // FIXME: Better auth. + if len(tmpBody.Field1) > 0 && len(tmpBody.Field2) > 0 { + cloneOptions.Auth = &http2.BasicAuth{ + + Username: tmpBody.Field1, + Password: tmpBody.Field2, + } + } + + storer := memory.NewStorage() + r, err := git.Clone(storer, fs, cloneOptions) + if err != nil { + log.Printf("Failed loading repo into memory: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + + dir, err := fs.ReadDir("/") + if err != nil { + log.Printf("FAiled reading folder: %s", err) + } + _ = r + + log.Printf("Starting workflow folder iteration") + iterateWorkflowGithubFolders(fs, dir, "", "") + + } else if strings.Contains(tmpBody.URL, "s3") { + //https://docs.aws.amazon.com/sdk-for-go/api/service/s3/ + + //sess := session.Must(session.NewSession()) + //downloader := s3manager.NewDownloader(sess) + + //// Write the contents of S3 Object to the file + //storer := memory.NewStorage() + //n, err := downloader.Download(storer, &s3.GetObjectInput{ + // Bucket: aws.String(myBucket), + // Key: aws.String(myString), + //}) + //if err != nil { + // return fmt.Errorf("failed to download file, %v", err) + //} + //fmt.Printf("file downloaded, %d bytes\n", n) + } else { + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s is unsupported. Try e.g. github"}`, tmpBody.URL))) + return + } + + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) +} + func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -3501,10 +3640,70 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, return nil } +// Onlyname is used to +func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error { + var err error + + for _, file := range dir { + if len(onlyname) > 0 && file.Name() != onlyname { + continue + } + + // Folder? + switch mode := file.Mode(); { + case mode.IsDir(): + tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name()) + dir, err := fs.ReadDir(tmpExtra) + if err != nil { + log.Printf("Failed to read dir: %s", err) + break + } + + // Go routine? Hmm, this can be super quick I guess + err = iterateWorkflowGithubFolders(fs, dir, tmpExtra, "") + if err != nil { + break + } + case mode.IsRegular(): + // Check the file + filename := file.Name() + if strings.HasSuffix(filename, ".json") { + path := fmt.Sprintf("%s%s", extra, file.Name()) + fileReader, err := fs.Open(path) + if err != nil { + log.Printf("Error reading file: %s", err) + continue + } + + readFile, err := ioutil.ReadAll(fileReader) + if err != nil { + log.Printf("Error reading file: %s", err) + continue + } + + var workflow Workflow + err = json.Unmarshal(readFile, &workflow) + if err != nil { + continue + } + + ctx := context.Background() + err = setWorkflow(ctx, workflow, workflow.ID) + if err != nil { + log.Printf("Failed setting (download) workflow: %s", err) + continue + } + log.Printf("Uploaded workflow %s!", filename) + } + } + } + + return err +} + // Onlyname is used to func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error { var err error - runUpload := false for _, file := range dir { if len(onlyname) > 0 && file.Name() != onlyname { continue @@ -3531,83 +3730,71 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin if filename == "Dockerfile" { log.Printf("Handle Dockerfile in location %s", extra) - extraSplit := strings.Split(extra, "/") - tags := []string{} - if len(extraSplit) > 1 { - tags = []string{ - fmt.Sprintf("%s:%s_%s", baseDockerName, strings.ReplaceAll(extraSplit[0], " ", "-"), extraSplit[1]), - // Version = folder of last part of extra - // Name = first folder of extra - } - } else { - // Skip - runUpload = false - log.Printf("Skipping folder %s because the extra variable is empty~", extra) - break - //return nil - } - - /// Only upload if successful and no errors - err := buildImageMemory(fs, tags, extra) - if err != nil { - log.Printf("Failed image build memory: %s", err) - runUpload = false - } else { - runUpload = true - } - } - } - } - - // Done sequentailly to prevent bad uploads - if runUpload && err == nil { - for _, file := range dir { - if file.Name() == "api.yaml" || file.Name() == "api.yaml" { - log.Printf("Run update of %sapi.yaml in backend if it doesn't exist!!", extra) - fullPath := fmt.Sprintf("%s%s", extra, file.Name()) - + // Try api.yaml and api.yml + fullPath := fmt.Sprintf("%s%s", extra, "api.yaml") fileReader, err := fs.Open(fullPath) if err != nil { - return err + fullPath = fmt.Sprintf("%s%s", extra, "api.yml") + fileReader, err = fs.Open(fullPath) + if err != nil { + log.Printf("Failed finding api.yaml/yml: %s", err) + continue + } } readFile, err := ioutil.ReadAll(fileReader) if err != nil { - log.Printf("Filereader error: %s", err) - return err + log.Printf("Failed reading %s: %s", fullPath, err) + continue } var workflowapp WorkflowApp err = gyaml.Unmarshal(readFile, &workflowapp) if err != nil { - log.Printf("Failed api.yaml unmarshal: %s", err) - return err + log.Printf("Failed unmarshaling %s: %s", fullPath, err) + continue } - log.Printf("APIName: %s", workflowapp.Name) - extraSplit := strings.Split(extra, "/") - appName := fmt.Sprintf("%s_%s", strings.ReplaceAll(extraSplit[0], " ", "-"), extraSplit[1]) + newName := workflowapp.Name + newName = strings.ReplaceAll(newName, " ", "-") + + tags := []string{ + fmt.Sprintf("%s:%s_%s", baseDockerName, newName, workflowapp.AppVersion), + } ctx := context.Background() allapps, err := getAllWorkflowApps(ctx) if err != nil { log.Printf("Failed getting apps to verify: %s", err) - return err + continue + //return err } log.Printf("APPS: %d", len(allapps)) + // Make an option to override existing apps? + removeApps := []string{} for _, app := range allapps { if app.Name == workflowapp.Name && app.AppVersion == workflowapp.AppVersion { - log.Printf("App upload for %s:%s already exists.", app.Name, app.AppVersion) - return errors.New(fmt.Sprintf("App %s already exists. ", appName)) + //log.Printf("App upload for %s:%s already exists.", app.Name, app.AppVersion) + log.Printf("Overriding app %s:%s as it exists.", app.Name, app.AppVersion) + removeApps = append(removeApps, app.ID) } } err = checkWorkflowApp(workflowapp) if err != nil { log.Printf("%s for app %s:%s", err, workflowapp.Name, workflowapp.AppVersion) - return err + continue + } + + if len(removeApps) > 0 { + for _, item := range removeApps { + err = DeleteKey(ctx, "workflowapp", item) + if err != nil { + log.Printf("Failed deleting %s", item) + } + } } //if workflowapp.Environment == "" { @@ -3623,7 +3810,8 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin err = setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err != nil { log.Printf("Failed setting workflowapp: %s", err) - return err + continue + //return err } err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1) @@ -3637,8 +3825,16 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion) - //memcache.Delete(ctx, "all_apps") - //os.Exit(3) + + /// Only upload if successful and no errors + err = buildImageMemory(fs, tags, extra) + if err != nil { + log.Printf("Failed image build memory: %s", err) + } else { + if len(tags) > 0 { + log.Printf("Successfully built image %s", tags[0]) + } + } } } } @@ -3773,7 +3969,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := getWorkflow(ctx, fileId) if err != nil { - log.Printf("Failed getting the workflow locally: %s", err) + log.Printf("Failed getting the workflow locally (get executions): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return diff --git a/docker-compose.yml b/docker-compose.yml index 0fbc7c41..c00daf34 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,18 +13,6 @@ services: restart: unless-stopped depends_on: - backend - database: - #build: ./backend/database - image: frikky/shuffle:database - container_name: shuffle-database - hostname: shuffle-database - ports: - - "8000:8000" - networks: - - shuffle - restart: unless-stopped - volumes: - - ${DB_LOCATION}:/etc/shuffle backend: #build: ./backend image: frikky/shuffle:backend @@ -59,6 +47,18 @@ services: - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} - DOCKER_API_VERSION=1.40 restart: unless-stopped + database: + #build: ./backend/database + image: frikky/shuffle:database + container_name: shuffle-database + hostname: shuffle-database + ports: + - "8000:8000" + networks: + - shuffle + restart: unless-stopped + volumes: + - ${DB_LOCATION}:/etc/shuffle networks: shuffle: driver: bridge diff --git a/frontend/src/Admin.js b/frontend/src/Admin.js index a0a1fa74..c3700307 100644 --- a/frontend/src/Admin.js +++ b/frontend/src/Admin.js @@ -8,7 +8,8 @@ import ListItem from '@material-ui/core/ListItem'; import Button from '@material-ui/core/Button'; import Tabs from '@material-ui/core/Tabs'; import Tab from '@material-ui/core/Tab'; - +import ListItemText from '@material-ui/core/ListItemText'; +import ListItemSecondaryAction from '@material-ui/core/ListItemSecondaryAction'; import { useAlert } from "react-alert"; @@ -29,9 +30,39 @@ const Admin = (props) => { const [curTab, setCurTab] = React.useState(0); const [users, setUsers] = React.useState([]); const [environments, setEnvironments] = React.useState([]); + const [schedules, setSchedules] = React.useState([]) const alert = useAlert() + const deleteSchedule = (data) => { + // FIXME - add some check here ROFL + console.log("INPUT: ", data) + + // Just use this one? + const url = globalUrl+'/api/v1/workflows/'+data["workflow_id datastore:"]+"/schedule/"+data.id + console.log("URL: ", url) + fetch(url, { + method: 'DELETE', + credentials: "include", + headers: { + 'Content-Type': 'application/json', + }, + }) + .then(response => + response.json().then(responseJson => { + if (responseJson["success"] === false) { + alert.error("Failed stopping schedule") + } else { + getSchedules() + alert.success("Successfully stopped schedule!") + } + }), + ) + .catch(error => { + console.log("Error in userdata: ", error) + }); + } + const submitUser = (data) => { // FIXME - add some check here ROFL console.log("INPUT: ", data) @@ -132,6 +163,31 @@ const Admin = (props) => { }); } + const getSchedules = () => { + fetch(globalUrl+"/api/v1/workflows/schedules", { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for apps :O!") + return + } + + return response.json() + }) + .then((responseJson) => { + setSchedules(responseJson) + }) + .catch(error => { + alert.error(error.toString()) + }); + } + const getEnvironments = () => { fetch(globalUrl+"/api/v1/getenvironments", { method: 'GET', @@ -329,6 +385,54 @@ const Admin = (props) => { : null + const schedulesView = curTab === 2 ? +
+

+ Schedules +

+ + + + + + + + {schedules === undefined || schedules === null ? null : schedules.map(schedule => { + return ( + + + + + + + + ) + })} + +
+ : null + const environmentView = curTab === 1 ?

@@ -359,6 +463,8 @@ const Admin = (props) => { const setConfig = (event, newValue) => { if (newValue === 1) { getEnvironments() + } else if (newValue === 2) { + getSchedules() } setModalUser({}) @@ -377,10 +483,12 @@ const Admin = (props) => { > +
{usersView} {environmentView} + {schedulesView}
diff --git a/frontend/src/AngularWorkflow.js b/frontend/src/AngularWorkflow.js index c1701c69..ccb2d3ab 100644 --- a/frontend/src/AngularWorkflow.js +++ b/frontend/src/AngularWorkflow.js @@ -10,6 +10,8 @@ import Drawer from '@material-ui/core/Drawer'; import Button from '@material-ui/core/Button'; import Paper from '@material-ui/core/Paper'; import Grid from '@material-ui/core/Grid'; +import Tabs from '@material-ui/core/Tabs'; +import Tab from '@material-ui/core/Tab'; import ButtonBase from '@material-ui/core/ButtonBase'; import Tooltip from '@material-ui/core/Tooltip'; import Select from '@material-ui/core/Select'; @@ -100,7 +102,7 @@ const AngularWorkflow = (props) => { const [cy, setCy] = React.useState() const [appSearch, setAppSearch] = React.useState("") - const [currentView, setCurrentView] = React.useState("apps") + const [currentView, setCurrentView] = React.useState(0) const [triggerAuthentication, setTriggerAuthentication] = React.useState({}) const [triggerFolders, setTriggerFolders] = React.useState([]) @@ -164,9 +166,6 @@ const AngularWorkflow = (props) => { const [lastSaved, setLastSaved] = React.useState(true) - const [AppsHoverColor, setAppsHoverColor] = useState(hoverOutColor); - const [VariablesHoverColor, setVariablesHoverColor] = useState(hoverOutColor); - const [HookHoverColor, setHookHoverColor] = useState(hoverOutColor); const [appAdded, setAppAdded] = useState(false) const [update, setUpdate] = useState(""); const [workflowExecutions, setWorkflowExecutions] = React.useState([]); @@ -262,7 +261,7 @@ const AngularWorkflow = (props) => { const abortExecution = () => { setExecutionRunning(false) - alert.success("Aborting execution") + alert.info("Aborting execution") fetch(globalUrl+"/api/v1/workflows/"+props.match.params.key+"/executions/"+executionRequest.execution_id+"/abort", { method: 'GET', headers: { @@ -274,7 +273,9 @@ const AngularWorkflow = (props) => { .then((response) => { if (response.status !== 200) { console.log("Status not 200 for WORKFLOW EXECUTION :O!") - } + } else { + alert.success("Execution aborted") + } return response.json() }) @@ -547,68 +548,6 @@ const AngularWorkflow = (props) => { return true } - const executeWorkflowWebsocket = () => { - if (!lastSaved) { - //alert.error("You might have forgotten to save before executing.") - console.log("FIXME: Might have forgotten to save before executing.") - } - - var returncheck = monitorUpdates() - if (!returncheck) { - alert.error("No startnode set.") - return - } - - setVisited([]) - setExecutionRunning(true) - setExecutionRequest({}) - - var curelements = cy.elements() - for (var i = 0; i < curelements.length; i++) { - curelements[i].addClass("not-executing-highlight") - } - - if (executionText.length > 0) { - alert.success("Starting execution with argument "+executionText) - } else { - alert.success("Starting execution") - } - - const data = {"execution_argument": executionText} - fetch(globalUrl+"/api/v1/workflows/"+props.match.params.key+"/execute_fs", { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - }, - credentials: "include", - body: JSON.stringify(data), - }) - .then((response) => { - if (response.status !== 200) { - console.log("Status not 200 for WORKFLOW EXECUTION :O!") - } - - return response.json() - }) - .then((responseJson) => { - if (!responseJson.success) { - alert.error("Failed to start: "+responseJson.reason) - stop() - return - } - - setExecutionRequest({ - "execution_id": responseJson.execution_id, - "authorization": responseJson.authorization, - }) - setExecutingNodes([workflow.start]) - }) - .catch(error => { - alert.error(error.toString()) - }); - } - const executeWorkflow = () => { if (!lastSaved) { //alert.error("You might have forgotten to save before executing.") @@ -639,14 +578,14 @@ const AngularWorkflow = (props) => { const data = {"execution_argument": executionText, "start": workflow.start} fetch(globalUrl+"/api/v1/workflows/"+props.match.params.key+"/execute", { - method: 'POST', + method: 'POST', headers: { 'Content-Type': 'application/json', 'Accept': 'application/json', }, credentials: "include", body: JSON.stringify(data), - }) + }) .then((response) => { if (response.status !== 200) { console.log("Status not 200 for WORKFLOW EXECUTION :O!") @@ -657,7 +596,13 @@ const AngularWorkflow = (props) => { .then((responseJson) => { if (!responseJson.success) { alert.error("Failed to start: "+responseJson.reason) + setExecutionRunning(false) + setExecutionRequestStarted(false) stop() + + for (var i = 0; i < curelements.length; i++) { + curelements[i].removeClass("not-executing-highlight") + } return } else { setExecutionRunning(true) @@ -1429,16 +1374,17 @@ const AngularWorkflow = (props) => { } const appViewStyle = { - marginLeft: "5px", - marginRight: "5px", + marginLeft: 5, + marginRight: 5, display: "flex", flexDirection: "column", + height: "100%", } const scrollStyle = { - marginTop: "10px", + marginTop: 10, overflow: "scroll", - height: "66vh", + height: "100%", overflowX: "auto", overflowY: "auto", } @@ -1455,37 +1401,6 @@ const AngularWorkflow = (props) => { display: "flex", } - // All this is stupid lmao - const handleHookHover = () => { - setHookHoverColor(hoverColor) - setAppsHoverColor(hoverOutColor) - setVariablesHoverColor(hoverOutColor) - } - - const handleHookHoverOut = () => { - setHookHoverColor(hoverOutColor) - } - - const handleAppsHover = () => { - setAppsHoverColor(hoverColor) - setHookHoverColor(hoverOutColor) - setVariablesHoverColor(hoverOutColor) - } - - const handleAppsHoverOut = () => { - setAppsHoverColor(hoverOutColor) - } - - const handleVariablesHover = () => { - setVariablesHoverColor(hoverColor) - setHookHoverColor(hoverOutColor) - setAppsHoverColor(hoverOutColor) - } - - const handleVariablesHoverOut = () => { - setVariablesHoverColor(hoverOutColor) - } - const paperVariableStyle = { minHeight: "50px", maxHeight: "50px", @@ -1560,6 +1475,7 @@ const AngularWorkflow = (props) => { aria-controls="long-menu" aria-haspopup="true" onClick={menuClick} + style={{color: "white"}} > @@ -1605,57 +1521,77 @@ const AngularWorkflow = (props) => {

) } + + const curTab = 0 + const handleSetTab = (event, newValue) => { + setCurrentView(newValue) + } const HandleLeftView = () => { // Defaults to apps. var thisview = - if (currentView === "triggers") { + if (currentView === 1) { thisview = - } else if (currentView === "variables") { + } else if (currentView === 2) { thisview = } + const tabStyle = { + maxWidth: leftBarSize/3, + minWidth: leftBarSize/3, + flex: 1, + textTransform: "none", + } + + const iconStyle = { + marginTop: 3, + marginRight: 5, + } + return(
- -
+
{thisview}
-
- -
-
{setCurrentView("apps")}}> - - - - + + + - Apps - + -
-
{setCurrentView("triggers")}}> - - - - + + Apps + + + } style={tabStyle} /> + - Triggers - + -
-
{setCurrentView("variables")}}> - - - - + + Triggers + + + } style={tabStyle} /> + - Variables - + -
-
-
+ + Variables + + + }style={tabStyle} /> +
) } @@ -1965,7 +1901,7 @@ const AngularWorkflow = (props) => { node.data.type = "ACTION" node.isStartNode = action["id"] === workflow.start - return node; + return node }) const tmpelements = [].concat(actions) @@ -2001,14 +1937,16 @@ const AngularWorkflow = (props) => { } } - const handleDragStop = (e) => { + const handleDragStop = (e, app) => { newNodeId = "" + console.log("STOP!: ", e) + console.log("APP!: ", app) } const appScrollStyle = { overflow: "scroll", - maxHeight: bodyHeight-appBarSize-150, - minHeight: bodyHeight-appBarSize-150, + maxHeight: bodyHeight-appBarSize-55, + minHeight: bodyHeight-appBarSize-55, overflowY: "auto", overflowX: "hidden", } @@ -2063,7 +2001,8 @@ const AngularWorkflow = (props) => { return( {handleAppDrag(e, app)}} - onStop={(e) => {handleDragStop(e)}} + onStop={(e) => {handleDragStop(e, app)}} + key={app.id} dragging={false} position={{ x: 0, @@ -2427,7 +2366,7 @@ const AngularWorkflow = (props) => { } else if (data.variant === "WORKFLOW_VARIABLE") { varcolor = "#f85a3e" if (workflow.workflow_variables === null || workflow.workflow_variables === undefined || workflow.workflow_variables.length === 0) { - setCurrentView("variables") + setCurrentView(2) datafield =
@@ -2703,11 +2642,11 @@ const AngularWorkflow = (props) => { } const setTriggerFolderWrapperMulti = event => { - const { options } = event.target; - const value = []; + const { options } = event.target + const value = [] for (let i = 0, l = options.length; i < l; i += 1) { if (options[i].selected) { - value.push(options[i].value); + value.push(options[i].value) } } @@ -2783,7 +2722,7 @@ const AngularWorkflow = (props) => { if (splitItems.includes(value)) { for( var i = 0; i < splitItems.length; i++){ if (splitItems[i] === value) { - splitItems.splice(i, 1); + splitItems.splice(i, 1) } } @@ -2793,7 +2732,7 @@ const AngularWorkflow = (props) => { for( var i = 0; i < splitItems.length; i++){ if (splitItems[i] === "") { - splitItems.splice(i, 1); + splitItems.splice(i, 1) } } @@ -2824,7 +2763,7 @@ const AngularWorkflow = (props) => { } const AppConditionHandler = (props) => { - const { tmpdata, type } = props; + const { tmpdata, type } = props if (tmpdata === undefined) { return tmpdata @@ -2933,7 +2872,7 @@ const AngularWorkflow = (props) => { } else if (data.variant === "WORKFLOW_VARIABLE") { varcolor = "#f85a3e" if (workflow.workflow_variables === null || workflow.workflow_variables === undefined || workflow.workflow_variables.length === 0) { - setCurrentView("variables") + setCurrentView(2) datafield =
@@ -3192,8 +3131,8 @@ const AngularWorkflow = (props) => { const EdgeSidebar = () => { const ConditionHandler = (condition, index) => { - const [open, setOpen] = React.useState(false); - const [anchorEl, setAnchorEl] = React.useState(null); + const [open, setOpen] = React.useState(false) + const [anchorEl, setAnchorEl] = React.useState(null) const deleteCondition = (conditionIndex) => { console.log(selectedEdge) @@ -3223,7 +3162,7 @@ const AngularWorkflow = (props) => { const menuClick = (event) => { console.log("MENU CLICK") setOpen(!open) - setAnchorEl(event.currentTarget); + setAnchorEl(event.currentTarget) } return ( @@ -3363,7 +3302,7 @@ const AngularWorkflow = (props) => { }) .catch(error => { console.log(error.toString()) - }); + }) } const getTriggerAuth = () => { @@ -3384,7 +3323,7 @@ const AngularWorkflow = (props) => { }) .catch(error => { console.log(error.toString()) - }); + }) } // Getting the triggers and the folders if they exist @@ -3426,8 +3365,8 @@ const AngularWorkflow = (props) => { }) .catch(error => { console.log(error.toString()) - }); - }, 2500); + }) + }, 2500) console.log(data) saveWorkflow(workflow) @@ -3801,7 +3740,7 @@ const AngularWorkflow = (props) => { }) .catch(error => { alert.error(error.toString()) - }); + }) } const startMailSub = (trigger, triggerindex) => { @@ -3857,7 +3796,7 @@ const AngularWorkflow = (props) => { }) .catch(error => { alert.error(error.toString()) - }); + }) } const newWebhook = (trigger) => { @@ -3903,7 +3842,7 @@ const AngularWorkflow = (props) => { }) .catch(error => { console.log(error.toString()) - }); + }) } const deleteWebhook = (trigger, triggerindex) => { @@ -3942,7 +3881,7 @@ const AngularWorkflow = (props) => { }) .catch(error => { alert.error(error.toString()) - }); + }) } const UserinputSidebar = () => { @@ -4356,7 +4295,7 @@ const AngularWorkflow = (props) => { } if (Object.getOwnPropertyNames(selectedAction).length > 0 && Object.getOwnPropertyNames(selectedApp).length > 0) { - //console.time('ACTIONSTART'); + //console.time('ACTIONSTART') return(
{appApiView} @@ -4479,11 +4418,13 @@ const AngularWorkflow = (props) => {
{timestamp}
- -
- {resultsLength}/{data.workflow.actions.length} -
- + {data.workflow.actions !== null ? + +
+ {resultsLength}/{data.workflow.actions.length} +
+ + : null}
@@ -4545,7 +4486,7 @@ const AngularWorkflow = (props) => { executionData.results.map(data => { var showResult = data.result.trim() showResult.split(" None").join(" \"None\"") - //showResult = replaceAll(showResult, " None", " \"None\""); + //showResult = replaceAll(showResult, " None", " \"None\"") var jsonvalid = true try { JSON.parse(showResult) @@ -4569,7 +4510,7 @@ const AngularWorkflow = (props) => { {jsonvalid ? @@ -4608,7 +4549,6 @@ const AngularWorkflow = (props) => { -
:
@@ -4872,4 +4812,4 @@ const AngularWorkflow = (props) => { ) } -export default AngularWorkflow; +export default AngularWorkflow diff --git a/frontend/src/App.js b/frontend/src/App.js index e40d7660..af034001 100644 --- a/frontend/src/App.js +++ b/frontend/src/App.js @@ -147,13 +147,13 @@ const App = (message, props) => { // This is a mess hahahah return ( - - - - {includedData} - - - + + + + {includedData} + + + ); }; diff --git a/frontend/src/AppCreator.js b/frontend/src/AppCreator.js index 7925246a..690b73ea 100644 --- a/frontend/src/AppCreator.js +++ b/frontend/src/AppCreator.js @@ -4,11 +4,13 @@ import {BrowserView, MobileView} from "react-device-detect"; import {Link} from 'react-router-dom'; import Paper from '@material-ui/core/Paper'; +import FormControlLabel from '@material-ui/core/FormControlLabel'; import Button from '@material-ui/core/Button'; import Divider from '@material-ui/core/Divider'; import Select from '@material-ui/core/Select'; import MenuItem from '@material-ui/core/MenuItem'; import FormControl from '@material-ui/core/FormControl'; +import Switch from '@material-ui/core/Switch'; import Dialog from '@material-ui/core/Dialog'; import DialogTitle from '@material-ui/core/DialogTitle'; import DialogContent from '@material-ui/core/DialogContent'; @@ -65,7 +67,7 @@ const useStyles = makeStyles({ const rewrite = (args) => { return args.reduce(function(args, a){ - if (0 == a.indexOf('-X')) { + if (0 === a.indexOf('-X')) { args.push('-X') args.push(a.slice(2)) } else { @@ -103,35 +105,35 @@ const parseCurl = (s) => { out.url = arg break; - case arg == '-A' || arg == '--user-agent': + case arg === '-A' || arg === '--user-agent': state = 'user-agent' break; - case arg == '-H' || arg == '--header': + case arg === '-H' || arg === '--header': state = 'header' break; - case arg == '-d' || arg == '--data' || arg == '--data-ascii': + case arg === '-d' || arg === '--data' || arg === '--data-ascii': state = 'data' break; - case arg == '-u' || arg == '--user': + case arg === '-u' || arg === '--user': state = 'user' break; - case arg == '-I' || arg == '--head': + case arg === '-I' || arg === '--head': out.method = 'HEAD' break; - case arg == '-X' || arg == '--request': + case arg === '-X' || arg === '--request': state = 'method' break; - case arg == '-b' || arg =='--cookie': + case arg === '-b' || arg === '--cookie': state = 'cookie' break; - case arg == '--compressed': + case arg === '--compressed': out.header['Accept-Encoding'] = out.header['Accept-Encoding'] || 'deflate, gzip' break; @@ -147,7 +149,7 @@ const parseCurl = (s) => { state = '' break; case 'data': - if (out.method == 'GET' || out.method == 'HEAD') out.method = 'POST' + if (out.method === 'GET' || out.method === 'HEAD') out.method = 'POST' out.header['Content-Type'] = out.header['Content-Type'] || 'application/x-www-form-urlencoded' out.body = out.body ? out.body + '&' + arg @@ -196,6 +198,7 @@ const AppCreator = (props) => { const [updater, setUpdater] = useState("tmp") const [baseUrl, setBaseUrl] = useState(""); const [actionsModalOpen, setActionsModalOpen] = useState(false); + const [authenticationRequired, setAuthenticationRequired] = useState(false); const [authenticationOption, setAuthenticationOption] = useState(authenticationOptions[0]); const [parameterName, setParameterName] = useState(""); const [parameterLocation, setParameterLocation] = useState(apikeySelection.length > 0 ? apikeySelection[0] : ""); @@ -372,30 +375,6 @@ const AppCreator = (props) => { securitySchemes = data.components.securitySchemes } - // FIXME: Have multiple authentication options? - if (securitySchemes !== undefined) { - for (const [key, value] of Object.entries(securitySchemes)) { - if (value.scheme === "bearer") { - setAuthenticationOption("Bearer auth") - break - } else if (value.type === "apiKey") { - setAuthenticationOption("API key") - - value.in = value.in.charAt(0).toUpperCase() + value.in.slice(1); - setParameterLocation(value.in) - if (!apikeySelection.includes(value.in)) { - console.log("APIKEY SELECT: ", apikeySelection) - alert.error("Might be error in setting up API key authentication") - } - - setParameterName(value.name) - break - } else if (value.scheme === "basic") { - setAuthenticationOption("Basic auth") - break - } - } - } console.log(data) @@ -451,9 +430,39 @@ const AppCreator = (props) => { } } + + // FIXME: Have multiple authentication options? + if (securitySchemes !== undefined) { + for (const [key, value] of Object.entries(securitySchemes)) { + if (value.scheme === "bearer") { + setAuthenticationOption("Bearer auth") + setAuthenticationRequired(true) + break + } else if (value.type === "apiKey") { + setAuthenticationOption("API key") + + value.in = value.in.charAt(0).toUpperCase() + value.in.slice(1); + setParameterLocation(value.in) + if (!apikeySelection.includes(value.in)) { + console.log("APIKEY SELECT: ", apikeySelection) + alert.error("Might be error in setting up API key authentication") + } + + console.log("PARAM NAME: ", value.name) + setParameterName(value.name) + setAuthenticationRequired(true) + break + } else if (value.scheme === "basic") { + setAuthenticationOption("Basic auth") + setAuthenticationRequired(true) + break + } + } + } + setActions(newActions) } - + // Saving the app that's been configured. const submitApp = () => { alert.info("Uploading and building app " + name) @@ -701,7 +710,6 @@ const AppCreator = (props) => { Users will be required to submit their API as the header "Authorization: Bearer APIKEY" -
: null @@ -714,7 +722,6 @@ const AppCreator = (props) => { Users will be required to submit a valid username and password before using the API -
: null @@ -810,7 +817,7 @@ const AppCreator = (props) => { id="standard-required" margin="normal" variant="outlined" - defaultValue={parameterName} + value={parameterName} helperText={
Can't be empty. Can't contain any of the following characters: !#$%&'^+-._~|]+$
} onChange={e => setParameterName(e.target.value)} InputProps={{ @@ -847,7 +854,6 @@ const AppCreator = (props) => { )} )} -
: null @@ -926,11 +932,13 @@ const AppCreator = (props) => { {data.method} - {url} - {data.name}
+ {/*
{testAction(index)}}> Test
+ */}
{deleteAction(index)}}> Delete @@ -1261,6 +1269,10 @@ const AppCreator = (props) => { if (request.header !== undefined && request.header !== null) { var headers = [] for (let [key, value] of Object.entries(request.header)) { + if (parameterName !== undefined && key.toLowerCase() === parameterName.toLowerCase()) { + continue + } + headers += key+"="+value+"\n" } @@ -1550,11 +1562,17 @@ const AppCreator = (props) => { }} /> -
Authentication
+
Authentication +
upload = ref} onChange={importFiles} />
@@ -962,10 +983,160 @@ const Workflows = (props) => { + + const importWorkflowsFromUrl = (url) => { + console.log("IMPORT WORKFLOWS FROM ", downloadUrl) + + const parsedData = { + "url": url, + } + + if (field1.length > 0) { + parsedData["field_1"] = field1 + } + + if (field2.length > 0) { + parsedData["field_2"] = field2 + } + + alert.success("Getting specific workflows from your URL.") + var cors = "cors" + fetch(globalUrl+"/api/v1/workflows/download_remote", { + method: "POST", + mode: "cors", + headers: { + 'Accept': 'application/json', + }, + body: JSON.stringify(parsedData), + credentials: "include", + }) + .then((response) => { + if (response.status === 200) { + response.text().then(function (text) { + console.log("RETURN: ", text) + alert.success("Loaded existing apps!") + }) + } + + return response.json() + }) + .then((responseJson) => { + console.log("DATA: ", responseJson) + if (responseJson.reason !== undefined) { + alert.error("Failed loading: "+responseJson.reason) + } else { + alert.error("Failed loading") + } + }) + .catch(error => { + alert.error(error.toString()) + }) + } + + const handleGithubValidation = () => { + importWorkflowsFromUrl(downloadUrl) + setLoadWorkflowsModalOpen(false) + } + + const workflowDownloadModalOpen = loadWorkflowsModalOpen ? + { + }} + PaperProps={{ + style: { + backgroundColor: surfaceColor, + color: "white", + minWidth: "800px", + minHeight: "320px", + }, + }} + > + +
+ Load workflows from github repo +
+ + + +
+
+ + + Repository (supported: github, gitlab, bitbucket) + setDownloadUrl(e.target.value)} + placeholder="https://github.com/frikky/shuffle-apps" + fullWidth + /> + + Authentication (optional - private repos etc): +
+ setField1(e.target.value)} + type="username" + placeholder="Username / APIkey (optional)" + fullWidth + /> + setField2(e.target.value)} + type="password" + placeholder="Password (optional)" + fullWidth + /> +
+ + + + + + + : null + const loadedCheck = isLoaded && isLoggedIn && workflowDone ?
{workflowView} {modalView} + {workflowDownloadModalOpen}
:
diff --git a/frontend/src/assets/img/logo.png b/frontend/src/assets/img/logo.png new file mode 100644 index 00000000..c6826517 Binary files /dev/null and b/frontend/src/assets/img/logo.png differ diff --git a/frontend/src/assets/img/logo.svg b/frontend/src/assets/img/logo.svg new file mode 100644 index 00000000..3c516967 --- /dev/null +++ b/frontend/src/assets/img/logo.svg @@ -0,0 +1,26 @@ + + + + + + + + + + + + + background + + + + Layer 1 + + + + + + + + + \ No newline at end of file diff --git a/frontend/src/assets/img/logo.webp b/frontend/src/assets/img/logo.webp new file mode 100644 index 00000000..d1dadedd Binary files /dev/null and b/frontend/src/assets/img/logo.webp differ diff --git a/shuffle_adminaccount.png b/frontend/src/assets/img/shuffle_adminaccount.png similarity index 100% rename from shuffle_adminaccount.png rename to frontend/src/assets/img/shuffle_adminaccount.png diff --git a/shuffle_webhook.png b/frontend/src/assets/img/shuffle_webhook.png similarity index 100% rename from shuffle_webhook.png rename to frontend/src/assets/img/shuffle_webhook.png diff --git a/setup.sh b/setup.sh deleted file mode 100644 index 6ceb1699..00000000 --- a/setup.sh +++ /dev/null @@ -1,33 +0,0 @@ -# Build script to make it work as an open source platform - -# 1. Grab builtin functions - Done in backend as a button click -# 2. Upload to database & build docker images -# 3. Run docker-compose: frontend, backend, database & orborus -# 4. Set up docker swarm for apps? - -# Basic overview of how it works: -# -# Backend: o -# | -# Orborus: o -# / \ -# Workers: o o -# / / \ -# Apps: o o o - -# 1. Grab builtin functions -# Where should I have these? Maybe OpenAPI github repo and just preload? - -echo "Building frontend" -cd frontend -npm run build -rm -rf ../backend/go-app/build -cp -r build/ ../backend/go-app/build - -echo "Setting up backend" -cd ../backend/go-app -go build -go test - -#gcloud app deploy $GOPATH/src/github.com/frikky/shuffle/app.yaml -echo "This script is not finished. Please follow the docker installation guide here: https://github.com/frikky/shuffle/blob/master/install-guide.md"