From 54ed729056ea4b9ccc5840d162c535fdba0ea736 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 13 Jun 2020 15:55:32 +0200 Subject: [PATCH 1/7] Added info to README --- README.md | 10 +++++----- docker-compose.yml | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 7262a756..4da0022e 100644 --- a/README.md +++ b/README.md @@ -13,17 +13,17 @@ [1. Introducing Shuffle](https://medium.com/security-operation-capybara/introducing-shuffle-an-open-source-soar-platform-part-1-58a529de7d12) [2. Getting started with Shuffle](https://medium.com/security-operation-capybara/getting-started-with-shuffle-an-open-source-soar-platform-part-2-1d7c67a64244) +## Documentation +[Documentation](https://shuffler.io/docs) can be found on https://shuffler.io/docs/about or in your own instance. Currently lacking: +* API documentation +* Updates after migrating from SaaS to open source + ## Related repositories * Apps: https://github.com/frikky/shuffle-apps * Workflows: https://github.com/frikky/shuffle-workflows (empty) * Security OpenAPI apps: https://github.com/frikky/OpenAPI-security-definitions * Documentation: https://github.com/frikky/shuffle-docs -## Documentation -Documentation can be found on https://shuffler.io/docs/about or in your own instance. Currently lacking: -* API documentation -* Updates after migrating from SaaS to open source - ## Features * Simple workflow automation editor * Premade apps for a number of security tools diff --git a/docker-compose.yml b/docker-compose.yml index c00daf34..9fbfea69 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - #build: ./frontend + build: ./frontend image: frikky/shuffle:frontend container_name: shuffle-frontend hostname: shuffle-frontend @@ -14,7 +14,7 @@ services: depends_on: - backend backend: - #build: ./backend + build: ./backend image: frikky/shuffle:backend container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} From 4f25d9c2e7dbb20fcacffd7e038dc8a130f78776 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 13 Jun 2020 16:52:38 +0200 Subject: [PATCH 2/7] Fixed code in docs --- README.md | 5 +++-- backend/go-app/codegen.go | 2 -- docker-compose.yml | 2 +- frontend/src/Docs.js | 19 +++++++++---------- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 4da0022e..4f180ec2 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,9 @@ * Cloud: Register at https://shuffler.io/register and get cooking (there are some differences!) ## Blogposts -[1. Introducing Shuffle](https://medium.com/security-operation-capybara/introducing-shuffle-an-open-source-soar-platform-part-1-58a529de7d12) -[2. Getting started with Shuffle](https://medium.com/security-operation-capybara/getting-started-with-shuffle-an-open-source-soar-platform-part-2-1d7c67a64244) +* [1. Introducing Shuffle](https://medium.com/security-operation-capybara/introducing-shuffle-an-open-source-soar-platform-part-1-58a529de7d12) +* [2. Getting started with Shuffle](https://medium.com/security-operation-capybara/getting-started-with-shuffle-an-open-source-soar-platform-part-2-1d7c67a64244) +* [3. Integrating Shuffle with Virustotal and TheHive](https://medium.com/@Frikkylikeme/integrating-shuffle-with-virustotal-and-thehive-open-source-soar-part-3-8e2e0d3396a9) ## Documentation [Documentation](https://shuffler.io/docs) can be found on https://shuffler.io/docs/about or in your own instance. Currently lacking: diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index 48711b1a..30567b16 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -379,8 +379,6 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet verifyAddin, ) - log.Printf("CODE: %s", data) - //log.Println(data) //log.Println(functionname) return functionname, data diff --git a/docker-compose.yml b/docker-compose.yml index 9fbfea69..77ee3a1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - build: ./frontend + #build: ./frontend image: frikky/shuffle:frontend container_name: shuffle-frontend hostname: shuffle-frontend diff --git a/frontend/src/Docs.js b/frontend/src/Docs.js index 9a0df2ea..69d0eb3f 100644 --- a/frontend/src/Docs.js +++ b/frontend/src/Docs.js @@ -58,8 +58,6 @@ const Docs = (props) => { // Continue this, and find the h2 with the data in it lol if (window.location.hash.length > 0) { - console.log("HELLO") - var parent = document.getElementById("markdown_wrapper") if (parent !== null) { var elements = parent.getElementsByTagName('h2') @@ -166,6 +164,7 @@ const Docs = (props) => { flex: "1", maxWidth: 750, overflow: "hidden", + paddingBottom: 200, } function OuterLink(props) { @@ -180,15 +179,20 @@ const Docs = (props) => { } function CodeHandler(props) { - return {props.value} + return ( + 
+				
+					{props.value}
+				
+
+ ) } function Heading(props) { const element = React.createElement(`h${props.level}`, {style: {marginTop: 25}}, props.children) - console.log(props) return ( - + {props.level !== 1 ? : null} {element} ) @@ -208,11 +212,6 @@ const Docs = (props) => {
    -
  • - -

    Home

    -     -
    • {list.map(item => { const path = "/docs/"+item const newname = item.charAt(0).toUpperCase()+item.substring(1).split("_").join(" ") From 302044b4f3446bcf1da18397b8ce81a71de41f26 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 13 Jun 2020 19:03:19 +0200 Subject: [PATCH 3/7] Added color change to edges in graph --- frontend/src/AngularWorkflow.js | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/frontend/src/AngularWorkflow.js b/frontend/src/AngularWorkflow.js index ccb2d3ab..8bf68950 100644 --- a/frontend/src/AngularWorkflow.js +++ b/frontend/src/AngularWorkflow.js @@ -62,8 +62,6 @@ import cxtmenu from 'cytoscape-cxtmenu'; import { w3cwebsocket as W3CWebSocket } from "websocket"; import { useAlert } from "react-alert"; -const hoverColor = "#f85a3e" -const hoverOutColor = "#e8eaf6" const surfaceColor = "#27292D" const inputColor = "#383B40" @@ -1142,7 +1140,6 @@ const AngularWorkflow = (props) => { } const onNodeHover = (event) => { - //event.target.style("border-width", "5px") event.target.animate({ style: { "border-width": "5px", @@ -1156,24 +1153,22 @@ const AngularWorkflow = (props) => { } const onEdgeHoverOut = (event) => { - //event.target.removeStyle() + event.target.removeStyle() } // This is here to have a proper transition for lines const onEdgeHover = (event) => { - - //console.log(event.target.data()) - //const sourcecolor = cy.getElementById(event.target.data("source")).style("border-color") - //const targetcolor = cy.getElementById(event.target.data("target")).style("border-color") - //event.target.animate({ - // style: { - // "line-fill": "linear-gradient", - // 'target-arrow-color': targetcolor, - // "line-gradient-stop-colors": [sourcecolor, targetcolor], - // "line-gradient-stop-positions": [0, 1], - // }, - // duration: 0, - //}) + const sourcecolor = cy.getElementById(event.target.data("source")).style("border-color") + const targetcolor = cy.getElementById(event.target.data("target")).style("border-color") + event.target.animate({ + style: { + "line-fill": "linear-gradient", + 'target-arrow-color': targetcolor, + "line-gradient-stop-colors": [sourcecolor, targetcolor], + "line-gradient-stop-positions": [0, 1], + }, + duration: 0, + }) } From 0c2e4966b0720a56e0b652b897bbab1bdebbb70a Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 14 Jun 2020 08:21:38 +0200 Subject: [PATCH 4/7] Fixed spaces and headers in app generation actions --- backend/go-app/codegen.go | 283 ++++++++++++++++++++++---------- frontend/src/AngularWorkflow.js | 20 ++- frontend/src/AppCreator.js | 67 +++++--- 3 files changed, 258 insertions(+), 112 deletions(-) diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index 30567b16..3094092c 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -243,7 +243,7 @@ func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) { } // This function generates the python code that's being used. -func makePythoncode(swagger *openapi3.Swagger, name, url, method string, parameters, optionalQueries []string) (string, string) { +func makePythoncode(swagger *openapi3.Swagger, name, url, method string, parameters, optionalQueries, headers []string) (string, string) { method = strings.ToLower(method) queryString := "" queryData := "" @@ -349,10 +349,33 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet } } + preparedHeaders := "headers={}" + if len(headers) > 0 { + preparedHeaders = "headers={" + for count, header := range headers { + headerSplit := strings.Split(header, "=") + added := false + if len(headerSplit) == 2 { + if strings.Contains(preparedHeaders, headerSplit[0]) { + continue + } + + preparedHeaders += fmt.Sprintf(`"%s": "%s"`, headerSplit[0], headerSplit[1]) + added = true + } + + if count != len(headers)-1 && added { + preparedHeaders += "," + } + } + + preparedHeaders += "}" + } + // Extra param for url if it's changeable // Extra param for authentication scheme(s) data := fmt.Sprintf(` async def %s(self%s%s%s%s%s%s): - headers={} + %s url=f"%s%s" %s %s @@ -367,6 +390,7 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet queryString, bodyParameter, verifyParam, + preparedHeaders, urlInline, url, verifyWrapper, @@ -531,7 +555,11 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, // Could just as well be go at this point lol pythonFunctions := []string{} for actualPath, path := range swagger.Paths { - // FIXME: Add everything from here: + actualPath = strings.Replace(actualPath, " ", "_", -1) + actualPath = strings.Replace(actualPath, ".", "", -1) + actualPath = strings.Replace(actualPath, ".", "", -1) + actualPath = strings.Replace(actualPath, "\\", "", -1) + // https://godoc.org/github.com/getkin/kin-openapi/openapi3#PathItem if path.Get != nil { action, curCode := handleGet(swagger, api, extraParameters, path, actualPath) @@ -808,13 +836,26 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ } } + headersFound := []string{} if len(path.Connect.Parameters) > 0 { - for _, param := range path.Connect.Parameters { - if param.Value.Schema == nil || param.Value.In == "header" { + for counter, param := range path.Connect.Parameters { + if param.Value.Schema == nil { + continue + } else if param.Value.In == "header" { + headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) continue } + + parsedName := param.Value.Name + parsedName = strings.ReplaceAll(parsedName, " ", "_") + parsedName = strings.ReplaceAll(parsedName, ",", "_") + parsedName = strings.ReplaceAll(parsedName, ".", "_") + parsedName = strings.ReplaceAll(parsedName, "|", "_") + param.Value.Name = parsedName + path.Connect.Parameters[counter].Value.Name = parsedName + curParam := WorkflowAppActionParameter{ - Name: param.Value.Name, + Name: parsedName, Description: param.Value.Description, Multiline: false, Required: param.Value.Required, @@ -841,14 +882,8 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ } } - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - if param.Value.In == "path" { - parameters = append(parameters, param.Value.Name) + parameters = append(parameters, curParam.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { //log.Printf("QUERY!: %s", param.Value.Name) @@ -871,6 +906,12 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ firstQuery = false } + if param.Value.Required { + action.Parameters = append(action.Parameters, curParam) + } else { + optionalParameters = append(optionalParameters, curParam) + } + } } @@ -880,7 +921,7 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "connect", parameters, optionalQueries) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "connect", parameters, optionalQueries, headersFound) if len(functionname) > 0 { action.Name = functionname @@ -905,8 +946,6 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor action.Returns.Schema.Type = "string" baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - //log.Println(path.Parameters) - // Parameters: []WorkflowAppActionParameter{}, // FIXME - add data for POST stuff firstQuery := true @@ -941,14 +980,26 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor } } + headersFound := []string{} if len(path.Get.Parameters) > 0 { - for _, param := range path.Get.Parameters { - if param.Value.Schema == nil || param.Value.In == "header" { + for counter, param := range path.Get.Parameters { + if param.Value.Schema == nil { + continue + } else if param.Value.In == "header" { + headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) continue } + parsedName := param.Value.Name + parsedName = strings.ReplaceAll(parsedName, " ", "_") + parsedName = strings.ReplaceAll(parsedName, ",", "_") + parsedName = strings.ReplaceAll(parsedName, ".", "_") + parsedName = strings.ReplaceAll(parsedName, "|", "_") + param.Value.Name = parsedName + path.Get.Parameters[counter].Value.Name = parsedName + curParam := WorkflowAppActionParameter{ - Name: param.Value.Name, + Name: parsedName, Description: param.Value.Description, Multiline: false, Required: param.Value.Required, @@ -975,14 +1026,8 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor } } - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - if param.Value.In == "path" { - parameters = append(parameters, param.Value.Name) + parameters = append(parameters, curParam.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { //log.Printf("QUERY!: %s", param.Value.Name) @@ -1005,6 +1050,11 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor } firstQuery = false } + if param.Value.Required { + action.Parameters = append(action.Parameters, curParam) + } else { + optionalParameters = append(optionalParameters, curParam) + } } } @@ -1015,7 +1065,7 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "get", parameters, optionalQueries) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "get", parameters, optionalQueries, headersFound) if len(functionname) > 0 { action.Name = functionname @@ -1073,13 +1123,27 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo }) } } + + headersFound := []string{} if len(path.Head.Parameters) > 0 { - for _, param := range path.Head.Parameters { - if param.Value.Schema == nil || param.Value.In == "header" { + for counter, param := range path.Head.Parameters { + if param.Value.Schema == nil { + continue + } else if param.Value.In == "header" { + headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) continue } + + parsedName := param.Value.Name + parsedName = strings.ReplaceAll(parsedName, " ", "_") + parsedName = strings.ReplaceAll(parsedName, ",", "_") + parsedName = strings.ReplaceAll(parsedName, ".", "_") + parsedName = strings.ReplaceAll(parsedName, "|", "_") + param.Value.Name = parsedName + path.Head.Parameters[counter].Value.Name = parsedName + curParam := WorkflowAppActionParameter{ - Name: param.Value.Name, + Name: parsedName, Description: param.Value.Description, Multiline: false, Required: param.Value.Required, @@ -1106,14 +1170,8 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo } } - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - if param.Value.In == "path" { - parameters = append(parameters, param.Value.Name) + parameters = append(parameters, curParam.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { //log.Printf("QUERY!: %s", param.Value.Name) @@ -1136,6 +1194,12 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo firstQuery = false } + if param.Value.Required { + action.Parameters = append(action.Parameters, curParam) + } else { + optionalParameters = append(optionalParameters, curParam) + } + } } @@ -1145,7 +1209,7 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "head", parameters, optionalQueries) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "head", parameters, optionalQueries, headersFound) if len(functionname) > 0 { action.Name = functionname @@ -1203,13 +1267,27 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] }) } } + + headersFound := []string{} if len(path.Delete.Parameters) > 0 { - for _, param := range path.Delete.Parameters { - if param.Value.Schema == nil || param.Value.In == "header" { + for counter, param := range path.Delete.Parameters { + if param.Value.Schema == nil { + continue + } else if param.Value.In == "header" { + headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) continue } + + parsedName := param.Value.Name + parsedName = strings.ReplaceAll(parsedName, " ", "_") + parsedName = strings.ReplaceAll(parsedName, ",", "_") + parsedName = strings.ReplaceAll(parsedName, ".", "_") + parsedName = strings.ReplaceAll(parsedName, "|", "_") + param.Value.Name = parsedName + path.Delete.Parameters[counter].Value.Name = parsedName + curParam := WorkflowAppActionParameter{ - Name: param.Value.Name, + Name: parsedName, Description: param.Value.Description, Multiline: false, Required: param.Value.Required, @@ -1236,14 +1314,8 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] } } - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - if param.Value.In == "path" { - parameters = append(parameters, param.Value.Name) + parameters = append(parameters, curParam.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { //log.Printf("QUERY!: %s", param.Value.Name) @@ -1266,6 +1338,12 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] firstQuery = false } + if param.Value.Required { + action.Parameters = append(action.Parameters, curParam) + } else { + optionalParameters = append(optionalParameters, curParam) + } + } } @@ -1275,7 +1353,7 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound) if len(functionname) > 0 { action.Name = functionname @@ -1298,10 +1376,6 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo Parameters: extraParameters, } - //if path.Post.RequestBody != nil { - // log.Printf("RequestBody: %#v", path.Post.RequestBody) - //} - action.Returns.Schema.Type = "string" baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) @@ -1337,14 +1411,26 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo } } + headersFound := []string{} if len(path.Post.Parameters) > 0 { - for _, param := range path.Post.Parameters { - if param.Value.Schema == nil || param.Value.In == "header" { + for counter, param := range path.Post.Parameters { + if param.Value.Schema == nil { + continue + } else if param.Value.In == "header" { + headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) continue } + parsedName := param.Value.Name + parsedName = strings.ReplaceAll(parsedName, " ", "_") + parsedName = strings.ReplaceAll(parsedName, ",", "_") + parsedName = strings.ReplaceAll(parsedName, ".", "_") + parsedName = strings.ReplaceAll(parsedName, "|", "_") + param.Value.Name = parsedName + path.Post.Parameters[counter].Value.Name = parsedName + curParam := WorkflowAppActionParameter{ - Name: param.Value.Name, + Name: parsedName, Description: param.Value.Description, Multiline: false, Required: param.Value.Required, @@ -1357,7 +1443,7 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo if param.Value.Example != nil { curParam.Example = param.Value.Example.(string) - if param.Value.Name == "body" { + if parsedName == "body" { curParam.Value = param.Value.Example.(string) } } @@ -1371,14 +1457,8 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo } } - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - if param.Value.In == "path" { - parameters = append(parameters, param.Value.Name) + parameters = append(parameters, curParam.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { //log.Printf("QUERY!: %s", param.Value.Name) @@ -1401,6 +1481,11 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo firstQuery = false } + if param.Value.Required { + action.Parameters = append(action.Parameters, curParam) + } else { + optionalParameters = append(optionalParameters, curParam) + } } } @@ -1410,7 +1495,7 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "post", parameters, optionalQueries) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "post", parameters, optionalQueries, headersFound) if len(functionname) > 0 { action.Name = functionname @@ -1468,13 +1553,27 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W }) } } + + headersFound := []string{} if len(path.Patch.Parameters) > 0 { - for _, param := range path.Patch.Parameters { - if param.Value.Schema == nil || param.Value.In == "header" { + for counter, param := range path.Patch.Parameters { + if param.Value.Schema == nil { + continue + } else if param.Value.In == "header" { + headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) continue } + + parsedName := param.Value.Name + parsedName = strings.ReplaceAll(parsedName, " ", "_") + parsedName = strings.ReplaceAll(parsedName, ",", "_") + parsedName = strings.ReplaceAll(parsedName, ".", "_") + parsedName = strings.ReplaceAll(parsedName, "|", "_") + param.Value.Name = parsedName + path.Patch.Parameters[counter].Value.Name = parsedName + curParam := WorkflowAppActionParameter{ - Name: param.Value.Name, + Name: parsedName, Description: param.Value.Description, Multiline: false, Required: param.Value.Required, @@ -1501,14 +1600,8 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W } } - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - if param.Value.In == "path" { - parameters = append(parameters, param.Value.Name) + parameters = append(parameters, curParam.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) } else if param.Value.In == "query" { //log.Printf("QUERY!: %s", param.Value.Name) @@ -1531,6 +1624,11 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W firstQuery = false } + if param.Value.Required { + action.Parameters = append(action.Parameters, curParam) + } else { + optionalParameters = append(optionalParameters, curParam) + } } } @@ -1540,7 +1638,7 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "patch", parameters, optionalQueries) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound) if len(functionname) > 0 { action.Name = functionname @@ -1599,13 +1697,26 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor } } + headersFound := []string{} if len(path.Put.Parameters) > 0 { - for _, param := range path.Put.Parameters { - if param.Value.Schema == nil || param.Value.In == "header" { + for counter, param := range path.Put.Parameters { + if param.Value.Schema == nil { + continue + } else if param.Value.In == "header" { + headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) continue } + + parsedName := param.Value.Name + parsedName = strings.ReplaceAll(parsedName, " ", "_") + parsedName = strings.ReplaceAll(parsedName, ",", "_") + parsedName = strings.ReplaceAll(parsedName, ".", "_") + parsedName = strings.ReplaceAll(parsedName, "|", "_") + param.Value.Name = parsedName + path.Put.Parameters[counter].Value.Name = parsedName + curParam := WorkflowAppActionParameter{ - Name: param.Value.Name, + Name: parsedName, Description: param.Value.Description, Multiline: false, Required: param.Value.Required, @@ -1632,12 +1743,6 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor } } - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - if param.Value.In == "path" { parameters = append(parameters, param.Value.Name) //baseUrl = fmt.Sprintf("%s%s", baseUrl) @@ -1662,6 +1767,12 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor firstQuery = false } + if param.Value.Required { + action.Parameters = append(action.Parameters, curParam) + } else { + optionalParameters = append(optionalParameters, curParam) + } + } } @@ -1671,7 +1782,7 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor action.Parameters = append(action.Parameters, optionalParam) } - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "put", parameters, optionalQueries) + functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound) if len(functionname) > 0 { action.Name = functionname diff --git a/frontend/src/AngularWorkflow.js b/frontend/src/AngularWorkflow.js index 8bf68950..854249f7 100644 --- a/frontend/src/AngularWorkflow.js +++ b/frontend/src/AngularWorkflow.js @@ -2411,7 +2411,7 @@ const AngularWorkflow = (props) => {
    - {data.name}: + {data.name}
    { @@ -2497,6 +2497,13 @@ const AngularWorkflow = (props) => { //setStartNode(selectedAction.id) } + function sortByKey(array, key) { + return array.sort(function(a, b) { + var x = a[key]; var y = b[key] + return ((x < y) ? -1 : ((x > y) ? 1 : 0)) + }) + } + const appApiView = Object.getOwnPropertyNames(selectedAction).length > 0 && Object.getOwnPropertyNames(selectedApp).length > 0 ?
    @@ -2593,7 +2600,7 @@ const AngularWorkflow = (props) => { } }} > - {selectedApp.actions.map(data => { + {sortByKey(selectedApp.actions, "label").map(data => { var newActionname = data.name if (data.label !== undefined && data.label !== null && data.label.length > 0) { newActionname = data.label @@ -2605,13 +2612,17 @@ const AngularWorkflow = (props) => { newActionname = newActionname.replace("_", " ") newActionname = newActionname.charAt(0).toUpperCase()+newActionname.substring(1) return ( - + {newActionname} ) })} + {selectedAction.description !== undefined && selectedAction.description.length > 0 ? +
    + {selectedAction.description} +
    : null}
    @@ -4536,6 +4547,9 @@ const AngularWorkflow = (props) => { boxSelectionEnabled={true} autounselectify={false} cy={(incy) => { + // FIXME: There's something specific loading when + // you do the first hover of a node. Why is this different? + console.log("CY: ", incy) setCy(incy) }} /> diff --git a/frontend/src/AppCreator.js b/frontend/src/AppCreator.js index 690b73ea..91266d7f 100644 --- a/frontend/src/AppCreator.js +++ b/frontend/src/AppCreator.js @@ -614,12 +614,25 @@ const AppCreator = (props) => { const headersSplit = item.headers.split("\n") for (var key in headersSplit) { const header = headersSplit[key] + console.log("HEADER: ", header) var key = "" var value = "" - if (header.length > 0 && header.includes("=")) { + if (header.length > 0 && header.includes("= ")) { + const headersplit = header.split("= ") + key = headersplit[0] + value = headersplit[1] + } else if (header.length > 0 && header.includes("=")) { const headersplit = header.split("=") key = headersplit[0] value = headersplit[1] + } else if (header.length > 0 && header.includes(": ")) { + const headersplit = header.split(": ") + key = headersplit[0] + value = headersplit[1] + } else if (header.length > 0 && header.includes(":")) { + const headersplit = header.split(":") + key = headersplit[0] + value = headersplit[1] } else { continue } @@ -1273,6 +1286,10 @@ const AppCreator = (props) => { continue } + if (key === "Authorization" && authenticationOption === "Bearer auth") { + continue + } + headers += key+"="+value+"\n" } @@ -1284,33 +1301,37 @@ const AppCreator = (props) => { } // Parse URL - parsedurl = request.url + if (request.url !== undefined) { + parsedurl = request.url + } } - if (parsedurl.includes("<") && parsedurl.includes(">")) { - parsedurl = parsedurl.split("<").join("{") - parsedurl = parsedurl.split(">").join("}") - } - - if (parsedurl.startsWith("http") || parsedurl.startsWith("ftp")) { - if (parsedurl !== undefined && parsedurl.includes(parameterName)) { - // Remove <> etc. - // - - console.log("IT HAS THE PARAM NAME!") - const newurl = new URL(encodeURI(parsedurl)) - newurl.searchParams.delete(parameterName) - parsedurl = decodeURI(newurl.href) + if (parsedurl !== undefined) { + if (parsedurl.includes("<") && parsedurl.includes(">")) { + parsedurl = parsedurl.split("<").join("{") + parsedurl = parsedurl.split(">").join("}") } - // Remove the base URL itself - if (parsedurl !== undefined && baseUrl !== undefined && baseUrl.length > 0 && parsedurl.includes(baseUrl)) { - parsedurl = parsedurl.replace(baseUrl, "") - } + if (parsedurl.startsWith("http") || parsedurl.startsWith("ftp")) { + if (parsedurl !== undefined && parsedurl.includes(parameterName)) { + // Remove <> etc. + // + + console.log("IT HAS THE PARAM NAME!") + const newurl = new URL(encodeURI(parsedurl)) + newurl.searchParams.delete(parameterName) + parsedurl = decodeURI(newurl.href) + } - // Check URL query && headers - setActionField("url", parsedurl) - setUrlPath(parsedurl) + // Remove the base URL itself + if (parsedurl !== undefined && baseUrl !== undefined && baseUrl.length > 0 && parsedurl.includes(baseUrl)) { + parsedurl = parsedurl.replace(baseUrl, "") + } + + // Check URL query && headers + setActionField("url", parsedurl) + setUrlPath(parsedurl) + } } //console.log("URL: ", request.url) From 76ce2bddeefb71d2e60bc5da25e844cdec583e08 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 14 Jun 2020 18:36:45 +0200 Subject: [PATCH 5/7] Updated database location because of OS X issues --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index a07a0644..33d6f11c 100644 --- a/.env +++ b/.env @@ -13,4 +13,4 @@ BACKEND_PORT=5001 FRONTEND_PORT=3001 FRONTEND_PORT_HTTPS=3443 OUTER_HOSTNAME=shuffle-backend -DB_LOCATION=/etc/shuffle +DB_LOCATION=./etc/shuffle From d2d89d0bb9bee845490771257090f92ca56fd5bb Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 14 Jun 2020 18:40:55 +0200 Subject: [PATCH 6/7] Updated code generator for Shuffle --- README.md | 4 ++-- backend/app_sdk/app_base.py | 9 ++------- backend/go-app/codegen.go | 1 + frontend/src/AngularWorkflow.js | 3 ++- frontend/src/AppCreator.js | 2 +- functions/onprem/orborus/orborus.go | 6 +++--- 6 files changed, 11 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 4f180ec2..697aa3f1 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,8 @@ ## Related repositories * Apps: https://github.com/frikky/shuffle-apps -* Workflows: https://github.com/frikky/shuffle-workflows (empty) -* Security OpenAPI apps: https://github.com/frikky/OpenAPI-security-definitions +* Workflows: https://github.com/frikky/shuffle-workflows +* Security OpenAPI apps: https://github.com/frikky/security-openapis * Documentation: https://github.com/frikky/shuffle-docs ## Features diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 33fa9314..8bf5736b 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -73,12 +73,9 @@ class AppBase: except requests.exceptions.ConnectionError as e: print("Connectionerror: %s" % e) return - #self.logger.info("AFTER initial stream result") - self.logger.info("THIS IS THE NEW UPDATE") # Verify whether there are any parameters with ACTION_RESULT required # If found, we get the full results list from backend - fullexecution = {} try: tmpdata = { @@ -195,10 +192,9 @@ class AppBase: # Regex to find all the things if parameter["variant"] == "STATIC_VALUE": data = parameter["value"] - self.logger.debug(f"\n\nHandle static data with JSON: {data}\n\n") - actualitem = re.findall(match, data, re.MULTILINE) - self.logger.info("STATIC PARSED: %s" % actualitem) + #self.logger.debug(f"\n\nHandle static data with JSON: {data}\n\n") + #self.logger.info("STATIC PARSED: %s" % actualitem) if len(actualitem) > 0: for replace in actualitem: try: @@ -487,7 +483,6 @@ class AppBase: # With this parameter ready, add it to... a greater list of parameters. Rofl multi_parameters[parameter["name"]] = resultarray else: - print("Hello, in here?: %s" % value) params[parameter["name"]] = value multi_parameters[parameter["name"]] = value diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index 3094092c..7a728aa9 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -243,6 +243,7 @@ func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) { } // This function generates the python code that's being used. +// This is really meta when you program it. Handling parameters is hard here. func makePythoncode(swagger *openapi3.Swagger, name, url, method string, parameters, optionalQueries, headers []string) (string, string) { method = strings.ToLower(method) queryString := "" diff --git a/frontend/src/AngularWorkflow.js b/frontend/src/AngularWorkflow.js index 854249f7..c74ddf19 100644 --- a/frontend/src/AngularWorkflow.js +++ b/frontend/src/AngularWorkflow.js @@ -2281,6 +2281,7 @@ const AngularWorkflow = (props) => { rows="5" color="primary" defaultValue={data.value} + type={placeholder.includes("***") ? "password" : "text"} placeholder={placeholder} onChange={(event) => { changeActionParameter(event, count) @@ -2585,7 +2586,7 @@ const AngularWorkflow = (props) => { : null*/}
    -
    +
    Actions