From 8fe024440169722fb3970351c8f46a341a94aeb0 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 30 May 2020 11:49:23 +0200 Subject: [PATCH] Updated OpenAPI app creation process --- backend/go-app/codegen.go | 192 ++++++++++++++++-- backend/go-app/main.go | 6 +- backend/go-app/walkoff.go | 1 + frontend/package.json | 1 + frontend/src/AppCreator.js | 402 +++++++++++++++++++++++++++++++------ frontend/src/Apps.js | 73 ++++--- 6 files changed, 555 insertions(+), 120 deletions(-) diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index c8e4e6ee..368ae854 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -4,12 +4,14 @@ import ( "archive/zip" "bytes" "context" + "encoding/json" "errors" "fmt" "io" "io/ioutil" "log" "os" + "strconv" "strings" "cloud.google.com/go/storage" @@ -365,6 +367,20 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (WorkflowApp, []stri // Setting up security schemes extraParameters := []WorkflowAppActionParameter{} + if val, ok := swagger.Info.ExtensionProps.Extensions["x-logo"]; ok { + j, err := json.Marshal(&val) + if err == nil { + if j[0] == 0x22 && j[len(j)-1] == 0x22 { + j = j[1 : len(j)-1] + } + + //log.Printf("%s", j) + api.SmallImage = string(j) + api.LargeImage = string(j) + log.Printf("Set images!") + } + } + securitySchemes := swagger.Components.SecuritySchemes if securitySchemes != nil { //log.Printf("%#v", securitySchemes) @@ -719,6 +735,24 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ }, } + // FIXME: Example & Multiline + if param.Value.Example != nil { + curParam.Example = param.Value.Example.(string) + + if param.Value.Name == "body" { + curParam.Value = param.Value.Example.(string) + } + } + if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { + j, err := json.Marshal(&val) + if err == nil { + b, err := strconv.ParseBool(string(j)) + if err == nil { + curParam.Multiline = b + } + } + } + if param.Value.Required { action.Parameters = append(action.Parameters, curParam) } else { @@ -809,6 +843,24 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor }, } + // FIXME: Example & Multiline + if param.Value.Example != nil { + curParam.Example = param.Value.Example.(string) + + if param.Value.Name == "body" { + curParam.Value = param.Value.Example.(string) + } + } + if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { + j, err := json.Marshal(&val) + if err == nil { + b, err := strconv.ParseBool(string(j)) + if err == nil { + curParam.Multiline = b + } + } + } + if param.Value.Required { action.Parameters = append(action.Parameters, curParam) } else { @@ -894,6 +946,24 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo }, } + // FIXME: Example & Multiline + if param.Value.Example != nil { + curParam.Example = param.Value.Example.(string) + + if param.Value.Name == "body" { + curParam.Value = param.Value.Example.(string) + } + } + if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { + j, err := json.Marshal(&val) + if err == nil { + b, err := strconv.ParseBool(string(j)) + if err == nil { + curParam.Multiline = b + } + } + } + if param.Value.Required { action.Parameters = append(action.Parameters, curParam) } else { @@ -979,6 +1049,24 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] }, } + // FIXME: Example & Multiline + if param.Value.Example != nil { + curParam.Example = param.Value.Example.(string) + + if param.Value.Name == "body" { + curParam.Value = param.Value.Example.(string) + } + } + if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { + j, err := json.Marshal(&val) + if err == nil { + b, err := strconv.ParseBool(string(j)) + if err == nil { + curParam.Multiline = b + } + } + } + if param.Value.Required { action.Parameters = append(action.Parameters, curParam) } else { @@ -1049,23 +1137,27 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo firstQuery = true optionalQueries := []string{} parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{ - WorkflowAppActionParameter{ - Name: "body", - Description: "The body to use", - Multiline: true, - Required: false, - Example: `{"username": "test"}`, - Schema: SchemaDefinition{ - Type: "string", + optionalParameters := []WorkflowAppActionParameter{} + /* + WorkflowAppActionParameter{ + Name: "body", + Description: "The body to use", + Multiline: true, + Required: false, + Example: `{"username": "test"}`, + Schema: SchemaDefinition{ + Type: "string", + }, }, - }, - } + } + */ if len(path.Post.Parameters) > 0 { for _, param := range path.Post.Parameters { if param.Value.Schema == nil { continue } + + log.Printf("PARAM: %#v", param.Value) curParam := WorkflowAppActionParameter{ Name: param.Value.Name, Description: param.Value.Description, @@ -1076,6 +1168,24 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo }, } + // FIXME: Example & Multiline + if param.Value.Example != nil { + curParam.Example = param.Value.Example.(string) + + if param.Value.Name == "body" { + curParam.Value = param.Value.Example.(string) + } + } + if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { + j, err := json.Marshal(&val) + if err == nil { + b, err := strconv.ParseBool(string(j)) + if err == nil { + curParam.Multiline = b + } + } + } + if param.Value.Required { action.Parameters = append(action.Parameters, curParam) } else { @@ -1172,6 +1282,24 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W }, } + // FIXME: Example & Multiline + if param.Value.Example != nil { + curParam.Example = param.Value.Example.(string) + + if param.Value.Name == "body" { + curParam.Value = param.Value.Example.(string) + } + } + if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { + j, err := json.Marshal(&val) + if err == nil { + b, err := strconv.ParseBool(string(j)) + if err == nil { + curParam.Multiline = b + } + } + } + if param.Value.Required { action.Parameters = append(action.Parameters, curParam) } else { @@ -1241,18 +1369,20 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor firstQuery = true optionalQueries := []string{} parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{ - WorkflowAppActionParameter{ - Name: "body", - Description: "The body to use", - Multiline: true, - Required: false, - Example: `{"username": "test"}`, - Schema: SchemaDefinition{ - Type: "string", + optionalParameters := []WorkflowAppActionParameter{} + /* + WorkflowAppActionParameter{ + Name: "body", + Description: "The body to use", + Multiline: true, + Required: false, + Example: `{"username": "test"}`, + Schema: SchemaDefinition{ + Type: "string", + }, }, - }, - } + } + */ if len(path.Put.Parameters) > 0 { for _, param := range path.Put.Parameters { if param.Value.Schema == nil { @@ -1268,6 +1398,24 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor }, } + // FIXME: Example & Multiline + if param.Value.Example != nil { + curParam.Example = param.Value.Example.(string) + + if param.Value.Name == "body" { + curParam.Value = param.Value.Example.(string) + } + } + if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { + j, err := json.Marshal(&val) + if err == nil { + b, err := strconv.ParseBool(string(j)) + if err == nil { + curParam.Multiline = b + } + } + } + if param.Value.Required { action.Parameters = append(action.Parameters, curParam) } else { diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 53d21219..356e0e4a 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5543,10 +5543,6 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { } api.Owner = user.Id - if len(test.Image) > 0 { - api.SmallImage = test.Image - api.LargeImage = test.Image - } err = dumpApi(basePath, api) if err != nil { @@ -5569,7 +5565,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { identifier = strings.Replace(identifier, " ", "-", -1) identifier = strings.Replace(identifier, "_", "-", -1) - log.Printf("Successfully uploaded %s to bucket. Proceeding to cloud function", identifier) + log.Printf("Successfully parsed %s. Proceeding to docker container", identifier) // Now that the baseline is setup, we need to make it into a cloud function // 1. Upload the API to datastore for use diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index af100687..3ddf336e 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2744,6 +2744,7 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": false}`)) return } + log.Printf("API: %#v", parsedApi) //log.Printf("Parsed API: %#v", parsedApi) if len(parsedApi.ID) > 0 { diff --git a/frontend/package.json b/frontend/package.json index cb2263b1..a7f6e0b1 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -51,6 +51,7 @@ "react-router-dom": "^4.3.1", "react-scripts": "^2.1.8", "reactstrap": "^7.1.0", + "shellwords": "^0.1.1", "simplebar": "^4.2.3", "styled-components": "^4.4.0", "websocket": "^1.0.30", diff --git a/frontend/src/AppCreator.js b/frontend/src/AppCreator.js index 6ccd9d35..c27008da 100644 --- a/frontend/src/AppCreator.js +++ b/frontend/src/AppCreator.js @@ -2,6 +2,7 @@ import React, {useState, useEffect} from 'react'; import { makeStyles } from '@material-ui/styles'; import {BrowserView, MobileView} from "react-device-detect"; +import {Link} from 'react-router-dom'; import Paper from '@material-ui/core/Paper'; import Button from '@material-ui/core/Button'; import Divider from '@material-ui/core/Divider'; @@ -18,6 +19,7 @@ import CheckCircleIcon from '@material-ui/icons/CheckCircle'; import ErrorOutline from '@material-ui/icons/ErrorOutline'; import { useAlert } from "react-alert"; +import words from "shellwords" const surfaceColor = "#27292D" const inputColor = "#383B40" @@ -55,7 +57,118 @@ const useStyles = makeStyles({ notchedOutline: { borderColor: "#f85a3e !important" }, -}); +}) + + +function rewrite(args) { + return args.reduce(function(args, a){ + if (0 == a.indexOf('-X')) { + args.push('-X') + args.push(a.slice(2)) + } else { + args.push(a) + } + + return args + }, []) +} + +function parseField(s) { + return s.split(/: (.+)/) +} + +function isURL(s) { + return /^https?:\/\//.test(s) +} + +const parseCurl = (s) => { + //console.log("CURL: ", s) + + if (0 != s.indexOf('curl ')) { + console.log("Not curl start") + return "" + } + + var args = rewrite(words.split(s)) + var out = { method: 'GET', header: {} } + var state = '' + + args.forEach(function(arg){ + switch (true) { + case isURL(arg): + out.url = arg + break; + + case arg == '-A' || arg == '--user-agent': + state = 'user-agent' + break; + + case arg == '-H' || arg == '--header': + state = 'header' + break; + + case arg == '-d' || arg == '--data' || arg == '--data-ascii': + state = 'data' + break; + + case arg == '-u' || arg == '--user': + state = 'user' + break; + + case arg == '-I' || arg == '--head': + out.method = 'HEAD' + break; + + case arg == '-X' || arg == '--request': + state = 'method' + break; + + case arg == '-b' || arg =='--cookie': + state = 'cookie' + break; + + case arg == '--compressed': + out.header['Accept-Encoding'] = out.header['Accept-Encoding'] || 'deflate, gzip' + break; + + case !!arg: + switch (state) { + case 'header': + var field = parseField(arg) + out.header[field[0]] = field[1] + state = '' + break; + case 'user-agent': + out.header['User-Agent'] = arg + state = '' + break; + case 'data': + if (out.method == 'GET' || out.method == 'HEAD') out.method = 'POST' + out.header['Content-Type'] = out.header['Content-Type'] || 'application/x-www-form-urlencoded' + out.body = out.body + ? out.body + '&' + arg + : arg + state = '' + break; + case 'user': + out.header['Authorization'] = 'Basic ' + btoa(arg) + state = '' + break; + case 'method': + out.method = arg + state = '' + break; + case 'cookie': + out.header['Set-Cookie'] = arg + state = '' + break; + } + break; + } + }) + + return out +} // Should be different if logged in :| const AppCreator = (props) => { @@ -144,13 +257,13 @@ const AppCreator = (props) => { const handleEditApp = () => { fetch(globalUrl+"/api/v1/apps/"+props.match.params.appid+"/config", { - method: 'GET', + method: 'GET', headers: { 'Content-Type': 'application/json', 'Accept': 'application/json', }, - credentials: "include", - }) + credentials: "include", + }) .then((response) => { if (response.status !== 200) { window.location.pathname = "/apps" @@ -164,9 +277,7 @@ const AppCreator = (props) => { alert.error("Failed to get the app") } else { const data = JSON.parse(responseJson.body) - console.log("LOADED IMAGE: ", data.image) - setFileBase64(data.image) - parseOpenapiData(data) + parseIncomingOpenapiData(data) } }) .catch(error => { @@ -184,13 +295,13 @@ const AppCreator = (props) => { } fetch(globalUrl+"/api/v1/get_openapi/"+urlParams.get("id"), { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - }, - credentials: "include", - }) + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) .then((response) => { if (response.status !== 200) { throw new Error("NOT 200 :O") @@ -199,12 +310,12 @@ const AppCreator = (props) => { return response.json() }) .then((responseJson) => { - setIsAppLoaded(true) + setIsAppLoaded(true) if (!responseJson.success) { alert.error("Failed to verify") } else { const data = JSON.parse(responseJson.body) - parseOpenapiData(data) + parseIncomingOpenapiData(data) } }) .catch(error => { @@ -227,14 +338,20 @@ const AppCreator = (props) => { } // Sets the data up as it should be at later points - const parseOpenapiData = (data) => { + // This is the data FROM the database, not what's being saved + const parseIncomingOpenapiData = (data) => { setBasedata(data) - setName(data.info.title) setDescription(data.info.description) document.title = "Apps - "+data.info.title + console.log(data.info["x-logo"]) + if (data.info !== null && data.info !== undefined && data.info["x-logo"] !== undefined) { + setFileBase64(data.info["x-logo"]) + console.log("IMG: ", data.inf0["x-logo"]) + } + if (data.info.contact != undefined) { setContact(data.info.contact) } @@ -243,7 +360,7 @@ const AppCreator = (props) => { setBaseUrl(data.servers[0].url) } - console.log(data) + // This is annoying (: var securitySchemes = data.components.securityDefinitions @@ -295,9 +412,6 @@ const AppCreator = (props) => { "errors": [], } - //console.log(`${path}: ${method}`); - //console.log(methodvalue) - for (var key in methodvalue.parameters) { const parameter = methodvalue.parameters[key] if (parameter.in === "query") { @@ -323,12 +437,12 @@ const AppCreator = (props) => { } } - console.log(newActions) setActions(newActions) } + // Saving the app that's been configured. const submitApp = () => { - alert.info("Uploading private app " + name) + alert.info("Uploading and building app " + name) setErrorCode("") // Format the information @@ -336,6 +450,7 @@ const AppCreator = (props) => { const host = splitBase[2] const schemes = [splitBase[0]] const basePath = "/"+(splitBase.slice(3, )).join("/") + console.log("IMAGE: ", fileBase64) const data = { "swagger": "3.0", @@ -343,6 +458,7 @@ const AppCreator = (props) => { "title": name, "description": description, "version": "1.0", + "x-logo": fileBase64, }, "servers": [{"url": baseUrl}], "host": host, @@ -353,7 +469,6 @@ const AppCreator = (props) => { "components": { "securitySchemes": {}, }, - "image": fileBase64, "id": props.match.params.appid, "securityDefinitions": {}, } @@ -368,7 +483,7 @@ const AppCreator = (props) => { data.info["contact"] = contact } - console.log("LOADED IMAGE: ", data.image) + //console.log("LOADED IMAGE: ", data.image) for (var key in actions) { const item = actions[key] @@ -440,6 +555,23 @@ const AppCreator = (props) => { //console.log(queryitem) } } + + if (item.body.length > 0) { + console.log("HANDLE BODY!") + newitem = { + "in": "body", + "name": "body", + "multiline": true, + "description": "Generated by shuffler.io OpenAPI", + "required": false, + "example": item.body, + "schema": { + "type": "string", + }, + } + + data.paths[item.url][item.method.toLowerCase()].parameters.push(newitem) + } } if (authenticationOption === "API key") { @@ -462,7 +594,7 @@ const AppCreator = (props) => { } fetch(globalUrl+"/api/v1/verify_openapi", { - method: 'POST', + method: 'POST', headers: { 'Content-Type': 'application/json', 'Accept': 'application/json', @@ -495,9 +627,9 @@ const AppCreator = (props) => { } const bearerAuth = authenticationOption === "Bearer auth" ? -
+

- + Bearer auth

@@ -508,9 +640,9 @@ const AppCreator = (props) => { // Basicauth const basicAuth = authenticationOption === "Basic auth" ? -
+

- + Basic authentication

@@ -596,9 +728,9 @@ const AppCreator = (props) => { setActions(actions) } - console.log("Option: ", authenticationOption) - console.log("Location: ", parameterLocation) - console.log("Name: ", parameterName) + //console.log("Option: ", authenticationOption) + //console.log("Location: ", parameterLocation) + //console.log("Name: ", parameterName) const apiKey = authenticationOption === "API key" ?

API key

@@ -818,13 +950,30 @@ const AppCreator = (props) => { } const UrlPathParameters = () => { + var paths = [] + var queries = [] + if (urlPath.includes("{") && urlPath.includes("}")) { - var values = [] var tmpWord = "" var record = false + + var query = false for (var key in urlPath) { + if (urlPath[key] === "?") { + query = true + } + if (urlPath[key] === "}") { - values.push(tmpWord) + if (tmpWord === parameterName) { + tmpWord = "" + record = false + continue + } else if (query) { + queries.push(tmpWord) + } else { + paths.push(tmpWord) + } + tmpWord = "" record = false } @@ -833,24 +982,75 @@ const AppCreator = (props) => { tmpWord += urlPath[key] } - if (urlPath[key] === "{" && urlPath[key-1] === "/") { + //if (urlPath[key] === "{" && urlPath[key-1] === "/") { + if (urlPath[key] === "{") { record = true } } - - if (!currentAction.paths === values) { - currentAction.paths = values - setCurrentAction(currentAction) - } - - return ( -
- Required parameters: {values.join(", ")} -
- ) } - return null + if (urlPath.includes("<") && urlPath.includes(">")) { + var tmpWord = "" + var record = false + + var query = false + for (var key in urlPath) { + if (urlPath[key] === "?") { + query = true + } + + if (urlPath[key] === ">") { + if (tmpWord === parameterName) { + tmpWord = "" + record = false + continue + } else if (query) { + queries.push(tmpWord) + } else { + paths.push(tmpWord) + } + + tmpWord = "" + record = false + } + + if (record) { + tmpWord += urlPath[key] + } + + //if (urlPath[key] === "{" && urlPath[key-1] === "/") { + if (urlPath[key] === "<") { + record = true + } + } + } + + if (currentAction.paths !== paths) { + setActionField("paths", paths) + } + + console.log("QUERIES: ", queries) + var tmpQueries = [] + + // No overlapping of names + for (var key in queries) { + const tmpquery = queries[key] + const found = tmpQueries.find(query => query.name === tmpquery) + if (found === undefined) { + tmpQueries.push({"name": queries[key], required: true}) + } + } + + // FIXME: Frontend isn't updating.. + if (JSON.stringify(tmpQueries) !== JSON.stringify(urlPathQueries)) { + setUrlPathQueries(tmpQueries) + } + + return paths.length > 0 ? +
+ Required parameters: {paths.join(", ")} +
+ : null } const newActionModal = @@ -878,7 +1078,7 @@ const AppCreator = (props) => {
New action
 - Learn more about app creation + Learn more about actions
Name { margin="normal" variant="outlined" defaultValue={currentAction["name"]} - onChange={e => setActionField("name", e.target.value)} + onChange={e => { + // Fix basic issues in frontend. Python functions run a-zA-Z0-9_ + const regex = /[A-Z-a-z0-9 _]/g; + const found = e.target.value.match(regex); + if (found !== null) { + setActionField("name", found.join("")) + } + }} key={currentAction} InputProps={{ classes: { @@ -951,7 +1158,7 @@ const AppCreator = (props) => { ))}
- URL path + URL path / Curl statement { setUrlPath(e.target.value) console.log(e.target.value) }} - helperText={
The path to use. Must start with /. Add {"{variable}"} to have path variables
} + helperText={
The path to use. Must start with /. Use {"{variablename}"} to have path variables
} InputProps={{ classes: { notchedOutline: classes.notchedOutline, @@ -976,6 +1183,55 @@ const AppCreator = (props) => { color: "white", }, }} + onBlur={event => { + var parsedurl = event.target.value + if (parsedurl.startsWith("curl")) { + const request = parseCurl(event.target.value) + console.log(request) + if (request.method.toUpperCase() !== currentAction.Method) { + setCurrentActionMethod(request.method.toUpperCase()) + setActionField("method", request.method.toUpperCase()) + } + + if (request.header !== undefined && request.header !== null) { + var headers = "" + for (let [key, value] of Object.entries(request.header)) { + headers += key+"="+value+"\n" + } + + setActionField("headers", headers) + } + + if (request.body !== undefined && request.body !== null) { + setActionField("body", request.body) + } + + // Parse URL + parsedurl = request.url + } + + if (parsedurl.startsWith("http") || parsedurl.startsWith("ftp")) { + if (parsedurl !== undefined && parsedurl.includes(parameterName)) { + // Remove <> etc. + // + + console.log("IT HAS THE PARAM NAME!") + const newurl = new URL(encodeURI(parsedurl)) + newurl.searchParams.delete(parameterName) + parsedurl = decodeURI(newurl.href) + } + + // Remove the base URL itself + if (parsedurl !== undefined && baseUrl !== undefined && baseUrl.length > 0 && parsedurl.includes(baseUrl)) { + parsedurl = parsedurl.replace(baseUrl, "") + } + + // Check URL query && headers + setActionField("url", parsedurl) + setUrlPath(parsedurl) + } + //console.log("URL: ", request.url) + }} /> {loopQueries} @@ -1029,7 +1285,7 @@ const AppCreator = (props) => {

Actions

Actions are the tasks performed by an app. Read more about actions and apps - here. + here.
{loopActions}