From 4d6984e23e6672ab88026edbec42600b7477d286 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 7 Sep 2020 20:03:53 +0200 Subject: [PATCH 1/2] Fiddled with apps --- frontend/src/views/AngularWorkflow.jsx | 127 ++++++++++++++----------- 1 file changed, 72 insertions(+), 55 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index b940d7b5..3052d471 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -157,6 +157,7 @@ const AngularWorkflow = (props) => { const [apps, setApps] = React.useState([]); const [filteredApps, setFilteredApps] = React.useState([]); + const [prioritizedApps, setPrioritizedApps] = React.useState([]); const [firstrequest, setFirstrequest] = React.useState(true) //const [apps, setApps] = React.useState(appdata); //const [filteredApps, setFilteredApps] = React.useState(); @@ -794,15 +795,20 @@ const AngularWorkflow = (props) => { }); } + const internalIds = [ + "80a1fdd2-95c2-49ab-81f6-e05689beb745", // Shuffle tools + "39c5f8fa-a088-4cdc-826f-19e2e61cb284", // Testing + ] + const getApps = () => { fetch(globalUrl+"/api/v1/workflows/apps", { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - }, - credentials: "include", - }) + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) .then((response) => { if (response.status !== 200) { console.log("Status not 200 for apps :O!") @@ -819,6 +825,9 @@ const AngularWorkflow = (props) => { setApps(responseJson) setFilteredApps(responseJson) getAppAuthentication() + + + setPrioritizedApps(responseJson.filter(app => internalIds.includes(app.id))) }) .catch(error => { alert.error(error.toString()) @@ -2206,6 +2215,54 @@ const AngularWorkflow = (props) => { setFilteredApps(apps.filter(app => app.name.toLowerCase().includes(event.target.value.trim().toLowerCase()))) } + const ParsedAppPaper = (props) => { + const app = props.app + + // FIXME - add label to apps, as this might be slow with A LOT of apps + var newAppname = app.name + newAppname = newAppname.replace("_", " ").charAt(0).toUpperCase()+newAppname.substring(1) + const maxlen = 24 + if (newAppname.length > maxlen) { + newAppname = newAppname.slice(0, maxlen)+".." + } + + const image = "url("+app.large_image+")" + + return ( + {handleAppDrag(e, app)}} + onStop={(e) => {handleDragStop(e, app)}} + key={app.id} + dragging={false} + position={{ + x: 0, + y: 0, + }} + > + +
+
+ + +
+ + + +

{newAppname}

+ + + Short description... + + + Version: {app.app_version} + + + + + + ) + } + const AppView = () => { return(
@@ -2232,55 +2289,14 @@ const AngularWorkflow = (props) => { }} /> */} - {filteredApps.map(app=> { - // FIXME - add label to apps, as this might be slow with A LOT of apps - var newAppname = app.name - newAppname = newAppname.replace("_", " ").charAt(0).toUpperCase()+newAppname.substring(1) - const maxlen = 24 - if (newAppname.length > maxlen) { - newAppname = newAppname.slice(0, maxlen)+".." - } - - // Description fucks this up - fix overflow :) - //const maxdesclen = 5 - //var desc = app.description - //if (newAppname.length > maxdesclen) { - // desc = desc.slice(0, maxdesclen)+".." - //} - - const image = "url("+app.large_image+")" + {prioritizedApps.map((app, index) => { return( - {handleAppDrag(e, app)}} - onStop={(e) => {handleDragStop(e, app)}} - key={app.id} - dragging={false} - position={{ - x: 0, - y: 0, - }} - > - -
-
- - -
- - - -

{newAppname}

- - - Short description... - - - Version: {app.app_version} - - - - - + + ) + })} + {filteredApps.filter(innerapp => !internalIds.includes(innerapp.id)).map((app, index) => { + return( + ) })}
@@ -3347,6 +3363,7 @@ const AngularWorkflow = (props) => { overflow: "scroll", overflowX: "auto", overflowY: "auto", + zIndex: 1000, } const setTriggerFolderWrapperMulti = event => { From 6ec4fed722ffe29ee05aaf03c783e859db9f5c99 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 13 Sep 2020 08:10:46 +0200 Subject: [PATCH 2/2] Major fixes to app creator --- backend/app_sdk/app_base.py | 8 +- backend/app_sdk/build.sh | 4 +- backend/go-app/codegen.go | 115 +++++++++++++++++++++++-- backend/go-app/main.go | 22 ++--- frontend/src/views/Admin.jsx | 35 ++++---- frontend/src/views/AngularWorkflow.jsx | 55 +++++++----- frontend/src/views/AppCreator.jsx | 60 +++++++++---- functions/onprem/worker/worker.go | 12 ++- 8 files changed, 225 insertions(+), 86 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 29573512..e2a7f4f5 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -270,7 +270,7 @@ class AppBase: except TypeError: return data - print("Running %s" % data) + #print("Running %s" % data) # Look for the INNER wrapper first, then move out wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length"] @@ -337,7 +337,7 @@ class AppBase: if len(newstring) > 0: newdata.append(newstring) - print(newdata) + #print(newdata) parsedlist = [] non_string = False for item in newdata: @@ -352,7 +352,7 @@ class AppBase: elif len(parsedlist) == 1 and non_string: return parsedlist[0] else: - print("Casting back to string because multi: ", parsedlist) + #print("Casting back to string because multi: ", parsedlist) newlist = [] for item in parsedlist: try: @@ -870,7 +870,7 @@ class AppBase: if not multiexecution: print("APP_SDK DONE: Starting normal execution of function") newres = await func(**params) - print("NEWRES: ", newres) + #print("NEWRES: ", newres) if isinstance(newres, str): result += newres else: diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 0c7af3f7..34180390 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,9 +1,9 @@ #!/bin/bash NAME=app_sdk -VERSION=0.6.1 +VERSION=0.6.2 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force -docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/app_sdk:0.6.0 +docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION #docker push frikky/$NAME:$VERSION #docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index 2b3be803..a5898e5d 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -253,9 +253,13 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet parameterData := "" if len(optionalQueries) > 0 { queryString += ", " - for _, query := range optionalQueries { + for index, query := range optionalQueries { // Check if it's a part of the URL already - queryString += fmt.Sprintf("%s=\"\", ", query) + queryString += fmt.Sprintf("%s=\"\"", query) + if index != len(optionalQueries)-1 { + queryString += ", " + } + queryData += fmt.Sprintf(` if %s: url += f"&%s={%s}"`, query, query, query) @@ -274,8 +278,8 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet authenticationParameter = ", apikey" authenticationSetup = "if apikey != \" \": headers[\"Authorization\"] = f\"Bearer {apikey}\"" } else if swagger.Components.SecuritySchemes["BasicAuth"] != nil { - authenticationParameter = ", username, password" - authenticationAddin = ", auth=(username, password)" + authenticationParameter = ", username_basic, password_basic" + authenticationAddin = ", auth=(username_basic, password_basic)" } else if swagger.Components.SecuritySchemes["ApiKeyAuth"] != nil { authenticationParameter = ", apikey" if swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.In == "header" { @@ -401,6 +405,11 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet verifyAddin, ) + if strings.Contains(functionname, "get_returns_the_vuln") { + log.Println(data) + log.Printf("Queries: %s", queryString) + } + //log.Printf(data) return functionname, data } @@ -553,7 +562,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, }) } else if securitySchemes["BasicAuth"] != nil { api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{ - Name: "username", + Name: "username_auth", Value: "", Example: "username", Description: securitySchemes["BasicAuth"].Value.Description, @@ -565,7 +574,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, }) api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{ - Name: "password", + Name: "password_auth", Value: "", Example: "*****", Description: securitySchemes["BasicAuth"].Value.Description, @@ -577,7 +586,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, }) extraParameters = append(extraParameters, WorkflowAppActionParameter{ - Name: "username", + Name: "username_basic", Description: "The username to use", Multiline: false, Required: true, @@ -588,7 +597,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, }, }) extraParameters = append(extraParameters, WorkflowAppActionParameter{ - Name: "password", + Name: "password_basic", Description: "The password to use", Multiline: false, Required: true, @@ -846,21 +855,102 @@ func deployAppToDatastore(ctx context.Context, workflowapp WorkflowApp) error { return nil } +// FIXME: +// https://docs.python.org/3.2/reference/lexical_analysis.html#identifiers +// This is used to build the python functions. func fixFunctionName(functionName, actualPath string) string { if len(functionName) == 0 { functionName = actualPath } + + // REGEX THIS SHIT + // ROFL + //log.Printf("Fixing function name for %s", functionName) - functionName = strings.Replace(functionName, " ", "_", -1) functionName = strings.Replace(functionName, ".", "", -1) + functionName = strings.Replace(functionName, ",", "", -1) functionName = strings.Replace(functionName, ".", "", -1) + functionName = strings.Replace(functionName, "&", "", -1) functionName = strings.Replace(functionName, "/", "", -1) functionName = strings.Replace(functionName, "\\", "", -1) + + functionName = strings.Replace(functionName, "!", "", -1) + functionName = strings.Replace(functionName, "?", "", -1) + functionName = strings.Replace(functionName, "@", "", -1) + functionName = strings.Replace(functionName, "#", "", -1) + functionName = strings.Replace(functionName, "$", "", -1) + functionName = strings.Replace(functionName, "&", "", -1) + functionName = strings.Replace(functionName, "*", "", -1) + functionName = strings.Replace(functionName, "(", "", -1) + functionName = strings.Replace(functionName, ")", "", -1) + functionName = strings.Replace(functionName, "[", "", -1) + functionName = strings.Replace(functionName, "]", "", -1) + functionName = strings.Replace(functionName, "{", "", -1) + functionName = strings.Replace(functionName, "}", "", -1) + functionName = strings.Replace(functionName, `"`, "", -1) + functionName = strings.Replace(functionName, `'`, "", -1) + functionName = strings.Replace(functionName, `|`, "", -1) + functionName = strings.Replace(functionName, `~`, "", -1) + + functionName = strings.Replace(functionName, " ", "_", -1) + functionName = strings.Replace(functionName, "-", "_", -1) + functionName = strings.ToLower(functionName) return functionName } +// Returns a valid param name +func validateParameterName(name string) string { + invalid := []string{"False", + "await", + "else", + "import", + "pass", + "None", + "break", + "except", + "in", + "raise", + "True", + "class", + "finally", + "is", + "return", + "and", + "continue", + "for", + "lambda", + "try", + "as", + "def", + "from", + "nonlocal", + "while", + "assert", + "del", + "global", + "not", + "with", + "async", + "elif", + "if", + "or", + "yield", + } + + newname := name + for _, item := range invalid { + if item == name { + //log.Printf("%s is NOT a valid parameter name!", item) + newname = fmt.Sprintf("%s_shuffle", item) + break + } + } + + return newname +} + func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { // What to do with this, hmm functionName := fixFunctionName(path.Connect.Summary, actualPath) @@ -926,6 +1016,7 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ parsedName = strings.ReplaceAll(parsedName, ",", "_") parsedName = strings.ReplaceAll(parsedName, ".", "_") parsedName = strings.ReplaceAll(parsedName, "|", "_") + parsedName = validateParameterName(parsedName) param.Value.Name = parsedName path.Connect.Parameters[counter].Value.Name = parsedName @@ -1074,6 +1165,7 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor parsedName = strings.ReplaceAll(parsedName, ",", "_") parsedName = strings.ReplaceAll(parsedName, ".", "_") parsedName = strings.ReplaceAll(parsedName, "|", "_") + parsedName = validateParameterName(parsedName) param.Value.Name = parsedName path.Get.Parameters[counter].Value.Name = parsedName @@ -1222,6 +1314,7 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo parsedName = strings.ReplaceAll(parsedName, ",", "_") parsedName = strings.ReplaceAll(parsedName, ".", "_") parsedName = strings.ReplaceAll(parsedName, "|", "_") + parsedName = validateParameterName(parsedName) param.Value.Name = parsedName path.Head.Parameters[counter].Value.Name = parsedName @@ -1370,6 +1463,7 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] parsedName = strings.ReplaceAll(parsedName, ",", "_") parsedName = strings.ReplaceAll(parsedName, ".", "_") parsedName = strings.ReplaceAll(parsedName, "|", "_") + parsedName = validateParameterName(parsedName) param.Value.Name = parsedName path.Delete.Parameters[counter].Value.Name = parsedName @@ -1517,6 +1611,7 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo parsedName = strings.ReplaceAll(parsedName, ",", "_") parsedName = strings.ReplaceAll(parsedName, ".", "_") parsedName = strings.ReplaceAll(parsedName, "|", "_") + parsedName = validateParameterName(parsedName) param.Value.Name = parsedName path.Post.Parameters[counter].Value.Name = parsedName @@ -1664,6 +1759,7 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W parsedName = strings.ReplaceAll(parsedName, ",", "_") parsedName = strings.ReplaceAll(parsedName, ".", "_") parsedName = strings.ReplaceAll(parsedName, "|", "_") + parsedName = validateParameterName(parsedName) param.Value.Name = parsedName path.Patch.Parameters[counter].Value.Name = parsedName @@ -1811,6 +1907,7 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor parsedName = strings.ReplaceAll(parsedName, ",", "_") parsedName = strings.ReplaceAll(parsedName, ".", "_") parsedName = strings.ReplaceAll(parsedName, "|", "_") + parsedName = validateParameterName(parsedName) param.Value.Name = parsedName path.Put.Parameters[counter].Value.Name = parsedName diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 5f5cf11f..3530ad73 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -47,7 +47,6 @@ import ( // Random xj "github.com/basgys/goxml2json" newscheduler "github.com/carlescere/scheduler" - gyaml "github.com/ghodss/yaml" "github.com/satori/go.uuid" "golang.org/x/crypto/bcrypt" "gopkg.in/yaml.v3" @@ -5545,7 +5544,7 @@ func handleSwaggerValidation(body []byte) (ParsedOpenApi, error) { //log.Printf("Json err: %s", err) err = yaml.Unmarshal(body, &version) if err != nil { - log.Printf("Yaml error: %s", err) + log.Printf("Yaml error (1): %s", err) } else { //log.Printf("Successfully parsed YAML!") } @@ -5583,9 +5582,9 @@ func handleSwaggerValidation(body []byte) (ParsedOpenApi, error) { err = json.Unmarshal(body, &swagger) if err != nil { //log.Printf("Json error? %s", err) - err = gyaml.Unmarshal(body, &swagger) + err = yaml.Unmarshal(body, &swagger) if err != nil { - log.Printf("Yaml error: %s", err) + log.Printf("Yaml error (2): %s", err) return ParsedOpenApi{}, err } else { //log.Printf("Valid yaml!") @@ -5677,7 +5676,7 @@ func validateSwagger(resp http.ResponseWriter, request *http.Request) { log.Printf("Json err: %s", err) err = yaml.Unmarshal(body, &version) if err != nil { - log.Printf("Yaml error: %s", err) + log.Printf("Yaml error (3): %s", err) //resp.WriteHeader(422) //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed reading openapi to json and yaml: %s"}`, err))) //return @@ -5736,17 +5735,18 @@ func validateSwagger(resp http.ResponseWriter, request *http.Request) { //log.Println(string(body)) err = json.Unmarshal(body, &swagger) if err != nil { - log.Printf("Json error? %s", err) - err = gyaml.Unmarshal(body, &swagger) + log.Printf("Json error for v2 - trying yaml: %s", err) + err = yaml.Unmarshal([]byte(body), &swagger) if err != nil { - log.Printf("Yaml error: %s", err) + log.Printf("Yaml error (4): %s", err) + + resp.WriteHeader(422) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed reading openapi2: %s"}`, err))) + return } else { log.Printf("Found valid yaml!") } - resp.WriteHeader(422) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed reading openapi2: %s"}`, err))) - return } swaggerv3, err := openapi2conv.ToV3Swagger(&swagger) diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 640c46e8..4de8c320 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -489,7 +489,6 @@ const Admin = (props) => { return response.json() }) .then((responseJson) => { - console.log(responseJson) setUsers(responseJson) }) .catch(error => { @@ -857,9 +856,9 @@ const Admin = (props) => { style={{ minWidth: 180, maxWidth: 180 }} /> - {users === undefined ? null : users.map(data => { + {users === undefined ? null : users.map((data, index) => { return ( - + { /> { + {