From 33ac163ef64ac93de73b928346034438d9091d80 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 13 Mar 2021 06:47:04 +0100 Subject: [PATCH] Made saving a workflow fast again --- backend/app_sdk/app_base.py | 12 +++++- backend/go-app/codegen.go | 33 +++++++++++++-- backend/go-app/walkoff.go | 31 +++++++------- docker-compose.yml | 4 +- frontend/src/components/AlertTemplate.js | 1 + frontend/src/views/Admin.jsx | 2 +- frontend/src/views/AppCreator.jsx | 12 +++--- frontend/src/views/Apps.jsx | 53 +++++++++++++++++------- 8 files changed, 104 insertions(+), 44 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index c369533b..c2667fec 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -481,10 +481,17 @@ class AppBase: new_params = self.validate_unique_fields(param_multiplier) print(f"NEW PARAMS: {new_params}") if len(new_params) == 0: - print(f"No ID's to handle for validation") + print("[WARNING] SHOULD STOP MULTI-EXECUTION BECAUSE FIELDS AREN'T UNIQUE") + action_result["status"] = "SKIPPED" + action_result["result"] = f"All values were non-unique" + action_result["completed_at"] = int(time.time()) + self.send_result(action_result, headers, stream_path) + exit() + #return else: #subparams = new_params print(f"NEW PARAMS: {new_params}") + param_multiplier = new_params #print("Returned with newparams of length %d", len(new_params)) #if isinstance(new_params, list) and len(new_params) == 1: @@ -1821,14 +1828,15 @@ class AppBase: if len(itemlist) > curminlength: curminlength = len(itemlist) + except json.decoder.JSONDecodeError as e: print("JSON Error: %s in %s" % (e, actualitem)) replacements[to_be_replaced] = actualitem + #print("In second part of else: %s" % (len(itemlist))) # This is a result array for JUST this value.. # What if there are more? - print("LENGTH: %d. In second part of else: %s" % (len(itemlist), replacements)) resultarray = [] for i in range(0, curminlength): tmpitem = json.loads(json.dumps(parameter["value"])) diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index 4389219f..b0b9b14d 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -1105,13 +1105,16 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ } } - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ Name: "ssl_verify", Description: "Check if you want to verify request", Multiline: false, Required: false, Example: "True", + Options: []string{ + "True", + "False", + }, Schema: SchemaDefinition{ Type: "string", }, @@ -1244,10 +1247,14 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ Name: "ssl_verify", - Description: "Check if you want to verify the SSL certificate request", + Description: "Check if you want to verify request", Multiline: false, Required: false, - Example: "False - default=True", + Example: "True", + Options: []string{ + "True", + "False", + }, Schema: SchemaDefinition{ Type: "string", }, @@ -1382,6 +1389,10 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo Multiline: false, Required: false, Example: "True", + Options: []string{ + "True", + "False", + }, Schema: SchemaDefinition{ Type: "string", }, @@ -1517,6 +1528,10 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] Multiline: false, Required: false, Example: "True", + Options: []string{ + "True", + "False", + }, Schema: SchemaDefinition{ Type: "string", }, @@ -1684,6 +1699,10 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo Multiline: false, Required: false, Example: "True", + Options: []string{ + "True", + "False", + }, Schema: SchemaDefinition{ Type: "string", }, @@ -1823,6 +1842,10 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W Multiline: false, Required: false, Example: "True", + Options: []string{ + "True", + "False", + }, Schema: SchemaDefinition{ Type: "string", }, @@ -1958,6 +1981,10 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor Multiline: false, Required: false, Example: "True", + Options: []string{ + "True", + "False", + }, Schema: SchemaDefinition{ Type: "string", }, diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 05990f8b..3d9923b5 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2224,6 +2224,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { return } + log.Printf("PRE BODY") body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("Failed hook unmarshaling: %s", err) @@ -2267,6 +2268,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { allNodes := []string{} workflow.Categories = Categories{} + log.Printf("PRE APPS") workflowapps, apperr := getAllWorkflowApps(ctx, 500) //log.Printf("Action: %#v", action.Authentication) @@ -2299,6 +2301,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { newActions = append(newActions, action) } + log.Printf("PRE SAVECHECK") if !workflow.PreviouslySaved { log.Printf("[WORKFLOW INIT] NOT PREVIOUSLY SAVED - SET ACTION AUTH!") //AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"` @@ -2434,6 +2437,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { workflow.PreviouslySaved = true } + log.Printf("PRE TRIGGERS") workflow.Actions = newActions newTriggers := []Trigger{} for _, trigger := range workflow.Triggers { @@ -2549,6 +2553,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { workflow.Triggers = newTriggers + log.Printf("PRE VARIABLES") for _, variable := range workflow.WorkflowVariables { if len(variable.Value) == 0 { log.Printf("Can't have an empty variable: %s", variable.Name) @@ -2596,6 +2601,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } // FIXME - append all nodes (actions, triggers etc) to one single array here + log.Printf("PRE VARIABLES") if len(foundNodes) != len(allNodes) || len(workflow.Actions) <= 0 { // This shit takes a few seconds lol if !workflow.IsValid { @@ -2637,18 +2643,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // Have to do it like this to add the user's apps //log.Println("Apps set starting") //log.Printf("EXIT ON ERROR: %#v", workflow.Configuration.ExitOnError) - workflowApps := []WorkflowApp{} - //memcacheName = "all_apps" - //if item, err := memcache.Get(ctx, memcacheName); err == memcache.ErrCacheMiss { - // // Not in cache - // log.Printf("Apps not in cache.") - workflowApps, err = getAllWorkflowApps(ctx, 100) - if err != nil { - log.Printf("Failed getting all workflow apps from database: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } + //workflowapps, apperr := getAllWorkflowApps(ctx, 500) // Started getting the single apps, but if it's weird, this is faster // 1. Check workflow.Start @@ -2680,6 +2675,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } // Check every app action and param to see whether they exist + log.Printf("PRE ACTIONS 2") newActions = []Action{} for _, action := range workflow.Actions { reservedApps := []string{ @@ -2731,7 +2727,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } else { curapp := WorkflowApp{} // FIXME - can this work with ONLY AppID? - for _, app := range workflowApps { + for _, app := range workflowapps { if app.ID == action.AppID { curapp = app break @@ -2860,10 +2856,11 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { Errors: workflow.Errors, } - cacheKey := fmt.Sprintf("workflowapps-sorted-100") - requestCache.Delete(cacheKey) - cacheKey = fmt.Sprintf("workflowapps-sorted-500") - requestCache.Delete(cacheKey) + // Really don't know why this was happening + //cacheKey := fmt.Sprintf("workflowapps-sorted-100") + //requestCache.Delete(cacheKey) + //cacheKey = fmt.Sprintf("workflowapps-sorted-500") + //requestCache.Delete(cacheKey) log.Printf("[INFO] Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId) resp.WriteHeader(200) diff --git a/docker-compose.yml b/docker-compose.yml index c8d2d2ab..b26f3c1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.62 + image: ghcr.io/frikky/shuffle-frontend:0.8.63 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.62 + image: ghcr.io/frikky/shuffle-backend:0.8.63 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: diff --git a/frontend/src/components/AlertTemplate.js b/frontend/src/components/AlertTemplate.js index a476afc7..58ff46cc 100644 --- a/frontend/src/components/AlertTemplate.js +++ b/frontend/src/components/AlertTemplate.js @@ -18,6 +18,7 @@ const alertStyle = { width: 400, boxSizing: 'border-box', zIndex: 100001, + overflow: "hidden", } const buttonStyle = { diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 7cee4bbe..c9f85d55 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -1958,7 +1958,7 @@ const Admin = (props) => { primary={new Date(file.created_at*1000).toISOString()} /> { } } - console.log(methodvalue["requestBody"]["content"]) + //console.log(methodvalue["requestBody"]["content"]) if (methodvalue["requestBody"]["content"]["multipart/form-data"] !== undefined) { if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"] !== null) { if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["type"] === "object") { @@ -1796,8 +1796,8 @@ const AppCreator = (props) => { ) })} - {actionBodyRequest.map(data => ( - + {actionBodyRequest.map((data, index) => ( + {data} ))} @@ -1833,8 +1833,8 @@ const AppCreator = (props) => { console.log("URL: ", parsedurl) if (parsedurl.includes("<") && parsedurl.includes(">")) { console.log("REPLACE") - parsedurl = parsedurl.replace("<", "{") - parsedurl = parsedurl.replace(">", "}") + parsedurl = parsedurl.replaceAll("<", "{") + parsedurl = parsedurl.replaceAll(">", "}") } if (parsedurl.startsWith("PUT ") || parsedurl.startsWith("GET ") ||parsedurl.startsWith("POST ") || parsedurl.startsWith("DELETE ") ||parsedurl.startsWith("PATCH ") || parsedurl.startsWith("CONNECT ")) { @@ -2208,8 +2208,10 @@ const AppCreator = (props) => { }}/> const zoomIn = () => { + console.log("ZOOOMING IN") setScale(scale+0.1); } + const zoomOut = () => { setScale(scale-0.1); } diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 04fb0a62..0b4cd611 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -202,14 +202,40 @@ const Apps = (props) => { .then((responseJson) => { //console.log("Apps: ", responseJson) //responseJson = sortByKey(responseJson, "large_image") - responseJson = sortByKey(responseJson, "generated") + //responseJson = sortByKey(responseJson, "is_valid") + //setFilteredApps(responseJson.filter(app => !internalIds.includes(app.name) && !(!app.activated && app.generated))) + + var privateapps = [] + var valid = [] + var invalid = [] + for (var key in responseJson) { + const app = responseJson[key] + if (app.is_valid && !(!app.activated && app.generated)) { + privateapps.push(app) + } else if (app.private_id !== undefined && app.private_id.length > 0) { + valid.push(app) + } else { + invalid.push(app) + } + } - setApps(responseJson) - setFilteredApps(responseJson) - if (responseJson.length > 0) { - setSelectedApp(responseJson[0]) - if (responseJson[0].actions !== null && responseJson[0].actions.length > 0) { - setSelectedAction(responseJson[0].actions[0]) + //console.log(privateapps) + //console.log(valid) + //console.log(invalid) + //console.log(privateapps) + //privateapps.reverse() + privateapps.push(...valid) + privateapps.push(...invalid) + + setApps(privateapps) + setFilteredApps(privateapps) + if (privateapps.length > 0) { + if (selectedApp.id === undefined || selectedApp.id === null) { + setSelectedApp(privateapps[0]) + } + + if (privateapps[0].actions !== null && privateapps[0].actions.length > 0) { + setSelectedAction(privateapps[0].actions[0]) } else { setSelectedAction({}) } @@ -358,8 +384,7 @@ const Apps = (props) => { {imageline} -
-
+
@@ -957,7 +982,7 @@ const Apps = (props) => { setValidation(true) setIsLoading(true) - start() + //start() const parsedData = { "url": url, @@ -989,10 +1014,10 @@ const Apps = (props) => { if (response.status === 200) { alert.success("Loaded existing apps!") } - setIsLoading(false) - stop() - setValidation(false) + //stop() + setIsLoading(false) + setValidation(false) return response.json() }) .then((responseJson) => { @@ -1005,7 +1030,7 @@ const Apps = (props) => { console.log("ERROR: ", error.toString()) alert.error(error.toString()) - stop() + //stop() setIsLoading(false) setValidation(false) })