From 5c14a3a202d7971af40dedef473c74caa11d6614 Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 18 Feb 2021 19:51:27 +0100 Subject: [PATCH] Lots of minor fixes --- backend/app_sdk/app_base.py | 8 +-- backend/go-app/codegen.go | 13 ++-- backend/go-app/docker.go | 2 +- backend/go-app/main.go | 8 +-- backend/go-app/walkoff.go | 82 ++++++++++++++++++++++---- frontend/src/views/AngularWorkflow.jsx | 2 +- frontend/src/views/AppCreator.jsx | 18 ++++-- frontend/src/views/Apps.jsx | 2 +- 8 files changed, 103 insertions(+), 32 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 47bb2aa5..ba35c635 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -66,7 +66,7 @@ class AppBase: self.logger.info(ret.text) except requests.exceptions.ConnectionError as e: #self.logger.exception("ConnectionError: %s" % e) - self.logger.exception("Expected connectionerror happened") + self.logger.exception("Expected ConnectionError happened") return except TypeError as e: #self.logger.exception(e) @@ -78,7 +78,7 @@ class AppBase: if ret.status_code != 200: self.logger.info(ret.text) except http.client.RemoteDisconnected as e: - self.logger.exception("Expected connectionerror happened") + self.logger.exception("Expected Remotedisconnect happened") return async def cartesian_product(self, L): @@ -432,7 +432,7 @@ class AppBase: content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"]) ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers) - print("RET2 (file get): %s" % ret2.text) + print("RET2 (file get) done") if ret2.status_code == 200: tmpdata = ret1.json() returndata = { @@ -1721,7 +1721,7 @@ class AppBase: #}) #print("[INFO] APP_SDK DONE: Starting NORMAL execution of function") - print("[INFO] Running execition\n") + print("[INFO] Running normal execution\n") newres = await func(**params) print("\n[INFO] Returned from execution:", newres) if isinstance(newres, tuple): diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index 4b20d2fa..74776e88 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -418,11 +418,14 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet fileBalance, ) - if strings.Contains(functionname, "filescan") { - //log.Printf("FUNCTION: %s", data) - log.Println(data) - log.Printf("Queries: %s", queryString) - } + // Use lowercase when checking + /* + if strings.Contains(functionname, "filter") { + //log.Printf("FUNCTION: %s", data) + log.Println(data) + log.Printf("Queries: %s", queryString) + } + */ //log.Printf(data) return functionname, data diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 0a0b4bae..776dbea0 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -299,7 +299,7 @@ func buildImage(tags []string, dockerfileFolder string) error { return err } - log.Printf("Tags: %s", tags) + log.Printf("[INFO] Docker Tags: %s", tags) dockerfileSplit := strings.Split(dockerfileFolder, "/") // Create a buffer diff --git a/backend/go-app/main.go b/backend/go-app/main.go index c27de623..230f50ae 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -6360,13 +6360,13 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { // FIXME: Check whether it's in use. if user.Id != app.Owner && user.Role != "admin" { - log.Printf("Wrong user (%s) for app %s when verifying swagger", user.Username, app.Name) + log.Printf("[WARNING] Wrong user (%s) for app %s when verifying swagger", user.Username, app.Name) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } - log.Printf("EDITING APP WITH ID %s", app.ID) + log.Printf("[INFO] EDITING APP WITH ID %s", app.ID) newmd5 = app.ID } @@ -6455,7 +6455,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { identifier = strings.Replace(identifier, " ", "-", -1) identifier = strings.Replace(identifier, "_", "-", -1) - log.Printf("Successfully parsed %s. Proceeding to docker container", identifier) + log.Printf("[INFO] Successfully parsed %s. Proceeding to docker container", identifier) // Now that the baseline is setup, we need to make it into a cloud function // 1. Upload the API to datastore for use @@ -7517,7 +7517,7 @@ func runInit(ctx context.Context) { } } - log.Printf("Downloading OpenAPI data for search - EXTRA APPS") + log.Printf("[INFO] Downloading OpenAPI data for search - EXTRA APPS") apis := "https://github.com/frikky/security-openapis" // THis gets memory problems hahah diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 5a241064..f3103954 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2062,7 +2062,7 @@ func updateAppAuth(auth AppAuthenticationStorage, workflowId, nodeId string, add // FIXME: Add a way to use !add to remove updateAuth := false if !workflowFound && add { - log.Printf("Adding workflow things to auth!") + log.Printf("[INFO] Adding workflow things to auth!") usageItem := AuthenticationUsage{ WorkflowId: workflowId, Nodes: []string{nodeId}, @@ -2073,14 +2073,14 @@ func updateAppAuth(auth AppAuthenticationStorage, workflowId, nodeId string, add auth.NodeCount += 1 updateAuth = true } else if !nodeFound && add { - log.Printf("Adding node things to auth!") + log.Printf("[INFO] Adding node things to auth!") auth.Usage[workflowIndex].Nodes = append(auth.Usage[workflowIndex].Nodes, nodeId) auth.NodeCount += 1 updateAuth = true } if updateAuth { - log.Printf("Updating auth!") + log.Printf("[INFO] Updating auth!") ctx := context.Background() err := setWorkflowAppAuthDatastore(ctx, auth, auth.Id) if err != nil { @@ -4825,7 +4825,7 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("Getting app %s (OpenAPI)", fileId) + log.Printf("[INFO] Getting app %s (OpenAPI)", fileId) parsedApi, err := getOpenApiDatastore(ctx, fileId) if err != nil { log.Printf("OpenApi doesn't exist for: %s - err: %s", fileId, err) @@ -6813,18 +6813,76 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { } // Query for the specifci workflowId - q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(30) + maxAmount := 30 + q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(maxAmount) var workflowExecutions []WorkflowExecution _, err = dbclient.GetAll(ctx, q, &workflowExecutions) if err != nil { + if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { - q = datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(15) - _, err = dbclient.GetAll(ctx, q, &workflowExecutions) - if err != nil { - log.Printf("Error getting workflowexec (2): %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting all workflowexecutions for %s"}`, fileId))) - return + q = datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(1) + /* + _, err = dbclient.GetAll(ctx, q, &workflowExecutions) + if err != nil { + log.Printf("Error getting workflowexec (2): %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting all workflowexecutions for %s"}`, fileId))) + return + } + */ + + cursorStr := "" + for { + it := dbclient.Run(ctx, q) + //_, err = it.Next(&app) + for { + var workflowExecution WorkflowExecution + _, err := it.Next(&workflowExecution) + if err != nil { + break + } + + workflowExecutions = append(workflowExecutions, workflowExecution) + } + + //log.Printf("Len: %d", len(workflowExecutions)) + if len(workflowExecutions) > maxAmount { + break + } + + nextCursor, err := it.Cursor() + if err != iterator.Done && err != nil { + if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { + //log.Printf("NEXT!") + nextStr := fmt.Sprintf("%s", nextCursor) + if cursorStr == nextStr { + break + } + + cursorStr = nextStr + + continue + } else { + log.Printf("BREAK: %s", err) + break + } + } + + if err != nil { + log.Printf("Cursorerror: %s", err) + break + } else { + //log.Printf("NEXTCURSOR: %s", nextCursor) + nextStr := fmt.Sprintf("%s", nextCursor) + if cursorStr == nextStr { + break + } + + cursorStr = nextStr + q = q.Start(nextCursor) + //cursorStr = nextCursor + //break + } } } else { log.Printf("Error getting workflowexec: %s", err) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 54eb4504..715f8bb0 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -785,7 +785,7 @@ const AngularWorkflow = (props) => { setSavingState(1) setTimeout(() => { setSavingState(0) - }, 2000); + }, 1500); } }) .catch(error => { diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 0026c5c4..645c77d9 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -926,6 +926,7 @@ const AppCreator = (props) => { //console.log(queryitem) } } + //data.paths[item.url][item.method.toLowerCase()].parameters.push(newitem) if (item.paths.length > 0) { for (querykey in item.paths) { @@ -1828,13 +1829,19 @@ const AppCreator = (props) => { }} onBlur={event => { var parsedurl = event.target.value + console.log("URL: ", parsedurl) + if (parsedurl.includes("<") && parsedurl.includes(">")) { + console.log("REPLACE") + parsedurl = parsedurl.replace("<", "{") + parsedurl = parsedurl.replace(">", "}") + } + if (parsedurl.startsWith("PUT ") || parsedurl.startsWith("GET ") ||parsedurl.startsWith("POST ") || parsedurl.startsWith("DELETE ") ||parsedurl.startsWith("PATCH ") || parsedurl.startsWith("CONNECT ")) { const tmp = parsedurl.split(" ") if (tmp.length > 1) { parsedurl = tmp[1] setActionField("url", parsedurl) - setUrlPath(parsedurl) setCurrentActionMethod(tmp[0].toUpperCase()) setActionField("method", tmp[0].toUpperCase()) @@ -1900,12 +1907,15 @@ const AppCreator = (props) => { } // Check URL query && headers - setActionField("url", parsedurl) - setUrlPath(parsedurl) + //setActionField("url", parsedurl) } } } + if (event.target.value !== parsedurl) { + setUrlPath(parsedurl) + setActionField("url", parsedurl) + } //console.log("URL: ", request.url) }} /> @@ -1980,7 +1990,7 @@ const AppCreator = (props) => {