marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Made saving a workflow fast again

Browse Source
This commit is contained in:
frikky
2021-03-13 06:47:04 +01:00
parent b6dcc01e8f
commit 33ac163ef6
8 changed files with 104 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app_sdk/app_base.py
+10 -2
View File
@@ -481,10 +481,17 @@ class AppBase:
new_params = self.validate_unique_fields(param_multiplier)
print(f"NEW PARAMS: {new_params}")
if len(new_params) == 0:
print(f"No ID's to handle for validation")
print("[WARNING] SHOULD STOP MULTI-EXECUTION BECAUSE FIELDS AREN'T UNIQUE")
action_result["status"] = "SKIPPED"
action_result["result"] = f"All values were non-unique"
action_result["completed_at"] = int(time.time())
self.send_result(action_result, headers, stream_path)
exit()
#return
else:
#subparams = new_params
print(f"NEW PARAMS: {new_params}")
param_multiplier = new_params
#print("Returned with newparams of length %d", len(new_params))
#if isinstance(new_params, list) and len(new_params) == 1:
@@ -1821,14 +1828,15 @@ class AppBase:
if len(itemlist) > curminlength:
curminlength = len(itemlist)
except json.decoder.JSONDecodeError as e:
print("JSON Error: %s in %s" % (e, actualitem))
replacements[to_be_replaced] = actualitem
#print("In second part of else: %s" % (len(itemlist)))
# This is a result array for JUST this value..
# What if there are more?
print("LENGTH: %d. In second part of else: %s" % (len(itemlist), replacements))
resultarray = []
for i in range(0, curminlength):
tmpitem = json.loads(json.dumps(parameter["value"]))
backend/go-app/codegen.go
+30 -3
View File
@@ -1105,13 +1105,16 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [
}
}
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Options: []string{
"True",
"False",
},
Schema: SchemaDefinition{
Type: "string",
},
@@ -1244,10 +1247,14 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify the SSL certificate request",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "False - default=True",
Example: "True",
Options: []string{
"True",
"False",
},
Schema: SchemaDefinition{
Type: "string",
},
@@ -1382,6 +1389,10 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
Multiline: false,
Required: false,
Example: "True",
Options: []string{
"True",
"False",
},
Schema: SchemaDefinition{
Type: "string",
},
@@ -1517,6 +1528,10 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []
Multiline: false,
Required: false,
Example: "True",
Options: []string{
"True",
"False",
},
Schema: SchemaDefinition{
Type: "string",
},
@@ -1684,6 +1699,10 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
Multiline: false,
Required: false,
Example: "True",
Options: []string{
"True",
"False",
},
Schema: SchemaDefinition{
Type: "string",
},
@@ -1823,6 +1842,10 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W
Multiline: false,
Required: false,
Example: "True",
Options: []string{
"True",
"False",
},
Schema: SchemaDefinition{
Type: "string",
},
@@ -1958,6 +1981,10 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
Multiline: false,
Required: false,
Example: "True",
Options: []string{
"True",
"False",
},
Schema: SchemaDefinition{
Type: "string",
},
backend/go-app/walkoff.go
+14 -17
View File
@@ -2224,6 +2224,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
return
}
log.Printf("PRE BODY")
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Failed hook unmarshaling: %s", err)
@@ -2267,6 +2268,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
allNodes := []string{}
workflow.Categories = Categories{}
log.Printf("PRE APPS")
workflowapps, apperr := getAllWorkflowApps(ctx, 500)
//log.Printf("Action: %#v", action.Authentication)
@@ -2299,6 +2301,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
newActions = append(newActions, action)
}
log.Printf("PRE SAVECHECK")
if !workflow.PreviouslySaved {
log.Printf("[WORKFLOW INIT] NOT PREVIOUSLY SAVED - SET ACTION AUTH!")
//AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"`
@@ -2434,6 +2437,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
workflow.PreviouslySaved = true
}
log.Printf("PRE TRIGGERS")
workflow.Actions = newActions
newTriggers := []Trigger{}
for _, trigger := range workflow.Triggers {
@@ -2549,6 +2553,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
workflow.Triggers = newTriggers
log.Printf("PRE VARIABLES")
for _, variable := range workflow.WorkflowVariables {
if len(variable.Value) == 0 {
log.Printf("Can't have an empty variable: %s", variable.Name)
@@ -2596,6 +2601,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
}
// FIXME - append all nodes (actions, triggers etc) to one single array here
log.Printf("PRE VARIABLES")
if len(foundNodes) != len(allNodes) || len(workflow.Actions) <= 0 {
// This shit takes a few seconds lol
if !workflow.IsValid {
@@ -2637,18 +2643,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
// Have to do it like this to add the user's apps
//log.Println("Apps set starting")
//log.Printf("EXIT ON ERROR: %#v", workflow.Configuration.ExitOnError)
workflowApps := []WorkflowApp{}
//memcacheName = "all_apps"
//if item, err := memcache.Get(ctx, memcacheName); err == memcache.ErrCacheMiss {
// // Not in cache
// log.Printf("Apps not in cache.")
workflowApps, err = getAllWorkflowApps(ctx, 100)
if err != nil {
log.Printf("Failed getting all workflow apps from database: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
//workflowapps, apperr := getAllWorkflowApps(ctx, 500)
// Started getting the single apps, but if it's weird, this is faster
// 1. Check workflow.Start
@@ -2680,6 +2675,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
}
// Check every app action and param to see whether they exist
log.Printf("PRE ACTIONS 2")
newActions = []Action{}
for _, action := range workflow.Actions {
reservedApps := []string{
@@ -2731,7 +2727,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
} else {
curapp := WorkflowApp{}
// FIXME - can this work with ONLY AppID?
for _, app := range workflowApps {
for _, app := range workflowapps {
if app.ID == action.AppID {
curapp = app
break
@@ -2860,10 +2856,11 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
Errors: workflow.Errors,
}
cacheKey := fmt.Sprintf("workflowapps-sorted-100")
requestCache.Delete(cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-500")
requestCache.Delete(cacheKey)
// Really don't know why this was happening
//cacheKey := fmt.Sprintf("workflowapps-sorted-100")
//requestCache.Delete(cacheKey)
//cacheKey = fmt.Sprintf("workflowapps-sorted-500")
//requestCache.Delete(cacheKey)
log.Printf("[INFO] Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId)
resp.WriteHeader(200)