marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Made saving a workflow fast again

Browse Source
This commit is contained in:
frikky
2021-03-13 06:47:04 +01:00
parent b6dcc01e8f
commit 33ac163ef6
8 changed files with 104 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app_sdk/app_base.py
+10 -2
View File
@@ -481,10 +481,17 @@ class AppBase:
new_params = self.validate_unique_fields(param_multiplier)
print(f"NEW PARAMS: {new_params}")
if len(new_params) == 0:
print(f"No ID's to handle for validation")
print("[WARNING] SHOULD STOP MULTI-EXECUTION BECAUSE FIELDS AREN'T UNIQUE")
action_result["status"] = "SKIPPED"
action_result["result"] = f"All values were non-unique"
action_result["completed_at"] = int(time.time())
self.send_result(action_result, headers, stream_path)
exit()
#return
else:
#subparams = new_params
print(f"NEW PARAMS: {new_params}")
param_multiplier = new_params
#print("Returned with newparams of length %d", len(new_params))
#if isinstance(new_params, list) and len(new_params) == 1:
@@ -1821,14 +1828,15 @@ class AppBase:
if len(itemlist) > curminlength:
curminlength = len(itemlist)
except json.decoder.JSONDecodeError as e:
print("JSON Error: %s in %s" % (e, actualitem))
replacements[to_be_replaced] = actualitem
#print("In second part of else: %s" % (len(itemlist)))
# This is a result array for JUST this value..
# What if there are more?
print("LENGTH: %d. In second part of else: %s" % (len(itemlist), replacements))
resultarray = []
for i in range(0, curminlength):
tmpitem = json.loads(json.dumps(parameter["value"]))
backend/go-app/codegen.go
+30 -3
View File
@@ -1105,13 +1105,16 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [
}
}
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "True",
Options: []string{
"True",
"False",
},
Schema: SchemaDefinition{
Type: "string",
},
@@ -1244,10 +1247,14 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
optionalParameters = append(optionalParameters, WorkflowAppActionParameter{
Name: "ssl_verify",
Description: "Check if you want to verify the SSL certificate request",
Description: "Check if you want to verify request",
Multiline: false,
Required: false,
Example: "False - default=True",
Example: "True",
Options: []string{
"True",
"False",
},
Schema: SchemaDefinition{
Type: "string",
},
@@ -1382,6 +1389,10 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
Multiline: false,
Required: false,
Example: "True",
Options: []string{
"True",
"False",
},
Schema: SchemaDefinition{
Type: "string",
},
@@ -1517,6 +1528,10 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []
Multiline: false,
Required: false,
Example: "True",
Options: []string{
"True",
"False",
},
Schema: SchemaDefinition{
Type: "string",
},
@@ -1684,6 +1699,10 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
Multiline: false,
Required: false,
Example: "True",
Options: []string{
"True",
"False",
},
Schema: SchemaDefinition{
Type: "string",
},
@@ -1823,6 +1842,10 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W
Multiline: false,
Required: false,
Example: "True",
Options: []string{
"True",
"False",
},
Schema: SchemaDefinition{
Type: "string",
},
@@ -1958,6 +1981,10 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
Multiline: false,
Required: false,
Example: "True",
Options: []string{
"True",
"False",
},
Schema: SchemaDefinition{
Type: "string",
},
backend/go-app/walkoff.go
+14 -17
View File
@@ -2224,6 +2224,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
return
}
log.Printf("PRE BODY")
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Failed hook unmarshaling: %s", err)
@@ -2267,6 +2268,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
allNodes := []string{}
workflow.Categories = Categories{}
log.Printf("PRE APPS")
workflowapps, apperr := getAllWorkflowApps(ctx, 500)
//log.Printf("Action: %#v", action.Authentication)
@@ -2299,6 +2301,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
newActions = append(newActions, action)
}
log.Printf("PRE SAVECHECK")
if !workflow.PreviouslySaved {
log.Printf("[WORKFLOW INIT] NOT PREVIOUSLY SAVED - SET ACTION AUTH!")
//AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"`
@@ -2434,6 +2437,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
workflow.PreviouslySaved = true
}
log.Printf("PRE TRIGGERS")
workflow.Actions = newActions
newTriggers := []Trigger{}
for _, trigger := range workflow.Triggers {
@@ -2549,6 +2553,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
workflow.Triggers = newTriggers
log.Printf("PRE VARIABLES")
for _, variable := range workflow.WorkflowVariables {
if len(variable.Value) == 0 {
log.Printf("Can't have an empty variable: %s", variable.Name)
@@ -2596,6 +2601,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
}
// FIXME - append all nodes (actions, triggers etc) to one single array here
log.Printf("PRE VARIABLES")
if len(foundNodes) != len(allNodes) || len(workflow.Actions) <= 0 {
// This shit takes a few seconds lol
if !workflow.IsValid {
@@ -2637,18 +2643,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
// Have to do it like this to add the user's apps
//log.Println("Apps set starting")
//log.Printf("EXIT ON ERROR: %#v", workflow.Configuration.ExitOnError)
workflowApps := []WorkflowApp{}
//memcacheName = "all_apps"
//if item, err := memcache.Get(ctx, memcacheName); err == memcache.ErrCacheMiss {
// // Not in cache
// log.Printf("Apps not in cache.")
workflowApps, err = getAllWorkflowApps(ctx, 100)
if err != nil {
log.Printf("Failed getting all workflow apps from database: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
//workflowapps, apperr := getAllWorkflowApps(ctx, 500)
// Started getting the single apps, but if it's weird, this is faster
// 1. Check workflow.Start
@@ -2680,6 +2675,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
}
// Check every app action and param to see whether they exist
log.Printf("PRE ACTIONS 2")
newActions = []Action{}
for _, action := range workflow.Actions {
reservedApps := []string{
@@ -2731,7 +2727,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
} else {
curapp := WorkflowApp{}
// FIXME - can this work with ONLY AppID?
for _, app := range workflowApps {
for _, app := range workflowapps {
if app.ID == action.AppID {
curapp = app
break
@@ -2860,10 +2856,11 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
Errors: workflow.Errors,
}
cacheKey := fmt.Sprintf("workflowapps-sorted-100")
requestCache.Delete(cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-500")
requestCache.Delete(cacheKey)
// Really don't know why this was happening
//cacheKey := fmt.Sprintf("workflowapps-sorted-100")
//requestCache.Delete(cacheKey)
//cacheKey = fmt.Sprintf("workflowapps-sorted-500")
//requestCache.Delete(cacheKey)
log.Printf("[INFO] Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId)
resp.WriteHeader(200)
docker-compose.yml
+2 -2
View File
@@ -2,7 +2,7 @@ version: '3'
services:
frontend:
#build: ./frontend
image: ghcr.io/frikky/shuffle-frontend:0.8.62
image: ghcr.io/frikky/shuffle-frontend:0.8.63
container_name: shuffle-frontend
hostname: shuffle-frontend
ports:
@@ -17,7 +17,7 @@ services:
- backend
backend:
#build: ./backend
image: ghcr.io/frikky/shuffle-backend:0.8.62
image: ghcr.io/frikky/shuffle-backend:0.8.63
container_name: shuffle-backend
hostname: ${BACKEND_HOSTNAME}
# Here for debugging:
frontend/src/components/AlertTemplate.js
+1
View File
@@ -18,6 +18,7 @@ const alertStyle = {
width: 400,
boxSizing: 'border-box',
zIndex: 100001,
overflow: "hidden",
}
const buttonStyle = {
frontend/src/views/Admin.jsx
+1 -1
View File
@@ -1958,7 +1958,7 @@ const Admin = (props) => {
primary={new Date(file.created_at*1000).toISOString()}
/>
<ListItemText
style={{maxWidth: 150, minWidth: 150}}
style={{maxWidth: 150, minWidth: 150, overflow: "hidden",}}
primary={file.filename}
/>
<ListItemText
frontend/src/views/AppCreator.jsx
+7 -5
View File
@@ -569,7 +569,7 @@ const AppCreator = (props) => {
}
}
console.log(methodvalue["requestBody"]["content"])
//console.log(methodvalue["requestBody"]["content"])
if (methodvalue["requestBody"]["content"]["multipart/form-data"] !== undefined) {
if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"] !== null) {
if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["type"] === "object") {
@@ -1796,8 +1796,8 @@ const AppCreator = (props) => {
</MenuItem>
)
})}
{actionBodyRequest.map(data => (
<MenuItem style={{backgroundColor: inputColor, color: "white"}} value={data}>
{actionBodyRequest.map((data, index) => (
<MenuItem key={index} style={{backgroundColor: inputColor, color: "white"}} value={data}>
{data}
</MenuItem>
))}
@@ -1833,8 +1833,8 @@ const AppCreator = (props) => {
console.log("URL: ", parsedurl)
if (parsedurl.includes("<") && parsedurl.includes(">")) {
console.log("REPLACE")
parsedurl = parsedurl.replace("<", "{")
parsedurl = parsedurl.replace(">", "}")
parsedurl = parsedurl.replaceAll("<", "{")
parsedurl = parsedurl.replaceAll(">", "}")
}
if (parsedurl.startsWith("PUT ") || parsedurl.startsWith("GET ") ||parsedurl.startsWith("POST ") || parsedurl.startsWith("DELETE ") ||parsedurl.startsWith("PATCH ") || parsedurl.startsWith("CONNECT ")) {
@@ -2208,8 +2208,10 @@ const AppCreator = (props) => {
}}/>
const zoomIn = () => {
console.log("ZOOOMING IN")
setScale(scale+0.1);
}
const zoomOut = () => {
setScale(scale-0.1);
}
frontend/src/views/Apps.jsx
+39 -14
View File
@@ -202,14 +202,40 @@ const Apps = (props) => {
.then((responseJson) => {
//console.log("Apps: ", responseJson)
//responseJson = sortByKey(responseJson, "large_image")
responseJson = sortByKey(responseJson, "generated")
//responseJson = sortByKey(responseJson, "is_valid")
//setFilteredApps(responseJson.filter(app => !internalIds.includes(app.name) && !(!app.activated && app.generated)))
var privateapps = []
var valid = []
var invalid = []
for (var key in responseJson) {
const app = responseJson[key]
if (app.is_valid && !(!app.activated && app.generated)) {
privateapps.push(app)
} else if (app.private_id !== undefined && app.private_id.length > 0) {
valid.push(app)
} else {
invalid.push(app)
}
}
setApps(responseJson)
setFilteredApps(responseJson)
if (responseJson.length > 0) {
setSelectedApp(responseJson[0])
if (responseJson[0].actions !== null && responseJson[0].actions.length > 0) {
setSelectedAction(responseJson[0].actions[0])
//console.log(privateapps)
//console.log(valid)
//console.log(invalid)
//console.log(privateapps)
//privateapps.reverse()
privateapps.push(...valid)
privateapps.push(...invalid)
setApps(privateapps)
setFilteredApps(privateapps)
if (privateapps.length > 0) {
if (selectedApp.id === undefined || selectedApp.id === null) {
setSelectedApp(privateapps[0])
}
if (privateapps[0].actions !== null && privateapps[0].actions.length > 0) {
setSelectedAction(privateapps[0].actions[0])
} else {
setSelectedAction({})
}
@@ -358,8 +384,7 @@ const Apps = (props) => {
<ButtonBase style={{backgroundColor: theme.palette.inputColor, border: 3}}>
{imageline}
</ButtonBase>
<div style={{marginLeft: "10px", marginTop: "5px", marginBottom: "5px", width: boxWidth, backgroundColor: boxColor}}>
</div>
<div style={{marginLeft: "10px", marginTop: "5px", marginBottom: "5px", width: boxWidth, backgroundColor: boxColor}}/>
<Grid container style={{margin: "0px 10px 10px 10px", flex: "1"}}>
<Grid style={{display: "flex", flexDirection: "column", width: "100%"}}>
<Grid item style={{flex: "1"}}>
@@ -957,7 +982,7 @@ const Apps = (props) => {
setValidation(true)
setIsLoading(true)
start()
//start()
const parsedData = {
"url": url,
@@ -989,10 +1014,10 @@ const Apps = (props) => {
if (response.status === 200) {
alert.success("Loaded existing apps!")
}
setIsLoading(false)
stop()
setValidation(false)
//stop()
setIsLoading(false)
setValidation(false)
return response.json()
})
.then((responseJson) => {
@@ -1005,7 +1030,7 @@ const Apps = (props) => {
console.log("ERROR: ", error.toString())
alert.error(error.toString())
stop()
//stop()
setIsLoading(false)
setValidation(false)
})