marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

#293: Entire fix in a single button

Browse Source
This commit is contained in:
frikky
2021-03-11 16:50:58 +01:00
parent 61f5159235
commit 2a825dd1ea
12 changed files with 594 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app_sdk/app_base.py
+163 -8
View File
@@ -9,6 +9,7 @@ import requests
import urllib.parse
import http.client
import urllib3
import hashlib
class AppBase:
__version__ = None
@@ -91,6 +92,135 @@ class AppBase:
else:
return {()}
# Handles unique fields by negoiating with the backend
def validate_unique_fields(self, params):
#print("IN THE UNIQUE FIELDS PLACE!")
newlist = [params]
if isinstance(params, list):
#print("ITS A LIST!")
newlist = params
#self.full_execution = os.getenv("FULL_EXECUTION", "")
#print(len(params))
#print(params.items())
#print(list(params.items()))
#print(f"PARAM: {params}")
#print(f"NEWLIST: {newlist}")
# FIXME: Also handle MULTI PARAM
values = []
param_names = []
all_values = {}
index = 0
for outerparam in newlist:
#print(f"INNERTYPE: {type(outerparam)}")
#print(f"HANDLING PARAM {key}")
param_value = ""
for key, value in outerparam.items():
#print("KEY: %s" % key)
#value = params[key]
for param in self.action["parameters"]:
try:
if param["name"] == key and param["unique_toggled"]:
print(f"FOUND: {key} with param {param}!")
if isinstance(value, dict) or isinstance(value, list):
try:
value = json.dumps(value)
except json.decoder.JSONDecodeError as e:
print(f"Error in json decode for param {value}: {e}")
continue
elif isinstance(value, int) or isinstance(value, float):
value = str(value)
elif value == False:
value = "False"
elif value == True:
value = "True"
print(f"VALUE APPEND: {value}")
param_value += value
if param["name"] not in param_names:
param_names.append(param["name"])
except (KeyError, NameError) as e:
print(f"Key/NameError in param handler: {e}")
print(f"OUTER VALUE: {param_value}")
if len(param_value) > 0:
md5 = hashlib.md5(param_value.encode('utf-8')).hexdigest()
values.append(md5)
all_values[md5] = {
"index": index,
}
index += 1
# When in here, it means it should be unique
# Should this be done by the backend? E.g. ask it if the value is valid?
# 1. Check if it's unique towards key:value store in org for action
# 2. Check if COMBINATION is unique towards key:value store of action for org
# 3. Have a workflow configuration for unique ID's in unison or per field? E.g. if toggled, then send a hash of all fields together alphabetically, but if not, send one field at a time
# org_id = full_execution["workflow"]["execution_org"]["id"]
# USE ARRAY?
new_params = []
if len(values) > 0:
org_id = self.full_execution["workflow"]["execution_org"]["id"]
data = {
"append": True,
"workflow_check": False,
"authorization": self.authorization,
"execution_ref": self.current_execution_id,
"org_id": org_id,
"values": [{
"app": self.action["app_name"],
"action": self.action["name"],
"parameternames": param_names,
"parametervalues": values,
}]
}
#print(f"DATA: {data}")
# 1594869a676630b397bc34f7dc0951a3
#print(f"VALUE URL: {url}")
#print(f"RET: {ret.text}")
#print(f"ID: {ret.status_code}")
url = f"{self.url}/api/v1/orgs/{org_id}/validate_app_values"
ret = requests.post(url, json=data)
if ret.status_code == 200:
json_value = ret.json()
if len(json_value["found"]) > 0:
modifier = 0
for item in json_value["found"]:
print(f"Should remove {item}")
try:
print(f"FOUND: {all_values[item]}")
print(f"SHOULD REMOVE INDEX: {all_values[item]['index']}")
try:
newlist.pop(all_values[item]["index"]-modifier)
modifier += 1
except IndexError as e:
print(f"Error popping value from array: {e}")
except (NameError, KeyError) as e:
print(f"Failed removal: {e}")
#return False
else:
print("None of the items were found!")
return newlist
else:
print(f"[WARNING] Failed checking values with status code {ret.status_code}!")
#return True
return newlist
# Returns a list of all the executions to be done in the inner loop
# FIXME: Doesn't take into account whether you actually WANT to loop or not
# Check if the last part of the value is #?
@@ -347,6 +477,26 @@ class AppBase:
ret = []
param_multiplier = await self.get_param_multipliers(newparams)
# FIXME: This does a deduplication of the data
new_params = self.validate_unique_fields(param_multiplier)
print(f"NEW PARAMS: {new_params}")
if len(new_params) == 0:
print(f"No ID's to handle for validation")
else:
#subparams = new_params
print(f"NEW PARAMS: {new_params}")
#print("Returned with newparams of length %d", len(new_params))
#if isinstance(new_params, list) and len(new_params) == 1:
# params = new_params[0]
#else:
# print("[WARNING] SHOULD STOP EXECUTION BECAUSE FIELDS AREN'T UNIQUE")
# action_result["status"] = "SKIPPED"
# action_result["result"] = f"A non-unique value was found"
# action_result["completed_at"] = int(time.time())
# self.send_result(action_result, headers, stream_path)
# return
print("[INFO] Multiplier length: %d" % len(param_multiplier))
for subparams in param_multiplier:
print(f"SUBPARAMS IN MULTI: {subparams}")
@@ -1398,6 +1548,7 @@ class AppBase:
return True, ""
# THE START IS ACTUALLY RIGHT HERE :O
# Checks whether conditions are met, otherwise set
branchcheck, tmpresult = check_branch_conditions(action, fullexecution)
@@ -1848,14 +1999,19 @@ class AppBase:
#print()
if not multiexecution:
#newparams.append({
# "name": val["key"],
# "value": val["value"],
# "variant": "STATIC_VALUE",
# "id": "body_replacement",
#})
# Runs a single iteration here
new_params = self.validate_unique_fields(params)
print(f"Returned with newparams of length {len(new_params)}")
if isinstance(new_params, list) and len(new_params) == 1:
params = new_params[0]
else:
print("[WARNING] SHOULD STOP EXECUTION BECAUSE FIELDS AREN'T UNIQUE")
action_result["status"] = "SKIPPED"
action_result["result"] = f"A non-unique value was found"
action_result["completed_at"] = int(time.time())
self.send_result(action_result, headers, stream_path)
return
#print("[INFO] APP_SDK DONE: Starting NORMAL execution of function")
print("[INFO] Running normal execution\n")
newres = await func(**params)
print("\n[INFO] Returned from execution!")#, newres)
@@ -2058,7 +2214,6 @@ class AppBase:
print("Normal result - no list?")
result = results
print("RESULT: %s" % result)
action_result["status"] = "SUCCESS"
action_result["result"] = str(result)
if action_result["result"] == "":
backend/app_sdk/build.sh
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/bash
NAME=shuffle-app_sdk
VERSION=0.8.61
VERSION=0.8.62
docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
backend/go-app/main.go
+245
View File
@@ -7722,6 +7722,247 @@ func handleStopCloudSync(syncUrl string, org Org) error {
return nil
}
func handleKeyValueCheck(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
return
}
// Append: Checks if the value should be appended
// WorkflowCheck: Checks if the value should only check for workflow in org, or entire org
// Authorization: the Authorization to use
// ExecutionRef: Ref for the execution
// Values: The values to use
type DataValues struct {
App string
Actions string
ParameterNames []string
ParameterValues []string
}
type ReturnData struct {
Append bool `json:"append"`
WorkflowCheck bool `json:"workflow_check"`
Authorization string `json:"authorization"`
ExecutionRef string `json:"execution_ref"`
OrgId string `json:"org_id"`
Values []DataValues `json:"values"`
}
//for key, value := range data.Apps {
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
var tmpData ReturnData
err = json.Unmarshal(body, &tmpData)
if err != nil {
log.Printf("Failed unmarshalling test: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if tmpData.OrgId != fileId {
log.Printf("[INFO] OrgId %s and %s don't match", tmpData.OrgId, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "Organization ID's don't match"}`))
return
}
ctx := context.Background()
org, err := getOrg(ctx, tmpData.OrgId)
if err != nil {
log.Printf("[INFO] Organization doesn't exist: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
workflowExecution, err := getWorkflowExecution(ctx, tmpData.ExecutionRef)
if err != nil {
log.Printf("[INFO] User can't edit the org")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "No permission to get execution"}`))
return
}
if workflowExecution.Authorization != tmpData.Authorization {
log.Printf("[INFO] Execution auth %s and %s don't match", workflowExecution.Authorization, tmpData.Authorization)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "Auth doesn't match"}`))
return
}
if workflowExecution.Status != "EXECUTING" {
log.Printf("[INFO] Workflow isn't executing and shouldn't be searching", workflowExecution.ExecutionId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "Workflow isn't executing"}`))
return
}
if workflowExecution.ExecutionOrg != org.Id {
log.Printf("[INFO] Org %s wasn't used to execute %s", org.Id, workflowExecution.ExecutionId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "Bad organization specified"}`))
return
}
// Prepared for the future~
if len(tmpData.Values) != 1 {
log.Printf("[INFO] Filter data can only hande 1 value right now, not %d", len(tmpData.Values))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "Can't handle multiple apps yet, just one"}`))
return
}
value := tmpData.Values[0]
// FIXME: Alphabetically sort the parameternames
// FIXME: Add organization wide search, not just workflow based
found := []string{}
notFound := []string{}
dbKey := fmt.Sprintf("app_execution_values")
parameterNames := fmt.Sprintf("%s_%s", value.App, strings.Join(value.ParameterNames, "_"))
log.Printf("[INFO] PARAMNAME: %s", parameterNames)
if tmpData.WorkflowCheck {
//for _, item := range tmpData.Values {
// log.Printf("[INFO] Should validate if values %#v in app parameter %#v exists WITH WORKFLOW %s", item.ParameterValues, item.ParameterNames, workflowExecution.Workflow.ID)
// FIXME: Make this alphabetical
for _, value := range value.ParameterValues {
if len(value) == 0 {
log.Printf("Shouldn't have value of length 0!")
continue
}
log.Printf("[INFO] Looking for value %s", value)
q := datastore.NewQuery(dbKey).Filter("org_id =", org.Id).Filter("workflow_id =", workflowExecution.Workflow.ID).Filter("parameter_name =", parameterNames).Filter("value =", value)
foundCount, err := dbclient.Count(ctx, q)
if err != nil {
log.Printf("[WARNING] Failed getting key %s: %s", dbKey, err)
notFound = append(notFound, value)
//found = append(found, value)
continue
}
if foundCount > 0 {
found = append(found, value)
} else {
log.Printf("[INFO] Found for %s: %d", dbKey, foundCount)
notFound = append(notFound, value)
}
}
} else {
log.Printf("[INFO] Should validate if value %s in app %s exists WITH ORG %s", workflowExecution.Workflow.ID)
for _, value := range value.ParameterValues {
if len(value) == 0 {
log.Printf("Shouldn't have value of length 0!")
continue
}
log.Printf("[INFO] Looking for value %s", value)
q := datastore.NewQuery(dbKey).Filter("org_id =", org.Id).Filter("workflow_id =", "").Filter("parameter_name =", parameterNames).Filter("value =", value)
foundCount, err := dbclient.Count(ctx, q)
if err != nil {
log.Printf("[WARNING] Failed getting key %s: %s", dbKey, err)
notFound = append(notFound, value)
//found = append(found, value)
continue
}
if foundCount > 0 {
found = append(found, value)
} else {
log.Printf("[INFO] Found for %s: %d", dbKey, foundCount)
notFound = append(notFound, value)
}
}
}
//App string
//Actions string
//ParameterNames string
//ParamererValues []string
appended := 0
if tmpData.Append {
log.Printf("[INFO] Should append %d values!", len(notFound))
dbKey := fmt.Sprintf("app_execution_values")
//q := datastore.NewQuery(dbKey).Filter("org_id =", org.Id).Filter("workflow_id", workflowExecution.Workflow.ID).Filter("app_name =", parameterNames).Filter("value =", value)
key := datastore.NameKey(dbKey, "", nil)
type NewValue struct {
OrgId string `json:"org_id" datastore:"org_id"`
WorkflowId string `json:"workflow_id" datastore:"workflow_id"`
WorkflowExecutionId string `json:"workflow_execution_id" datastore:"workflow_execution_id"`
ParameterName string `json:"parameter_name" datastore:"parameter_name"`
Value string `json:"value" datastore:"value"`
}
//parameterNames := strings.Join(value.ParameterNames, "_")
for _, notFoundValue := range notFound {
newRequest := NewValue{
OrgId: org.Id,
WorkflowExecutionId: workflowExecution.ExecutionId,
ParameterName: parameterNames,
Value: notFoundValue,
}
if tmpData.WorkflowCheck {
newRequest.WorkflowId = workflowExecution.Workflow.ID
}
if _, err := dbclient.Put(ctx, key, &newRequest); err != nil {
log.Printf("Error adding %s to appvalue: %s", notFoundValue, err)
continue
}
appended += 1
log.Printf("[INFO] Added %s as new appvalue to datastore", notFoundValue)
}
}
type returnStruct struct {
Success bool `json:"success"`
Appended int `json:"appended"`
Found []string `json:"found"`
}
returnData := returnStruct{
Success: true,
Appended: appended,
Found: found,
}
b, _ := json.Marshal(returnData)
resp.WriteHeader(200)
resp.Write(b)
}
func handleEditOrg(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
@@ -8432,6 +8673,10 @@ func initHandlers() {
r.HandleFunc("/api/v1/orgs/", handleGetOrgs).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}", handleGetOrg).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}", handleEditOrg).Methods("POST", "OPTIONS")
// This is a new API that validates if a key has been seen before.
// Not sure what the best course of action is for it.
r.HandleFunc("/api/v1/orgs/{orgId}/validate_app_values", handleKeyValueCheck).Methods("POST", "OPTIONS")
//r.HandleFunc("/api/v1/orgs/{orgId}", handleEditOrg).Methods("POST", "OPTIONS")
// Docker orborus specific
backend/go-app/walkoff.go
+17 -14
View File
@@ -229,6 +229,7 @@ type WorkflowAppActionParameter struct {
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
SkipMulticheck bool `json:"skip_multicheck" datastore:"skip_multicheck" yaml:"skip_multicheck"`
ValueReplace []Valuereplace `json:"value_replace" datastore:"value_replace,noindex" yaml:"value_replace,omitempty"`
UniqueToggled bool `json:"unique_toggled" datastore:"unique_toggled" yaml:"unique_toggled"`
}
type Valuereplace struct {
@@ -1187,6 +1188,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl
// Find underlying nodes and add them
} else {
log.Printf("[WARNING] Actionresult is %s for node %s in %s. Continuing anyway because of workflow configuration.", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId)
// Finds ALL childnodes to set them to SKIPPED
childNodes = findChildNodes(*workflowExecution, actionResult.Action.ID)
@@ -6119,8 +6121,6 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) {
return
}
log.Printf("[INFO] Starting app hotloading")
cacheKey := fmt.Sprintf("workflowapps-sorted-100")
requestCache.Delete(cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-500")
@@ -6149,7 +6149,7 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) {
return
}
log.Printf("[INFO] Hotloading from %s", location)
log.Printf("[INFO] Starting hotloading from %s", location)
err = handleAppHotload(location, true)
if err != nil {
log.Printf("Failed app hotload: %s", err)
@@ -6251,6 +6251,20 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) {
} else {
log.Printf("Updating apps with updates")
}
if tmpBody.ForceUpdate {
ctx := context.Background()
dockercli, err := client.NewEnvClient()
if err == nil {
_, err := dockercli.ImagePull(ctx, "frikky/shuffle:app_sdk", types.ImagePullOptions{})
if err != nil {
log.Printf("[WARNING] Failed to download apps with the new App SDK: %s", err)
}
} else {
log.Printf("[WARNING] Failed to download apps with the new App SDK because of docker cli: %s", err)
}
}
iterateAppGithubFolders(fs, dir, "", "", tmpBody.ForceUpdate)
} else if strings.Contains(tmpBody.URL, "s3") {
@@ -6568,17 +6582,6 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
buildLaterList := []buildLaterStruct{}
ctx := context.Background()
if forceUpdate {
dockercli, err := client.NewEnvClient()
if err == nil {
_, err := dockercli.ImagePull(ctx, "frikky/shuffle:app_sdk", types.ImagePullOptions{})
if err != nil {
log.Printf("[WARNING] Failed to download apps with the new App SDK: %s", err)
}
} else {
log.Printf("[WARNING] Failed to download apps with the new App SDK because of docker cli: %s", err)
}
}
// It's here to prevent getting them in every iteration
for _, file := range dir {