marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

#141: Fixed more general use-case issues for files

Browse Source
This commit is contained in:
frikky
2020-11-25 17:21:16 +01:00
parent acc6378dc2
commit 66707c8a7f
7 changed files with 180 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app_sdk/app_base.py
+31 -6
View File
@@ -148,6 +148,7 @@ class AppBase:
print("")
self.full_execution = fullexecution
self.logger.info("AFTER FULLEXEC stream result")
# Gets the value at the parenthesis level you want
@@ -478,7 +479,7 @@ class AppBase:
#Actionname: Start_node
print(f"Actionname: {actionname_lower}")
print(f"\nActionname: {actionname_lower}")
# 1. Find the action
baseresult = ""
@@ -549,22 +550,30 @@ class AppBase:
except KeyError as error:
print(f"KeyError in JSON: {error}")
print(f"After first trycatch")
print(f"After first trycatch. Baseresult: ", baseresult)
# 2. Find the JSON data
if len(baseresult) == 0:
return ""+appendresult, False
print("After second return")
if len(parsersplit) == 1:
return str(baseresult)+str(appendresult), False
baseresult = baseresult.replace("\'", "\"")
baseresult = baseresult.replace(" True,", " true,")
baseresult = baseresult.replace(" False", " false,")
print("After third parser return - Formatted: ", baseresult)
basejson = {}
try:
basejson = json.loads(baseresult)
except json.decoder.JSONDecodeError as e:
print("Parser issue with JSON: %s" % e)
return str(baseresult)+str(appendresult), False
print("After fourth parser return as JSON")
data, is_loop = recurse_json(basejson, parsersplit[1:])
parseditem = data
if is_loop:
@@ -577,6 +586,7 @@ class AppBase:
print("SET DATA WRAPPER TO %s!" % parsersplit[-1])
parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data))
print("Before last return")
return str(parseditem)+str(appendresult), is_loop
# Parses parameters sent to it and returns whether it did it successfully with the values found
@@ -854,8 +864,9 @@ class AppBase:
returndata = {
"success": True,
"filename": tmpdata["filename"],
"data": ret2.text,
"data": ret2.content,
}
# open('facebook.ico', 'wb').write(r.content)
return returndata
@@ -1093,7 +1104,21 @@ class AppBase:
#replacement = parse_wrapper_start(replacement)
tmpitem = tmpitem.replace(key, replacement, -1)
resultarray.append(tmpitem)
# This code handles files.
isfile = False
try:
if parameter["schema"]["type"] == "file" and len(value) > 0:
print("SHOULD HANDLE FILE IN MULTI. Get based on value %s" % parameter["value"])
file_value = get_files(fullexecution, tmpitem)
print("FILE VALUE FOR VAL %s: %s" % (tmpitem, file_value))
resultarray.append(file_value)
except KeyError as e:
print("SCHEMA ERROR IN FILE HANDLING: %s" % e)
if not isfile:
resultarray.append(tmpitem)
# With this parameter ready, add it to... a greater list of parameters. Rofl
print("LENGTH OF ARR: %d" % len(resultarray))
@@ -1114,9 +1139,9 @@ class AppBase:
# This code handles files.
try:
if parameter["schema"]["type"] == "file" and len(value) > 0:
print("SHOULD HANDLE FILE. Get based on value %s" % parameter["value"])
print("\n SHOULD HANDLE FILE. Get based on value %s. <--- is this a valid ID?" % parameter["value"])
file_value = get_files(fullexecution, value)
print("FILE VALUE: %s" % file_value)
print("FILE VALUE: %s \n" % file_value)
params[parameter["name"]] = file_value
multi_parameters[parameter["name"]] = file_value
backend/go-app/files.go
+55 -15
View File
@@ -37,6 +37,7 @@ type File struct {
WorkflowId string `json:"workflow_id" datastore:"workflow_id"`
Workflows []string `json:"workflows" datastore:"workflows"`
DownloadPath string `json:"download_path" datastore:"download_path"`
Md5sum string `json:"md5_sum" datastore:"md5_sum"`
}
var basepath = os.Getenv("SHUFFLE_FILE_LOCATION")
@@ -66,11 +67,14 @@ func fileAuthentication(request *http.Request) (string, error) {
// This is annoying af and is done because of maxlength lol
newApikey := apikeyCheck[1]
if newApikey != workflowExecution.Authorization {
log.Printf("[ERROR] Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization)
//log.Printf("[ERROR] Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization)
log.Printf("[ERROR] Bad apikey for execution %s.", executionId[0])
//%s vs %s", executionId[0], apikey, workflowExecution.Authorization)
return "", errors.New("Bad authorization key")
}
log.Printf("[INFO] Authorization is correct for execution %s! %s vs %s. Setting Org", executionId, apikey, workflowExecution.Authorization)
log.Printf("[INFO] Authorization is correct for execution %s!", executionId[0])
//%s vs %s. Setting Org", executionId, apikey, workflowExecution.Authorization)
if len(workflowExecution.ExecutionOrg) > 0 {
return workflowExecution.ExecutionOrg, nil
} else if len(workflowExecution.Workflow.ExecutingOrg.Id) > 0 {
@@ -115,7 +119,14 @@ func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) {
fileId = strings.Split(fileId, "?")[0]
}
log.Printf("\n\n[INFO] User is trying to delete file %s\n\n", fileId)
if len(fileId) != 36 {
log.Printf("Bad format for fileId %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
return
}
log.Printf("\n\n[INFO] User is trying to GET File Meta for %s\n\n", fileId)
// 1. Check user directly
// 2. Check workflow execution authorization
@@ -136,7 +147,7 @@ func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) {
}
// 1. Verify if the user has access to the file: org_id and workflow
log.Printf("[INFO] Should DELETE file %s if user has access", fileId)
log.Printf("[INFO] Should GET FILE META for %s if user has access", fileId)
ctx := context.Background()
file, err := getFile(ctx, fileId)
if err != nil {
@@ -172,7 +183,7 @@ func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) {
return
}
log.Printf("[INFO] Successfully deleted file %s", fileId)
log.Printf("[INFO] Successfully got file meta for %s", fileId)
resp.WriteHeader(200)
resp.Write([]byte(newBody))
}
@@ -200,6 +211,13 @@ func handleDeleteFile(resp http.ResponseWriter, request *http.Request) {
fileId = strings.Split(fileId, "?")[0]
}
if len(fileId) != 36 {
log.Printf("Bad format for fileId %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
return
}
log.Printf("\n\n[INFO] User is trying to delete file %s\n\n", fileId)
// 1. Check user directly
@@ -318,7 +336,14 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
fileId = location[4]
}
log.Printf("\n\nUser is trying to get file %s\n\n", fileId)
if len(fileId) != 36 {
log.Printf("Bad format for fileId %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
return
}
log.Printf("\n\nUser is trying to download file %s\n\n", fileId)
// 1. Check user directly
// 2. Check workflow execution authorization
@@ -346,11 +371,11 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
}
// 1. Verify if the user has access to the file: org_id and workflow
log.Printf("Should get file %s", fileId)
log.Printf("[INFO] Should get file %s", fileId)
ctx := context.Background()
file, err := getFile(ctx, fileId)
if err != nil {
log.Printf("File %s not found: %s", fileId, err)
log.Printf("[ERROR] File %s not found: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
@@ -376,7 +401,7 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
}
if file.Status != "active" {
log.Printf("File status isn't active. Can't continue.")
log.Printf("[ERROR] File status isn't active, but %s. Can't continue.", file.Status)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "The file isn't ready to be downloaded yet. Status required: active"}`))
return
@@ -384,7 +409,7 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
// Fixme: More auth: org and workflow!
downloadPath := file.DownloadPath
log.Printf("Downloadpath: %s", downloadPath)
log.Printf("[INFO] Downloadpath: %s", downloadPath)
Openfile, err := os.Open(downloadPath)
defer Openfile.Close() //Close after function return
if err != nil {
@@ -438,6 +463,13 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) {
fileId = location[4]
}
if len(fileId) != 36 {
log.Printf("Bad format for fileId %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
return
}
// 1. Check user directly
// 2. Check workflow execution authorization
user, err := handleApiAuthentication(resp, request)
@@ -494,7 +526,6 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) {
}
request.ParseMultipartForm(32 << 20)
//var buf bytes.Buffer
parsedFile, _, err := request.FormFile("shuffle_file")
if err != nil {
log.Printf("[ERROR] Couldn't upload file: %s", err)
@@ -513,16 +544,23 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) {
return
}
//io.Copy(&buf, parsedFile)
//contents := buf.String()
//buf.Reset()
// Can be used for validation files for change
/*
var buf bytes.Buffer
io.Copy(&buf, parsedFile)
contents := buf.Bytes()
md5 := md5sum(contents)
buf.Reset()
*/
md5 := ""
f, err := os.OpenFile(file.DownloadPath, os.O_WRONLY|os.O_CREATE, os.ModePerm)
if err != nil {
// Rolling back file
file.Status = "created"
setFile(ctx, *file)
log.Printf("Failed creating file: %s", err)
log.Printf("[ERROR] Failed uploading and creating file: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
@@ -532,7 +570,9 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) {
io.Copy(f, parsedFile)
// FIXME: Set this one to 200 anyway? Can't download file then tho..
log.Printf("[INFO] MD5 for file %s is %s", file.Filename, md5)
file.Status = "active"
file.Md5sum = md5
err = setFile(ctx, *file)
if err != nil {
log.Printf("[ERROR] Failed setting file back to active")
backend/go-app/main.go
+31 -29
View File
@@ -7119,37 +7119,39 @@ func runInit(ctx context.Context) {
}
}
fileq := datastore.NewQuery("Files").Limit(1)
count, err := dbclient.Count(ctx, fileq)
log.Printf("FILECOUNT: %d", count)
if err == nil && count < 10 {
basepath := "."
filename := "testfile.txt"
fileId := uuid.NewV4().String()
log.Printf("Creating new file reference %s because none exist!", fileId)
workflowId := "2cf1169d-b460-41de-8c36-28b2092866f8"
downloadPath := fmt.Sprintf("%s/%s/%s/%s", basepath, activeOrgs[0].Id, workflowId, fileId)
/*
fileq := datastore.NewQuery("Files").Limit(1)
count, err := dbclient.Count(ctx, fileq)
log.Printf("FILECOUNT: %d", count)
if err == nil && count < 10 {
basepath := "."
filename := "testfile.txt"
fileId := uuid.NewV4().String()
log.Printf("Creating new file reference %s because none exist!", fileId)
workflowId := "2cf1169d-b460-41de-8c36-28b2092866f8"
downloadPath := fmt.Sprintf("%s/%s/%s/%s", basepath, activeOrgs[0].Id, workflowId, fileId)
timeNow := time.Now().Unix()
newFile := File{
Id: fileId,
CreatedAt: timeNow,
UpdatedAt: timeNow,
Description: "Created by system for testing",
Status: "active",
Filename: filename,
OrgId: activeOrgs[0].Id,
WorkflowId: workflowId,
DownloadPath: downloadPath,
}
timeNow := time.Now().Unix()
newFile := File{
Id: fileId,
CreatedAt: timeNow,
UpdatedAt: timeNow,
Description: "Created by system for testing",
Status: "active",
Filename: filename,
OrgId: activeOrgs[0].Id,
WorkflowId: workflowId,
DownloadPath: downloadPath,
}
err = setFile(ctx, newFile)
if err != nil {
log.Printf("Failed setting file: %s", err)
} else {
log.Printf("Created file %s in init", newFile.DownloadPath)
err = setFile(ctx, newFile)
if err != nil {
log.Printf("Failed setting file: %s", err)
} else {
log.Printf("Created file %s in init", newFile.DownloadPath)
}
}
}
*/
var allworkflowapps []AppAuthenticationStorage
q = datastore.NewQuery("workflowappauth")
@@ -8045,10 +8047,10 @@ func initHandlers() {
// https://developer.box.com/reference/get-files-id-content/
// 1. Creating the "get file" option. Make it possible to run this in the frontend.
r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/files/create", handleCreateFile).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}", handleGetFileMeta).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS")
http.Handle("/", r)
}
backend/go-app/walkoff.go
+1 -1
View File
@@ -2795,7 +2795,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
continue
}
log.Printf("Should set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID)
log.Printf("[WARNING] Set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID)
defaultResults = append(defaultResults, ActionResult{
Action: action,
ExecutionId: workflowExecution.ExecutionId,
docker-compose.yml
+3 -3
View File
@@ -1,7 +1,7 @@
version: '3'
services:
frontend:
build: ./frontend
#build: ./frontend
image: ghcr.io/frikky/shuffle-frontend:0.8.0
container_name: shuffle-frontend
hostname: shuffle-frontend
@@ -16,8 +16,8 @@ services:
depends_on:
- backend
backend:
#build: ./backend
image: ghcr.io/frikky/shuffle-backend:0.8.1
build: ./backend
image: ghcr.io/frikky/shuffle-backend:0.8.11
container_name: shuffle-backend
hostname: ${BACKEND_HOSTNAME}
# Here for debugging:
frontend/src/views/AngularWorkflow.jsx
+58 -1
View File
@@ -5354,7 +5354,7 @@ const AngularWorkflow = (props) => {
return null
}
const cytoscapeViewWidths = 700
const cytoscapeViewWidths = 750
const bottomBarStyle = {
position: "fixed",
right: 20,
@@ -5412,6 +5412,62 @@ const AngularWorkflow = (props) => {
return null
}
const FileMenu = () => {
const [newAnchor, setNewAnchor] = React.useState(null);
const [showShuffleMenu, setShowShuffleMenu] = React.useState(false)
{ /*const [showShuffleMenu, setShowShuffleMenu] = React.useState(true) */}
return (
<div style={{"display": "inline-block"}}>
<Menu
id="long-menu"
anchorEl={newAnchor}
open={showShuffleMenu}
onClose={() => {
setShowShuffleMenu(false)
}}
>
<div style={{margin: 15, color: "white", maxWidth: 250, minWidth: 250, }}>
<h4>This menu is used to control the workflow itself.</h4>
<Divider style={{backgroundColor: "white", marginTop: 10, marginBottom: 10,}}/>
<FormControlLabel
style={{marginBottom: 15, color: "white",}}
label={<div style={{color: "white"}}>Exit on Error</div>}
control={
<Switch checked={workflow.configuration.exit_on_error} onChange={() => {
workflow.configuration.exit_on_error = !workflow.configuration.exit_on_error
setWorkflow(workflow)
setUpdate("exit_on_error_"+workflow.configuration.exit_on_error ? "true" : "false")
setShowShuffleMenu(false)
}} />
}
/>
<FormControlLabel
style={{marginBottom: 15, color: "white",}}
label={<div style={{color: "white"}}>Start from top</div>}
control={
<Switch checked={workflow.configuration.start_from_top} onChange={() => {
workflow.configuration.start_from_top = !workflow.configuration.start_from_top
setWorkflow(workflow)
setUpdate("start_from_top_"+workflow.configuration.start_from_top ? "true" : "false")
setShowShuffleMenu(false)
}} />
}
/>
</div>
</Menu>
<Tooltip color="secondary" title="Workflow settings" placement="top-start">
<Button color="primary" style={{height: 50, marginLeft: 10, }} variant="outlined" onClick={(event) => {
setShowShuffleMenu(!showShuffleMenu)
setNewAnchor(event.currentTarget)
}}>
<SettingsIcon />
</Button>
</Tooltip>
</div>
)
}
const WorkflowMenu = () => {
const [newAnchor, setNewAnchor] = React.useState(null);
const [showShuffleMenu, setShowShuffleMenu] = React.useState(false)
@@ -5539,6 +5595,7 @@ const AngularWorkflow = (props) => {
<DirectionsRunIcon />
</Button>
</Tooltip>
{/* <FileMenu /> */}
<WorkflowMenu />
</div>
</div>
frontend/src/views/Workflows.jsx
+1 -1
View File
@@ -360,7 +360,7 @@ const Workflows = (props) => {
data.triggers[key].status = "stopped"
}
}
data["org"] = []
data.execution_org = {"id": ""}
console.log(data)