marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

#204: Added file listing and downloads for org

Browse Source
This commit is contained in:
frikky
2020-12-18 03:44:35 +01:00
parent 50bcf3bf3d
commit be9fef6a85
5 changed files with 340 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/go-app/files.go
+81 -1
View File
@@ -42,6 +42,7 @@ type File struct {
DownloadPath string `json:"download_path" datastore:"download_path"`
Md5sum string `json:"md5_sum" datastore:"md5_sum"`
Sha256sum string `json:"sha256_sum" datastore:"sha256_sum"`
FileSize int64 `json:"filesize" datastore:"filesize"`
}
var basepath = os.Getenv("SHUFFLE_FILE_LOCATION")
@@ -100,6 +101,51 @@ func fileExists(filename string) bool {
return !info.IsDir()
}
func handleGetFiles(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
// 1. Check user directly
// 2. Check workflow execution authorization
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("[INFO] INITIAL Api authentication failed in file LIST: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("[AUTH] User isn't admin")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Need to be admin"}`)))
return
}
ctx := context.Background()
files, err := getAllFiles(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed to get files: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error getting files."}`)))
return
}
log.Printf("Got %d files for org %s", len(files), user.ActiveOrg.Id)
newBody, err := json.Marshal(files)
if err != nil {
log.Printf("[ERROR] Failed marshaling files: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to marshal files"}`))
return
}
resp.WriteHeader(200)
resp.Write([]byte(newBody))
}
func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
@@ -417,8 +463,18 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
Openfile, err := os.Open(downloadPath)
defer Openfile.Close() //Close after function return
if err != nil {
file.Status = "deleted"
err = setFile(ctx, *file)
if err != nil {
log.Printf("Failed setting file to uploading")
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed setting file to uploading"}`))
return
}
//File not found, send 404
http.Error(resp, "File not found.", 404)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "File doesn't exist locally"}`))
return
}
@@ -552,6 +608,7 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) {
var buf bytes.Buffer
io.Copy(&buf, parsedFile)
contents := buf.Bytes()
file.FileSize = int64(len(contents))
md5 := md5sum(contents)
buf.Reset()
@@ -754,6 +811,9 @@ func getFile(ctx context.Context, id string) (*File, error) {
func setFile(ctx context.Context, file File) error {
// clear session_token and API_token for user
timeNow := time.Now().Unix()
file.UpdatedAt = timeNow
k := datastore.NameKey("Files", file.Id, nil)
if _, err := dbclient.Put(ctx, k, &file); err != nil {
log.Println(err)
@@ -762,3 +822,23 @@ func setFile(ctx context.Context, file File) error {
return nil
}
func getAllFiles(ctx context.Context, orgId string) ([]File, error) {
var files []File
q := datastore.NewQuery("Files").Filter("org_id =", orgId).Order("-updated_at").Limit(100)
_, err := dbclient.GetAll(ctx, q, &files)
if err != nil {
if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") {
q = q.Limit(50)
_, err := dbclient.GetAll(ctx, q, &files)
if err != nil {
return []File{}, err
}
} else {
return []File{}, err
}
}
return files, nil
}
backend/go-app/main.go
+8 -6
View File
@@ -108,6 +108,7 @@ type StatisticsItem struct {
Total int64 `json:"total" datastore:"total"`
Fieldname string `json:"field_name" datastore:"field_name"`
Data []StatisticsData `json:"data" datastore:"data"`
OrgId string `json:"org_id" datastore:"org_id"`
}
// "Execution by status"
@@ -1148,7 +1149,7 @@ func createNewUser(username, password, role, apikey string, org Org) error {
}
}
err = increaseStatisticsField(ctx, "successful_register", username, 1)
err = increaseStatisticsField(ctx, "successful_register", username, 1, org.Id)
if err != nil {
log.Printf("Failed to increase total apps loaded stats: %s", err)
}
@@ -3471,7 +3472,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
// OrgId: activeOrgs[0].Id,
workflowExecution, executionResp, err := handleExecution(item, workflow, newRequest)
if err == nil {
err = increaseStatisticsField(ctx, "total_webhooks_ran", workflowExecution.Workflow.ID, 1)
err = increaseStatisticsField(ctx, "total_webhooks_ran", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg)
if err != nil {
log.Printf("Failed to increase total apps loaded stats: %s", err)
}
@@ -3679,7 +3680,7 @@ func handleNewHook(resp http.ResponseWriter, request *http.Request) {
return
}
err = increaseStatisticsField(ctx, "total_workflow_triggers", requestdata.Workflow, 1)
err = increaseStatisticsField(ctx, "total_workflow_triggers", requestdata.Workflow, 1, user.ActiveOrg.Id)
if err != nil {
log.Printf("[INFO] Failed to increase total workflows: %s", err)
}
@@ -6481,12 +6482,12 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) {
// Backup every single one
setOpenApiDatastore(ctx, api.ID, parsed)
err = increaseStatisticsField(ctx, "total_apps_created", newmd5, 1)
err = increaseStatisticsField(ctx, "total_apps_created", newmd5, 1, user.ActiveOrg.Id)
if err != nil {
log.Printf("Failed to increase success execution stats: %s", err)
}
err = increaseStatisticsField(ctx, "openapi_apps_created", newmd5, 1)
err = increaseStatisticsField(ctx, "openapi_apps_created", newmd5, 1, user.ActiveOrg.Id)
if err != nil {
log.Printf("Failed to increase success execution stats: %s", err)
}
@@ -6873,7 +6874,7 @@ func remoteOrgJobHandler(org Org, interval int) error {
func runInit(ctx context.Context) {
// Setting stats for backend starts (failure count as well)
log.Printf("Starting INIT setup")
err := increaseStatisticsField(ctx, "backend_executions", "", 1)
err := increaseStatisticsField(ctx, "backend_executions", "", 1, "")
if err != nil {
log.Printf("Failed increasing local stats: %s", err)
}
@@ -8125,6 +8126,7 @@ func initHandlers() {
r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}", handleGetFileMeta).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/files", handleGetFiles).Methods("GET", "OPTIONS")
http.Handle("/", r)
}
backend/go-app/walkoff.go
+16 -15
View File
@@ -440,7 +440,7 @@ type AppExecutionExample struct {
// This might be... a bit off, but that's fine :)
// This might also be stupid, as we want timelines and such
// Anyway, these are super basic stupid stats.
func increaseStatisticsField(ctx context.Context, fieldname, id string, amount int64) error {
func increaseStatisticsField(ctx context.Context, fieldname, id string, amount int64, orgId string) error {
// 1. Get current stats
// 2. Increase field(s)
@@ -461,6 +461,7 @@ func increaseStatisticsField(ctx context.Context, fieldname, id string, amount i
if strings.Contains(fmt.Sprintf("%s", err), "entity") {
statisticsItem = StatisticsItem{
Total: amount,
OrgId: orgId,
Fieldname: fieldname,
Data: []StatisticsData{
newData,
@@ -1069,7 +1070,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl
}
newResults = append(newResults, newResult)
increaseStatisticsField(ctx, "workflow_execution_actions_skipped", workflowExecution.Workflow.ID, 1)
increaseStatisticsField(ctx, "workflow_execution_actions_skipped", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg)
}
}
}
@@ -1094,12 +1095,12 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl
workflowExecution.Results = newResults
if workflowExecution.Status == "ABORTED" {
err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1)
err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg)
if err != nil {
log.Printf("Failed to increase aborted execution stats: %s", err)
}
} else if workflowExecution.Status == "FAILURE" {
err = increaseStatisticsField(ctx, "workflow_executions_failure", workflowExecution.Workflow.ID, 1)
err = increaseStatisticsField(ctx, "workflow_executions_failure", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg)
if err != nil {
log.Printf("Failed to increase failure execution stats: %s", err)
}
@@ -1237,7 +1238,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl
workflowExecution.LastNode = actionResult.Action.ID
}
err = increaseStatisticsField(ctx, "workflow_executions_success", workflowExecution.Workflow.ID, 1)
err = increaseStatisticsField(ctx, "workflow_executions_success", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg)
if err != nil {
log.Printf("Failed to increase success execution stats: %s", err)
}
@@ -1525,7 +1526,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) {
ctx := context.Background()
log.Printf("Saved new workflow %s with name %s", workflow.ID, workflow.Name)
err = increaseStatisticsField(ctx, "total_workflows", workflow.ID, 1)
err = increaseStatisticsField(ctx, "total_workflows", workflow.ID, 1, workflow.OrgId)
if err != nil {
log.Printf("Failed to increase total workflows stats: %s", err)
}
@@ -1725,7 +1726,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) {
}
}
err = increaseStatisticsField(ctx, "total_workflow_triggers", workflow.ID, -1)
err = increaseStatisticsField(ctx, "total_workflow_triggers", workflow.ID, -1, workflow.OrgId)
if err != nil {
log.Printf("Failed to increase total workflows: %s", err)
}
@@ -1741,7 +1742,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) {
return
}
err = increaseStatisticsField(ctx, "total_workflows", fileId, -1)
err = increaseStatisticsField(ctx, "total_workflows", fileId, -1, workflow.OrgId)
if err != nil {
log.Printf("Failed to increase total workflows: %s", err)
}
@@ -2314,7 +2315,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
totalOldActions := len(tmpworkflow.Actions)
totalNewActions := len(workflow.Actions)
err = increaseStatisticsField(ctx, "total_workflow_actions", workflow.ID, int64(totalNewActions-totalOldActions))
err = increaseStatisticsField(ctx, "total_workflow_actions", workflow.ID, int64(totalNewActions-totalOldActions), workflow.OrgId)
if err != nil {
log.Printf("Failed to change total actions data: %s", err)
}
@@ -2486,7 +2487,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) {
return
}
err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1)
err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg)
if err != nil {
log.Printf("Failed to increase aborted execution stats: %s", err)
}
@@ -3069,7 +3070,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
}
}
err = increaseStatisticsField(ctx, "workflow_executions", workflow.ID, 1)
err = increaseStatisticsField(ctx, "workflow_executions", workflow.ID, 1, workflowExecution.ExecutionOrg)
if err != nil {
log.Printf("Failed to increase stats execution stats: %s", err)
}
@@ -4130,7 +4131,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) {
return
}
err = increaseStatisticsField(ctx, "total_apps_deleted", fileId, 1)
err = increaseStatisticsField(ctx, "total_apps_deleted", fileId, 1, user.ActiveOrg.Id)
if err != nil {
log.Printf("Failed to increase total apps loaded stats: %s", err)
}
@@ -5706,12 +5707,12 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
continue
}
err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1)
err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1, "")
if err != nil {
log.Printf("Failed to increase total apps created stats: %s", err)
}
err = increaseStatisticsField(ctx, "total_apps_loaded", workflowapp.ID, 1)
err = increaseStatisticsField(ctx, "total_apps_loaded", workflowapp.ID, 1, "")
if err != nil {
log.Printf("Failed to increase total apps loaded stats: %s", err)
}
@@ -6185,7 +6186,7 @@ func handleDeleteHook(resp http.ResponseWriter, request *http.Request) {
}
if len(hook.Workflows) > 0 {
err = increaseStatisticsField(ctx, "total_workflow_triggers", hook.Workflows[0], -1)
err = increaseStatisticsField(ctx, "total_workflow_triggers", hook.Workflows[0], -1, user.ActiveOrg.Id)
if err != nil {
log.Printf("Failed to increase total workflows: %s", err)
}