diff --git a/backend/go-app/files.go b/backend/go-app/files.go index a2856e2b..b9b30a6d 100644 --- a/backend/go-app/files.go +++ b/backend/go-app/files.go @@ -42,6 +42,7 @@ type File struct { DownloadPath string `json:"download_path" datastore:"download_path"` Md5sum string `json:"md5_sum" datastore:"md5_sum"` Sha256sum string `json:"sha256_sum" datastore:"sha256_sum"` + FileSize int64 `json:"filesize" datastore:"filesize"` } var basepath = os.Getenv("SHUFFLE_FILE_LOCATION") @@ -100,6 +101,51 @@ func fileExists(filename string) bool { return !info.IsDir() } +func handleGetFiles(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + // 1. Check user directly + // 2. Check workflow execution authorization + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("[INFO] INITIAL Api authentication failed in file LIST: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if user.Role != "admin" { + log.Printf("[AUTH] User isn't admin") + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Need to be admin"}`))) + return + } + + ctx := context.Background() + files, err := getAllFiles(ctx, user.ActiveOrg.Id) + if err != nil { + log.Printf("[ERROR] Failed to get files: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error getting files."}`))) + return + } + + log.Printf("Got %d files for org %s", len(files), user.ActiveOrg.Id) + newBody, err := json.Marshal(files) + if err != nil { + log.Printf("[ERROR] Failed marshaling files: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false, "reason": "Failed to marshal files"}`)) + return + } + + resp.WriteHeader(200) + resp.Write([]byte(newBody)) +} + func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -417,8 +463,18 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) { Openfile, err := os.Open(downloadPath) defer Openfile.Close() //Close after function return if err != nil { + file.Status = "deleted" + err = setFile(ctx, *file) + if err != nil { + log.Printf("Failed setting file to uploading") + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false, "reason": "Failed setting file to uploading"}`)) + return + } + //File not found, send 404 - http.Error(resp, "File not found.", 404) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "File doesn't exist locally"}`)) return } @@ -552,6 +608,7 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) { var buf bytes.Buffer io.Copy(&buf, parsedFile) contents := buf.Bytes() + file.FileSize = int64(len(contents)) md5 := md5sum(contents) buf.Reset() @@ -754,6 +811,9 @@ func getFile(ctx context.Context, id string) (*File, error) { func setFile(ctx context.Context, file File) error { // clear session_token and API_token for user + timeNow := time.Now().Unix() + file.UpdatedAt = timeNow + k := datastore.NameKey("Files", file.Id, nil) if _, err := dbclient.Put(ctx, k, &file); err != nil { log.Println(err) @@ -762,3 +822,23 @@ func setFile(ctx context.Context, file File) error { return nil } + +func getAllFiles(ctx context.Context, orgId string) ([]File, error) { + var files []File + q := datastore.NewQuery("Files").Filter("org_id =", orgId).Order("-updated_at").Limit(100) + + _, err := dbclient.GetAll(ctx, q, &files) + if err != nil { + if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { + q = q.Limit(50) + _, err := dbclient.GetAll(ctx, q, &files) + if err != nil { + return []File{}, err + } + } else { + return []File{}, err + } + } + + return files, nil +} diff --git a/backend/go-app/main.go b/backend/go-app/main.go index d68218e9..7257fa6f 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -108,6 +108,7 @@ type StatisticsItem struct { Total int64 `json:"total" datastore:"total"` Fieldname string `json:"field_name" datastore:"field_name"` Data []StatisticsData `json:"data" datastore:"data"` + OrgId string `json:"org_id" datastore:"org_id"` } // "Execution by status" @@ -1148,7 +1149,7 @@ func createNewUser(username, password, role, apikey string, org Org) error { } } - err = increaseStatisticsField(ctx, "successful_register", username, 1) + err = increaseStatisticsField(ctx, "successful_register", username, 1, org.Id) if err != nil { log.Printf("Failed to increase total apps loaded stats: %s", err) } @@ -3471,7 +3472,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { // OrgId: activeOrgs[0].Id, workflowExecution, executionResp, err := handleExecution(item, workflow, newRequest) if err == nil { - err = increaseStatisticsField(ctx, "total_webhooks_ran", workflowExecution.Workflow.ID, 1) + err = increaseStatisticsField(ctx, "total_webhooks_ran", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) if err != nil { log.Printf("Failed to increase total apps loaded stats: %s", err) } @@ -3679,7 +3680,7 @@ func handleNewHook(resp http.ResponseWriter, request *http.Request) { return } - err = increaseStatisticsField(ctx, "total_workflow_triggers", requestdata.Workflow, 1) + err = increaseStatisticsField(ctx, "total_workflow_triggers", requestdata.Workflow, 1, user.ActiveOrg.Id) if err != nil { log.Printf("[INFO] Failed to increase total workflows: %s", err) } @@ -6481,12 +6482,12 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { // Backup every single one setOpenApiDatastore(ctx, api.ID, parsed) - err = increaseStatisticsField(ctx, "total_apps_created", newmd5, 1) + err = increaseStatisticsField(ctx, "total_apps_created", newmd5, 1, user.ActiveOrg.Id) if err != nil { log.Printf("Failed to increase success execution stats: %s", err) } - err = increaseStatisticsField(ctx, "openapi_apps_created", newmd5, 1) + err = increaseStatisticsField(ctx, "openapi_apps_created", newmd5, 1, user.ActiveOrg.Id) if err != nil { log.Printf("Failed to increase success execution stats: %s", err) } @@ -6873,7 +6874,7 @@ func remoteOrgJobHandler(org Org, interval int) error { func runInit(ctx context.Context) { // Setting stats for backend starts (failure count as well) log.Printf("Starting INIT setup") - err := increaseStatisticsField(ctx, "backend_executions", "", 1) + err := increaseStatisticsField(ctx, "backend_executions", "", 1, "") if err != nil { log.Printf("Failed increasing local stats: %s", err) } @@ -8125,6 +8126,7 @@ func initHandlers() { r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}", handleGetFileMeta).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS") + r.HandleFunc("/api/v1/files", handleGetFiles).Methods("GET", "OPTIONS") http.Handle("/", r) } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 0437b7a0..2045cf82 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -440,7 +440,7 @@ type AppExecutionExample struct { // This might be... a bit off, but that's fine :) // This might also be stupid, as we want timelines and such // Anyway, these are super basic stupid stats. -func increaseStatisticsField(ctx context.Context, fieldname, id string, amount int64) error { +func increaseStatisticsField(ctx context.Context, fieldname, id string, amount int64, orgId string) error { // 1. Get current stats // 2. Increase field(s) @@ -461,6 +461,7 @@ func increaseStatisticsField(ctx context.Context, fieldname, id string, amount i if strings.Contains(fmt.Sprintf("%s", err), "entity") { statisticsItem = StatisticsItem{ Total: amount, + OrgId: orgId, Fieldname: fieldname, Data: []StatisticsData{ newData, @@ -1069,7 +1070,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } newResults = append(newResults, newResult) - increaseStatisticsField(ctx, "workflow_execution_actions_skipped", workflowExecution.Workflow.ID, 1) + increaseStatisticsField(ctx, "workflow_execution_actions_skipped", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) } } } @@ -1094,12 +1095,12 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl workflowExecution.Results = newResults if workflowExecution.Status == "ABORTED" { - err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1) + err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) if err != nil { log.Printf("Failed to increase aborted execution stats: %s", err) } } else if workflowExecution.Status == "FAILURE" { - err = increaseStatisticsField(ctx, "workflow_executions_failure", workflowExecution.Workflow.ID, 1) + err = increaseStatisticsField(ctx, "workflow_executions_failure", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) if err != nil { log.Printf("Failed to increase failure execution stats: %s", err) } @@ -1237,7 +1238,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl workflowExecution.LastNode = actionResult.Action.ID } - err = increaseStatisticsField(ctx, "workflow_executions_success", workflowExecution.Workflow.ID, 1) + err = increaseStatisticsField(ctx, "workflow_executions_success", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) if err != nil { log.Printf("Failed to increase success execution stats: %s", err) } @@ -1525,7 +1526,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() log.Printf("Saved new workflow %s with name %s", workflow.ID, workflow.Name) - err = increaseStatisticsField(ctx, "total_workflows", workflow.ID, 1) + err = increaseStatisticsField(ctx, "total_workflows", workflow.ID, 1, workflow.OrgId) if err != nil { log.Printf("Failed to increase total workflows stats: %s", err) } @@ -1725,7 +1726,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { } } - err = increaseStatisticsField(ctx, "total_workflow_triggers", workflow.ID, -1) + err = increaseStatisticsField(ctx, "total_workflow_triggers", workflow.ID, -1, workflow.OrgId) if err != nil { log.Printf("Failed to increase total workflows: %s", err) } @@ -1741,7 +1742,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { return } - err = increaseStatisticsField(ctx, "total_workflows", fileId, -1) + err = increaseStatisticsField(ctx, "total_workflows", fileId, -1, workflow.OrgId) if err != nil { log.Printf("Failed to increase total workflows: %s", err) } @@ -2314,7 +2315,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { totalOldActions := len(tmpworkflow.Actions) totalNewActions := len(workflow.Actions) - err = increaseStatisticsField(ctx, "total_workflow_actions", workflow.ID, int64(totalNewActions-totalOldActions)) + err = increaseStatisticsField(ctx, "total_workflow_actions", workflow.ID, int64(totalNewActions-totalOldActions), workflow.OrgId) if err != nil { log.Printf("Failed to change total actions data: %s", err) } @@ -2486,7 +2487,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { return } - err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1) + err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) if err != nil { log.Printf("Failed to increase aborted execution stats: %s", err) } @@ -3069,7 +3070,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } } - err = increaseStatisticsField(ctx, "workflow_executions", workflow.ID, 1) + err = increaseStatisticsField(ctx, "workflow_executions", workflow.ID, 1, workflowExecution.ExecutionOrg) if err != nil { log.Printf("Failed to increase stats execution stats: %s", err) } @@ -4130,7 +4131,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { return } - err = increaseStatisticsField(ctx, "total_apps_deleted", fileId, 1) + err = increaseStatisticsField(ctx, "total_apps_deleted", fileId, 1, user.ActiveOrg.Id) if err != nil { log.Printf("Failed to increase total apps loaded stats: %s", err) } @@ -5706,12 +5707,12 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin continue } - err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1) + err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1, "") if err != nil { log.Printf("Failed to increase total apps created stats: %s", err) } - err = increaseStatisticsField(ctx, "total_apps_loaded", workflowapp.ID, 1) + err = increaseStatisticsField(ctx, "total_apps_loaded", workflowapp.ID, 1, "") if err != nil { log.Printf("Failed to increase total apps loaded stats: %s", err) } @@ -6185,7 +6186,7 @@ func handleDeleteHook(resp http.ResponseWriter, request *http.Request) { } if len(hook.Workflows) > 0 { - err = increaseStatisticsField(ctx, "total_workflow_triggers", hook.Workflows[0], -1) + err = increaseStatisticsField(ctx, "total_workflow_triggers", hook.Workflows[0], -1, user.ActiveOrg.Id) if err != nil { log.Printf("Failed to increase total workflows: %s", err) } diff --git a/frontend/package.json b/frontend/package.json index a4c40596..b627526f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "shuffler", "homepage": "https://shuffler.io", - "version": "0.6.0", + "version": "0.8.3", "private": true, "dependencies": { "@material-ui/core": "^4.5.2", diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index ad857ad6..252af7ef 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -31,6 +31,9 @@ import { useTheme } from '@material-ui/core/styles'; import HandlePayment from './HandlePayment' import OrgHeader from '../components/OrgHeader' +import OpenInNewIcon from '@material-ui/icons/OpenInNew'; +import CloudDownloadIcon from '@material-ui/icons/CloudDownload'; +import DescriptionIcon from '@material-ui/icons/Description'; import PolymerIcon from '@material-ui/icons/Polymer'; import CheckCircleIcon from '@material-ui/icons/CheckCircle'; import CloseIcon from '@material-ui/icons/Close'; @@ -78,6 +81,7 @@ const Admin = (props) => { const [environments, setEnvironments] = React.useState([]); const [authentication, setAuthentication] = React.useState([]); const [schedules, setSchedules] = React.useState([]) + const [files, setFiles] = React.useState([]) const [selectedUser, setSelectedUser] = React.useState({}) const [newPassword, setNewPassword] = React.useState(""); const [selectedUserModalOpen, setSelectedUserModalOpen] = React.useState(false) @@ -550,6 +554,78 @@ const Admin = (props) => { }); } + const getFiles = () => { + fetch(globalUrl + "/api/v1/files", { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for apps :O!") + return + } + + return response.json() + }) + .then((responseJson) => { + console.log(responseJson) + setFiles(responseJson) + }) + .catch(error => { + alert.error(error.toString()) + }); + } + + const downloadFile = (file) => { + fetch(globalUrl + "/api/v1/files/"+file.id+"/content", { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for apps :O!") + return "" + } + + return response.text() + }) + .then((respdata) => { + if (respdata.length === 0) { + alert.error("Failed getting file") + return + } + + var blob = new Blob( [ respdata ], { + type: 'application/octet-stream' + }) + + var url = URL.createObjectURL( blob ) + var link = document.createElement( 'a' ) + link.setAttribute( 'href', url ) + link.setAttribute( 'download', `${file.filename}` ) + var event = document.createEvent( 'MouseEvents' ) + event.initMouseEvent( 'click', true, true, window, 1, 0, 0, 0, 0, false, false, false, false, 0, null) + link.dispatchEvent( event ) + + //return response.json() + }) + .then((responseJson) => { + //console.log(responseJson) + //setSchedules(responseJson) + }) + .catch(error => { + alert.error(error.toString()) + }); + } + const getSchedules = () => { fetch(globalUrl + "/api/v1/workflows/schedules", { method: 'GET', @@ -705,6 +781,48 @@ const Admin = (props) => { }); } + const setConfig = (event, newValue) => { + if (newValue === 1) { + getUsers() + } else if (newValue === 2) { + getAppAuthentication() + } else if (newValue === 3) { + getEnvironments() + } else if (newValue === 4) { + getSchedules() + } else if (newValue === 5) { + getFiles() + } else if (newValue === 6) { + getOrgs() + } + + if (newValue === 6) { + console.log("Should get apps for categories.") + } + + const views = { + 0: "organization", + 1: "users", + 2: "app_auth", + 3: "environments", + 4: "schedules", + 5: "files", + 6: "categories", + } + + //var theURL = window.location.pathname + //FIXME: Add url edits + //var theURL = window.location + //theURL.replace(`/${views[curTab]}`, `/${views[newValue]}`) + //window.history.pushState({"html":response.html,"pageTitle":response.pageTitle},"", urlPath); + + //console.log(newpath) + //window.location.pathame = newpath + + setModalUser({}) + setCurTab(newValue) + } + if (firstRequest) { setFirstRequest(false) @@ -720,13 +838,14 @@ const Admin = (props) => { "app_auth": 2, "environments": 3, "schedules": 4, - "categories": 5, + "files": 5, } if (props.match.params.key !== undefined) { const tmpitem = views[props.match.params.key] if (tmpitem !== undefined) { - setCurTab(tmpitem) + //setCurTab(tmpitem) + setConfig("", tmpitem) } } } @@ -1532,6 +1651,113 @@ const Admin = (props) => { : null + const filesView = curTab === 5 ? +