marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #214 from frikky/launch

Browse Source 
0.8.3
This commit is contained in:
Frikky
2020-12-06 18:08:55 +01:00
committed by GitHub
parent cca56a4c59 b13c3f3765
commit 7899f9764d
12 changed files with 1659 additions and 492 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/go-app/docker.go
+8 -8
View File
@@ -181,11 +181,11 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
// Dockerfile is inside the TAR itself. Not local context
// docker build --build-arg http_proxy=http://my.proxy.url
buildOptions := types.ImageBuildOptions{
Remove: true,
Tags: tags,
BuildArgs: map[string]*string{},
NetworkMode: "host",
Remove: true,
Tags: tags,
BuildArgs: map[string]*string{},
}
// NetworkMode: "host",
httpProxy := os.Getenv("HTTP_PROXY")
if len(httpProxy) > 0 {
@@ -244,11 +244,11 @@ func buildImage(tags []string, dockerfileFolder string) error {
dockerFileTarReader := bytes.NewReader(buf.Bytes())
buildOptions := types.ImageBuildOptions{
Remove: true,
Tags: tags,
BuildArgs: map[string]*string{},
NetworkMode: "host",
Remove: true,
Tags: tags,
BuildArgs: map[string]*string{},
}
//NetworkMode: "host",
httpProxy := os.Getenv("HTTP_PROXY")
if len(httpProxy) > 0 {
backend/go-app/files.go
+764
View File
@@ -0,0 +1,764 @@
package main
/*
Handles files within Workflows.of Shuffle
*/
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"strconv"
"strings"
"time"
"cloud.google.com/go/datastore"
"github.com/satori/go.uuid"
)
type File struct {
Id string `json:"id" datastore:"id"`
Type string `json:"type" datastore:"type"`
CreatedAt int64 `json:"created_at" datastore:"created_at"`
UpdatedAt int64 `json:"updated_at" datastore:"updated_at"`
MetaAccessAt int64 `json:"meta_access_at" datastore:"meta_access_at"`
DownloadAt int64 `json:"last_downloaded" datastore:"last_downloaded"`
Description string `json:"description" datastore:"description"`
ExpiresAt string `json:"expires_at" datastore:"expires_at"`
Status string `json:"status" datastore:"status"`
Filename string `json:"filename" datastore:"filename"`
URL string `json:"url" datastore:"org"`
OrgId string `json:"org_id" datastore:"org_id"`
WorkflowId string `json:"workflow_id" datastore:"workflow_id"`
Workflows []string `json:"workflows" datastore:"workflows"`
DownloadPath string `json:"download_path" datastore:"download_path"`
Md5sum string `json:"md5_sum" datastore:"md5_sum"`
Sha256sum string `json:"sha256_sum" datastore:"sha256_sum"`
}
var basepath = os.Getenv("SHUFFLE_FILE_LOCATION")
func fileAuthentication(request *http.Request) (string, error) {
executionId, ok := request.URL.Query()["execution_id"]
if ok && len(executionId) > 0 {
ctx := context.Background()
workflowExecution, err := getWorkflowExecution(ctx, executionId[0])
if err != nil {
log.Printf("[ERROR] Couldn't find execution ID %s", executionId[0])
return "", err
}
apikey := request.Header.Get("Authorization")
if !strings.HasPrefix(apikey, "Bearer ") {
log.Printf("[ERROR} Apikey doesn't start with bearer (2)")
return "", errors.New("No auth key found")
}
apikeyCheck := strings.Split(apikey, " ")
if len(apikeyCheck) != 2 {
log.Printf("[ERROR] Invalid format for apikey (2)")
return "", errors.New("No space in authkey")
}
// This is annoying af and is done because of maxlength lol
newApikey := apikeyCheck[1]
if newApikey != workflowExecution.Authorization {
//log.Printf("[ERROR] Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization)
log.Printf("[ERROR] Bad apikey for execution %s.", executionId[0])
//%s vs %s", executionId[0], apikey, workflowExecution.Authorization)
return "", errors.New("Bad authorization key")
}
log.Printf("[INFO] Authorization is correct for execution %s!", executionId[0])
//%s vs %s. Setting Org", executionId, apikey, workflowExecution.Authorization)
if len(workflowExecution.ExecutionOrg) > 0 {
return workflowExecution.ExecutionOrg, nil
} else if len(workflowExecution.Workflow.ExecutingOrg.Id) > 0 {
return workflowExecution.ExecutionOrg, nil
} else {
log.Printf("[ERROR] Couldn't find org for workflow execution, but auth was correct.")
}
}
return "", errors.New("No execution id specified")
}
// https://golangcode.com/check-if-a-file-exists/
func fileExists(filename string) bool {
info, err := os.Stat(filename)
if os.IsNotExist(err) {
return false
}
return !info.IsDir()
}
func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("[INFO] Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if strings.Contains(fileId, "?") {
fileId = strings.Split(fileId, "?")[0]
}
if len(fileId) != 36 {
log.Printf("Bad format for fileId %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
return
}
log.Printf("\n\n[INFO] User is trying to GET File Meta for %s\n\n", fileId)
// 1. Check user directly
// 2. Check workflow execution authorization
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("[INFO] INITIAL Api authentication failed in file deletion: %s", err)
orgId, err := fileAuthentication(request)
if err != nil {
log.Printf("[ERROR] Bad file authentication in get: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
user.ActiveOrg.Id = orgId
user.Username = "Execution File API"
}
// 1. Verify if the user has access to the file: org_id and workflow
log.Printf("[INFO] Should GET FILE META for %s if user has access", fileId)
ctx := context.Background()
file, err := getFile(ctx, fileId)
if err != nil {
log.Printf("[INFO] File %s not found: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
found := false
if file.OrgId == user.ActiveOrg.Id {
found = true
} else {
for _, item := range user.Orgs {
if item == file.OrgId {
found = true
break
}
}
}
if !found {
log.Printf("[INFO] User %s doesn't have access to %s", user.Username, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
newBody, err := json.Marshal(file)
if err != nil {
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to marshal filedata"}`))
return
}
log.Printf("[INFO] Successfully got file meta for %s", fileId)
resp.WriteHeader(200)
resp.Write([]byte(newBody))
}
func handleDeleteFile(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("[INFO] Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if strings.Contains(fileId, "?") {
fileId = strings.Split(fileId, "?")[0]
}
if len(fileId) != 36 {
log.Printf("Bad format for fileId %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
return
}
log.Printf("\n\n[INFO] User is trying to delete file %s\n\n", fileId)
// 1. Check user directly
// 2. Check workflow execution authorization
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("[INFO] INITIAL Api authentication failed in file deletion: %s", err)
orgId, err := fileAuthentication(request)
if err != nil {
log.Printf("[ERROR] Bad file authentication in get: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
user.ActiveOrg.Id = orgId
user.Username = "Execution File API"
}
// 1. Verify if the user has access to the file: org_id and workflow
log.Printf("[INFO] Should DELETE file %s if user has access", fileId)
ctx := context.Background()
file, err := getFile(ctx, fileId)
if err != nil {
log.Printf("[INFO] File %s not found: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
found := false
if file.OrgId == user.ActiveOrg.Id {
found = true
} else {
for _, item := range user.Orgs {
if item == file.OrgId {
found = true
break
}
}
}
if !found {
log.Printf("[INFO] User %s doesn't have access to %s", user.Username, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if file.Status == "deleted" {
log.Printf("[INFO] File with ID %s is already deleted.", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if fileExists(file.DownloadPath) {
err = os.Remove(file.DownloadPath)
if err != nil {
log.Printf("[ERROR] Failed deleting file locally: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting filein path %s"}`, file.DownloadPath)))
return
}
log.Printf("[INFO] Deleted file %s locally. Next is database.", file.DownloadPath)
} else {
log.Printf("[ERROR] File doesn't exist. Can't delete. Should maybe delete file anyway?")
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "File in location %s doesn't exist"}`, file.DownloadPath)))
return
}
file.Status = "deleted"
err = setFile(ctx, *file)
if err != nil {
log.Printf("[ERROR] Failed setting file to deleted")
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed setting file to deleted"}`))
return
}
/*
//Actually delete it?
err = DeleteKey(ctx, "files", fileId)
if err != nil {
log.Printf("Failed deleting file with ID %s: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
*/
log.Printf("[INFO] Successfully deleted file %s", fileId)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if len(fileId) != 36 {
log.Printf("Bad format for fileId %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
return
}
log.Printf("\n\nUser is trying to download file %s\n\n", fileId)
// 1. Check user directly
// 2. Check workflow execution authorization
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("INITIAL Api authentication failed in file download: %s", err)
orgId, err := fileAuthentication(request)
if err != nil {
log.Printf("Bad file authentication in get: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
user.ActiveOrg.Id = orgId
user.Username = "Execution File API"
/*
} else {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
*/
}
// 1. Verify if the user has access to the file: org_id and workflow
log.Printf("[INFO] Should get file %s", fileId)
ctx := context.Background()
file, err := getFile(ctx, fileId)
if err != nil {
log.Printf("[ERROR] File %s not found: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
found := false
if file.OrgId == user.ActiveOrg.Id {
found = true
} else {
for _, item := range user.Orgs {
if item == file.OrgId {
found = true
break
}
}
}
if !found {
log.Printf("User %s doesn't have access to %s", user.Username, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if file.Status != "active" {
log.Printf("[ERROR] File status isn't active, but %s. Can't continue.", file.Status)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "The file isn't ready to be downloaded yet. Status required: active"}`))
return
}
// Fixme: More auth: org and workflow!
downloadPath := file.DownloadPath
log.Printf("[INFO] Downloadpath: %s", downloadPath)
Openfile, err := os.Open(downloadPath)
defer Openfile.Close() //Close after function return
if err != nil {
//File not found, send 404
http.Error(resp, "File not found.", 404)
return
}
//File is found, create and send the correct headers
//Get the Content-Type of the file
//Create a buffer to store the header of the file in
FileHeader := make([]byte, 512)
//Copy the headers into the FileHeader buffer
Openfile.Read(FileHeader)
//Get content type of file
FileContentType := http.DetectContentType(FileHeader)
//Get the file size
FileStat, _ := Openfile.Stat() //Get info from file
FileSize := strconv.FormatInt(FileStat.Size(), 10) //Get file size as a string
//Send the headers
resp.Header().Set("Content-Disposition", "attachment; filename="+fileId)
resp.Header().Set("Content-Type", FileContentType)
resp.Header().Set("Content-Length", FileSize)
//Send the file
//We read 512 bytes from the file already, so we reset the offset back to 0
Openfile.Seek(0, 0)
io.Copy(resp, Openfile) //'Copy' the file to the client
return
//log.Printf("Should download file %s", downloadPath)
}
func handleUploadFile(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if len(fileId) != 36 {
log.Printf("Bad format for fileId %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
return
}
// 1. Check user directly
// 2. Check workflow execution authorization
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("INITIAL Api authentication failed in file upload: %s", err)
orgId, err := fileAuthentication(request)
if err != nil {
log.Printf("Bad file authentication in create file: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
user.ActiveOrg.Id = orgId
user.Username = "Execution File API"
}
log.Printf("[INFO] Should UPLOAD file %s if user has access", fileId)
ctx := context.Background()
file, err := getFile(ctx, fileId)
if err != nil {
log.Printf("File %s not found: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
found := false
if file.OrgId == user.ActiveOrg.Id {
found = true
} else {
for _, item := range user.Orgs {
if item == file.OrgId {
found = true
break
}
}
}
if !found {
log.Printf("User %s doesn't have access to %s", user.Username, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
log.Printf("[INFO] STATUS: %s", file.Status)
if file.Status != "created" {
log.Printf("File status isn't created. Can't upload.")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "This file already has data."}`))
return
}
request.ParseMultipartForm(32 << 20)
parsedFile, _, err := request.FormFile("shuffle_file")
if err != nil {
log.Printf("[ERROR] Couldn't upload file: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed uploading file"}`))
return
}
defer parsedFile.Close()
file.Status = "uploading"
err = setFile(ctx, *file)
if err != nil {
log.Printf("Failed setting file to uploading")
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed setting file to uploading"}`))
return
}
// Can be used for validation files for change
var buf bytes.Buffer
io.Copy(&buf, parsedFile)
contents := buf.Bytes()
md5 := md5sum(contents)
buf.Reset()
sha256Sum := sha256.Sum256(contents)
//parsedFile.Reset()
f, err := os.OpenFile(file.DownloadPath, os.O_WRONLY|os.O_CREATE, os.ModePerm)
if err != nil {
// Rolling back file
file.Status = "created"
setFile(ctx, *file)
log.Printf("[ERROR] Failed uploading and creating file: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
defer f.Close()
parsedFile.Seek(0, io.SeekStart)
io.Copy(f, parsedFile)
// FIXME: Set this one to 200 anyway? Can't download file then tho..
file.Status = "active"
file.Md5sum = md5
file.Sha256sum = fmt.Sprintf("%x", sha256Sum)
log.Printf("[INFO] MD5 for file %s (%s) is %s and SHA256 is %s", file.Filename, file.Id, file.Md5sum, file.Sha256sum)
err = setFile(ctx, *file)
if err != nil {
log.Printf("[ERROR] Failed setting file back to active")
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed setting file to active"}`))
return
}
log.Printf("[INFO] Successfully uploaded file ID %s", file.Id)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func handleCreateFile(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
// 1. Check user directly
// 2. Check workflow execution authorization
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("[INFO] INITIAL Api authentication failed in file creation: %s", err)
orgId, err := fileAuthentication(request)
if err != nil {
log.Printf("[ERROR] Bad file authentication in create file: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
user.ActiveOrg.Id = orgId
user.Username = "Execution File API"
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Println("Failed reading body")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to read data"}`)))
return
}
type FileStructure struct {
Filename string `json:"filename"`
OrgId string `json:"org_id"`
WorkflowId string `json:"workflow_id"`
}
var curfile FileStructure
err = json.Unmarshal(body, &curfile)
if err != nil {
log.Printf("[ERROR] Failed unmarshaling: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to unmarshal data"}`)))
return
}
// Loads of validation below
if len(curfile.Filename) == 0 || len(curfile.OrgId) == 0 || len(curfile.WorkflowId) == 0 {
log.Printf("[ERROR] Missing field during upload.")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Missing field. Required: filename, org_id, workflow_id"}`)))
return
}
ctx := context.Background()
if user.ActiveOrg.Id != curfile.OrgId {
log.Printf("[ERROR] User can't access org %s", curfile.OrgId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Error with organization"}`))
return
}
// Try to get the org and workflow in case they don't exist
workflow, err := getWorkflow(ctx, curfile.WorkflowId)
if err != nil {
log.Printf("[ERROR] Workflow %s doesn't exist.", curfile.WorkflowId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`))
return
}
_, err = getOrg(ctx, curfile.OrgId)
if err != nil {
log.Printf("[ERROR] Org %s doesn't exist.", curfile.OrgId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`))
return
}
if workflow.ExecutingOrg.Id != curfile.OrgId {
found := false
for _, curorg := range workflow.Org {
if curorg.Id == curfile.OrgId {
found = true
break
}
}
if !found {
log.Printf("[ERROR] Org %s doesn't have access to %s.", curfile.OrgId, curfile.WorkflowId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`))
return
}
}
if strings.Contains(curfile.Filename, "/") || strings.Contains(curfile.Filename, `"`) || strings.Contains(curfile.Filename, "..") || strings.Contains(curfile.Filename, "~") {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Invalid characters in filename"}`))
return
}
// 1. Create the file object.
if len(basepath) == 0 {
basepath = "shuffle-files"
}
folderPath := fmt.Sprintf("%s/%s/%s", basepath, curfile.OrgId, curfile.WorkflowId)
// Try to make the full file location
err = os.MkdirAll(folderPath, os.ModePerm)
if err != nil {
log.Printf("[ERROR] Writing issue for file location creation: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed creating upload location"}`))
return
}
filename := curfile.Filename
fileId := uuid.NewV4().String()
downloadPath := fmt.Sprintf("%s/%s", folderPath, fileId)
timeNow := time.Now().Unix()
newFile := File{
Id: fileId,
CreatedAt: timeNow,
UpdatedAt: timeNow,
Description: "",
Status: "created",
Filename: filename,
OrgId: curfile.OrgId,
WorkflowId: curfile.WorkflowId,
DownloadPath: downloadPath,
}
err = setFile(ctx, newFile)
if err != nil {
log.Printf("[ERROR] Failed setting file: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed setting file reference"}`))
return
} else {
log.Printf("[INFO] Created file %s", newFile.DownloadPath)
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, fileId)))
}
func getFile(ctx context.Context, id string) (*File, error) {
key := datastore.NameKey("Files", id, nil)
curFile := &File{}
if err := dbclient.Get(ctx, key, curFile); err != nil {
return &File{}, err
}
return curFile, nil
}
func setFile(ctx context.Context, file File) error {
// clear session_token and API_token for user
k := datastore.NameKey("Files", file.Id, nil)
if _, err := dbclient.Put(ctx, k, &file); err != nil {
log.Println(err)
return err
}
return nil
}
backend/go-app/main.go
+40 -226
View File
@@ -2422,7 +2422,7 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) {
}
if count == 0 {
log.Printf("No users - redirecting for management user")
log.Printf("[WARNING] No users - redirecting for management user")
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "stay"}`)))
return
@@ -2555,12 +2555,12 @@ func getApikey(ctx context.Context, apikey string) (User, error) {
var users []User
_, err := dbclient.GetAll(ctx, q, &users)
if err != nil {
log.Printf("Error getting users apikey (getapikey): %s", err)
log.Printf("[ERROR] Error getting users apikey (getapikey): %s", err)
return User{}, err
}
if len(users) == 0 {
log.Printf("No users found for apikey %s", apikey)
log.Printf("[WARNING] No users found for apikey %s", apikey)
return User{}, err
}
@@ -2577,28 +2577,6 @@ func getSession(ctx context.Context, thissession string) (*session, error) {
return curUser, nil
}
// ListBooks returns a list of books, ordered by title.
func getFile(ctx context.Context, id string) (*File, error) {
key := datastore.NameKey("Files", id, nil)
curFile := &File{}
if err := dbclient.Get(ctx, key, curFile); err != nil {
return &File{}, err
}
return curFile, nil
}
func setFile(ctx context.Context, file File) error {
// clear session_token and API_token for user
k := datastore.NameKey("Files", file.Id, nil)
if _, err := dbclient.Put(ctx, k, &file); err != nil {
log.Println(err)
return err
}
return nil
}
// ListBooks returns a list of books, ordered by title.
func getOrg(ctx context.Context, id string) (*Org, error) {
key := datastore.NameKey("Organizations", id, nil)
@@ -6552,7 +6530,7 @@ func handleAppHotload(location string, forceUpdate bool) error {
}
//log.Printf("Reading app folder: %#v", dir)
err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate)
_, _, err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate)
if err != nil {
log.Printf("Err: %s", err)
return err
@@ -6835,7 +6813,7 @@ func remoteOrgJobHandler(org Org, interval int) error {
respBody, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("Failed body read in job sync: %s", err)
log.Printf("[ERROR] Failed body read in job sync: %s", err)
return err
}
@@ -6843,7 +6821,7 @@ func remoteOrgJobHandler(org Org, interval int) error {
err = remoteOrgJobController(org, respBody)
if err != nil {
log.Printf("Failed job controller run: %s", err)
log.Printf("[ERROR] Failed job controller run for %s: %s", respBody, err)
return err
}
return nil
@@ -7141,37 +7119,39 @@ func runInit(ctx context.Context) {
}
}
fileq := datastore.NewQuery("Files").Limit(1)
count, err := dbclient.Count(ctx, fileq)
log.Printf("FILECOUNT: %d", count)
if err == nil && count < 10 {
basepath := "."
filename := "testfile.txt"
fileId := uuid.NewV4().String()
log.Printf("Creating new file reference %s because none exist!", fileId)
workflowId := "2cf1169d-b460-41de-8c36-28b2092866f8"
downloadPath := fmt.Sprintf("%s/%s/%s/%s", basepath, activeOrgs[0].Id, workflowId, fileId)
/*
fileq := datastore.NewQuery("Files").Limit(1)
count, err := dbclient.Count(ctx, fileq)
log.Printf("FILECOUNT: %d", count)
if err == nil && count < 10 {
basepath := "."
filename := "testfile.txt"
fileId := uuid.NewV4().String()
log.Printf("Creating new file reference %s because none exist!", fileId)
workflowId := "2cf1169d-b460-41de-8c36-28b2092866f8"
downloadPath := fmt.Sprintf("%s/%s/%s/%s", basepath, activeOrgs[0].Id, workflowId, fileId)
timeNow := time.Now().Unix()
newFile := File{
Id: fileId,
CreatedAt: timeNow,
UpdatedAt: timeNow,
Description: "Created by system for testing",
Status: "active",
Filename: filename,
OrgId: activeOrgs[0].Id,
WorkflowId: workflowId,
DownloadPath: downloadPath,
}
timeNow := time.Now().Unix()
newFile := File{
Id: fileId,
CreatedAt: timeNow,
UpdatedAt: timeNow,
Description: "Created by system for testing",
Status: "active",
Filename: filename,
OrgId: activeOrgs[0].Id,
WorkflowId: workflowId,
DownloadPath: downloadPath,
}
err = setFile(ctx, newFile)
if err != nil {
log.Printf("Failed setting file: %s", err)
} else {
log.Printf("Created file %s in init", newFile.DownloadPath)
err = setFile(ctx, newFile)
if err != nil {
log.Printf("Failed setting file: %s", err)
} else {
log.Printf("Created file %s in init", newFile.DownloadPath)
}
}
}
*/
var allworkflowapps []AppAuthenticationStorage
q = datastore.NewQuery("workflowappauth")
@@ -7934,172 +7914,6 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) {
resp.Write(respBody)
}
type File struct {
Id string `json:"id" datastore:"id"`
Type string `json:"type" datastore:"type"`
CreatedAt int64 `json:"created_at" datastore:"created_at"`
UpdatedAt int64 `json:"updated_at" datastore:"updated_at"`
Description string `json:"description" datastore:"description"`
ExpiresAt string `json:"expires_at" datastore:"expires_at"`
Status string `json:"status" datastore:"status"`
Filename string `json:"filename" datastore:"filename"`
URL string `json:"url" datastore:"org"`
OrgId string `json:"org_id" datastore:"org_id"`
WorkflowId string `json:"workflow_id" datastore:"workflow_id"`
Workflows []string `json:"workflows" datastore:"workflows"`
DownloadPath string `json:"download_path" datastore:"download_path"`
}
func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
log.Printf("\n\nUser is trying to get file %s\n\n", fileId)
// 1. Check user directly
// 2. Check workflow execution authorization
setOrgId := false
user, err := handleApiAuthentication(resp, request)
if err != nil {
// r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS")
log.Printf("INITIAL Api authentication failed in file download: %s", err)
executionId, ok := request.URL.Query()["execution_id"]
if ok && len(executionId) > 0 {
ctx := context.Background()
workflowExecution, err := getWorkflowExecution(ctx, executionId[0])
if err != nil {
log.Printf("Couldn't find execution ID %s", executionId[0])
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
apikey := request.Header.Get("Authorization")
if !strings.HasPrefix(apikey, "Bearer ") {
log.Printf("Apikey doesn't start with bearer (2)")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
apikeyCheck := strings.Split(apikey, " ")
if len(apikeyCheck) != 2 {
log.Printf("Invalid format for apikey (2)")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// This is annoying af and is done because of maxlength lol
newApikey := apikeyCheck[1]
if newApikey != workflowExecution.Authorization {
log.Printf("Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
log.Printf("Authorization is correct for execution %s! %s vs %s", executionId, apikey, workflowExecution.Authorization)
setOrgId = true
} else {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
// 1. Verify if the user has access to the file: org_id and workflow
log.Printf("Should get file %s", fileId)
ctx := context.Background()
file, err := getFile(ctx, fileId)
if err != nil {
log.Printf("File %s not found: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// This is a workaround for file grabs from an app
if setOrgId == true {
user.ActiveOrg.Id = file.OrgId
}
found := false
if file.OrgId == user.ActiveOrg.Id {
found = true
} else {
for _, item := range user.Orgs {
if item == file.OrgId {
found = true
break
}
}
}
if !found {
log.Printf("User %s doesn't have access to %s", user.Username, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Fixme: More auth: org and workflow!
downloadPath := file.DownloadPath
log.Printf("Downloadpath: %s", downloadPath)
Openfile, err := os.Open(downloadPath)
defer Openfile.Close() //Close after function return
if err != nil {
//File not found, send 404
http.Error(resp, "File not found.", 404)
return
}
//File is found, create and send the correct headers
//Get the Content-Type of the file
//Create a buffer to store the header of the file in
FileHeader := make([]byte, 512)
//Copy the headers into the FileHeader buffer
Openfile.Read(FileHeader)
//Get content type of file
FileContentType := http.DetectContentType(FileHeader)
//Get the file size
FileStat, _ := Openfile.Stat() //Get info from file
FileSize := strconv.FormatInt(FileStat.Size(), 10) //Get file size as a string
//Send the headers
resp.Header().Set("Content-Disposition", "attachment; filename="+fileId)
resp.Header().Set("Content-Type", FileContentType)
resp.Header().Set("Content-Length", FileSize)
//Send the file
//We read 512 bytes from the file already, so we reset the offset back to 0
Openfile.Seek(0, 0)
io.Copy(resp, Openfile) //'Copy' the file to the client
return
//log.Printf("Should download file %s", downloadPath)
//resp.WriteHeader(200)
//resp.Write([]byte("OK"))
}
func initHandlers() {
var err error
ctx := context.Background()
@@ -8233,10 +8047,10 @@ func initHandlers() {
// https://developer.box.com/reference/get-files-id-content/
// 1. Creating the "get file" option. Make it possible to run this in the frontend.
r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS")
//r.HandleFunc("/api/v1/files/create", handleGetFile).Methods("POST", "OPTIONS")
//r.HandleFunc("/api/v1/files/upload", handleGetFile).Methods("POST", "OPTIONS")
//r.HandleFunc("/api/v1/files/{fileId}", handleGetFile).Methods("GET", "OPTIONS")
//r.HandleFunc("/api/v1/files/{fileId}", handleGetFile).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/files/create", handleCreateFile).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}", handleGetFileMeta).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS")
http.Handle("/", r)
}
backend/go-app/walkoff.go
+150 -26
View File
@@ -954,6 +954,30 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
return
}
runWorkflowExecutionTransaction(ctx, 0, workflowExecution.ExecutionId, actionResult, resp)
}
// Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times
func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult ActionResult, resp http.ResponseWriter) {
// Should start a tx for the execution here
tx, err := dbclient.NewTransaction(ctx)
if err != nil {
log.Printf("client.NewTransaction: %v", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed creating transaction"}`)))
return
}
key := datastore.NameKey("workflowexecution", workflowExecutionId, nil)
workflowExecution := &WorkflowExecution{}
if err := tx.Get(key, workflowExecution); err != nil {
log.Printf("tx.Get bug: %v", err)
tx.Rollback()
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting the workflow key"}`)))
return
}
if actionResult.Status == "ABORTED" || actionResult.Status == "FAILURE" {
log.Printf("Actionresult is %s. Should set workflowExecution and exit all running functions", actionResult.Status)
@@ -1228,18 +1252,37 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
}
}
err = setWorkflowExecution(ctx, *workflowExecution)
if err != nil {
//workflowExecution.Result = "Error setting workflow: result too large"
//workflowExecution.Status = "FINISHED"
//workflowExecution.CompletedAt = int64(time.Now().Unix())
// Transactions: https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-go
// Prevents timing issues
//ExecutionId
if _, err := tx.Put(key, workflowExecution); err != nil {
tx.Rollback()
log.Printf("[ERROR] tx.Put bug: %v", err)
log.Printf("Error saving workflow execution actionresult setting: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution actionresult: %s"}`, err)))
return
}
if _, err = tx.Commit(); err != nil {
if attempts >= 5 {
log.Printf("[ERROR] QUITTING: tx.Commit %d: %v", attempts, err)
tx.Rollback()
workflowExecution.Status = "ABORTED"
setWorkflowExecution(ctx, *workflowExecution)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
log.Printf("[WARNING] tx.Commit %d: %v", attempts, err)
attempts += 1
runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp)
return
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
@@ -1360,6 +1403,7 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) {
q := datastore.NewQuery("workflow").Filter("owner =", user.Id)
if user.Role == "admin" {
q = datastore.NewQuery("workflow").Filter("org_id =", user.ActiveOrg.Id)
log.Printf("[INFO] Getting workflows (ADMIN) for organization %s", user.ActiveOrg.Id)
}
var workflows []Workflow
@@ -1830,7 +1874,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
for _, action := range workflow.Actions {
allNodes = append(allNodes, action.ID)
if len(action.Errors) > 0 {
if len(action.Errors) > 0 || !action.IsValid {
action.IsValid = true
action.Errors = []string{}
}
@@ -2192,9 +2236,11 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
// Handles check for required params
if !found && param.Required {
log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name)))
return
action.Errors = append(action.Errors, "Parameter %s is required", param.Name)
//newActions = append(newActions, action)
//resp.WriteHeader(401)
//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name)))
//return
}
}
@@ -2209,6 +2255,15 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
workflow.IsValid = true
log.Printf("Tags: %#v", workflow.Tags)
// FIXME: Is this too drastic? May lead to issues in the future.
// Should maybe make a copy for the old org.
if workflow.OrgId != user.ActiveOrg.Id {
log.Printf("[WARNING] Editing workflow to be owned by %s", user.ActiveOrg.Id)
workflow.OrgId = user.ActiveOrg.Id
workflow.ExecutingOrg = user.ActiveOrg
workflow.Org = append(workflow.Org, user.ActiveOrg)
}
err = setWorkflow(ctx, workflow, fileId)
if err != nil {
log.Printf("Failed saving workflow to database: %s", err)
@@ -2234,7 +2289,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
Errors: workflow.Errors,
}
log.Printf("Saved new version of workflow %s (%s)", workflow.Name, fileId)
log.Printf("Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId)
resp.WriteHeader(200)
newBody, err := json.Marshal(returndata)
if err != nil {
@@ -2795,7 +2850,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
continue
}
log.Printf("Should set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID)
log.Printf("[WARNING] Set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID)
defaultResults = append(defaultResults, ActionResult{
Action: action,
ExecutionId: workflowExecution.ExecutionId,
@@ -2836,7 +2891,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
var allEnvs []Environment
if len(workflowExecution.ExecutionOrg) > 0 {
log.Printf("Executing ORG: %s", workflowExecution.ExecutionOrg)
log.Printf("[INFO] Executing ORG: %s", workflowExecution.ExecutionOrg)
allEnvironments, err := getEnvironments(ctx, workflowExecution.ExecutionOrg)
if err != nil {
@@ -5336,11 +5391,24 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra
return err
}
type buildLaterStruct struct {
Tags []string
Extra string
Id string
}
// Onlyname is used to
func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) error {
func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) ([]buildLaterStruct, []buildLaterStruct, error) {
var err error
allapps := []WorkflowApp{}
reservedNames := []string{
"OWA",
"NLP",
}
buildLaterFirst := []buildLaterStruct{}
buildLaterList := []buildLaterStruct{}
// It's here to prevent getting them in every iteration
ctx := context.Background()
@@ -5360,11 +5428,20 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
}
// Go routine? Hmm, this can be super quick I guess
err = iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate)
buildFirst, buildLast, err := iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate)
if err != nil {
log.Printf("Error reading folder: %s", err)
continue
}
for _, item := range buildFirst {
buildLaterFirst = append(buildLaterFirst, item)
}
for _, item := range buildLast {
buildLaterList = append(buildLaterList, item)
}
case mode.IsRegular():
// Check the file
filename := file.Name()
@@ -5562,22 +5639,69 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
//log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion)
/// Only upload if successful and no errors
err = buildImageMemory(fs, tags, extra)
if err != nil {
log.Printf("Failed image build memory: %s", err)
} else {
if len(tags) > 0 {
log.Printf("Successfully built image %s", tags[0])
} else {
log.Printf("Successfully built Docker image")
// ID can be used to e.g. set a build status.
buildLater := buildLaterStruct{
Tags: tags,
Extra: extra,
Id: workflowapp.ID,
}
reservedFound := false
for _, appname := range reservedNames {
if strings.ToUpper(workflowapp.Name) == strings.ToUpper(appname) {
buildLaterList = append(buildLaterList, buildLater)
reservedFound = true
break
}
}
/// Only upload if successful and no errors
if !reservedFound {
buildLaterFirst = append(buildLaterFirst, buildLater)
} else {
log.Printf("\n\n[WARNING] Skipping build of %s to later\n\n", workflowapp.Name)
}
}
}
}
return err
if len(buildLaterFirst) == 0 && len(buildLaterList) == 0 {
return buildLaterFirst, buildLaterList, err
}
//log.Printf("BUILDLATERFIRST: %d, BUILDLATERLIST: %d", len(buildLaterFirst), len(buildLaterList))
if len(extra) == 0 {
log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst))
for _, item := range buildLaterFirst {
err = buildImageMemory(fs, item.Tags, item.Extra)
if err != nil {
log.Printf("Failed image build memory: %s", err)
} else {
if len(item.Tags) > 0 {
log.Printf("Successfully built image %s", item.Tags[0])
} else {
log.Printf("Successfully built Docker image")
}
}
}
log.Printf("Starting build of %d skipped docker images", len(buildLaterList))
for _, item := range buildLaterList {
err = buildImageMemory(fs, item.Tags, item.Extra)
if err != nil {
log.Printf("Failed image build memory: %s", err)
} else {
if len(item.Tags) > 0 {
log.Printf("Successfully built image %s", item.Tags[0])
} else {
log.Printf("Successfully built Docker image")
}
}
}
}
return buildLaterFirst, buildLaterList, err
}
func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) {
@@ -5722,7 +5846,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) {
}
// Query for the specifci workflowId
q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(20)
q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(30)
var workflowExecutions []WorkflowExecution
_, err = dbclient.GetAll(ctx, q, &workflowExecutions)
if err != nil {