marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #214 from frikky/launch

Browse Source 
0.8.3
This commit is contained in:
Frikky
2020-12-06 18:08:55 +01:00
committed by GitHub
parent cca56a4c59 b13c3f3765
commit 7899f9764d
12 changed files with 1659 additions and 492 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/Dockerfile
+1
View File
@@ -8,6 +8,7 @@ ADD ./go-app/main.go /app
ADD ./go-app/walkoff.go /app
ADD ./go-app/docker.go /app
ADD ./go-app/codegen.go /app
ADD ./go-app/files.go /app
ADD ./go-app/go.mod /app
backend/app_sdk/app_base.py
+277 -54
View File
@@ -1,4 +1,5 @@
import os
import copy
import sys
import re
import time
@@ -51,6 +52,139 @@ class AppBase:
self.logger.info("Result: %d" % ret.status_code)
if ret.status_code != 200:
self.logger.info(ret.text)
# Things to consider for files:
# - How can you download / stream a file?
# - Can you decide if you want a stream or the files directly?
def get_file(self, value):
full_execution = self.full_execution
org_id = full_execution["workflow"]["execution_org"]["id"]
print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value))
if isinstance(value, list):
print("IS LIST!")
#if len(value) == 1:
# value = value[0]
else:
value = [value]
returns = []
for item in value:
print("VALUE: %s" % item)
if len(item) != 36:
print("Bad length for value")
continue
#return {
# "filename": "",
# "data": "",
# "success": False,
#}
get_path = "/api/v1/files/%s?execution_id=%s" % (item, full_execution["execution_id"])
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer %s" % self.authorization
}
ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers)
print("RET1: %s" % ret1.text)
if ret1.status_code != 200:
returns.append({
"filename": "",
"data": "",
"success": False,
})
continue
content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"])
ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers)
print("Ret2: %s" % ret2.text)
if ret2.status_code == 200:
tmpdata = ret1.json()
returndata = {
"success": True,
"filename": tmpdata["filename"],
"data": ret2.content,
}
returns.append(returndata)
if len(returns) == 0:
return {
"success": False,
"filename": "",
"data": b"",
}
elif len(returns) == 1:
return returns[0]
else:
return returns
# Sets files in the backend
def set_files(self, infiles):
full_execution = self.full_execution
workflow_id = full_execution["workflow"]["id"]
org_id = full_execution["workflow"]["execution_org"]["id"]
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer %s" % self.authorization
}
create_path = "/api/v1/files/create?execution_id=%s" % full_execution["execution_id"]
file_ids = []
for curfile in infiles:
filename = "unspecified"
data = {
"filename": filename,
"workflow_id": workflow_id,
"org_id": org_id,
}
try:
data["filename"] = curfile["filename"]
filename = curfile["filename"]
except KeyError as e:
print("KeyError in file setup: %s" % e)
pass
ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data)
print("Ret CREATE: %s" % ret.text)
cur_id = ""
if ret.status_code == 200:
print("RET: %s" % ret.text)
ret_json = ret.json()
if not ret_json["success"]:
print("Not success in file upload creation.")
continue
print("Should handle ID %s" % ret_json["id"])
file_ids.append(ret_json["id"])
cur_id = ret_json["id"]
else:
print("Bad status code: %d" % ret.status_code)
continue
if len(cur_id) == 0:
print("No file ID specified from backend")
continue
new_headers = {
"Authorization": "Bearer %s" % self.authorization,
}
upload_path = "/api/v1/files/%s/upload?execution_id=%s" % (cur_id, full_execution["execution_id"])
print("Create path: %s" % create_path)
# FIXME: Typical failure here if data is returned badly formatted
files={"shuffle_file": (filename, curfile["data"])}
#open(filename,'rb')}
ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers)
print("Ret UPLOAD: %s" % ret.text)
print("Ret2 UPLOAD: %d" % ret.status_code)
print("IDS TO RETURN: %s" % file_ids)
return file_ids
async def execute_action(self, action):
# FIXME - add request for the function STARTING here. Use "results stream" or something
@@ -148,6 +282,7 @@ class AppBase:
print("")
self.full_execution = fullexecution
self.logger.info("AFTER FULLEXEC stream result")
# Gets the value at the parenthesis level you want
@@ -207,7 +342,7 @@ class AppBase:
try:
return int(data)
except ValueError:
print("ValueError while casting %s" % data)
print("ValueError while casting %s to int" % data)
return data
if "lower" in thistype:
return data.lower()
@@ -219,15 +354,19 @@ class AppBase:
return data.strip()
if "split" in thistype:
return data.split()
if "len" in thistype or "length" in thistype:
if "len" in thistype or "length" in thistype or "lenght" in thistype:
tmp = ""
try:
tmp = json.loads(data)
tmpdata = data.replace("\'", "\"")
tmp = json.loads(tmpdata)
except:
print("Passing bug")
pass
if isinstance(tmp, list):
return str(len(tmp))
return len(tmp)
elif isinstance(tmp, object):
return len(tmp)
return str(len(data))
if "parse" in thistype:
@@ -273,7 +412,7 @@ class AppBase:
#print("Running %s" % data)
# Look for the INNER wrapper first, then move out
wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length"]
wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length", "lenght"]
found = False
for wrapper in wrappers:
if wrapper not in data.lower():
@@ -303,6 +442,7 @@ class AppBase:
continue
parsed_value = parse_type(innervalue[0], thistype.lower())
print("Parsed value from %s: %s" % (thistype, parsed_value))
return parsed_value
print("DATA: %s\n" % data)
@@ -403,6 +543,7 @@ class AppBase:
# Magical way of returning which makes app sdk identify
# it as multi execution
return newvalue, True
elif len(actualitem) > 0:
# FIXME: This is absolutely not perfect.
print("In recursion v2: ", actualitem)
@@ -478,7 +619,7 @@ class AppBase:
#Actionname: Start_node
print(f"Actionname: {actionname_lower}")
print(f"\nActionname: {actionname_lower}")
# 1. Find the action
baseresult = ""
@@ -549,21 +690,29 @@ class AppBase:
except KeyError as error:
print(f"KeyError in JSON: {error}")
print(f"After first trycatch")
print(f"After first trycatch. Baseresult: ", baseresult)
# 2. Find the JSON data
if len(baseresult) == 0:
return ""+appendresult, False
print("After second return")
if len(parsersplit) == 1:
return str(baseresult)+str(appendresult), False
baseresult = baseresult.replace("\'", "\"")
baseresult = baseresult.replace(" True,", " true,")
baseresult = baseresult.replace(" False", " false,")
print("After third parser return - Formatted: ", baseresult)
basejson = {}
try:
basejson = json.loads(baseresult)
except json.decoder.JSONDecodeError as e:
print("Parser issue with JSON: %s" % e)
return str(baseresult)+str(appendresult), False
print("After fourth parser return as JSON")
data, is_loop = recurse_json(basejson, parsersplit[1:])
parseditem = data
@@ -577,6 +726,7 @@ class AppBase:
print("SET DATA WRAPPER TO %s!" % parsersplit[-1])
parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data))
print("Before last return with %s" % appendresult)
return str(parseditem)+str(appendresult), is_loop
# Parses parameters sent to it and returns whether it did it successfully with the values found
@@ -824,26 +974,6 @@ class AppBase:
return True, ""
# Things to consider for files:
# - How can you download / stream a file?
# - Can you decide if you want a stream or the files directly?
def get_files(full_execution, value):
print("FULL EXEC: %s" % full_execution)
org_id = full_execution["workflow"]["execution_org"]["id"]
print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value))
get_path = "/api/v1/files/%s/content?execution_id=%s" % (value, full_execution["execution_id"])
print("PATH: %s" % get_path)
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer %s" % self.authorization
}
ret = requests.get("%s%s" % (self.url, get_path), headers=headers)
print("RET CONTENT: %s" % ret.text)
print("RET CODE FILE: %d" % ret.status_code)
# r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS")
# Checks whether conditions are met, otherwise set
branchcheck, tmpresult = check_branch_conditions(action, fullexecution)
if not branchcheck:
@@ -911,15 +1041,6 @@ class AppBase:
multiexecution = False
multi_execution_lists = []
for parameter in action["parameters"]:
is_file = False
try:
if parameter["schema"]["type"] == "file":
print("SHOULD HANDLE FILE. Get based on value %s" % parameter["value"])
get_files(fullexecution, parameter["value"])
is_file = True
except KeyError as e:
print("SCHEMA ERROR: %s" % e)
check, value, is_loop = parse_params(action, fullexecution, parameter)
if check:
raise "Value check error: %s" % Exception(check)
@@ -967,13 +1088,40 @@ class AppBase:
minlength = len(json_replacement)
tmpitem = tmpitem.replace(actualitem[0][0], replacement, 1)
params[parameter["name"]] = tmpitem
multi_execution_lists.append(json_replacement)
multi_parameters[parameter["name"]] = json_replacement
#print("LENGTH OF ARR: %d" % len(resultarray))
#print("RESULTARRAY: %s" % resultarray)
print("MULTI finished: %s" % replacement)
# This code handles files.
print("(1) ------------ PARAM: %s" % parameter["schema"]["type"])
resultarray = []
isfile = False
try:
if parameter["schema"]["type"] == "file" and len(value) > 0:
print("(1) SHOULD HANDLE FILE IN MULTI. Get based on value %s" % tmpitem)
# This is silly :)
# Q: Is there something wrong with the download system?
# It seems to return "FILE CONTENT: %s" with the ID as %s
for tmp_file_split in json.loads(tmpitem):
print("(1) PRE GET FILE %s" % tmp_file_split)
file_value = self.get_file(tmp_file_split)
print("(1) POST AWAIT %s" % file_value)
resultarray.append(file_value)
print("(1) FILE VALUE FOR VAL %s: %s" % (tmp_file_split, file_value))
isfile = True
except KeyError as e:
print("(1) SCHEMA ERROR IN FILE HANDLING: %s" % e)
except json.decoder.JSONDecodeError as e:
print("(1) JSON ERROR IN FILE HANDLING: %s" % e)
if not isfile:
params[parameter["name"]] = tmpitem
multi_parameters[parameter["name"]] = json_replacement
else:
print("Resultarray: %s" % resultarray)
params[parameter["name"]] = resultarray
multi_parameters[parameter["name"]] = resultarray
multi_execution_lists.append(json_replacement)
print("MULTI finished: %s" % json_replacement)
else:
# This is here to handle for loops within variables.. kindof
# 1. Find the length of the longest array
@@ -1017,7 +1165,30 @@ class AppBase:
#replacement = parse_wrapper_start(replacement)
tmpitem = tmpitem.replace(key, replacement, -1)
resultarray.append(tmpitem)
# This code handles files.
print("(2) ------------ PARAM: %s" % parameter["schema"]["type"])
isfile = False
try:
if parameter["schema"]["type"] == "file" and len(value) > 0:
print("(2) SHOULD HANDLE FILE IN MULTI. Get based on value %s" % parameter["value"])
for tmp_file_split in json.loads(parameter["value"]):
print("(2) PRE GET FILE %s" % tmp_file_split)
file_value = self.get_file(tmp_file_split)
print("(2) POST AWAIT %s" % file_value)
resultarray.append(file_value)
print("(2) FILE VALUE FOR VAL %s: %s" % (tmp_file_split, file_value))
isfile = True
except KeyError as e:
print("(2) SCHEMA ERROR IN FILE HANDLING: %s" % e)
except json.decoder.JSONDecodeError as e:
print("(2) JSON ERROR IN FILE HANDLING: %s" % e)
if not isfile:
resultarray.append(tmpitem)
# With this parameter ready, add it to... a greater list of parameters. Rofl
print("LENGTH OF ARR: %d" % len(resultarray))
@@ -1035,6 +1206,18 @@ class AppBase:
params[parameter["name"]] = value
multi_parameters[parameter["name"]] = value
# This code handles files.
try:
if parameter["schema"]["type"] == "file" and len(value) > 0:
print("\n SHOULD HANDLE FILE. Get based on value %s. <--- is this a valid ID?" % parameter["value"])
file_value = self.get_file(value)
print("FILE VALUE: %s \n" % file_value)
params[parameter["name"]] = file_value
multi_parameters[parameter["name"]] = file_value
except KeyError as e:
print("SCHEMA ERROR IN FILE HANDLING: %s" % e)
# Fix lists here
print("CHECKING multi execution list!")
if len(multi_execution_lists) > 0:
@@ -1044,28 +1227,58 @@ class AppBase:
if listitem in filteredlist:
continue
filteredlist.append(listitem)
# FIXME: Subsub required?. Recursion!
# Basically multiply what we have with the outer loop?
#
if isinstance(listitem, list):
for subitem in listitem:
filteredlist.append(subitem)
else:
filteredlist.append(listitem)
#print("New list length: %d" % len(filteredlist))
if len(filteredlist) > 1:
print("Calculating new multi-loop length with %d lists" % len(filteredlist))
tmplength = 1
for innerlist in filteredlist:
print("List length: %d. %d*%d" % (len(innerlist), len(innerlist), tmplength))
tmplength = len(innerlist)*tmplength
print("List length: %d. %d*%d" % (tmplength, len(innerlist), tmplength))
minlength = tmplength
print("New multi execution length: %d\n" % tmplength)
# FIXME - this is horrible, but works for now
#for i in range(calltimes):
if not multiexecution:
print("APP_SDK DONE: Starting NORMAL execution of function")
print("Running with params %s" % params)
print("Running with params (0): %s" % params)
newres = await func(**params)
print("Return from execution: %s" % newres)
if isinstance(newres, str):
print("Returned from execution.")
if isinstance(newres, tuple):
print("Handling return as tuple")
# Handles files.
filedata = ""
file_ids = []
print("TUPLE: %s" % newres[1])
if isinstance(newres[1], list):
print("HANDLING LIST FROM RET")
file_ids = self.set_files(newres[1])
elif isinstance(newres[1], object):
print("Handling JSON from ret")
file_ids = self.set_files([newres[1]])
elif isinstance(newres[1], str):
print("Handling STRING from ret")
file_ids = self.set_files([newres[1]])
else:
print("NO FILES TO HANDLE")
tmp_result = {
"result": newres[0],
"file_ids": file_ids
}
result = json.dumps(tmp_result)
elif isinstance(newres, str):
print("Handling return as string")
result += newres
else:
try:
@@ -1073,16 +1286,22 @@ class AppBase:
except ValueError:
result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres)
print("Can't handle type %s value from function" % (type(newres)))
print("POST NEWRES RESULT: ", result)
else:
print("APP_SDK DONE: Starting MULTI execution with values %s of length %d" % (multi_parameters, minlength))
print("APP_SDK DONE: Starting MULTI execution (length: %d) with values %s" % (minlength, multi_parameters))
# 1. Use number of executions based on the arrays being similar
# 2. Find the right value from the parsed multi_params
results = []
json_object = False
for i in range(0, minlength):
# To be able to use the results as a list:
baseparams = json.loads(json.dumps(multi_parameters))
print("1: %s" % multi_parameters)
#baseparams = json.loads(json.dumps(multi_parameters))
baseparams = copy.deepcopy(multi_parameters)
print("2: %s: %s" % (type(baseparams), baseparams))
# {'call': ['GoogleSafebrowsing_2_0', 'VirusTotal_GetReport_3_0']}
# 1. Check if list length is same as minlength
# 2. If NOT same length, duplicate based on length of array
@@ -1093,7 +1312,7 @@ class AppBase:
try:
firstlist = True
for key, value in baseparams.items():
print("Itemtype: %s" % type(value))
if isinstance(value, list):
try:
newvalue = value[i]
@@ -1136,6 +1355,8 @@ class AppBase:
firstlist = False
baseparams[key] = newvalue
print("3")
except IndexError as e:
print("IndexError: %s" % e)
baseparams[key] = "IndexError: %s" % e
@@ -1143,7 +1364,9 @@ class AppBase:
print("KeyError: %s" % e)
baseparams[key] = "KeyError: %s" % e
print("Running with params %s" % baseparams)
print("4")
print("Running with params (1): %s" % baseparams)
ret = await func(**baseparams)
print("Return from execution: %s" % ret)
if isinstance(ret, dict) or isinstance(ret, list):
backend/app_sdk/build.sh
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/bash
NAME=shuffle-app_sdk
VERSION=0.8.0
VERSION=0.8.2
docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
backend/go-app/docker.go
+8 -8
View File
@@ -181,11 +181,11 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
// Dockerfile is inside the TAR itself. Not local context
// docker build --build-arg http_proxy=http://my.proxy.url
buildOptions := types.ImageBuildOptions{
Remove: true,
Tags: tags,
BuildArgs: map[string]*string{},
NetworkMode: "host",
Remove: true,
Tags: tags,
BuildArgs: map[string]*string{},
}
// NetworkMode: "host",
httpProxy := os.Getenv("HTTP_PROXY")
if len(httpProxy) > 0 {
@@ -244,11 +244,11 @@ func buildImage(tags []string, dockerfileFolder string) error {
dockerFileTarReader := bytes.NewReader(buf.Bytes())
buildOptions := types.ImageBuildOptions{
Remove: true,
Tags: tags,
BuildArgs: map[string]*string{},
NetworkMode: "host",
Remove: true,
Tags: tags,
BuildArgs: map[string]*string{},
}
//NetworkMode: "host",
httpProxy := os.Getenv("HTTP_PROXY")
if len(httpProxy) > 0 {
backend/go-app/files.go
+764
View File
@@ -0,0 +1,764 @@
package main
/*
Handles files within Workflows.of Shuffle
*/
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"strconv"
"strings"
"time"
"cloud.google.com/go/datastore"
"github.com/satori/go.uuid"
)
type File struct {
Id string `json:"id" datastore:"id"`
Type string `json:"type" datastore:"type"`
CreatedAt int64 `json:"created_at" datastore:"created_at"`
UpdatedAt int64 `json:"updated_at" datastore:"updated_at"`
MetaAccessAt int64 `json:"meta_access_at" datastore:"meta_access_at"`
DownloadAt int64 `json:"last_downloaded" datastore:"last_downloaded"`
Description string `json:"description" datastore:"description"`
ExpiresAt string `json:"expires_at" datastore:"expires_at"`
Status string `json:"status" datastore:"status"`
Filename string `json:"filename" datastore:"filename"`
URL string `json:"url" datastore:"org"`
OrgId string `json:"org_id" datastore:"org_id"`
WorkflowId string `json:"workflow_id" datastore:"workflow_id"`
Workflows []string `json:"workflows" datastore:"workflows"`
DownloadPath string `json:"download_path" datastore:"download_path"`
Md5sum string `json:"md5_sum" datastore:"md5_sum"`
Sha256sum string `json:"sha256_sum" datastore:"sha256_sum"`
}
var basepath = os.Getenv("SHUFFLE_FILE_LOCATION")
func fileAuthentication(request *http.Request) (string, error) {
executionId, ok := request.URL.Query()["execution_id"]
if ok && len(executionId) > 0 {
ctx := context.Background()
workflowExecution, err := getWorkflowExecution(ctx, executionId[0])
if err != nil {
log.Printf("[ERROR] Couldn't find execution ID %s", executionId[0])
return "", err
}
apikey := request.Header.Get("Authorization")
if !strings.HasPrefix(apikey, "Bearer ") {
log.Printf("[ERROR} Apikey doesn't start with bearer (2)")
return "", errors.New("No auth key found")
}
apikeyCheck := strings.Split(apikey, " ")
if len(apikeyCheck) != 2 {
log.Printf("[ERROR] Invalid format for apikey (2)")
return "", errors.New("No space in authkey")
}
// This is annoying af and is done because of maxlength lol
newApikey := apikeyCheck[1]
if newApikey != workflowExecution.Authorization {
//log.Printf("[ERROR] Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization)
log.Printf("[ERROR] Bad apikey for execution %s.", executionId[0])
//%s vs %s", executionId[0], apikey, workflowExecution.Authorization)
return "", errors.New("Bad authorization key")
}
log.Printf("[INFO] Authorization is correct for execution %s!", executionId[0])
//%s vs %s. Setting Org", executionId, apikey, workflowExecution.Authorization)
if len(workflowExecution.ExecutionOrg) > 0 {
return workflowExecution.ExecutionOrg, nil
} else if len(workflowExecution.Workflow.ExecutingOrg.Id) > 0 {
return workflowExecution.ExecutionOrg, nil
} else {
log.Printf("[ERROR] Couldn't find org for workflow execution, but auth was correct.")
}
}
return "", errors.New("No execution id specified")
}
// https://golangcode.com/check-if-a-file-exists/
func fileExists(filename string) bool {
info, err := os.Stat(filename)
if os.IsNotExist(err) {
return false
}
return !info.IsDir()
}
func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("[INFO] Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if strings.Contains(fileId, "?") {
fileId = strings.Split(fileId, "?")[0]
}
if len(fileId) != 36 {
log.Printf("Bad format for fileId %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
return
}
log.Printf("\n\n[INFO] User is trying to GET File Meta for %s\n\n", fileId)
// 1. Check user directly
// 2. Check workflow execution authorization
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("[INFO] INITIAL Api authentication failed in file deletion: %s", err)
orgId, err := fileAuthentication(request)
if err != nil {
log.Printf("[ERROR] Bad file authentication in get: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
user.ActiveOrg.Id = orgId
user.Username = "Execution File API"
}
// 1. Verify if the user has access to the file: org_id and workflow
log.Printf("[INFO] Should GET FILE META for %s if user has access", fileId)
ctx := context.Background()
file, err := getFile(ctx, fileId)
if err != nil {
log.Printf("[INFO] File %s not found: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
found := false
if file.OrgId == user.ActiveOrg.Id {
found = true
} else {
for _, item := range user.Orgs {
if item == file.OrgId {
found = true
break
}
}
}
if !found {
log.Printf("[INFO] User %s doesn't have access to %s", user.Username, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
newBody, err := json.Marshal(file)
if err != nil {
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to marshal filedata"}`))
return
}
log.Printf("[INFO] Successfully got file meta for %s", fileId)
resp.WriteHeader(200)
resp.Write([]byte(newBody))
}
func handleDeleteFile(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("[INFO] Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if strings.Contains(fileId, "?") {
fileId = strings.Split(fileId, "?")[0]
}
if len(fileId) != 36 {
log.Printf("Bad format for fileId %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
return
}
log.Printf("\n\n[INFO] User is trying to delete file %s\n\n", fileId)
// 1. Check user directly
// 2. Check workflow execution authorization
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("[INFO] INITIAL Api authentication failed in file deletion: %s", err)
orgId, err := fileAuthentication(request)
if err != nil {
log.Printf("[ERROR] Bad file authentication in get: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
user.ActiveOrg.Id = orgId
user.Username = "Execution File API"
}
// 1. Verify if the user has access to the file: org_id and workflow
log.Printf("[INFO] Should DELETE file %s if user has access", fileId)
ctx := context.Background()
file, err := getFile(ctx, fileId)
if err != nil {
log.Printf("[INFO] File %s not found: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
found := false
if file.OrgId == user.ActiveOrg.Id {
found = true
} else {
for _, item := range user.Orgs {
if item == file.OrgId {
found = true
break
}
}
}
if !found {
log.Printf("[INFO] User %s doesn't have access to %s", user.Username, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if file.Status == "deleted" {
log.Printf("[INFO] File with ID %s is already deleted.", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if fileExists(file.DownloadPath) {
err = os.Remove(file.DownloadPath)
if err != nil {
log.Printf("[ERROR] Failed deleting file locally: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting filein path %s"}`, file.DownloadPath)))
return
}
log.Printf("[INFO] Deleted file %s locally. Next is database.", file.DownloadPath)
} else {
log.Printf("[ERROR] File doesn't exist. Can't delete. Should maybe delete file anyway?")
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "File in location %s doesn't exist"}`, file.DownloadPath)))
return
}
file.Status = "deleted"
err = setFile(ctx, *file)
if err != nil {
log.Printf("[ERROR] Failed setting file to deleted")
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed setting file to deleted"}`))
return
}
/*
//Actually delete it?
err = DeleteKey(ctx, "files", fileId)
if err != nil {
log.Printf("Failed deleting file with ID %s: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
*/
log.Printf("[INFO] Successfully deleted file %s", fileId)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if len(fileId) != 36 {
log.Printf("Bad format for fileId %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
return
}
log.Printf("\n\nUser is trying to download file %s\n\n", fileId)
// 1. Check user directly
// 2. Check workflow execution authorization
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("INITIAL Api authentication failed in file download: %s", err)
orgId, err := fileAuthentication(request)
if err != nil {
log.Printf("Bad file authentication in get: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
user.ActiveOrg.Id = orgId
user.Username = "Execution File API"
/*
} else {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
*/
}
// 1. Verify if the user has access to the file: org_id and workflow
log.Printf("[INFO] Should get file %s", fileId)
ctx := context.Background()
file, err := getFile(ctx, fileId)
if err != nil {
log.Printf("[ERROR] File %s not found: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
found := false
if file.OrgId == user.ActiveOrg.Id {
found = true
} else {
for _, item := range user.Orgs {
if item == file.OrgId {
found = true
break
}
}
}
if !found {
log.Printf("User %s doesn't have access to %s", user.Username, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if file.Status != "active" {
log.Printf("[ERROR] File status isn't active, but %s. Can't continue.", file.Status)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "The file isn't ready to be downloaded yet. Status required: active"}`))
return
}
// Fixme: More auth: org and workflow!
downloadPath := file.DownloadPath
log.Printf("[INFO] Downloadpath: %s", downloadPath)
Openfile, err := os.Open(downloadPath)
defer Openfile.Close() //Close after function return
if err != nil {
//File not found, send 404
http.Error(resp, "File not found.", 404)
return
}
//File is found, create and send the correct headers
//Get the Content-Type of the file
//Create a buffer to store the header of the file in
FileHeader := make([]byte, 512)
//Copy the headers into the FileHeader buffer
Openfile.Read(FileHeader)
//Get content type of file
FileContentType := http.DetectContentType(FileHeader)
//Get the file size
FileStat, _ := Openfile.Stat() //Get info from file
FileSize := strconv.FormatInt(FileStat.Size(), 10) //Get file size as a string
//Send the headers
resp.Header().Set("Content-Disposition", "attachment; filename="+fileId)
resp.Header().Set("Content-Type", FileContentType)
resp.Header().Set("Content-Length", FileSize)
//Send the file
//We read 512 bytes from the file already, so we reset the offset back to 0
Openfile.Seek(0, 0)
io.Copy(resp, Openfile) //'Copy' the file to the client
return
//log.Printf("Should download file %s", downloadPath)
}
func handleUploadFile(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if len(fileId) != 36 {
log.Printf("Bad format for fileId %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
return
}
// 1. Check user directly
// 2. Check workflow execution authorization
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("INITIAL Api authentication failed in file upload: %s", err)
orgId, err := fileAuthentication(request)
if err != nil {
log.Printf("Bad file authentication in create file: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
user.ActiveOrg.Id = orgId
user.Username = "Execution File API"
}
log.Printf("[INFO] Should UPLOAD file %s if user has access", fileId)
ctx := context.Background()
file, err := getFile(ctx, fileId)
if err != nil {
log.Printf("File %s not found: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
found := false
if file.OrgId == user.ActiveOrg.Id {
found = true
} else {
for _, item := range user.Orgs {
if item == file.OrgId {
found = true
break
}
}
}
if !found {
log.Printf("User %s doesn't have access to %s", user.Username, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
log.Printf("[INFO] STATUS: %s", file.Status)
if file.Status != "created" {
log.Printf("File status isn't created. Can't upload.")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "This file already has data."}`))
return
}
request.ParseMultipartForm(32 << 20)
parsedFile, _, err := request.FormFile("shuffle_file")
if err != nil {
log.Printf("[ERROR] Couldn't upload file: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed uploading file"}`))
return
}
defer parsedFile.Close()
file.Status = "uploading"
err = setFile(ctx, *file)
if err != nil {
log.Printf("Failed setting file to uploading")
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed setting file to uploading"}`))
return
}
// Can be used for validation files for change
var buf bytes.Buffer
io.Copy(&buf, parsedFile)
contents := buf.Bytes()
md5 := md5sum(contents)
buf.Reset()
sha256Sum := sha256.Sum256(contents)
//parsedFile.Reset()
f, err := os.OpenFile(file.DownloadPath, os.O_WRONLY|os.O_CREATE, os.ModePerm)
if err != nil {
// Rolling back file
file.Status = "created"
setFile(ctx, *file)
log.Printf("[ERROR] Failed uploading and creating file: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
defer f.Close()
parsedFile.Seek(0, io.SeekStart)
io.Copy(f, parsedFile)
// FIXME: Set this one to 200 anyway? Can't download file then tho..
file.Status = "active"
file.Md5sum = md5
file.Sha256sum = fmt.Sprintf("%x", sha256Sum)
log.Printf("[INFO] MD5 for file %s (%s) is %s and SHA256 is %s", file.Filename, file.Id, file.Md5sum, file.Sha256sum)
err = setFile(ctx, *file)
if err != nil {
log.Printf("[ERROR] Failed setting file back to active")
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed setting file to active"}`))
return
}
log.Printf("[INFO] Successfully uploaded file ID %s", file.Id)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func handleCreateFile(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
// 1. Check user directly
// 2. Check workflow execution authorization
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("[INFO] INITIAL Api authentication failed in file creation: %s", err)
orgId, err := fileAuthentication(request)
if err != nil {
log.Printf("[ERROR] Bad file authentication in create file: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
user.ActiveOrg.Id = orgId
user.Username = "Execution File API"
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Println("Failed reading body")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to read data"}`)))
return
}
type FileStructure struct {
Filename string `json:"filename"`
OrgId string `json:"org_id"`
WorkflowId string `json:"workflow_id"`
}
var curfile FileStructure
err = json.Unmarshal(body, &curfile)
if err != nil {
log.Printf("[ERROR] Failed unmarshaling: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to unmarshal data"}`)))
return
}
// Loads of validation below
if len(curfile.Filename) == 0 || len(curfile.OrgId) == 0 || len(curfile.WorkflowId) == 0 {
log.Printf("[ERROR] Missing field during upload.")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Missing field. Required: filename, org_id, workflow_id"}`)))
return
}
ctx := context.Background()
if user.ActiveOrg.Id != curfile.OrgId {
log.Printf("[ERROR] User can't access org %s", curfile.OrgId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Error with organization"}`))
return
}
// Try to get the org and workflow in case they don't exist
workflow, err := getWorkflow(ctx, curfile.WorkflowId)
if err != nil {
log.Printf("[ERROR] Workflow %s doesn't exist.", curfile.WorkflowId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`))
return
}
_, err = getOrg(ctx, curfile.OrgId)
if err != nil {
log.Printf("[ERROR] Org %s doesn't exist.", curfile.OrgId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`))
return
}
if workflow.ExecutingOrg.Id != curfile.OrgId {
found := false
for _, curorg := range workflow.Org {
if curorg.Id == curfile.OrgId {
found = true
break
}
}
if !found {
log.Printf("[ERROR] Org %s doesn't have access to %s.", curfile.OrgId, curfile.WorkflowId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`))
return
}
}
if strings.Contains(curfile.Filename, "/") || strings.Contains(curfile.Filename, `"`) || strings.Contains(curfile.Filename, "..") || strings.Contains(curfile.Filename, "~") {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Invalid characters in filename"}`))
return
}
// 1. Create the file object.
if len(basepath) == 0 {
basepath = "shuffle-files"
}
folderPath := fmt.Sprintf("%s/%s/%s", basepath, curfile.OrgId, curfile.WorkflowId)
// Try to make the full file location
err = os.MkdirAll(folderPath, os.ModePerm)
if err != nil {
log.Printf("[ERROR] Writing issue for file location creation: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed creating upload location"}`))
return
}
filename := curfile.Filename
fileId := uuid.NewV4().String()
downloadPath := fmt.Sprintf("%s/%s", folderPath, fileId)
timeNow := time.Now().Unix()
newFile := File{
Id: fileId,
CreatedAt: timeNow,
UpdatedAt: timeNow,
Description: "",
Status: "created",
Filename: filename,
OrgId: curfile.OrgId,
WorkflowId: curfile.WorkflowId,
DownloadPath: downloadPath,
}
err = setFile(ctx, newFile)
if err != nil {
log.Printf("[ERROR] Failed setting file: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed setting file reference"}`))
return
} else {
log.Printf("[INFO] Created file %s", newFile.DownloadPath)
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, fileId)))
}
func getFile(ctx context.Context, id string) (*File, error) {
key := datastore.NameKey("Files", id, nil)
curFile := &File{}
if err := dbclient.Get(ctx, key, curFile); err != nil {
return &File{}, err
}
return curFile, nil
}
func setFile(ctx context.Context, file File) error {
// clear session_token and API_token for user
k := datastore.NameKey("Files", file.Id, nil)
if _, err := dbclient.Put(ctx, k, &file); err != nil {
log.Println(err)
return err
}
return nil
}
backend/go-app/main.go
+40 -226
View File
@@ -2422,7 +2422,7 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) {
}
if count == 0 {
log.Printf("No users - redirecting for management user")
log.Printf("[WARNING] No users - redirecting for management user")
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "stay"}`)))
return
@@ -2555,12 +2555,12 @@ func getApikey(ctx context.Context, apikey string) (User, error) {
var users []User
_, err := dbclient.GetAll(ctx, q, &users)
if err != nil {
log.Printf("Error getting users apikey (getapikey): %s", err)
log.Printf("[ERROR] Error getting users apikey (getapikey): %s", err)
return User{}, err
}
if len(users) == 0 {
log.Printf("No users found for apikey %s", apikey)
log.Printf("[WARNING] No users found for apikey %s", apikey)
return User{}, err
}
@@ -2577,28 +2577,6 @@ func getSession(ctx context.Context, thissession string) (*session, error) {
return curUser, nil
}
// ListBooks returns a list of books, ordered by title.
func getFile(ctx context.Context, id string) (*File, error) {
key := datastore.NameKey("Files", id, nil)
curFile := &File{}
if err := dbclient.Get(ctx, key, curFile); err != nil {
return &File{}, err
}
return curFile, nil
}
func setFile(ctx context.Context, file File) error {
// clear session_token and API_token for user
k := datastore.NameKey("Files", file.Id, nil)
if _, err := dbclient.Put(ctx, k, &file); err != nil {
log.Println(err)
return err
}
return nil
}
// ListBooks returns a list of books, ordered by title.
func getOrg(ctx context.Context, id string) (*Org, error) {
key := datastore.NameKey("Organizations", id, nil)
@@ -6552,7 +6530,7 @@ func handleAppHotload(location string, forceUpdate bool) error {
}
//log.Printf("Reading app folder: %#v", dir)
err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate)
_, _, err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate)
if err != nil {
log.Printf("Err: %s", err)
return err
@@ -6835,7 +6813,7 @@ func remoteOrgJobHandler(org Org, interval int) error {
respBody, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("Failed body read in job sync: %s", err)
log.Printf("[ERROR] Failed body read in job sync: %s", err)
return err
}
@@ -6843,7 +6821,7 @@ func remoteOrgJobHandler(org Org, interval int) error {
err = remoteOrgJobController(org, respBody)
if err != nil {
log.Printf("Failed job controller run: %s", err)
log.Printf("[ERROR] Failed job controller run for %s: %s", respBody, err)
return err
}
return nil
@@ -7141,37 +7119,39 @@ func runInit(ctx context.Context) {
}
}
fileq := datastore.NewQuery("Files").Limit(1)
count, err := dbclient.Count(ctx, fileq)
log.Printf("FILECOUNT: %d", count)
if err == nil && count < 10 {
basepath := "."
filename := "testfile.txt"
fileId := uuid.NewV4().String()
log.Printf("Creating new file reference %s because none exist!", fileId)
workflowId := "2cf1169d-b460-41de-8c36-28b2092866f8"
downloadPath := fmt.Sprintf("%s/%s/%s/%s", basepath, activeOrgs[0].Id, workflowId, fileId)
/*
fileq := datastore.NewQuery("Files").Limit(1)
count, err := dbclient.Count(ctx, fileq)
log.Printf("FILECOUNT: %d", count)
if err == nil && count < 10 {
basepath := "."
filename := "testfile.txt"
fileId := uuid.NewV4().String()
log.Printf("Creating new file reference %s because none exist!", fileId)
workflowId := "2cf1169d-b460-41de-8c36-28b2092866f8"
downloadPath := fmt.Sprintf("%s/%s/%s/%s", basepath, activeOrgs[0].Id, workflowId, fileId)
timeNow := time.Now().Unix()
newFile := File{
Id: fileId,
CreatedAt: timeNow,
UpdatedAt: timeNow,
Description: "Created by system for testing",
Status: "active",
Filename: filename,
OrgId: activeOrgs[0].Id,
WorkflowId: workflowId,
DownloadPath: downloadPath,
}
timeNow := time.Now().Unix()
newFile := File{
Id: fileId,
CreatedAt: timeNow,
UpdatedAt: timeNow,
Description: "Created by system for testing",
Status: "active",
Filename: filename,
OrgId: activeOrgs[0].Id,
WorkflowId: workflowId,
DownloadPath: downloadPath,
}
err = setFile(ctx, newFile)
if err != nil {
log.Printf("Failed setting file: %s", err)
} else {
log.Printf("Created file %s in init", newFile.DownloadPath)
err = setFile(ctx, newFile)
if err != nil {
log.Printf("Failed setting file: %s", err)
} else {
log.Printf("Created file %s in init", newFile.DownloadPath)
}
}
}
*/
var allworkflowapps []AppAuthenticationStorage
q = datastore.NewQuery("workflowappauth")
@@ -7934,172 +7914,6 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) {
resp.Write(respBody)
}
type File struct {
Id string `json:"id" datastore:"id"`
Type string `json:"type" datastore:"type"`
CreatedAt int64 `json:"created_at" datastore:"created_at"`
UpdatedAt int64 `json:"updated_at" datastore:"updated_at"`
Description string `json:"description" datastore:"description"`
ExpiresAt string `json:"expires_at" datastore:"expires_at"`
Status string `json:"status" datastore:"status"`
Filename string `json:"filename" datastore:"filename"`
URL string `json:"url" datastore:"org"`
OrgId string `json:"org_id" datastore:"org_id"`
WorkflowId string `json:"workflow_id" datastore:"workflow_id"`
Workflows []string `json:"workflows" datastore:"workflows"`
DownloadPath string `json:"download_path" datastore:"download_path"`
}
func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
log.Printf("\n\nUser is trying to get file %s\n\n", fileId)
// 1. Check user directly
// 2. Check workflow execution authorization
setOrgId := false
user, err := handleApiAuthentication(resp, request)
if err != nil {
// r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS")
log.Printf("INITIAL Api authentication failed in file download: %s", err)
executionId, ok := request.URL.Query()["execution_id"]
if ok && len(executionId) > 0 {
ctx := context.Background()
workflowExecution, err := getWorkflowExecution(ctx, executionId[0])
if err != nil {
log.Printf("Couldn't find execution ID %s", executionId[0])
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
apikey := request.Header.Get("Authorization")
if !strings.HasPrefix(apikey, "Bearer ") {
log.Printf("Apikey doesn't start with bearer (2)")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
apikeyCheck := strings.Split(apikey, " ")
if len(apikeyCheck) != 2 {
log.Printf("Invalid format for apikey (2)")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// This is annoying af and is done because of maxlength lol
newApikey := apikeyCheck[1]
if newApikey != workflowExecution.Authorization {
log.Printf("Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
log.Printf("Authorization is correct for execution %s! %s vs %s", executionId, apikey, workflowExecution.Authorization)
setOrgId = true
} else {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
// 1. Verify if the user has access to the file: org_id and workflow
log.Printf("Should get file %s", fileId)
ctx := context.Background()
file, err := getFile(ctx, fileId)
if err != nil {
log.Printf("File %s not found: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// This is a workaround for file grabs from an app
if setOrgId == true {
user.ActiveOrg.Id = file.OrgId
}
found := false
if file.OrgId == user.ActiveOrg.Id {
found = true
} else {
for _, item := range user.Orgs {
if item == file.OrgId {
found = true
break
}
}
}
if !found {
log.Printf("User %s doesn't have access to %s", user.Username, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Fixme: More auth: org and workflow!
downloadPath := file.DownloadPath
log.Printf("Downloadpath: %s", downloadPath)
Openfile, err := os.Open(downloadPath)
defer Openfile.Close() //Close after function return
if err != nil {
//File not found, send 404
http.Error(resp, "File not found.", 404)
return
}
//File is found, create and send the correct headers
//Get the Content-Type of the file
//Create a buffer to store the header of the file in
FileHeader := make([]byte, 512)
//Copy the headers into the FileHeader buffer
Openfile.Read(FileHeader)
//Get content type of file
FileContentType := http.DetectContentType(FileHeader)
//Get the file size
FileStat, _ := Openfile.Stat() //Get info from file
FileSize := strconv.FormatInt(FileStat.Size(), 10) //Get file size as a string
//Send the headers
resp.Header().Set("Content-Disposition", "attachment; filename="+fileId)
resp.Header().Set("Content-Type", FileContentType)
resp.Header().Set("Content-Length", FileSize)
//Send the file
//We read 512 bytes from the file already, so we reset the offset back to 0
Openfile.Seek(0, 0)
io.Copy(resp, Openfile) //'Copy' the file to the client
return
//log.Printf("Should download file %s", downloadPath)
//resp.WriteHeader(200)
//resp.Write([]byte("OK"))
}
func initHandlers() {
var err error
ctx := context.Background()
@@ -8233,10 +8047,10 @@ func initHandlers() {
// https://developer.box.com/reference/get-files-id-content/
// 1. Creating the "get file" option. Make it possible to run this in the frontend.
r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS")
//r.HandleFunc("/api/v1/files/create", handleGetFile).Methods("POST", "OPTIONS")
//r.HandleFunc("/api/v1/files/upload", handleGetFile).Methods("POST", "OPTIONS")
//r.HandleFunc("/api/v1/files/{fileId}", handleGetFile).Methods("GET", "OPTIONS")
//r.HandleFunc("/api/v1/files/{fileId}", handleGetFile).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/files/create", handleCreateFile).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}", handleGetFileMeta).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS")
http.Handle("/", r)
}
backend/go-app/walkoff.go
+150 -26
View File
@@ -954,6 +954,30 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
return
}
runWorkflowExecutionTransaction(ctx, 0, workflowExecution.ExecutionId, actionResult, resp)
}
// Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times
func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult ActionResult, resp http.ResponseWriter) {
// Should start a tx for the execution here
tx, err := dbclient.NewTransaction(ctx)
if err != nil {
log.Printf("client.NewTransaction: %v", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed creating transaction"}`)))
return
}
key := datastore.NameKey("workflowexecution", workflowExecutionId, nil)
workflowExecution := &WorkflowExecution{}
if err := tx.Get(key, workflowExecution); err != nil {
log.Printf("tx.Get bug: %v", err)
tx.Rollback()
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting the workflow key"}`)))
return
}
if actionResult.Status == "ABORTED" || actionResult.Status == "FAILURE" {
log.Printf("Actionresult is %s. Should set workflowExecution and exit all running functions", actionResult.Status)
@@ -1228,18 +1252,37 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
}
}
err = setWorkflowExecution(ctx, *workflowExecution)
if err != nil {
//workflowExecution.Result = "Error setting workflow: result too large"
//workflowExecution.Status = "FINISHED"
//workflowExecution.CompletedAt = int64(time.Now().Unix())
// Transactions: https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-go
// Prevents timing issues
//ExecutionId
if _, err := tx.Put(key, workflowExecution); err != nil {
tx.Rollback()
log.Printf("[ERROR] tx.Put bug: %v", err)
log.Printf("Error saving workflow execution actionresult setting: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution actionresult: %s"}`, err)))
return
}
if _, err = tx.Commit(); err != nil {
if attempts >= 5 {
log.Printf("[ERROR] QUITTING: tx.Commit %d: %v", attempts, err)
tx.Rollback()
workflowExecution.Status = "ABORTED"
setWorkflowExecution(ctx, *workflowExecution)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
log.Printf("[WARNING] tx.Commit %d: %v", attempts, err)
attempts += 1
runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp)
return
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
@@ -1360,6 +1403,7 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) {
q := datastore.NewQuery("workflow").Filter("owner =", user.Id)
if user.Role == "admin" {
q = datastore.NewQuery("workflow").Filter("org_id =", user.ActiveOrg.Id)
log.Printf("[INFO] Getting workflows (ADMIN) for organization %s", user.ActiveOrg.Id)
}
var workflows []Workflow
@@ -1830,7 +1874,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
for _, action := range workflow.Actions {
allNodes = append(allNodes, action.ID)
if len(action.Errors) > 0 {
if len(action.Errors) > 0 || !action.IsValid {
action.IsValid = true
action.Errors = []string{}
}
@@ -2192,9 +2236,11 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
// Handles check for required params
if !found && param.Required {
log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name)))
return
action.Errors = append(action.Errors, "Parameter %s is required", param.Name)
//newActions = append(newActions, action)
//resp.WriteHeader(401)
//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name)))
//return
}
}
@@ -2209,6 +2255,15 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
workflow.IsValid = true
log.Printf("Tags: %#v", workflow.Tags)
// FIXME: Is this too drastic? May lead to issues in the future.
// Should maybe make a copy for the old org.
if workflow.OrgId != user.ActiveOrg.Id {
log.Printf("[WARNING] Editing workflow to be owned by %s", user.ActiveOrg.Id)
workflow.OrgId = user.ActiveOrg.Id
workflow.ExecutingOrg = user.ActiveOrg
workflow.Org = append(workflow.Org, user.ActiveOrg)
}
err = setWorkflow(ctx, workflow, fileId)
if err != nil {
log.Printf("Failed saving workflow to database: %s", err)
@@ -2234,7 +2289,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
Errors: workflow.Errors,
}
log.Printf("Saved new version of workflow %s (%s)", workflow.Name, fileId)
log.Printf("Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId)
resp.WriteHeader(200)
newBody, err := json.Marshal(returndata)
if err != nil {
@@ -2795,7 +2850,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
continue
}
log.Printf("Should set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID)
log.Printf("[WARNING] Set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID)
defaultResults = append(defaultResults, ActionResult{
Action: action,
ExecutionId: workflowExecution.ExecutionId,
@@ -2836,7 +2891,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
var allEnvs []Environment
if len(workflowExecution.ExecutionOrg) > 0 {
log.Printf("Executing ORG: %s", workflowExecution.ExecutionOrg)
log.Printf("[INFO] Executing ORG: %s", workflowExecution.ExecutionOrg)
allEnvironments, err := getEnvironments(ctx, workflowExecution.ExecutionOrg)
if err != nil {
@@ -5336,11 +5391,24 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra
return err
}
type buildLaterStruct struct {
Tags []string
Extra string
Id string
}
// Onlyname is used to
func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) error {
func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) ([]buildLaterStruct, []buildLaterStruct, error) {
var err error
allapps := []WorkflowApp{}
reservedNames := []string{
"OWA",
"NLP",
}
buildLaterFirst := []buildLaterStruct{}
buildLaterList := []buildLaterStruct{}
// It's here to prevent getting them in every iteration
ctx := context.Background()
@@ -5360,11 +5428,20 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
}
// Go routine? Hmm, this can be super quick I guess
err = iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate)
buildFirst, buildLast, err := iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate)
if err != nil {
log.Printf("Error reading folder: %s", err)
continue
}
for _, item := range buildFirst {
buildLaterFirst = append(buildLaterFirst, item)
}
for _, item := range buildLast {
buildLaterList = append(buildLaterList, item)
}
case mode.IsRegular():
// Check the file
filename := file.Name()
@@ -5562,22 +5639,69 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
//log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion)
/// Only upload if successful and no errors
err = buildImageMemory(fs, tags, extra)
if err != nil {
log.Printf("Failed image build memory: %s", err)
} else {
if len(tags) > 0 {
log.Printf("Successfully built image %s", tags[0])
} else {
log.Printf("Successfully built Docker image")
// ID can be used to e.g. set a build status.
buildLater := buildLaterStruct{
Tags: tags,
Extra: extra,
Id: workflowapp.ID,
}
reservedFound := false
for _, appname := range reservedNames {
if strings.ToUpper(workflowapp.Name) == strings.ToUpper(appname) {
buildLaterList = append(buildLaterList, buildLater)
reservedFound = true
break
}
}
/// Only upload if successful and no errors
if !reservedFound {
buildLaterFirst = append(buildLaterFirst, buildLater)
} else {
log.Printf("\n\n[WARNING] Skipping build of %s to later\n\n", workflowapp.Name)
}
}
}
}
return err
if len(buildLaterFirst) == 0 && len(buildLaterList) == 0 {
return buildLaterFirst, buildLaterList, err
}
//log.Printf("BUILDLATERFIRST: %d, BUILDLATERLIST: %d", len(buildLaterFirst), len(buildLaterList))
if len(extra) == 0 {
log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst))
for _, item := range buildLaterFirst {
err = buildImageMemory(fs, item.Tags, item.Extra)
if err != nil {
log.Printf("Failed image build memory: %s", err)
} else {
if len(item.Tags) > 0 {
log.Printf("Successfully built image %s", item.Tags[0])
} else {
log.Printf("Successfully built Docker image")
}
}
}
log.Printf("Starting build of %d skipped docker images", len(buildLaterList))
for _, item := range buildLaterList {
err = buildImageMemory(fs, item.Tags, item.Extra)
if err != nil {
log.Printf("Failed image build memory: %s", err)
} else {
if len(item.Tags) > 0 {
log.Printf("Successfully built image %s", item.Tags[0])
} else {
log.Printf("Successfully built Docker image")
}
}
}
}
return buildLaterFirst, buildLaterList, err
}
func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) {
@@ -5722,7 +5846,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) {
}
// Query for the specifci workflowId
q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(20)
q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(30)
var workflowExecutions []WorkflowExecution
_, err = dbclient.GetAll(ctx, q, &workflowExecutions)
if err != nil {