marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #214 from frikky/launch

Browse Source 
0.8.3
This commit is contained in:
Frikky
2020-12-06 18:08:55 +01:00
committed by GitHub
parent cca56a4c59 b13c3f3765
commit 7899f9764d
12 changed files with 1659 additions and 492 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+2
View File
@@ -19,7 +19,9 @@ SHUFFLE_DEFAULT_PASSWORD=
SHUFFLE_DEFAULT_APIKEY=
# Local location of your app directory. Can't use ~/
# Files will get better at some point. Right now: local saving.
SHUFFLE_APP_HOTLOAD_LOCATION=./shuffle-apps
SHUFFLE_FILE_LOCATION=./shuffle-files
# Other configs
BACKEND_HOSTNAME=shuffle-backend
backend/Dockerfile
+1
View File
@@ -8,6 +8,7 @@ ADD ./go-app/main.go /app
ADD ./go-app/walkoff.go /app
ADD ./go-app/docker.go /app
ADD ./go-app/codegen.go /app
ADD ./go-app/files.go /app
ADD ./go-app/go.mod /app
backend/app_sdk/app_base.py
+277 -54
View File
@@ -1,4 +1,5 @@
import os
import copy
import sys
import re
import time
@@ -51,6 +52,139 @@ class AppBase:
self.logger.info("Result: %d" % ret.status_code)
if ret.status_code != 200:
self.logger.info(ret.text)
# Things to consider for files:
# - How can you download / stream a file?
# - Can you decide if you want a stream or the files directly?
def get_file(self, value):
full_execution = self.full_execution
org_id = full_execution["workflow"]["execution_org"]["id"]
print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value))
if isinstance(value, list):
print("IS LIST!")
#if len(value) == 1:
# value = value[0]
else:
value = [value]
returns = []
for item in value:
print("VALUE: %s" % item)
if len(item) != 36:
print("Bad length for value")
continue
#return {
# "filename": "",
# "data": "",
# "success": False,
#}
get_path = "/api/v1/files/%s?execution_id=%s" % (item, full_execution["execution_id"])
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer %s" % self.authorization
}
ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers)
print("RET1: %s" % ret1.text)
if ret1.status_code != 200:
returns.append({
"filename": "",
"data": "",
"success": False,
})
continue
content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"])
ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers)
print("Ret2: %s" % ret2.text)
if ret2.status_code == 200:
tmpdata = ret1.json()
returndata = {
"success": True,
"filename": tmpdata["filename"],
"data": ret2.content,
}
returns.append(returndata)
if len(returns) == 0:
return {
"success": False,
"filename": "",
"data": b"",
}
elif len(returns) == 1:
return returns[0]
else:
return returns
# Sets files in the backend
def set_files(self, infiles):
full_execution = self.full_execution
workflow_id = full_execution["workflow"]["id"]
org_id = full_execution["workflow"]["execution_org"]["id"]
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer %s" % self.authorization
}
create_path = "/api/v1/files/create?execution_id=%s" % full_execution["execution_id"]
file_ids = []
for curfile in infiles:
filename = "unspecified"
data = {
"filename": filename,
"workflow_id": workflow_id,
"org_id": org_id,
}
try:
data["filename"] = curfile["filename"]
filename = curfile["filename"]
except KeyError as e:
print("KeyError in file setup: %s" % e)
pass
ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data)
print("Ret CREATE: %s" % ret.text)
cur_id = ""
if ret.status_code == 200:
print("RET: %s" % ret.text)
ret_json = ret.json()
if not ret_json["success"]:
print("Not success in file upload creation.")
continue
print("Should handle ID %s" % ret_json["id"])
file_ids.append(ret_json["id"])
cur_id = ret_json["id"]
else:
print("Bad status code: %d" % ret.status_code)
continue
if len(cur_id) == 0:
print("No file ID specified from backend")
continue
new_headers = {
"Authorization": "Bearer %s" % self.authorization,
}
upload_path = "/api/v1/files/%s/upload?execution_id=%s" % (cur_id, full_execution["execution_id"])
print("Create path: %s" % create_path)
# FIXME: Typical failure here if data is returned badly formatted
files={"shuffle_file": (filename, curfile["data"])}
#open(filename,'rb')}
ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers)
print("Ret UPLOAD: %s" % ret.text)
print("Ret2 UPLOAD: %d" % ret.status_code)
print("IDS TO RETURN: %s" % file_ids)
return file_ids
async def execute_action(self, action):
# FIXME - add request for the function STARTING here. Use "results stream" or something
@@ -148,6 +282,7 @@ class AppBase:
print("")
self.full_execution = fullexecution
self.logger.info("AFTER FULLEXEC stream result")
# Gets the value at the parenthesis level you want
@@ -207,7 +342,7 @@ class AppBase:
try:
return int(data)
except ValueError:
print("ValueError while casting %s" % data)
print("ValueError while casting %s to int" % data)
return data
if "lower" in thistype:
return data.lower()
@@ -219,15 +354,19 @@ class AppBase:
return data.strip()
if "split" in thistype:
return data.split()
if "len" in thistype or "length" in thistype:
if "len" in thistype or "length" in thistype or "lenght" in thistype:
tmp = ""
try:
tmp = json.loads(data)
tmpdata = data.replace("\'", "\"")
tmp = json.loads(tmpdata)
except:
print("Passing bug")
pass
if isinstance(tmp, list):
return str(len(tmp))
return len(tmp)
elif isinstance(tmp, object):
return len(tmp)
return str(len(data))
if "parse" in thistype:
@@ -273,7 +412,7 @@ class AppBase:
#print("Running %s" % data)
# Look for the INNER wrapper first, then move out
wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length"]
wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length", "lenght"]
found = False
for wrapper in wrappers:
if wrapper not in data.lower():
@@ -303,6 +442,7 @@ class AppBase:
continue
parsed_value = parse_type(innervalue[0], thistype.lower())
print("Parsed value from %s: %s" % (thistype, parsed_value))
return parsed_value
print("DATA: %s\n" % data)
@@ -403,6 +543,7 @@ class AppBase:
# Magical way of returning which makes app sdk identify
# it as multi execution
return newvalue, True
elif len(actualitem) > 0:
# FIXME: This is absolutely not perfect.
print("In recursion v2: ", actualitem)
@@ -478,7 +619,7 @@ class AppBase:
#Actionname: Start_node
print(f"Actionname: {actionname_lower}")
print(f"\nActionname: {actionname_lower}")
# 1. Find the action
baseresult = ""
@@ -549,21 +690,29 @@ class AppBase:
except KeyError as error:
print(f"KeyError in JSON: {error}")
print(f"After first trycatch")
print(f"After first trycatch. Baseresult: ", baseresult)
# 2. Find the JSON data
if len(baseresult) == 0:
return ""+appendresult, False
print("After second return")
if len(parsersplit) == 1:
return str(baseresult)+str(appendresult), False
baseresult = baseresult.replace("\'", "\"")
baseresult = baseresult.replace(" True,", " true,")
baseresult = baseresult.replace(" False", " false,")
print("After third parser return - Formatted: ", baseresult)
basejson = {}
try:
basejson = json.loads(baseresult)
except json.decoder.JSONDecodeError as e:
print("Parser issue with JSON: %s" % e)
return str(baseresult)+str(appendresult), False
print("After fourth parser return as JSON")
data, is_loop = recurse_json(basejson, parsersplit[1:])
parseditem = data
@@ -577,6 +726,7 @@ class AppBase:
print("SET DATA WRAPPER TO %s!" % parsersplit[-1])
parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data))
print("Before last return with %s" % appendresult)
return str(parseditem)+str(appendresult), is_loop
# Parses parameters sent to it and returns whether it did it successfully with the values found
@@ -824,26 +974,6 @@ class AppBase:
return True, ""
# Things to consider for files:
# - How can you download / stream a file?
# - Can you decide if you want a stream or the files directly?
def get_files(full_execution, value):
print("FULL EXEC: %s" % full_execution)
org_id = full_execution["workflow"]["execution_org"]["id"]
print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value))
get_path = "/api/v1/files/%s/content?execution_id=%s" % (value, full_execution["execution_id"])
print("PATH: %s" % get_path)
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer %s" % self.authorization
}
ret = requests.get("%s%s" % (self.url, get_path), headers=headers)
print("RET CONTENT: %s" % ret.text)
print("RET CODE FILE: %d" % ret.status_code)
# r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS")
# Checks whether conditions are met, otherwise set
branchcheck, tmpresult = check_branch_conditions(action, fullexecution)
if not branchcheck:
@@ -911,15 +1041,6 @@ class AppBase:
multiexecution = False
multi_execution_lists = []
for parameter in action["parameters"]:
is_file = False
try:
if parameter["schema"]["type"] == "file":
print("SHOULD HANDLE FILE. Get based on value %s" % parameter["value"])
get_files(fullexecution, parameter["value"])
is_file = True
except KeyError as e:
print("SCHEMA ERROR: %s" % e)
check, value, is_loop = parse_params(action, fullexecution, parameter)
if check:
raise "Value check error: %s" % Exception(check)
@@ -967,13 +1088,40 @@ class AppBase:
minlength = len(json_replacement)
tmpitem = tmpitem.replace(actualitem[0][0], replacement, 1)
params[parameter["name"]] = tmpitem
multi_execution_lists.append(json_replacement)
multi_parameters[parameter["name"]] = json_replacement
#print("LENGTH OF ARR: %d" % len(resultarray))
#print("RESULTARRAY: %s" % resultarray)
print("MULTI finished: %s" % replacement)
# This code handles files.
print("(1) ------------ PARAM: %s" % parameter["schema"]["type"])
resultarray = []
isfile = False
try:
if parameter["schema"]["type"] == "file" and len(value) > 0:
print("(1) SHOULD HANDLE FILE IN MULTI. Get based on value %s" % tmpitem)
# This is silly :)
# Q: Is there something wrong with the download system?
# It seems to return "FILE CONTENT: %s" with the ID as %s
for tmp_file_split in json.loads(tmpitem):
print("(1) PRE GET FILE %s" % tmp_file_split)
file_value = self.get_file(tmp_file_split)
print("(1) POST AWAIT %s" % file_value)
resultarray.append(file_value)
print("(1) FILE VALUE FOR VAL %s: %s" % (tmp_file_split, file_value))
isfile = True
except KeyError as e:
print("(1) SCHEMA ERROR IN FILE HANDLING: %s" % e)
except json.decoder.JSONDecodeError as e:
print("(1) JSON ERROR IN FILE HANDLING: %s" % e)
if not isfile:
params[parameter["name"]] = tmpitem
multi_parameters[parameter["name"]] = json_replacement
else:
print("Resultarray: %s" % resultarray)
params[parameter["name"]] = resultarray
multi_parameters[parameter["name"]] = resultarray
multi_execution_lists.append(json_replacement)
print("MULTI finished: %s" % json_replacement)
else:
# This is here to handle for loops within variables.. kindof
# 1. Find the length of the longest array
@@ -1017,7 +1165,30 @@ class AppBase:
#replacement = parse_wrapper_start(replacement)
tmpitem = tmpitem.replace(key, replacement, -1)
resultarray.append(tmpitem)
# This code handles files.
print("(2) ------------ PARAM: %s" % parameter["schema"]["type"])
isfile = False
try:
if parameter["schema"]["type"] == "file" and len(value) > 0:
print("(2) SHOULD HANDLE FILE IN MULTI. Get based on value %s" % parameter["value"])
for tmp_file_split in json.loads(parameter["value"]):
print("(2) PRE GET FILE %s" % tmp_file_split)
file_value = self.get_file(tmp_file_split)
print("(2) POST AWAIT %s" % file_value)
resultarray.append(file_value)
print("(2) FILE VALUE FOR VAL %s: %s" % (tmp_file_split, file_value))
isfile = True
except KeyError as e:
print("(2) SCHEMA ERROR IN FILE HANDLING: %s" % e)
except json.decoder.JSONDecodeError as e:
print("(2) JSON ERROR IN FILE HANDLING: %s" % e)
if not isfile:
resultarray.append(tmpitem)
# With this parameter ready, add it to... a greater list of parameters. Rofl
print("LENGTH OF ARR: %d" % len(resultarray))
@@ -1035,6 +1206,18 @@ class AppBase:
params[parameter["name"]] = value
multi_parameters[parameter["name"]] = value
# This code handles files.
try:
if parameter["schema"]["type"] == "file" and len(value) > 0:
print("\n SHOULD HANDLE FILE. Get based on value %s. <--- is this a valid ID?" % parameter["value"])
file_value = self.get_file(value)
print("FILE VALUE: %s \n" % file_value)
params[parameter["name"]] = file_value
multi_parameters[parameter["name"]] = file_value
except KeyError as e:
print("SCHEMA ERROR IN FILE HANDLING: %s" % e)
# Fix lists here
print("CHECKING multi execution list!")
if len(multi_execution_lists) > 0:
@@ -1044,28 +1227,58 @@ class AppBase:
if listitem in filteredlist:
continue
filteredlist.append(listitem)
# FIXME: Subsub required?. Recursion!
# Basically multiply what we have with the outer loop?
#
if isinstance(listitem, list):
for subitem in listitem:
filteredlist.append(subitem)
else:
filteredlist.append(listitem)
#print("New list length: %d" % len(filteredlist))
if len(filteredlist) > 1:
print("Calculating new multi-loop length with %d lists" % len(filteredlist))
tmplength = 1
for innerlist in filteredlist:
print("List length: %d. %d*%d" % (len(innerlist), len(innerlist), tmplength))
tmplength = len(innerlist)*tmplength
print("List length: %d. %d*%d" % (tmplength, len(innerlist), tmplength))
minlength = tmplength
print("New multi execution length: %d\n" % tmplength)
# FIXME - this is horrible, but works for now
#for i in range(calltimes):
if not multiexecution:
print("APP_SDK DONE: Starting NORMAL execution of function")
print("Running with params %s" % params)
print("Running with params (0): %s" % params)
newres = await func(**params)
print("Return from execution: %s" % newres)
if isinstance(newres, str):
print("Returned from execution.")
if isinstance(newres, tuple):
print("Handling return as tuple")
# Handles files.
filedata = ""
file_ids = []
print("TUPLE: %s" % newres[1])
if isinstance(newres[1], list):
print("HANDLING LIST FROM RET")
file_ids = self.set_files(newres[1])
elif isinstance(newres[1], object):
print("Handling JSON from ret")
file_ids = self.set_files([newres[1]])
elif isinstance(newres[1], str):
print("Handling STRING from ret")
file_ids = self.set_files([newres[1]])
else:
print("NO FILES TO HANDLE")
tmp_result = {
"result": newres[0],
"file_ids": file_ids
}
result = json.dumps(tmp_result)
elif isinstance(newres, str):
print("Handling return as string")
result += newres
else:
try:
@@ -1073,16 +1286,22 @@ class AppBase:
except ValueError:
result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres)
print("Can't handle type %s value from function" % (type(newres)))
print("POST NEWRES RESULT: ", result)
else:
print("APP_SDK DONE: Starting MULTI execution with values %s of length %d" % (multi_parameters, minlength))
print("APP_SDK DONE: Starting MULTI execution (length: %d) with values %s" % (minlength, multi_parameters))
# 1. Use number of executions based on the arrays being similar
# 2. Find the right value from the parsed multi_params
results = []
json_object = False
for i in range(0, minlength):
# To be able to use the results as a list:
baseparams = json.loads(json.dumps(multi_parameters))
print("1: %s" % multi_parameters)
#baseparams = json.loads(json.dumps(multi_parameters))
baseparams = copy.deepcopy(multi_parameters)
print("2: %s: %s" % (type(baseparams), baseparams))
# {'call': ['GoogleSafebrowsing_2_0', 'VirusTotal_GetReport_3_0']}
# 1. Check if list length is same as minlength
# 2. If NOT same length, duplicate based on length of array
@@ -1093,7 +1312,7 @@ class AppBase:
try:
firstlist = True
for key, value in baseparams.items():
print("Itemtype: %s" % type(value))
if isinstance(value, list):
try:
newvalue = value[i]
@@ -1136,6 +1355,8 @@ class AppBase:
firstlist = False
baseparams[key] = newvalue
print("3")
except IndexError as e:
print("IndexError: %s" % e)
baseparams[key] = "IndexError: %s" % e
@@ -1143,7 +1364,9 @@ class AppBase:
print("KeyError: %s" % e)
baseparams[key] = "KeyError: %s" % e
print("Running with params %s" % baseparams)
print("4")
print("Running with params (1): %s" % baseparams)
ret = await func(**baseparams)
print("Return from execution: %s" % ret)
if isinstance(ret, dict) or isinstance(ret, list):
backend/app_sdk/build.sh
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/bash
NAME=shuffle-app_sdk
VERSION=0.8.0
VERSION=0.8.2
docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
backend/go-app/docker.go
+8 -8
View File
@@ -181,11 +181,11 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
// Dockerfile is inside the TAR itself. Not local context
// docker build --build-arg http_proxy=http://my.proxy.url
buildOptions := types.ImageBuildOptions{
Remove: true,
Tags: tags,
BuildArgs: map[string]*string{},
NetworkMode: "host",
Remove: true,
Tags: tags,
BuildArgs: map[string]*string{},
}
// NetworkMode: "host",
httpProxy := os.Getenv("HTTP_PROXY")
if len(httpProxy) > 0 {
@@ -244,11 +244,11 @@ func buildImage(tags []string, dockerfileFolder string) error {
dockerFileTarReader := bytes.NewReader(buf.Bytes())
buildOptions := types.ImageBuildOptions{
Remove: true,
Tags: tags,
BuildArgs: map[string]*string{},
NetworkMode: "host",
Remove: true,
Tags: tags,
BuildArgs: map[string]*string{},
}
//NetworkMode: "host",
httpProxy := os.Getenv("HTTP_PROXY")
if len(httpProxy) > 0 {
backend/go-app/files.go
+764
View File
@@ -0,0 +1,764 @@
package main
/*
Handles files within Workflows.of Shuffle
*/
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"strconv"
"strings"
"time"
"cloud.google.com/go/datastore"
"github.com/satori/go.uuid"
)
type File struct {
Id string `json:"id" datastore:"id"`
Type string `json:"type" datastore:"type"`
CreatedAt int64 `json:"created_at" datastore:"created_at"`
UpdatedAt int64 `json:"updated_at" datastore:"updated_at"`
MetaAccessAt int64 `json:"meta_access_at" datastore:"meta_access_at"`
DownloadAt int64 `json:"last_downloaded" datastore:"last_downloaded"`
Description string `json:"description" datastore:"description"`
ExpiresAt string `json:"expires_at" datastore:"expires_at"`
Status string `json:"status" datastore:"status"`
Filename string `json:"filename" datastore:"filename"`
URL string `json:"url" datastore:"org"`
OrgId string `json:"org_id" datastore:"org_id"`
WorkflowId string `json:"workflow_id" datastore:"workflow_id"`
Workflows []string `json:"workflows" datastore:"workflows"`
DownloadPath string `json:"download_path" datastore:"download_path"`
Md5sum string `json:"md5_sum" datastore:"md5_sum"`
Sha256sum string `json:"sha256_sum" datastore:"sha256_sum"`
}
var basepath = os.Getenv("SHUFFLE_FILE_LOCATION")
func fileAuthentication(request *http.Request) (string, error) {
executionId, ok := request.URL.Query()["execution_id"]
if ok && len(executionId) > 0 {
ctx := context.Background()
workflowExecution, err := getWorkflowExecution(ctx, executionId[0])
if err != nil {
log.Printf("[ERROR] Couldn't find execution ID %s", executionId[0])
return "", err
}
apikey := request.Header.Get("Authorization")
if !strings.HasPrefix(apikey, "Bearer ") {
log.Printf("[ERROR} Apikey doesn't start with bearer (2)")
return "", errors.New("No auth key found")
}
apikeyCheck := strings.Split(apikey, " ")
if len(apikeyCheck) != 2 {
log.Printf("[ERROR] Invalid format for apikey (2)")
return "", errors.New("No space in authkey")
}
// This is annoying af and is done because of maxlength lol
newApikey := apikeyCheck[1]
if newApikey != workflowExecution.Authorization {
//log.Printf("[ERROR] Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization)
log.Printf("[ERROR] Bad apikey for execution %s.", executionId[0])
//%s vs %s", executionId[0], apikey, workflowExecution.Authorization)
return "", errors.New("Bad authorization key")
}
log.Printf("[INFO] Authorization is correct for execution %s!", executionId[0])
//%s vs %s. Setting Org", executionId, apikey, workflowExecution.Authorization)
if len(workflowExecution.ExecutionOrg) > 0 {
return workflowExecution.ExecutionOrg, nil
} else if len(workflowExecution.Workflow.ExecutingOrg.Id) > 0 {
return workflowExecution.ExecutionOrg, nil
} else {
log.Printf("[ERROR] Couldn't find org for workflow execution, but auth was correct.")
}
}
return "", errors.New("No execution id specified")
}
// https://golangcode.com/check-if-a-file-exists/
func fileExists(filename string) bool {
info, err := os.Stat(filename)
if os.IsNotExist(err) {
return false
}
return !info.IsDir()
}
func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("[INFO] Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if strings.Contains(fileId, "?") {
fileId = strings.Split(fileId, "?")[0]
}
if len(fileId) != 36 {
log.Printf("Bad format for fileId %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
return
}
log.Printf("\n\n[INFO] User is trying to GET File Meta for %s\n\n", fileId)
// 1. Check user directly
// 2. Check workflow execution authorization
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("[INFO] INITIAL Api authentication failed in file deletion: %s", err)
orgId, err := fileAuthentication(request)
if err != nil {
log.Printf("[ERROR] Bad file authentication in get: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
user.ActiveOrg.Id = orgId
user.Username = "Execution File API"
}
// 1. Verify if the user has access to the file: org_id and workflow
log.Printf("[INFO] Should GET FILE META for %s if user has access", fileId)
ctx := context.Background()
file, err := getFile(ctx, fileId)
if err != nil {
log.Printf("[INFO] File %s not found: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
found := false
if file.OrgId == user.ActiveOrg.Id {
found = true
} else {
for _, item := range user.Orgs {
if item == file.OrgId {
found = true
break
}
}
}
if !found {
log.Printf("[INFO] User %s doesn't have access to %s", user.Username, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
newBody, err := json.Marshal(file)
if err != nil {
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to marshal filedata"}`))
return
}
log.Printf("[INFO] Successfully got file meta for %s", fileId)
resp.WriteHeader(200)
resp.Write([]byte(newBody))
}
func handleDeleteFile(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("[INFO] Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if strings.Contains(fileId, "?") {
fileId = strings.Split(fileId, "?")[0]
}
if len(fileId) != 36 {
log.Printf("Bad format for fileId %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
return
}
log.Printf("\n\n[INFO] User is trying to delete file %s\n\n", fileId)
// 1. Check user directly
// 2. Check workflow execution authorization
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("[INFO] INITIAL Api authentication failed in file deletion: %s", err)
orgId, err := fileAuthentication(request)
if err != nil {
log.Printf("[ERROR] Bad file authentication in get: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
user.ActiveOrg.Id = orgId
user.Username = "Execution File API"
}
// 1. Verify if the user has access to the file: org_id and workflow
log.Printf("[INFO] Should DELETE file %s if user has access", fileId)
ctx := context.Background()
file, err := getFile(ctx, fileId)
if err != nil {
log.Printf("[INFO] File %s not found: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
found := false
if file.OrgId == user.ActiveOrg.Id {
found = true
} else {
for _, item := range user.Orgs {
if item == file.OrgId {
found = true
break
}
}
}
if !found {
log.Printf("[INFO] User %s doesn't have access to %s", user.Username, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if file.Status == "deleted" {
log.Printf("[INFO] File with ID %s is already deleted.", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if fileExists(file.DownloadPath) {
err = os.Remove(file.DownloadPath)
if err != nil {
log.Printf("[ERROR] Failed deleting file locally: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting filein path %s"}`, file.DownloadPath)))
return
}
log.Printf("[INFO] Deleted file %s locally. Next is database.", file.DownloadPath)
} else {
log.Printf("[ERROR] File doesn't exist. Can't delete. Should maybe delete file anyway?")
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "File in location %s doesn't exist"}`, file.DownloadPath)))
return
}
file.Status = "deleted"
err = setFile(ctx, *file)
if err != nil {
log.Printf("[ERROR] Failed setting file to deleted")
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed setting file to deleted"}`))
return
}
/*
//Actually delete it?
err = DeleteKey(ctx, "files", fileId)
if err != nil {
log.Printf("Failed deleting file with ID %s: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
*/
log.Printf("[INFO] Successfully deleted file %s", fileId)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if len(fileId) != 36 {
log.Printf("Bad format for fileId %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
return
}
log.Printf("\n\nUser is trying to download file %s\n\n", fileId)
// 1. Check user directly
// 2. Check workflow execution authorization
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("INITIAL Api authentication failed in file download: %s", err)
orgId, err := fileAuthentication(request)
if err != nil {
log.Printf("Bad file authentication in get: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
user.ActiveOrg.Id = orgId
user.Username = "Execution File API"
/*
} else {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
*/
}
// 1. Verify if the user has access to the file: org_id and workflow
log.Printf("[INFO] Should get file %s", fileId)
ctx := context.Background()
file, err := getFile(ctx, fileId)
if err != nil {
log.Printf("[ERROR] File %s not found: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
found := false
if file.OrgId == user.ActiveOrg.Id {
found = true
} else {
for _, item := range user.Orgs {
if item == file.OrgId {
found = true
break
}
}
}
if !found {
log.Printf("User %s doesn't have access to %s", user.Username, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if file.Status != "active" {
log.Printf("[ERROR] File status isn't active, but %s. Can't continue.", file.Status)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "The file isn't ready to be downloaded yet. Status required: active"}`))
return
}
// Fixme: More auth: org and workflow!
downloadPath := file.DownloadPath
log.Printf("[INFO] Downloadpath: %s", downloadPath)
Openfile, err := os.Open(downloadPath)
defer Openfile.Close() //Close after function return
if err != nil {
//File not found, send 404
http.Error(resp, "File not found.", 404)
return
}
//File is found, create and send the correct headers
//Get the Content-Type of the file
//Create a buffer to store the header of the file in
FileHeader := make([]byte, 512)
//Copy the headers into the FileHeader buffer
Openfile.Read(FileHeader)
//Get content type of file
FileContentType := http.DetectContentType(FileHeader)
//Get the file size
FileStat, _ := Openfile.Stat() //Get info from file
FileSize := strconv.FormatInt(FileStat.Size(), 10) //Get file size as a string
//Send the headers
resp.Header().Set("Content-Disposition", "attachment; filename="+fileId)
resp.Header().Set("Content-Type", FileContentType)
resp.Header().Set("Content-Length", FileSize)
//Send the file
//We read 512 bytes from the file already, so we reset the offset back to 0
Openfile.Seek(0, 0)
io.Copy(resp, Openfile) //'Copy' the file to the client
return
//log.Printf("Should download file %s", downloadPath)
}
func handleUploadFile(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if len(fileId) != 36 {
log.Printf("Bad format for fileId %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
return
}
// 1. Check user directly
// 2. Check workflow execution authorization
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("INITIAL Api authentication failed in file upload: %s", err)
orgId, err := fileAuthentication(request)
if err != nil {
log.Printf("Bad file authentication in create file: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
user.ActiveOrg.Id = orgId
user.Username = "Execution File API"
}
log.Printf("[INFO] Should UPLOAD file %s if user has access", fileId)
ctx := context.Background()
file, err := getFile(ctx, fileId)
if err != nil {
log.Printf("File %s not found: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
found := false
if file.OrgId == user.ActiveOrg.Id {
found = true
} else {
for _, item := range user.Orgs {
if item == file.OrgId {
found = true
break
}
}
}
if !found {
log.Printf("User %s doesn't have access to %s", user.Username, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
log.Printf("[INFO] STATUS: %s", file.Status)
if file.Status != "created" {
log.Printf("File status isn't created. Can't upload.")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "This file already has data."}`))
return
}
request.ParseMultipartForm(32 << 20)
parsedFile, _, err := request.FormFile("shuffle_file")
if err != nil {
log.Printf("[ERROR] Couldn't upload file: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed uploading file"}`))
return
}
defer parsedFile.Close()
file.Status = "uploading"
err = setFile(ctx, *file)
if err != nil {
log.Printf("Failed setting file to uploading")
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed setting file to uploading"}`))
return
}
// Can be used for validation files for change
var buf bytes.Buffer
io.Copy(&buf, parsedFile)
contents := buf.Bytes()
md5 := md5sum(contents)
buf.Reset()
sha256Sum := sha256.Sum256(contents)
//parsedFile.Reset()
f, err := os.OpenFile(file.DownloadPath, os.O_WRONLY|os.O_CREATE, os.ModePerm)
if err != nil {
// Rolling back file
file.Status = "created"
setFile(ctx, *file)
log.Printf("[ERROR] Failed uploading and creating file: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
defer f.Close()
parsedFile.Seek(0, io.SeekStart)
io.Copy(f, parsedFile)
// FIXME: Set this one to 200 anyway? Can't download file then tho..
file.Status = "active"
file.Md5sum = md5
file.Sha256sum = fmt.Sprintf("%x", sha256Sum)
log.Printf("[INFO] MD5 for file %s (%s) is %s and SHA256 is %s", file.Filename, file.Id, file.Md5sum, file.Sha256sum)
err = setFile(ctx, *file)
if err != nil {
log.Printf("[ERROR] Failed setting file back to active")
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed setting file to active"}`))
return
}
log.Printf("[INFO] Successfully uploaded file ID %s", file.Id)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func handleCreateFile(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
// 1. Check user directly
// 2. Check workflow execution authorization
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("[INFO] INITIAL Api authentication failed in file creation: %s", err)
orgId, err := fileAuthentication(request)
if err != nil {
log.Printf("[ERROR] Bad file authentication in create file: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
user.ActiveOrg.Id = orgId
user.Username = "Execution File API"
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Println("Failed reading body")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to read data"}`)))
return
}
type FileStructure struct {
Filename string `json:"filename"`
OrgId string `json:"org_id"`
WorkflowId string `json:"workflow_id"`
}
var curfile FileStructure
err = json.Unmarshal(body, &curfile)
if err != nil {
log.Printf("[ERROR] Failed unmarshaling: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to unmarshal data"}`)))
return
}
// Loads of validation below
if len(curfile.Filename) == 0 || len(curfile.OrgId) == 0 || len(curfile.WorkflowId) == 0 {
log.Printf("[ERROR] Missing field during upload.")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Missing field. Required: filename, org_id, workflow_id"}`)))
return
}
ctx := context.Background()
if user.ActiveOrg.Id != curfile.OrgId {
log.Printf("[ERROR] User can't access org %s", curfile.OrgId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Error with organization"}`))
return
}
// Try to get the org and workflow in case they don't exist
workflow, err := getWorkflow(ctx, curfile.WorkflowId)
if err != nil {
log.Printf("[ERROR] Workflow %s doesn't exist.", curfile.WorkflowId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`))
return
}
_, err = getOrg(ctx, curfile.OrgId)
if err != nil {
log.Printf("[ERROR] Org %s doesn't exist.", curfile.OrgId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`))
return
}
if workflow.ExecutingOrg.Id != curfile.OrgId {
found := false
for _, curorg := range workflow.Org {
if curorg.Id == curfile.OrgId {
found = true
break
}
}
if !found {
log.Printf("[ERROR] Org %s doesn't have access to %s.", curfile.OrgId, curfile.WorkflowId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`))
return
}
}
if strings.Contains(curfile.Filename, "/") || strings.Contains(curfile.Filename, `"`) || strings.Contains(curfile.Filename, "..") || strings.Contains(curfile.Filename, "~") {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Invalid characters in filename"}`))
return
}
// 1. Create the file object.
if len(basepath) == 0 {
basepath = "shuffle-files"
}
folderPath := fmt.Sprintf("%s/%s/%s", basepath, curfile.OrgId, curfile.WorkflowId)
// Try to make the full file location
err = os.MkdirAll(folderPath, os.ModePerm)
if err != nil {
log.Printf("[ERROR] Writing issue for file location creation: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed creating upload location"}`))
return
}
filename := curfile.Filename
fileId := uuid.NewV4().String()
downloadPath := fmt.Sprintf("%s/%s", folderPath, fileId)
timeNow := time.Now().Unix()
newFile := File{
Id: fileId,
CreatedAt: timeNow,
UpdatedAt: timeNow,
Description: "",
Status: "created",
Filename: filename,
OrgId: curfile.OrgId,
WorkflowId: curfile.WorkflowId,
DownloadPath: downloadPath,
}
err = setFile(ctx, newFile)
if err != nil {
log.Printf("[ERROR] Failed setting file: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed setting file reference"}`))
return
} else {
log.Printf("[INFO] Created file %s", newFile.DownloadPath)
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, fileId)))
}
func getFile(ctx context.Context, id string) (*File, error) {
key := datastore.NameKey("Files", id, nil)
curFile := &File{}
if err := dbclient.Get(ctx, key, curFile); err != nil {
return &File{}, err
}
return curFile, nil
}
func setFile(ctx context.Context, file File) error {
// clear session_token and API_token for user
k := datastore.NameKey("Files", file.Id, nil)
if _, err := dbclient.Put(ctx, k, &file); err != nil {
log.Println(err)
return err
}
return nil
}
backend/go-app/main.go
+40 -226
View File
@@ -2422,7 +2422,7 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) {
}
if count == 0 {
log.Printf("No users - redirecting for management user")
log.Printf("[WARNING] No users - redirecting for management user")
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "stay"}`)))
return
@@ -2555,12 +2555,12 @@ func getApikey(ctx context.Context, apikey string) (User, error) {
var users []User
_, err := dbclient.GetAll(ctx, q, &users)
if err != nil {
log.Printf("Error getting users apikey (getapikey): %s", err)
log.Printf("[ERROR] Error getting users apikey (getapikey): %s", err)
return User{}, err
}
if len(users) == 0 {
log.Printf("No users found for apikey %s", apikey)
log.Printf("[WARNING] No users found for apikey %s", apikey)
return User{}, err
}
@@ -2577,28 +2577,6 @@ func getSession(ctx context.Context, thissession string) (*session, error) {
return curUser, nil
}
// ListBooks returns a list of books, ordered by title.
func getFile(ctx context.Context, id string) (*File, error) {
key := datastore.NameKey("Files", id, nil)
curFile := &File{}
if err := dbclient.Get(ctx, key, curFile); err != nil {
return &File{}, err
}
return curFile, nil
}
func setFile(ctx context.Context, file File) error {
// clear session_token and API_token for user
k := datastore.NameKey("Files", file.Id, nil)
if _, err := dbclient.Put(ctx, k, &file); err != nil {
log.Println(err)
return err
}
return nil
}
// ListBooks returns a list of books, ordered by title.
func getOrg(ctx context.Context, id string) (*Org, error) {
key := datastore.NameKey("Organizations", id, nil)
@@ -6552,7 +6530,7 @@ func handleAppHotload(location string, forceUpdate bool) error {
}
//log.Printf("Reading app folder: %#v", dir)
err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate)
_, _, err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate)
if err != nil {
log.Printf("Err: %s", err)
return err
@@ -6835,7 +6813,7 @@ func remoteOrgJobHandler(org Org, interval int) error {
respBody, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("Failed body read in job sync: %s", err)
log.Printf("[ERROR] Failed body read in job sync: %s", err)
return err
}
@@ -6843,7 +6821,7 @@ func remoteOrgJobHandler(org Org, interval int) error {
err = remoteOrgJobController(org, respBody)
if err != nil {
log.Printf("Failed job controller run: %s", err)
log.Printf("[ERROR] Failed job controller run for %s: %s", respBody, err)
return err
}
return nil
@@ -7141,37 +7119,39 @@ func runInit(ctx context.Context) {
}
}
fileq := datastore.NewQuery("Files").Limit(1)
count, err := dbclient.Count(ctx, fileq)
log.Printf("FILECOUNT: %d", count)
if err == nil && count < 10 {
basepath := "."
filename := "testfile.txt"
fileId := uuid.NewV4().String()
log.Printf("Creating new file reference %s because none exist!", fileId)
workflowId := "2cf1169d-b460-41de-8c36-28b2092866f8"
downloadPath := fmt.Sprintf("%s/%s/%s/%s", basepath, activeOrgs[0].Id, workflowId, fileId)
/*
fileq := datastore.NewQuery("Files").Limit(1)
count, err := dbclient.Count(ctx, fileq)
log.Printf("FILECOUNT: %d", count)
if err == nil && count < 10 {
basepath := "."
filename := "testfile.txt"
fileId := uuid.NewV4().String()
log.Printf("Creating new file reference %s because none exist!", fileId)
workflowId := "2cf1169d-b460-41de-8c36-28b2092866f8"
downloadPath := fmt.Sprintf("%s/%s/%s/%s", basepath, activeOrgs[0].Id, workflowId, fileId)
timeNow := time.Now().Unix()
newFile := File{
Id: fileId,
CreatedAt: timeNow,
UpdatedAt: timeNow,
Description: "Created by system for testing",
Status: "active",
Filename: filename,
OrgId: activeOrgs[0].Id,
WorkflowId: workflowId,
DownloadPath: downloadPath,
}
timeNow := time.Now().Unix()
newFile := File{
Id: fileId,
CreatedAt: timeNow,
UpdatedAt: timeNow,
Description: "Created by system for testing",
Status: "active",
Filename: filename,
OrgId: activeOrgs[0].Id,
WorkflowId: workflowId,
DownloadPath: downloadPath,
}
err = setFile(ctx, newFile)
if err != nil {
log.Printf("Failed setting file: %s", err)
} else {
log.Printf("Created file %s in init", newFile.DownloadPath)
err = setFile(ctx, newFile)
if err != nil {
log.Printf("Failed setting file: %s", err)
} else {
log.Printf("Created file %s in init", newFile.DownloadPath)
}
}
}
*/
var allworkflowapps []AppAuthenticationStorage
q = datastore.NewQuery("workflowappauth")
@@ -7934,172 +7914,6 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) {
resp.Write(respBody)
}
type File struct {
Id string `json:"id" datastore:"id"`
Type string `json:"type" datastore:"type"`
CreatedAt int64 `json:"created_at" datastore:"created_at"`
UpdatedAt int64 `json:"updated_at" datastore:"updated_at"`
Description string `json:"description" datastore:"description"`
ExpiresAt string `json:"expires_at" datastore:"expires_at"`
Status string `json:"status" datastore:"status"`
Filename string `json:"filename" datastore:"filename"`
URL string `json:"url" datastore:"org"`
OrgId string `json:"org_id" datastore:"org_id"`
WorkflowId string `json:"workflow_id" datastore:"workflow_id"`
Workflows []string `json:"workflows" datastore:"workflows"`
DownloadPath string `json:"download_path" datastore:"download_path"`
}
func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
log.Printf("\n\nUser is trying to get file %s\n\n", fileId)
// 1. Check user directly
// 2. Check workflow execution authorization
setOrgId := false
user, err := handleApiAuthentication(resp, request)
if err != nil {
// r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS")
log.Printf("INITIAL Api authentication failed in file download: %s", err)
executionId, ok := request.URL.Query()["execution_id"]
if ok && len(executionId) > 0 {
ctx := context.Background()
workflowExecution, err := getWorkflowExecution(ctx, executionId[0])
if err != nil {
log.Printf("Couldn't find execution ID %s", executionId[0])
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
apikey := request.Header.Get("Authorization")
if !strings.HasPrefix(apikey, "Bearer ") {
log.Printf("Apikey doesn't start with bearer (2)")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
apikeyCheck := strings.Split(apikey, " ")
if len(apikeyCheck) != 2 {
log.Printf("Invalid format for apikey (2)")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// This is annoying af and is done because of maxlength lol
newApikey := apikeyCheck[1]
if newApikey != workflowExecution.Authorization {
log.Printf("Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
log.Printf("Authorization is correct for execution %s! %s vs %s", executionId, apikey, workflowExecution.Authorization)
setOrgId = true
} else {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
// 1. Verify if the user has access to the file: org_id and workflow
log.Printf("Should get file %s", fileId)
ctx := context.Background()
file, err := getFile(ctx, fileId)
if err != nil {
log.Printf("File %s not found: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// This is a workaround for file grabs from an app
if setOrgId == true {
user.ActiveOrg.Id = file.OrgId
}
found := false
if file.OrgId == user.ActiveOrg.Id {
found = true
} else {
for _, item := range user.Orgs {
if item == file.OrgId {
found = true
break
}
}
}
if !found {
log.Printf("User %s doesn't have access to %s", user.Username, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Fixme: More auth: org and workflow!
downloadPath := file.DownloadPath
log.Printf("Downloadpath: %s", downloadPath)
Openfile, err := os.Open(downloadPath)
defer Openfile.Close() //Close after function return
if err != nil {
//File not found, send 404
http.Error(resp, "File not found.", 404)
return
}
//File is found, create and send the correct headers
//Get the Content-Type of the file
//Create a buffer to store the header of the file in
FileHeader := make([]byte, 512)
//Copy the headers into the FileHeader buffer
Openfile.Read(FileHeader)
//Get content type of file
FileContentType := http.DetectContentType(FileHeader)
//Get the file size
FileStat, _ := Openfile.Stat() //Get info from file
FileSize := strconv.FormatInt(FileStat.Size(), 10) //Get file size as a string
//Send the headers
resp.Header().Set("Content-Disposition", "attachment; filename="+fileId)
resp.Header().Set("Content-Type", FileContentType)
resp.Header().Set("Content-Length", FileSize)
//Send the file
//We read 512 bytes from the file already, so we reset the offset back to 0
Openfile.Seek(0, 0)
io.Copy(resp, Openfile) //'Copy' the file to the client
return
//log.Printf("Should download file %s", downloadPath)
//resp.WriteHeader(200)
//resp.Write([]byte("OK"))
}
func initHandlers() {
var err error
ctx := context.Background()
@@ -8233,10 +8047,10 @@ func initHandlers() {
// https://developer.box.com/reference/get-files-id-content/
// 1. Creating the "get file" option. Make it possible to run this in the frontend.
r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS")
//r.HandleFunc("/api/v1/files/create", handleGetFile).Methods("POST", "OPTIONS")
//r.HandleFunc("/api/v1/files/upload", handleGetFile).Methods("POST", "OPTIONS")
//r.HandleFunc("/api/v1/files/{fileId}", handleGetFile).Methods("GET", "OPTIONS")
//r.HandleFunc("/api/v1/files/{fileId}", handleGetFile).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/files/create", handleCreateFile).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}", handleGetFileMeta).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS")
http.Handle("/", r)
}
backend/go-app/walkoff.go
+150 -26
View File
@@ -954,6 +954,30 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
return
}
runWorkflowExecutionTransaction(ctx, 0, workflowExecution.ExecutionId, actionResult, resp)
}
// Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times
func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult ActionResult, resp http.ResponseWriter) {
// Should start a tx for the execution here
tx, err := dbclient.NewTransaction(ctx)
if err != nil {
log.Printf("client.NewTransaction: %v", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed creating transaction"}`)))
return
}
key := datastore.NameKey("workflowexecution", workflowExecutionId, nil)
workflowExecution := &WorkflowExecution{}
if err := tx.Get(key, workflowExecution); err != nil {
log.Printf("tx.Get bug: %v", err)
tx.Rollback()
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting the workflow key"}`)))
return
}
if actionResult.Status == "ABORTED" || actionResult.Status == "FAILURE" {
log.Printf("Actionresult is %s. Should set workflowExecution and exit all running functions", actionResult.Status)
@@ -1228,18 +1252,37 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
}
}
err = setWorkflowExecution(ctx, *workflowExecution)
if err != nil {
//workflowExecution.Result = "Error setting workflow: result too large"
//workflowExecution.Status = "FINISHED"
//workflowExecution.CompletedAt = int64(time.Now().Unix())
// Transactions: https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-go
// Prevents timing issues
//ExecutionId
if _, err := tx.Put(key, workflowExecution); err != nil {
tx.Rollback()
log.Printf("[ERROR] tx.Put bug: %v", err)
log.Printf("Error saving workflow execution actionresult setting: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution actionresult: %s"}`, err)))
return
}
if _, err = tx.Commit(); err != nil {
if attempts >= 5 {
log.Printf("[ERROR] QUITTING: tx.Commit %d: %v", attempts, err)
tx.Rollback()
workflowExecution.Status = "ABORTED"
setWorkflowExecution(ctx, *workflowExecution)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
log.Printf("[WARNING] tx.Commit %d: %v", attempts, err)
attempts += 1
runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp)
return
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
@@ -1360,6 +1403,7 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) {
q := datastore.NewQuery("workflow").Filter("owner =", user.Id)
if user.Role == "admin" {
q = datastore.NewQuery("workflow").Filter("org_id =", user.ActiveOrg.Id)
log.Printf("[INFO] Getting workflows (ADMIN) for organization %s", user.ActiveOrg.Id)
}
var workflows []Workflow
@@ -1830,7 +1874,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
for _, action := range workflow.Actions {
allNodes = append(allNodes, action.ID)
if len(action.Errors) > 0 {
if len(action.Errors) > 0 || !action.IsValid {
action.IsValid = true
action.Errors = []string{}
}
@@ -2192,9 +2236,11 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
// Handles check for required params
if !found && param.Required {
log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name)))
return
action.Errors = append(action.Errors, "Parameter %s is required", param.Name)
//newActions = append(newActions, action)
//resp.WriteHeader(401)
//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name)))
//return
}
}
@@ -2209,6 +2255,15 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
workflow.IsValid = true
log.Printf("Tags: %#v", workflow.Tags)
// FIXME: Is this too drastic? May lead to issues in the future.
// Should maybe make a copy for the old org.
if workflow.OrgId != user.ActiveOrg.Id {
log.Printf("[WARNING] Editing workflow to be owned by %s", user.ActiveOrg.Id)
workflow.OrgId = user.ActiveOrg.Id
workflow.ExecutingOrg = user.ActiveOrg
workflow.Org = append(workflow.Org, user.ActiveOrg)
}
err = setWorkflow(ctx, workflow, fileId)
if err != nil {
log.Printf("Failed saving workflow to database: %s", err)
@@ -2234,7 +2289,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
Errors: workflow.Errors,
}
log.Printf("Saved new version of workflow %s (%s)", workflow.Name, fileId)
log.Printf("Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId)
resp.WriteHeader(200)
newBody, err := json.Marshal(returndata)
if err != nil {
@@ -2795,7 +2850,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
continue
}
log.Printf("Should set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID)
log.Printf("[WARNING] Set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID)
defaultResults = append(defaultResults, ActionResult{
Action: action,
ExecutionId: workflowExecution.ExecutionId,
@@ -2836,7 +2891,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
var allEnvs []Environment
if len(workflowExecution.ExecutionOrg) > 0 {
log.Printf("Executing ORG: %s", workflowExecution.ExecutionOrg)
log.Printf("[INFO] Executing ORG: %s", workflowExecution.ExecutionOrg)
allEnvironments, err := getEnvironments(ctx, workflowExecution.ExecutionOrg)
if err != nil {
@@ -5336,11 +5391,24 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra
return err
}
type buildLaterStruct struct {
Tags []string
Extra string
Id string
}
// Onlyname is used to
func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) error {
func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) ([]buildLaterStruct, []buildLaterStruct, error) {
var err error
allapps := []WorkflowApp{}
reservedNames := []string{
"OWA",
"NLP",
}
buildLaterFirst := []buildLaterStruct{}
buildLaterList := []buildLaterStruct{}
// It's here to prevent getting them in every iteration
ctx := context.Background()
@@ -5360,11 +5428,20 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
}
// Go routine? Hmm, this can be super quick I guess
err = iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate)
buildFirst, buildLast, err := iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate)
if err != nil {
log.Printf("Error reading folder: %s", err)
continue
}
for _, item := range buildFirst {
buildLaterFirst = append(buildLaterFirst, item)
}
for _, item := range buildLast {
buildLaterList = append(buildLaterList, item)
}
case mode.IsRegular():
// Check the file
filename := file.Name()
@@ -5562,22 +5639,69 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
//log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion)
/// Only upload if successful and no errors
err = buildImageMemory(fs, tags, extra)
if err != nil {
log.Printf("Failed image build memory: %s", err)
} else {
if len(tags) > 0 {
log.Printf("Successfully built image %s", tags[0])
} else {
log.Printf("Successfully built Docker image")
// ID can be used to e.g. set a build status.
buildLater := buildLaterStruct{
Tags: tags,
Extra: extra,
Id: workflowapp.ID,
}
reservedFound := false
for _, appname := range reservedNames {
if strings.ToUpper(workflowapp.Name) == strings.ToUpper(appname) {
buildLaterList = append(buildLaterList, buildLater)
reservedFound = true
break
}
}
/// Only upload if successful and no errors
if !reservedFound {
buildLaterFirst = append(buildLaterFirst, buildLater)
} else {
log.Printf("\n\n[WARNING] Skipping build of %s to later\n\n", workflowapp.Name)
}
}
}
}
return err
if len(buildLaterFirst) == 0 && len(buildLaterList) == 0 {
return buildLaterFirst, buildLaterList, err
}
//log.Printf("BUILDLATERFIRST: %d, BUILDLATERLIST: %d", len(buildLaterFirst), len(buildLaterList))
if len(extra) == 0 {
log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst))
for _, item := range buildLaterFirst {
err = buildImageMemory(fs, item.Tags, item.Extra)
if err != nil {
log.Printf("Failed image build memory: %s", err)
} else {
if len(item.Tags) > 0 {
log.Printf("Successfully built image %s", item.Tags[0])
} else {
log.Printf("Successfully built Docker image")
}
}
}
log.Printf("Starting build of %d skipped docker images", len(buildLaterList))
for _, item := range buildLaterList {
err = buildImageMemory(fs, item.Tags, item.Extra)
if err != nil {
log.Printf("Failed image build memory: %s", err)
} else {
if len(item.Tags) > 0 {
log.Printf("Successfully built image %s", item.Tags[0])
} else {
log.Printf("Successfully built Docker image")
}
}
}
}
return buildLaterFirst, buildLaterList, err
}
func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) {
@@ -5722,7 +5846,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) {
}
// Query for the specifci workflowId
q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(20)
q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(30)
var workflowExecutions []WorkflowExecution
_, err = dbclient.GetAll(ctx, q, &workflowExecutions)
if err != nil {
docker-compose.yml
+4 -2
View File
@@ -2,7 +2,7 @@ version: '3'
services:
frontend:
#build: ./frontend
image: ghcr.io/frikky/shuffle-frontend:0.8.0
image: ghcr.io/frikky/shuffle-frontend:0.8.3
container_name: shuffle-frontend
hostname: shuffle-frontend
ports:
@@ -17,7 +17,7 @@ services:
- backend
backend:
#build: ./backend
image: ghcr.io/frikky/shuffle-backend:0.8.0
image: ghcr.io/frikky/shuffle-backend:0.8.3
container_name: shuffle-backend
hostname: ${BACKEND_HOSTNAME}
# Here for debugging:
@@ -28,9 +28,11 @@ services:
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps
- ${SHUFFLE_FILE_LOCATION}:/shuffle-files
environment:
- DATASTORE_EMULATOR_HOST=shuffle-database:8000
- SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps
- SHUFFLE_FILE_LOCATION=/shuffle-files
- ORG_ID=${ORG_ID}
- SHUFFLE_APP_DOWNLOAD_LOCATION=${SHUFFLE_APP_DOWNLOAD_LOCATION}
- SHUFFLE_DEFAULT_USERNAME=${SHUFFLE_DEFAULT_USERNAME}
frontend/src/views/AngularWorkflow.jsx
+239 -59
View File
@@ -191,7 +191,7 @@ const AngularWorkflow = (props) => {
const cloudSyncEnabled = props.userdata !== undefined && props.userdata.active_org !== null && props.userdata.active_org !== undefined ? props.userdata.active_org.cloud_sync === true : false
//const triggerEnvironments = cloudSyncEnabled ? ["cloud", "onprem"] : environments
const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"
const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"
const triggerEnvironments = isCloud ? ["cloud"] : ["cloud", "onprem"]
const unloadText = 'Are you sure you want to leave without saving (CTRL+S)?'
@@ -899,6 +899,41 @@ const AngularWorkflow = (props) => {
const onUnselect = (event) => {
console.time("UNSELECT")
// Attempt at rewrite of name in other actions in following nodes.
// Should probably be done in the onBlur for the textfield instead
/*
if (event.target.data().type === "ACTION") {
const nodeaction = event.target.data()
const curaction = workflow.actions.find(a => a.id === nodeaction.id)
console.log("workflowaction: ", curaction)
console.log("nodeaction: ", nodeaction)
if (nodeaction.label !== curaction.label) {
console.log("BEACH!")
var params = []
const fixedName = "$"+curaction.label.toLowerCase().replace(" ", "_")
for (var actionkey in workflow.actions) {
if (workflow.actions[actionkey].id === curaction.id) {
continue
}
for (var paramkey in workflow.actions[actionkey].parameters) {
const param = workflow.actions[actionkey].parameters[paramkey]
if (param.value === null || param.value === undefined || !param.value.includes("$")) {
continue
}
const innername = param.value.toLowerCase().replace(" ", "_")
if (innername.includes(fixedName)) {
//workflow.actions[actionkey].parameters[paramkey].replace(
//console.log("FOUND!: ", innername)
}
}
}
}
}
*/
// FIXME - check if they have value before overriding like this for no reason.
// Would save a lot of time (400~ ms -> 30ms)
//console.log("ACTION: ", selectedAction)
@@ -1009,8 +1044,29 @@ const AngularWorkflow = (props) => {
}
setSelectedActionName(curaction.name)
setSelectedAction(curaction)
/*
var params = []
const fixedName = "$"+curaction.label.toLowerCase().replace(" ", "_")
for (var actionkey in workflow.actions) {
if (workflow.actions[actionkey].id === curaction.id) {
continue
}
for (var paramkey in workflow.actions[actionkey].parameters) {
const param = workflow.actions[actionkey].parameters[paramkey]
if (param.value === null || param.value === undefined || !param.value.includes("$")) {
continue
}
const innername = param.value.toLowerCase().replace(" ", "_")
if (innername.includes(fixedName)) {
console.log("FOUND!: ", innername)
}
}
}
*/
} else if (data.type === "TRIGGER") {
//console.log("Should handle trigger "+data.triggertype)
//console.log(data)
@@ -1278,6 +1334,7 @@ const AngularWorkflow = (props) => {
} else {
setEnvironments({"name": "Onprem", "type": "onprem"})
}
return
}
@@ -2419,14 +2476,43 @@ const AngularWorkflow = (props) => {
// ACTION select
//
const selectedNameChange = (event) => {
console.log("OLDNAME: ", selectedActionName)
event.target.value = event.target.value.replace("(", "")
event.target.value = event.target.value.replace(")", "")
event.target.value = event.target.value.replace("$", "")
event.target.value = event.target.value.replace("#", "")
event.target.value = event.target.value.replace(".", "")
event.target.value = event.target.value.replace(",", "")
event.target.value = event.target.value.replace(" ", "_")
selectedAction.label = event.target.value
setSelectedAction(selectedAction)
/*
if (nodeaction.label !== curaction.label) {
console.log("BEACH!")
var params = []
const fixedName = "$"+curaction.label.toLowerCase().replace(" ", "_")
for (var actionkey in workflow.actions) {
if (workflow.actions[actionkey].id === curaction.id) {
continue
}
for (var paramkey in workflow.actions[actionkey].parameters) {
const param = workflow.actions[actionkey].parameters[paramkey]
if (param.value === null || param.value === undefined || !param.value.includes("$")) {
continue
}
const innername = param.value.toLowerCase().replace(" ", "_")
if (innername.includes(fixedName)) {
//workflow.actions[actionkey].parameters[paramkey].replace(
//console.log("FOUND!: ", innername)
}
}
}
}
*/
}
const selectedTriggerChange = (event) => {
@@ -2836,15 +2922,56 @@ const AngularWorkflow = (props) => {
}}
/>
console.log(selectedActionParameters[count])
if (selectedActionParameters[count].schema !== undefined && selectedActionParameters[count].schema !== null && selectedActionParameters[count].schema.type === "file") {
const fileId = "6daabec1-892b-469c-b603-c902e47223a9"
datafield = `SHOW FILES FROM OTHER NODES? Filename: ${selectedActionParameters[count].value}`
datafield =
<TextField
style={{backgroundColor: inputColor, borderRadius: borderRadius,}}
InputProps={{
style:{
color: "white",
minHeight: "50px",
marginLeft: "5px",
maxWidth: "95%",
fontSize: "1em",
},
endAdornment: (
<InputAdornment position="end">
<Tooltip title="Autocomplete text" placement="top">
<AddCircleOutlineIcon style={{cursor: "pointer"}} onClick={(event) => {
setMenuPosition({
top: event.pageY,
left: event.pageX,
})
setShowDropdownNumber(count)
setShowDropdown(true)
setShowAutocomplete(true)
}}/>
</Tooltip>
</InputAdornment>
)
}}
fullWidth
multiline={multiline}
rows="5"
color="primary"
defaultValue={data.value}
type={"text"}
placeholder={"The file ID to get"}
onChange={(event) => {
changeActionParameter(event, count)
}}
onBlur={(event) => {
}}
/>
//const fileId = "6daabec1-892b-469c-b603-c902e47223a9"
//datafield = `SHOW FILES FROM OTHER NODES? Filename: ${selectedActionParameters[count].value}`
/*
if (selectedActionParameters[count].value != fileId) {
changeActionParameter(fileId, count)
setUpdate(Math.random())
}
*/
} else if (selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0) {
if (selectedActionParameters[count].value === "" && selectedActionParameters[count].required) {
// Rofl, dirty workaround :)
@@ -4551,7 +4678,6 @@ const AngularWorkflow = (props) => {
placeholder={selectedTrigger.label}
onChange={selectedTriggerChange}
/>
{showEnvironment ?
<div style={{marginTop: "20px"}}>
<Typography>
Environment
@@ -4599,7 +4725,6 @@ const AngularWorkflow = (props) => {
})}
</Select>
</div>
: null}
<Divider style={{marginTop: "20px", height: "1px", width: "100%", backgroundColor: "rgb(91, 96, 100)"}}/>
<div style={{flex: "6", marginTop: "20px"}}>
<div>
@@ -5095,59 +5220,57 @@ const AngularWorkflow = (props) => {
placeholder={selectedTrigger.label}
onChange={selectedTriggerChange}
/>
{showEnvironment ?
<div style={{marginTop: "20px"}}>
<Typography>
Environment
</Typography>
<Select
value={selectedTrigger.environment}
disabled={selectedTrigger.status === "running"}
SelectDisplayProps={{
style: {
marginLeft: 10,
<div style={{marginTop: "20px"}}>
<Typography>
Environment
</Typography>
<Select
value={selectedTrigger.environment}
disabled={selectedTrigger.status === "running"}
SelectDisplayProps={{
style: {
marginLeft: 10,
}
}}
fullWidth
onChange={(e) => {
selectedTrigger.environment = e.target.value
setSelectedTrigger(selectedTrigger)
if (e.target.value === "cloud") {
console.log("Set cloud config")
workflow.triggers[selectedTriggerIndex].parameters[0].value = "*/2 * * * *"
}
}}
fullWidth
onChange={(e) => {
selectedTrigger.environment = e.target.value
setSelectedTrigger(selectedTrigger)
if (e.target.value === "cloud") {
console.log("Set cloud config")
workflow.triggers[selectedTriggerIndex].parameters[0].value = "*/2 * * * *"
//var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/")
//const urlpath = tmpvalue.slice(3, tmpvalue.length)
//const newurl = "https://shuffler.io/"+urlpath.join("/")
//workflow.triggers[selectedTriggerIndex].parameters[0].value = newurl
} else {
console.log("Set cloud config")
//var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/")
//const urlpath = tmpvalue.slice(3, tmpvalue.length)
//const newurl = window.location.origin+"/"+urlpath.join("/")
workflow.triggers[selectedTriggerIndex].parameters[0].value = "120"
}
//var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/")
//const urlpath = tmpvalue.slice(3, tmpvalue.length)
//const newurl = "https://shuffler.io/"+urlpath.join("/")
//workflow.triggers[selectedTriggerIndex].parameters[0].value = newurl
} else {
console.log("Set cloud config")
//var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/")
//const urlpath = tmpvalue.slice(3, tmpvalue.length)
//const newurl = window.location.origin+"/"+urlpath.join("/")
workflow.triggers[selectedTriggerIndex].parameters[0].value = "120"
}
setWorkflow(workflow)
setUpdate(Math.random())
}}
style={{backgroundColor: inputColor, color: "white", height: "50px"}}
>
{triggerEnvironments.map(data => {
if (data.archived) {
return null
}
return (
<MenuItem key={data} style={{backgroundColor: inputColor, color: "white"}} value={data}>
{data}
</MenuItem>
)
})}
</Select>
</div>
: null}
setWorkflow(workflow)
setUpdate(Math.random())
}}
style={{backgroundColor: inputColor, color: "white", height: "50px"}}
>
{triggerEnvironments.map(data => {
if (data.archived) {
return null
}
return (
<MenuItem key={data} style={{backgroundColor: inputColor, color: "white"}} value={data}>
{data}
</MenuItem>
)
})}
</Select>
</div>
<Divider style={{marginTop: "20px", height: "1px", width: "100%", backgroundColor: "rgb(91, 96, 100)"}}/>
<div style={{flex: "6", marginTop: "20px"}}>
<div>
@@ -5228,7 +5351,7 @@ const AngularWorkflow = (props) => {
return null
}
const cytoscapeViewWidths = 700
const cytoscapeViewWidths = 750
const bottomBarStyle = {
position: "fixed",
right: 20,
@@ -5286,6 +5409,62 @@ const AngularWorkflow = (props) => {
return null
}
const FileMenu = () => {
const [newAnchor, setNewAnchor] = React.useState(null);
const [showShuffleMenu, setShowShuffleMenu] = React.useState(false)
{ /*const [showShuffleMenu, setShowShuffleMenu] = React.useState(true) */}
return (
<div style={{"display": "inline-block"}}>
<Menu
id="long-menu"
anchorEl={newAnchor}
open={showShuffleMenu}
onClose={() => {
setShowShuffleMenu(false)
}}
>
<div style={{margin: 15, color: "white", maxWidth: 250, minWidth: 250, }}>
<h4>This menu is used to control the workflow itself.</h4>
<Divider style={{backgroundColor: "white", marginTop: 10, marginBottom: 10,}}/>
<FormControlLabel
style={{marginBottom: 15, color: "white",}}
label={<div style={{color: "white"}}>Exit on Error</div>}
control={
<Switch checked={workflow.configuration.exit_on_error} onChange={() => {
workflow.configuration.exit_on_error = !workflow.configuration.exit_on_error
setWorkflow(workflow)
setUpdate("exit_on_error_"+workflow.configuration.exit_on_error ? "true" : "false")
setShowShuffleMenu(false)
}} />
}
/>
<FormControlLabel
style={{marginBottom: 15, color: "white",}}
label={<div style={{color: "white"}}>Start from top</div>}
control={
<Switch checked={workflow.configuration.start_from_top} onChange={() => {
workflow.configuration.start_from_top = !workflow.configuration.start_from_top
setWorkflow(workflow)
setUpdate("start_from_top_"+workflow.configuration.start_from_top ? "true" : "false")
setShowShuffleMenu(false)
}} />
}
/>
</div>
</Menu>
<Tooltip color="secondary" title="Workflow settings" placement="top-start">
<Button color="primary" style={{height: 50, marginLeft: 10, }} variant="outlined" onClick={(event) => {
setShowShuffleMenu(!showShuffleMenu)
setNewAnchor(event.currentTarget)
}}>
<SettingsIcon />
</Button>
</Tooltip>
</div>
)
}
const WorkflowMenu = () => {
const [newAnchor, setNewAnchor] = React.useState(null);
const [showShuffleMenu, setShowShuffleMenu] = React.useState(false)
@@ -5413,6 +5592,7 @@ const AngularWorkflow = (props) => {
<DirectionsRunIcon />
</Button>
</Tooltip>
{/* <FileMenu /> */}
<WorkflowMenu />
</div>
</div>
@@ -5719,7 +5899,7 @@ const AngularWorkflow = (props) => {
try {
const tmp = String(JSON.parse(showResult))
if (!showResult.includes("{") && !showResult.includes("[")) {
console.log("IN HERE: ", tmp)
//console.log("IN HERE: ", tmp)
jsonvalid = false
}
} catch (e) {
frontend/src/views/Apps.jsx
+12 -7
View File
@@ -116,7 +116,7 @@ const Apps = (props) => {
const [apps, setApps] = React.useState([])
const [filteredApps, setFilteredApps] = React.useState([])
const [validation, setValidation] = React.useState(false)
const [isLoading, setIsLoading] = React.useState(false)
const [isLoading, setIsLoading] = React.useState(true)
const [appSearchLoading, setAppSearchLoading] = React.useState(false)
const [selectedAction, setSelectedAction] = React.useState({})
const [searchBackend, setSearchBackend] = React.useState(false)
@@ -207,6 +207,7 @@ const Apps = (props) => {
credentials: "include",
})
.then((response) => {
setIsLoading(false)
if (response.status !== 200) {
console.log("Status not 200 for apps :O!")
}
@@ -231,6 +232,7 @@ const Apps = (props) => {
})
.catch(error => {
alert.error(error.toString())
setIsLoading(false)
});
}
@@ -742,7 +744,7 @@ const Apps = (props) => {
<Divider style={{height: 1, backgroundColor: dividerColor, marginTop: 20, marginBottom: 20}} />
<div style={{}}>
<Button
variant="text"
variant="contained"
component="label"
color="primary"
style={{marginRight: 10, }}
@@ -932,11 +934,14 @@ const Apps = (props) => {
}
</Paper>
:
<Paper square style={uploadViewPaperStyle}>
<h4 style={{margin: 10}}>
No apps have been created, uploaded or downloaded yet. Click "Load existing apps" above to get the baseline. This may take a while as its building docker images.
</h4>
</Paper>
isLoading ?
<CircularProgress style={{width: 40, height: 40, margin: "auto"}}/>
:
<Paper square style={uploadViewPaperStyle}>
<h4 style={{margin: 10}}>
No apps have been created, uploaded or downloaded yet. Click "Load existing apps" above to get the baseline. This may take a while as its building docker images.
</h4>
</Paper>
}
</div>
</div>
frontend/src/views/Workflows.jsx
+161 -109
View File
@@ -15,6 +15,7 @@ import FormControlLabel from '@material-ui/core/FormControlLabel';
import Chip from '@material-ui/core/Chip';
import Switch from '@material-ui/core/Switch';
import Typography from '@material-ui/core/Typography';
import Zoom from '@material-ui/core/Zoom';
import CircularProgress from '@material-ui/core/CircularProgress';
import CachedIcon from '@material-ui/icons/Cached';
@@ -74,6 +75,7 @@ const Workflows = (props) => {
const [update, setUpdate] = React.useState("test");
const [deleteModalOpen, setDeleteModalOpen] = React.useState(false);
const [editingWorkflow, setEditingWorkflow] = React.useState({})
const [executionLoading, setExecutionLoading] = React.useState(false)
const { start, stop } = useInterval({
duration: 5000,
startImmediate: false,
@@ -245,6 +247,7 @@ const Workflows = (props) => {
}
const getWorkflowExecution = (id) => {
setExecutionLoading(true)
fetch(globalUrl+"/api/v1/workflows/"+id+"/executions", {
method: 'GET',
headers: {
@@ -254,6 +257,7 @@ const Workflows = (props) => {
credentials: "include",
})
.then((response) => {
setExecutionLoading(false)
if (response.status !== 200) {
console.log("Status not 200 for WORKFLOW EXECUTION :O!")
}
@@ -275,6 +279,7 @@ const Workflows = (props) => {
}
})
.catch(error => {
setExecutionLoading(false)
alert.error(error.toString())
});
}
@@ -360,7 +365,8 @@ const Workflows = (props) => {
data.triggers[key].status = "stopped"
}
}
data["org"] = []
data["org_id"] = ""
data.execution_org = {"id": ""}
console.log(data)
@@ -858,10 +864,17 @@ const Workflows = (props) => {
</div>
)
}
return (
<h4>
There are no executiondetails yet. Click "execute" to run your first one.
</h4>
executionLoading ?
<div style={{marginTop: 25, textAlign: "center"}}>
<CircularProgress />
</div>
:
<h4>
There are no executiondetails yet. Click "execute" to run your first one.
</h4>
)
}
@@ -880,9 +893,14 @@ const Workflows = (props) => {
)
}
return (
<h4>
There are no executions for this workflow yet
</h4>
executionLoading ?
<div style={{marginTop: 25, textAlign: "center"}}>
<CircularProgress />
</div>
:
<h4>
There are no executions for this workflow yet
</h4>
)
}
@@ -1012,8 +1030,19 @@ const Workflows = (props) => {
},
}}
>
<DialogTitle>
<div style={{color: "rgba(255,255,255,0.9)"}}>
{editingWorkflow.id !== undefined ? "Editing" : "New"} workflow
<div style={{float: "right"}}>
<Tooltip color="primary" title={"Import manually"} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
<PublishIcon />
</Button>
</Tooltip>
</div>
</div>
</DialogTitle>
<FormControl>
<DialogTitle><div style={{color: "white"}}>{editingWorkflow.id !== undefined ? "Editing" : "New"} workflow</div></DialogTitle>
<DialogContent>
<TextField
onBlur={(event) => setNewWorkflowName(event.target.value)}
@@ -1111,107 +1140,126 @@ const Workflows = (props) => {
workflowViewStyle.display = "none"
}
const workflowView = workflows.length > 0 ?
<div style={viewStyle}>
<div style={workflowViewStyle}>
<div style={{display: "flex"}}>
<div style={{flex: "4"}}>
<h2>Workflows</h2>
</div>
<div style={{marginTop: 20}}>
<Tooltip color="primary" title={"Create new workflow"} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => setModalOpen(true)}><AddIcon /></Button>
</Tooltip>
{/*
<Tooltip color="primary" title={"Import workflows"} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
<PublishIcon />
</Button>
</Tooltip>
*/}
<Tooltip color="primary" title={`Download ALL workflows (${workflows.length})`} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => {
exportAllWorkflows()
}}>
<GetAppIcon />
</Button>
</Tooltip>
<Tooltip color="primary" title={"Import workflows"} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => setLoadWorkflowsModalOpen(true)}>
<CloudDownloadIcon />
</Button>
</Tooltip>
<input hidden type="file" multiple="multiple" ref={(ref) => upload = ref} onChange={importFiles} />
</div>
</div>
<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
<div style={scrollStyle}>
{workflows.map((data, index) => {
return (
<WorkflowPaper key={index} data={data} />
)
})}
</div>
</div>
<div style={{flex: viewSize.executionsView, marginLeft: "10px", marginRight: "10px"}}>
<div style={{display: "flex"}}>
<div style={{flex: "10"}}>
<h2>Executions: {selectedWorkflow.name}</h2>
</div>
<div style={{flex: "1"}}>
<Button color="primary" style={{marginTop: "20px"}} variant="text" onClick={() => {
alert.info("Refreshing executions");
getWorkflowExecution(selectedWorkflow.id)
}}>
<CachedIcon />
const workflowButtons =
<span>
{workflows.length > 0 ?
<Tooltip color="primary" title={"Create new workflow"} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => setModalOpen(true)}><AddIcon /></Button>
</Tooltip>
: null}
<Tooltip color="primary" title={"Import workflows"} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
<PublishIcon />
</Button>
</Tooltip>
<input hidden type="file" multiple="multiple" ref={(ref) => upload = ref} onChange={importFiles} />
{workflows.length > 0 ?
<Tooltip color="primary" title={`Download ALL workflows (${workflows.length})`} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => {
exportAllWorkflows()
}}>
<GetAppIcon />
</Button>
</div>
</div>
<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
<div style={scrollStyle}>
<ExecutionsView />
</div>
</div>
<div style={{flex: viewSize.executionResults, marginLeft: "10px", marginRight: "10px", minWidth: "33%"}}>
<div style={{display: "flex"}}>
<div style={{flex: "1"}}>
<h2>Execution Timeline</h2>
</div>
<div style={{flex: 1}}>
<FormControlLabel
style={{color: "white", marginBottom: "0px", marginTop: "10px"}}
label={<div style={{color: "white"}}>Collapse results</div>}
control={<Switch checked={collapseJson} onChange={() => {setCollapseJson(!collapseJson)}} />}
/>
</div>
</div>
<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
<div style={scrollStyle}>
<ExecutionDetails />
</div>
</div>
</div>
:
<div style={emptyWorkflowStyle}>
<Paper style={boxStyle}>
<div>
<h2>Welcome to Shuffle</h2>
</div>
<div>
<p>
<b>Shuffle</b> is a flexible, easy to use, automation platform allowing users to integrate their services and devices freely. It's made to significantly reduce the amount of manual labor, and is focused on security applications. <a href="/docs/about" style={{textDecoration: "none", color: "#f85a3e"}}>Click here to learn more.</a>
</p>
</div>
<div>
If you want to jump straight into it, click here to create your first workflow:
</div>
<div>
<Button color="primary" style={{marginTop: "20px",}} variant="outlined" onClick={() => setModalOpen(true)}>New workflow</Button>
</div>
</Paper>
</div>
</Tooltip>
: null}
<Tooltip color="primary" title={"Download workflows"} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => setLoadWorkflowsModalOpen(true)}>
<CloudDownloadIcon />
</Button>
</Tooltip>
</span>
const WorkflowView = () => {
if (workflows.length === 0) {
return (
<div style={emptyWorkflowStyle}>
<Paper style={boxStyle}>
<div>
<h2>Welcome to Shuffle</h2>
</div>
<div>
<p>
<b>Shuffle</b> is a flexible, easy to use, automation platform allowing users to integrate their services and devices freely. It's made to significantly reduce the amount of manual labor, and is focused on security applications. <a href="/docs/about" style={{textDecoration: "none", color: "#f85a3e"}}>Click here to learn more.</a>
</p>
</div>
<div>
If you want to jump straight into it, click here to create your first workflow:
</div>
<div style={{display: "flex"}}>
<Button color="primary" style={{marginTop: "20px",}} variant="outlined" onClick={() => setModalOpen(true)}>New workflow</Button>
<span style={{paddingTop: 20, display: "flex",}}>
<Typography style={{marginTop: 5, marginLeft: 30, marginRight: 15}}>
..OR
</Typography>
{workflowButtons}
</span>
</div>
</Paper>
</div>
)
}
return (
<div style={viewStyle}>
<div style={workflowViewStyle}>
<div style={{display: "flex"}}>
<div style={{flex: "4"}}>
<h2>Workflows</h2>
</div>
<div style={{marginTop: 20}}>
{workflowButtons}
</div>
</div>
<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
<div style={scrollStyle}>
{workflows.map((data, index) => {
return (
<WorkflowPaper key={index} data={data} />
)
})}
</div>
</div>
<div style={{flex: viewSize.executionsView, marginLeft: "10px", marginRight: "10px"}}>
<div style={{display: "flex"}}>
<div style={{flex: "10"}}>
<h2>Executions: {selectedWorkflow.name}</h2>
</div>
<div style={{flex: "1"}}>
<Button color="primary" style={{marginTop: "20px"}} variant="text" onClick={() => {
alert.info("Refreshing executions");
getWorkflowExecution(selectedWorkflow.id)
}}>
<CachedIcon />
</Button>
</div>
</div>
<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
<div style={scrollStyle}>
<ExecutionsView />
</div>
</div>
<div style={{flex: viewSize.executionResults, marginLeft: "10px", marginRight: "10px", minWidth: "33%"}}>
<div style={{display: "flex"}}>
<div style={{flex: "1"}}>
<h2>Execution Timeline</h2>
</div>
<div style={{flex: 1}}>
<FormControlLabel
style={{color: "white", marginBottom: "0px", marginTop: "10px"}}
label={<div style={{color: "white"}}>Collapse results</div>}
control={<Switch checked={collapseJson} onChange={() => {setCollapseJson(!collapseJson)}} />}
/>
</div>
</div>
<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
<div style={scrollStyle}>
<ExecutionDetails />
</div>
</div>
</div>
)
}
const importWorkflowsFromUrl = (url) => {
console.log("IMPORT WORKFLOWS FROM ", downloadUrl)
@@ -1384,13 +1432,17 @@ const Workflows = (props) => {
const loadedCheck = isLoaded && isLoggedIn && workflowDone ?
<div>
{workflowView}
<WorkflowView />
{modalView}
{deleteModal}
{workflowDownloadModalOpen}
</div>
:
<div>
<div style={{paddingTop: 250, width: 250, margin: "auto", textAlign: "center"}}>
<CircularProgress />
<Typography>
Loading Workflows
</Typography>
</div>