Merge pull request #214 from frikky/launch

0.8.3

.env

@@ -19,7 +19,9 @@ SHUFFLE_DEFAULT_PASSWORD=
 SHUFFLE_DEFAULT_APIKEY=
 
 # Local location of your app directory. Can't use ~/
+# Files will get better at some point. Right now: local saving.
 SHUFFLE_APP_HOTLOAD_LOCATION=./shuffle-apps
+SHUFFLE_FILE_LOCATION=./shuffle-files
 
 # Other configs
 BACKEND_HOSTNAME=shuffle-backend

backend/Dockerfile

@@ -8,6 +8,7 @@ ADD ./go-app/main.go /app
 ADD ./go-app/walkoff.go /app
 ADD ./go-app/docker.go /app
 ADD ./go-app/codegen.go /app
+ADD ./go-app/files.go /app
 
 ADD ./go-app/go.mod /app
 

backend/app_sdk/app_base.py

@@ -1,4 +1,5 @@
 import os
+import copy
 import sys
 import re
 import time 
@@ -52,6 +53,139 @@ class AppBase:
             if ret.status_code != 200:
                 self.logger.info(ret.text)
 
+    # Things to consider for files:
+    # - How can you download / stream a file? 
+    # - Can you decide if you want a stream or the files directly?
+    def get_file(self, value):
+        full_execution = self.full_execution
+        org_id = full_execution["workflow"]["execution_org"]["id"]
+
+        print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value))
+
+        if isinstance(value, list):
+            print("IS LIST!")
+            #if len(value) == 1:
+            #    value = value[0]
+        else:
+            value = [value]
+
+        returns = []
+        for item in value:
+            print("VALUE: %s" % item)
+            if len(item) != 36:
+                print("Bad length for value")
+                continue
+                #return {
+                #    "filename": "",
+                #    "data": "",
+                #    "success": False,
+                #}
+
+            get_path = "/api/v1/files/%s?execution_id=%s" % (item, full_execution["execution_id"])
+            headers = {
+                "Content-Type": "application/json",     
+                "Authorization": "Bearer %s" % self.authorization
+            }
+
+            ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers)
+            print("RET1: %s" % ret1.text)
+            if ret1.status_code != 200:
+                returns.append({
+                    "filename": "",
+                    "data": "",
+                    "success": False,
+                })
+                continue
+
+            content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"])
+            ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers)
+            print("Ret2: %s" % ret2.text)
+            if ret2.status_code == 200:
+                tmpdata = ret1.json()
+                returndata = {
+                    "success": True,
+                    "filename": tmpdata["filename"],
+                    "data": ret2.content,
+                }
+                returns.append(returndata)
+
+        if len(returns) == 0:
+            return {
+                "success": False,
+                "filename": "",
+                "data": b"",
+            }
+        elif len(returns) == 1:
+            return returns[0]
+        else:
+            return returns
+
+    # Sets files in the backend
+    def set_files(self, infiles):
+        full_execution = self.full_execution
+        workflow_id = full_execution["workflow"]["id"]
+        org_id = full_execution["workflow"]["execution_org"]["id"]
+        headers = {
+            "Content-Type": "application/json",     
+            "Authorization": "Bearer %s" % self.authorization
+        }
+
+        create_path = "/api/v1/files/create?execution_id=%s" % full_execution["execution_id"]
+        file_ids = []
+        for curfile in infiles:
+            filename = "unspecified"
+            data = {
+                "filename": filename,
+                "workflow_id": workflow_id,
+                "org_id": org_id,
+            }
+
+            try:
+                data["filename"] = curfile["filename"]
+                filename = curfile["filename"]
+            except KeyError as e:
+                print("KeyError in file setup: %s" % e)
+                pass
+
+            ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data)
+            print("Ret CREATE: %s" % ret.text)
+            cur_id = ""
+            if ret.status_code == 200:
+                print("RET: %s" % ret.text)
+                ret_json = ret.json()
+                if not ret_json["success"]:
+                    print("Not success in file upload creation.")
+                    continue
+
+                print("Should handle ID %s" % ret_json["id"])
+                file_ids.append(ret_json["id"])
+                cur_id = ret_json["id"]
+            else:
+                print("Bad status code: %d" % ret.status_code)
+                continue
+
+            if len(cur_id) == 0:
+                print("No file ID specified from backend")
+                continue
+
+            new_headers = {
+                "Authorization": "Bearer %s" % self.authorization,
+            }
+
+            upload_path = "/api/v1/files/%s/upload?execution_id=%s" % (cur_id, full_execution["execution_id"])
+            print("Create path: %s" % create_path)
+
+            # FIXME: Typical failure here if data is returned badly formatted
+            files={"shuffle_file": (filename, curfile["data"])}
+            #open(filename,'rb')}
+
+            ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers)
+            print("Ret UPLOAD: %s" % ret.text)
+            print("Ret2 UPLOAD: %d" % ret.status_code)
+
+        print("IDS TO RETURN: %s" % file_ids)
+        return file_ids
+    
     async def execute_action(self, action):
         # FIXME - add request for the function STARTING here. Use "results stream" or something
         # PAUSED, AWAITING_DATA, PENDING, COMPLETED, ABORTED, EXECUTING, SUCCESS, FAILURE
@@ -148,6 +282,7 @@ class AppBase:
             print("")
 
 
+        self.full_execution = fullexecution
         self.logger.info("AFTER FULLEXEC stream result")
 
         # Gets the value at the parenthesis level you want
@@ -207,7 +342,7 @@ class AppBase:
                 try:
                     return int(data)
                 except ValueError:
-                    print("ValueError while casting %s" % data)
+                    print("ValueError while casting %s to int" % data)
                     return data
             if "lower" in thistype:
                 return data.lower()
@@ -219,15 +354,19 @@ class AppBase:
                 return data.strip()
             if "split" in thistype:
                 return data.split()
-            if "len" in thistype or "length" in thistype:
+            if "len" in thistype or "length" in thistype or "lenght" in thistype:
                 tmp = "" 
                 try:
-                    tmp = json.loads(data)
+                    tmpdata = data.replace("\'", "\"")
+                    tmp = json.loads(tmpdata)
                 except: 
+                    print("Passing bug")
                     pass
 
                 if isinstance(tmp, list):
-                    return str(len(tmp))
+                    return len(tmp)
+                elif isinstance(tmp, object):
+                    return len(tmp)
 
                 return str(len(data))
             if "parse" in thistype:
@@ -273,7 +412,7 @@ class AppBase:
             #print("Running %s" % data)
         
             # Look for the INNER wrapper first, then move out
-            wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length"]
+            wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length", "lenght"]
             found = False
             for wrapper in wrappers:
                 if wrapper not in data.lower():
@@ -303,6 +442,7 @@ class AppBase:
                         continue
         
                     parsed_value = parse_type(innervalue[0], thistype.lower())
+                    print("Parsed value from %s: %s" % (thistype, parsed_value))
                     return parsed_value
         
             print("DATA: %s\n" % data)
@@ -403,6 +543,7 @@ class AppBase:
                         # Magical way of returning which makes app sdk identify 
                         # it as multi execution
                         return newvalue, True
+
                     elif len(actualitem) > 0:
                         # FIXME: This is absolutely not perfect. 
                         print("In recursion v2: ", actualitem)
@@ -478,7 +619,7 @@ class AppBase:
 
             #Actionname: Start_node
 
-            print(f"Actionname: {actionname_lower}")
+            print(f"\nActionname: {actionname_lower}")
 
             # 1. Find the action
             baseresult = ""
@@ -549,22 +690,30 @@ class AppBase:
             except KeyError as error:
                 print(f"KeyError in JSON: {error}")
         
-            print(f"After first trycatch")
+            print(f"After first trycatch. Baseresult: ", baseresult)
         
             # 2. Find the JSON data
             if len(baseresult) == 0:
                 return ""+appendresult, False
         
+            print("After second return")
             if len(parsersplit) == 1:
                 return str(baseresult)+str(appendresult), False
         
             baseresult = baseresult.replace("\'", "\"")
+            baseresult = baseresult.replace(" True,", " true,")
+            baseresult = baseresult.replace(" False", " false,")
+
+            print("After third parser return - Formatted: ", baseresult)
             basejson = {}
             try:
                 basejson = json.loads(baseresult)
             except json.decoder.JSONDecodeError as e:
+                print("Parser issue with JSON: %s" % e)
                 return str(baseresult)+str(appendresult), False
 
+            print("After fourth parser return as JSON")
+        
             data, is_loop = recurse_json(basejson, parsersplit[1:])
             parseditem = data
             if is_loop:
@@ -577,6 +726,7 @@ class AppBase:
                     print("SET DATA WRAPPER TO %s!" % parsersplit[-1])
                     parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data))
 
+            print("Before last return with %s" % appendresult)
             return str(parseditem)+str(appendresult), is_loop
 
         # Parses parameters sent to it and returns whether it did it successfully with the values found
@@ -824,26 +974,6 @@ class AppBase:
     
             return True, ""
 
-        # Things to consider for files:
-        # - How can you download / stream a file? 
-        # - Can you decide if you want a stream or the files directly?
-        def get_files(full_execution, value):
-            print("FULL EXEC: %s" % full_execution)
-            org_id = full_execution["workflow"]["execution_org"]["id"]
-            print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value))
-            get_path = "/api/v1/files/%s/content?execution_id=%s" % (value, full_execution["execution_id"])
-
-            print("PATH: %s" % get_path)
-            headers = {
-                "Content-Type": "application/json",     
-                "Authorization": "Bearer %s" % self.authorization
-            }
-            ret = requests.get("%s%s" % (self.url, get_path), headers=headers)
-            print("RET CONTENT: %s" % ret.text)
-            print("RET CODE FILE: %d" % ret.status_code)
-
-	    # r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS")
-
         # Checks whether conditions are met, otherwise set 
         branchcheck, tmpresult = check_branch_conditions(action, fullexecution)
         if not branchcheck:
@@ -911,15 +1041,6 @@ class AppBase:
                         multiexecution = False
                         multi_execution_lists = []
                         for parameter in action["parameters"]:
-                            is_file = False
-                            try:
-                                if parameter["schema"]["type"] == "file":
-                                    print("SHOULD HANDLE FILE. Get based on value %s" % parameter["value"]) 
-                                    get_files(fullexecution, parameter["value"])
-                                    is_file = True
-                            except KeyError as e:
-                                print("SCHEMA ERROR: %s" % e)
-
                             check, value, is_loop = parse_params(action, fullexecution, parameter)
                             if check:
                                 raise "Value check error: %s" % Exception(check)
@@ -967,13 +1088,40 @@ class AppBase:
                                         minlength = len(json_replacement)
 
                                     tmpitem = tmpitem.replace(actualitem[0][0], replacement, 1)
-                                    params[parameter["name"]] = tmpitem
-                                    multi_execution_lists.append(json_replacement)
-                                    multi_parameters[parameter["name"]] = json_replacement 
 
-                                    #print("LENGTH OF ARR: %d" % len(resultarray))
+                                    # This code handles files.
-                                    #print("RESULTARRAY: %s" % resultarray)
+                                    print("(1) ------------ PARAM: %s" % parameter["schema"]["type"])
-                                    print("MULTI finished: %s" % replacement)
+                                    resultarray = []
+                                    isfile = False
+                                    try:
+                                        if parameter["schema"]["type"] == "file" and len(value) > 0:
+                                            print("(1) SHOULD HANDLE FILE IN MULTI. Get based on value %s" % tmpitem) 
+                                            # This is silly :)
+                                            # Q: Is there something wrong with the download system?
+                                            # It seems to return "FILE CONTENT: %s" with the ID as %s
+                                            for tmp_file_split in json.loads(tmpitem):
+                                                print("(1) PRE GET FILE %s" % tmp_file_split)
+                                                file_value = self.get_file(tmp_file_split)
+                                                print("(1) POST AWAIT %s" % file_value)
+                                                resultarray.append(file_value)
+                                                print("(1) FILE VALUE FOR VAL %s: %s" % (tmp_file_split, file_value))
+
+                                            isfile = True
+                                    except KeyError as e:
+                                        print("(1) SCHEMA ERROR IN FILE HANDLING: %s" % e)
+                                    except json.decoder.JSONDecodeError as e:
+                                        print("(1) JSON ERROR IN FILE HANDLING: %s" % e)
+
+                                    if not isfile:
+                                        params[parameter["name"]] = tmpitem
+                                        multi_parameters[parameter["name"]] = json_replacement 
+                                    else:
+                                        print("Resultarray: %s" % resultarray)
+                                        params[parameter["name"]] = resultarray 
+                                        multi_parameters[parameter["name"]] = resultarray 
+
+                                    multi_execution_lists.append(json_replacement)
+                                    print("MULTI finished: %s" % json_replacement)
                                 else:
                                     # This is here to handle for loops within variables.. kindof
                                     # 1. Find the length of the longest array
@@ -1017,7 +1165,30 @@ class AppBase:
                                             #replacement = parse_wrapper_start(replacement)
                                             tmpitem = tmpitem.replace(key, replacement, -1)
 
-                                        resultarray.append(tmpitem)
+
+                                        # This code handles files.
+                                        print("(2) ------------ PARAM: %s" % parameter["schema"]["type"])
+                                        isfile = False
+                                        try:
+                                            if parameter["schema"]["type"] == "file" and len(value) > 0:
+                                                print("(2) SHOULD HANDLE FILE IN MULTI. Get based on value %s" % parameter["value"]) 
+
+                                                for tmp_file_split in json.loads(parameter["value"]):
+                                                    print("(2) PRE GET FILE %s" % tmp_file_split)
+                                                    file_value = self.get_file(tmp_file_split)
+                                                    print("(2) POST AWAIT %s" % file_value)
+                                                    resultarray.append(file_value)
+                                                    print("(2) FILE VALUE FOR VAL %s: %s" % (tmp_file_split, file_value))
+
+
+                                                isfile = True
+                                        except KeyError as e:
+                                            print("(2) SCHEMA ERROR IN FILE HANDLING: %s" % e)
+                                        except json.decoder.JSONDecodeError as e:
+                                            print("(2) JSON ERROR IN FILE HANDLING: %s" % e)
+
+                                        if not isfile:
+                                            resultarray.append(tmpitem)
 
                                     # With this parameter ready, add it to... a greater list of parameters. Rofl
                                     print("LENGTH OF ARR: %d" % len(resultarray))
@@ -1035,6 +1206,18 @@ class AppBase:
                                 params[parameter["name"]] = value
                                 multi_parameters[parameter["name"]] = value 
 
+                                # This code handles files.
+                                try:
+                                    if parameter["schema"]["type"] == "file" and len(value) > 0:
+                                        print("\n SHOULD HANDLE FILE. Get based on value %s. <--- is this a valid ID?" % parameter["value"]) 
+                                        file_value = self.get_file(value)
+                                        print("FILE VALUE: %s \n" % file_value)
+
+                                        params[parameter["name"]] = file_value 
+                                        multi_parameters[parameter["name"]] = file_value 
+                                except KeyError as e:
+                                    print("SCHEMA ERROR IN FILE HANDLING: %s" % e)
+
                         # Fix lists here
                         print("CHECKING multi execution list!")
                         if len(multi_execution_lists) > 0:
@@ -1044,28 +1227,58 @@ class AppBase:
                                 if listitem in filteredlist:
                                     continue
 
-                                filteredlist.append(listitem)
+                                # FIXME: Subsub required?. Recursion! 
+                                # Basically multiply what we have with the outer loop?
+                                # 
+                                if isinstance(listitem, list):
+                                    for subitem in listitem:
+                                        filteredlist.append(subitem)
+                                else:
+                                    filteredlist.append(listitem)
 
                             #print("New list length: %d" % len(filteredlist))
                             if len(filteredlist) > 1:
                                 print("Calculating new multi-loop length with %d lists" % len(filteredlist))
                                 tmplength = 1
                                 for innerlist in filteredlist:
-                                    print("List length: %d. %d*%d" % (len(innerlist), len(innerlist), tmplength))
                                     tmplength = len(innerlist)*tmplength
+                                    print("List length: %d. %d*%d" % (tmplength, len(innerlist), tmplength))
 
                                 minlength = tmplength
 
                                 print("New multi execution length: %d\n" % tmplength)
                         
-                        # FIXME - this is horrible, but works for now
-                        #for i in range(calltimes):
                         if not multiexecution:
                             print("APP_SDK DONE: Starting NORMAL execution of function")
-                            print("Running with params %s" % params) 
+                            print("Running with params (0): %s" % params) 
                             newres = await func(**params)
-                            print("Return from execution: %s" % newres)
+                            print("Returned from execution.")
-                            if isinstance(newres, str):
+                            if isinstance(newres, tuple):
+                                print("Handling return as tuple")
+                                # Handles files.
+                                filedata = ""
+                                file_ids = []
+                                print("TUPLE: %s" % newres[1])
+                                if isinstance(newres[1], list):
+                                    print("HANDLING LIST FROM RET")
+                                    file_ids = self.set_files(newres[1])
+                                elif isinstance(newres[1], object):
+                                    print("Handling JSON from ret")
+                                    file_ids = self.set_files([newres[1]])
+                                elif isinstance(newres[1], str):
+                                    print("Handling STRING from ret")
+                                    file_ids = self.set_files([newres[1]])
+                                else:
+                                    print("NO FILES TO HANDLE")
+
+                                tmp_result = {
+                                    "result": newres[0], 
+                                    "file_ids": file_ids
+                                }
+                                
+                                result = json.dumps(tmp_result)
+                            elif isinstance(newres, str):
+                                print("Handling return as string")
                                 result += newres
                             else:
                                 try:
@@ -1073,16 +1286,22 @@ class AppBase:
                                 except ValueError:
                                     result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres)
                                     print("Can't handle type %s value from function" % (type(newres)))
+
                             print("POST NEWRES RESULT: ", result)
                         else:
-                            print("APP_SDK DONE: Starting MULTI execution with values %s of length %d" % (multi_parameters, minlength))
+                            print("APP_SDK DONE: Starting MULTI execution (length: %d) with values %s" % (minlength, multi_parameters))
                             # 1. Use number of executions based on the arrays being similar
                             # 2. Find the right value from the parsed multi_params
                             results = []
                             json_object = False
                             for i in range(0, minlength):
                                 # To be able to use the results as a list:
-                                baseparams = json.loads(json.dumps(multi_parameters))
+                                print("1: %s" % multi_parameters)
+                                #baseparams = json.loads(json.dumps(multi_parameters))
+                                baseparams = copy.deepcopy(multi_parameters)
+
+                                print("2: %s: %s" % (type(baseparams), baseparams))
+
                                 # {'call': ['GoogleSafebrowsing_2_0', 'VirusTotal_GetReport_3_0']}
                                 # 1. Check if list length is same as minlength
                                 # 2. If NOT same length, duplicate based on length of array
@@ -1093,7 +1312,7 @@ class AppBase:
                                 try:
                                     firstlist = True
                                     for key, value in baseparams.items():
-
+                                        print("Itemtype: %s" % type(value))
                                         if isinstance(value, list):
                                             try:
                                                 newvalue = value[i]
@@ -1136,6 +1355,8 @@ class AppBase:
                                                 firstlist = False
 
                                             baseparams[key] = newvalue
+
+                                    print("3")
                                 except IndexError as e:
                                     print("IndexError: %s" % e)
                                     baseparams[key] = "IndexError: %s" % e
@@ -1143,7 +1364,9 @@ class AppBase:
                                     print("KeyError: %s" % e)
                                     baseparams[key] = "KeyError: %s" % e
 
-                                print("Running with params %s" % baseparams) 
+
+                                print("4")
+                                print("Running with params (1): %s" % baseparams) 
                                 ret = await func(**baseparams)
                                 print("Return from execution: %s" % ret)
                                 if isinstance(ret, dict) or isinstance(ret, list):

backend/app_sdk/build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 NAME=shuffle-app_sdk
-VERSION=0.8.0
+VERSION=0.8.2
 
 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
 docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION

backend/go-app/docker.go

@@ -181,11 +181,11 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
 	// Dockerfile is inside the TAR itself. Not local context
 	// docker build --build-arg http_proxy=http://my.proxy.url
 	buildOptions := types.ImageBuildOptions{
-		Remove:      true,
+		Remove:    true,
-		Tags:        tags,
+		Tags:      tags,
-		BuildArgs:   map[string]*string{},
+		BuildArgs: map[string]*string{},
-		NetworkMode: "host",
 	}
+	// NetworkMode: "host",
 
 	httpProxy := os.Getenv("HTTP_PROXY")
 	if len(httpProxy) > 0 {
@@ -244,11 +244,11 @@ func buildImage(tags []string, dockerfileFolder string) error {
 
 	dockerFileTarReader := bytes.NewReader(buf.Bytes())
 	buildOptions := types.ImageBuildOptions{
-		Remove:      true,
+		Remove:    true,
-		Tags:        tags,
+		Tags:      tags,
-		BuildArgs:   map[string]*string{},
+		BuildArgs: map[string]*string{},
-		NetworkMode: "host",
 	}
+	//NetworkMode: "host",
 
 	httpProxy := os.Getenv("HTTP_PROXY")
 	if len(httpProxy) > 0 {

backend/go-app/files.go

@@ -0,0 +1,764 @@
+package main
+
+/*
+	Handles files within Workflows.of Shuffle
+*/
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"cloud.google.com/go/datastore"
+	"github.com/satori/go.uuid"
+)
+
+type File struct {
+	Id           string   `json:"id" datastore:"id"`
+	Type         string   `json:"type" datastore:"type"`
+	CreatedAt    int64    `json:"created_at" datastore:"created_at"`
+	UpdatedAt    int64    `json:"updated_at" datastore:"updated_at"`
+	MetaAccessAt int64    `json:"meta_access_at" datastore:"meta_access_at"`
+	DownloadAt   int64    `json:"last_downloaded" datastore:"last_downloaded"`
+	Description  string   `json:"description" datastore:"description"`
+	ExpiresAt    string   `json:"expires_at" datastore:"expires_at"`
+	Status       string   `json:"status" datastore:"status"`
+	Filename     string   `json:"filename" datastore:"filename"`
+	URL          string   `json:"url" datastore:"org"`
+	OrgId        string   `json:"org_id" datastore:"org_id"`
+	WorkflowId   string   `json:"workflow_id" datastore:"workflow_id"`
+	Workflows    []string `json:"workflows" datastore:"workflows"`
+	DownloadPath string   `json:"download_path" datastore:"download_path"`
+	Md5sum       string   `json:"md5_sum" datastore:"md5_sum"`
+	Sha256sum    string   `json:"sha256_sum" datastore:"sha256_sum"`
+}
+
+var basepath = os.Getenv("SHUFFLE_FILE_LOCATION")
+
+func fileAuthentication(request *http.Request) (string, error) {
+	executionId, ok := request.URL.Query()["execution_id"]
+	if ok && len(executionId) > 0 {
+		ctx := context.Background()
+		workflowExecution, err := getWorkflowExecution(ctx, executionId[0])
+		if err != nil {
+			log.Printf("[ERROR] Couldn't find execution ID %s", executionId[0])
+			return "", err
+		}
+
+		apikey := request.Header.Get("Authorization")
+		if !strings.HasPrefix(apikey, "Bearer ") {
+			log.Printf("[ERROR} Apikey doesn't start with bearer (2)")
+			return "", errors.New("No auth key found")
+		}
+
+		apikeyCheck := strings.Split(apikey, " ")
+		if len(apikeyCheck) != 2 {
+			log.Printf("[ERROR] Invalid format for apikey (2)")
+			return "", errors.New("No space in authkey")
+		}
+
+		// This is annoying af and is done because of maxlength lol
+		newApikey := apikeyCheck[1]
+		if newApikey != workflowExecution.Authorization {
+			//log.Printf("[ERROR] Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization)
+			log.Printf("[ERROR] Bad apikey for execution %s.", executionId[0])
+			//%s vs %s", executionId[0], apikey, workflowExecution.Authorization)
+			return "", errors.New("Bad authorization key")
+		}
+
+		log.Printf("[INFO] Authorization is correct for execution %s!", executionId[0])
+		//%s vs %s. Setting Org", executionId, apikey, workflowExecution.Authorization)
+		if len(workflowExecution.ExecutionOrg) > 0 {
+			return workflowExecution.ExecutionOrg, nil
+		} else if len(workflowExecution.Workflow.ExecutingOrg.Id) > 0 {
+			return workflowExecution.ExecutionOrg, nil
+		} else {
+			log.Printf("[ERROR] Couldn't find org for workflow execution, but auth was correct.")
+		}
+	}
+
+	return "", errors.New("No execution id specified")
+}
+
+// https://golangcode.com/check-if-a-file-exists/
+func fileExists(filename string) bool {
+	info, err := os.Stat(filename)
+	if os.IsNotExist(err) {
+		return false
+	}
+	return !info.IsDir()
+}
+
+func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) {
+	cors := handleCors(resp, request)
+	if cors {
+		return
+	}
+
+	var fileId string
+	location := strings.Split(request.URL.String(), "/")
+	if location[1] == "api" {
+		if len(location) <= 4 {
+			log.Printf("[INFO] Path too short: %d", len(location))
+			resp.WriteHeader(401)
+			resp.Write([]byte(`{"success": false}`))
+			return
+		}
+
+		fileId = location[4]
+	}
+
+	if strings.Contains(fileId, "?") {
+		fileId = strings.Split(fileId, "?")[0]
+	}
+
+	if len(fileId) != 36 {
+		log.Printf("Bad format for fileId %s", fileId)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
+		return
+	}
+
+	log.Printf("\n\n[INFO] User is trying to GET File Meta for %s\n\n", fileId)
+
+	// 1. Check user directly
+	// 2. Check workflow execution authorization
+	user, err := handleApiAuthentication(resp, request)
+	if err != nil {
+		log.Printf("[INFO] INITIAL Api authentication failed in file deletion: %s", err)
+
+		orgId, err := fileAuthentication(request)
+		if err != nil {
+			log.Printf("[ERROR] Bad file authentication in get: %s", err)
+			resp.WriteHeader(401)
+			resp.Write([]byte(`{"success": false}`))
+			return
+		}
+
+		user.ActiveOrg.Id = orgId
+		user.Username = "Execution File API"
+	}
+
+	// 1. Verify if the user has access to the file: org_id and workflow
+	log.Printf("[INFO] Should GET FILE META for %s if user has access", fileId)
+	ctx := context.Background()
+	file, err := getFile(ctx, fileId)
+	if err != nil {
+		log.Printf("[INFO] File %s not found: %s", fileId, err)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false}`))
+		return
+	}
+
+	found := false
+	if file.OrgId == user.ActiveOrg.Id {
+		found = true
+	} else {
+		for _, item := range user.Orgs {
+			if item == file.OrgId {
+				found = true
+				break
+			}
+		}
+	}
+
+	if !found {
+		log.Printf("[INFO] User %s doesn't have access to %s", user.Username, fileId)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false}`))
+		return
+	}
+
+	newBody, err := json.Marshal(file)
+	if err != nil {
+		resp.WriteHeader(500)
+		resp.Write([]byte(`{"success": false, "reason": "Failed to marshal filedata"}`))
+		return
+	}
+
+	log.Printf("[INFO] Successfully got file meta for %s", fileId)
+	resp.WriteHeader(200)
+	resp.Write([]byte(newBody))
+}
+
+func handleDeleteFile(resp http.ResponseWriter, request *http.Request) {
+	cors := handleCors(resp, request)
+	if cors {
+		return
+	}
+
+	var fileId string
+	location := strings.Split(request.URL.String(), "/")
+	if location[1] == "api" {
+		if len(location) <= 4 {
+			log.Printf("[INFO] Path too short: %d", len(location))
+			resp.WriteHeader(401)
+			resp.Write([]byte(`{"success": false}`))
+			return
+		}
+
+		fileId = location[4]
+	}
+
+	if strings.Contains(fileId, "?") {
+		fileId = strings.Split(fileId, "?")[0]
+	}
+
+	if len(fileId) != 36 {
+		log.Printf("Bad format for fileId %s", fileId)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
+		return
+	}
+
+	log.Printf("\n\n[INFO] User is trying to delete file %s\n\n", fileId)
+
+	// 1. Check user directly
+	// 2. Check workflow execution authorization
+	user, err := handleApiAuthentication(resp, request)
+	if err != nil {
+		log.Printf("[INFO] INITIAL Api authentication failed in file deletion: %s", err)
+
+		orgId, err := fileAuthentication(request)
+		if err != nil {
+			log.Printf("[ERROR] Bad file authentication in get: %s", err)
+			resp.WriteHeader(401)
+			resp.Write([]byte(`{"success": false}`))
+			return
+		}
+
+		user.ActiveOrg.Id = orgId
+		user.Username = "Execution File API"
+	}
+
+	// 1. Verify if the user has access to the file: org_id and workflow
+	log.Printf("[INFO] Should DELETE file %s if user has access", fileId)
+	ctx := context.Background()
+	file, err := getFile(ctx, fileId)
+	if err != nil {
+		log.Printf("[INFO] File %s not found: %s", fileId, err)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false}`))
+		return
+	}
+
+	found := false
+	if file.OrgId == user.ActiveOrg.Id {
+		found = true
+	} else {
+		for _, item := range user.Orgs {
+			if item == file.OrgId {
+				found = true
+				break
+			}
+		}
+	}
+
+	if !found {
+		log.Printf("[INFO] User %s doesn't have access to %s", user.Username, fileId)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false}`))
+		return
+	}
+
+	if file.Status == "deleted" {
+		log.Printf("[INFO] File with ID %s is already deleted.", fileId)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false}`))
+		return
+	}
+
+	if fileExists(file.DownloadPath) {
+		err = os.Remove(file.DownloadPath)
+		if err != nil {
+			log.Printf("[ERROR] Failed deleting file locally: %s", err)
+			resp.WriteHeader(401)
+			resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting filein path %s"}`, file.DownloadPath)))
+			return
+		}
+
+		log.Printf("[INFO] Deleted file %s locally. Next is database.", file.DownloadPath)
+	} else {
+		log.Printf("[ERROR] File doesn't exist. Can't delete. Should maybe delete file anyway?")
+		resp.WriteHeader(200)
+		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "File in location %s doesn't exist"}`, file.DownloadPath)))
+		return
+	}
+
+	file.Status = "deleted"
+	err = setFile(ctx, *file)
+	if err != nil {
+		log.Printf("[ERROR] Failed setting file to deleted")
+		resp.WriteHeader(500)
+		resp.Write([]byte(`{"success": false, "reason": "Failed setting file to deleted"}`))
+		return
+	}
+
+	/*
+		//Actually delete it?
+		err = DeleteKey(ctx, "files", fileId)
+		if err != nil {
+			log.Printf("Failed deleting file with ID %s: %s", fileId, err)
+			resp.WriteHeader(401)
+			resp.Write([]byte(`{"success": false}`))
+			return
+		}
+	*/
+
+	log.Printf("[INFO] Successfully deleted file %s", fileId)
+	resp.WriteHeader(200)
+	resp.Write([]byte(`{"success": true}`))
+}
+
+func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
+	cors := handleCors(resp, request)
+	if cors {
+		return
+	}
+
+	var fileId string
+	location := strings.Split(request.URL.String(), "/")
+	if location[1] == "api" {
+		if len(location) <= 4 {
+			log.Printf("Path too short: %d", len(location))
+			resp.WriteHeader(401)
+			resp.Write([]byte(`{"success": false}`))
+			return
+		}
+
+		fileId = location[4]
+	}
+
+	if len(fileId) != 36 {
+		log.Printf("Bad format for fileId %s", fileId)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
+		return
+	}
+
+	log.Printf("\n\nUser is trying to download file %s\n\n", fileId)
+
+	// 1. Check user directly
+	// 2. Check workflow execution authorization
+	user, err := handleApiAuthentication(resp, request)
+	if err != nil {
+		log.Printf("INITIAL Api authentication failed in file download: %s", err)
+
+		orgId, err := fileAuthentication(request)
+		if err != nil {
+			log.Printf("Bad file authentication in get: %s", err)
+			resp.WriteHeader(401)
+			resp.Write([]byte(`{"success": false}`))
+			return
+		}
+
+		user.ActiveOrg.Id = orgId
+		user.Username = "Execution File API"
+		/*
+			} else {
+				resp.WriteHeader(401)
+				resp.Write([]byte(`{"success": false}`))
+				return
+			}
+		*/
+	}
+
+	// 1. Verify if the user has access to the file: org_id and workflow
+	log.Printf("[INFO] Should get file %s", fileId)
+	ctx := context.Background()
+	file, err := getFile(ctx, fileId)
+	if err != nil {
+		log.Printf("[ERROR] File %s not found: %s", fileId, err)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false}`))
+		return
+	}
+
+	found := false
+	if file.OrgId == user.ActiveOrg.Id {
+		found = true
+	} else {
+		for _, item := range user.Orgs {
+			if item == file.OrgId {
+				found = true
+				break
+			}
+		}
+	}
+
+	if !found {
+		log.Printf("User %s doesn't have access to %s", user.Username, fileId)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false}`))
+		return
+	}
+
+	if file.Status != "active" {
+		log.Printf("[ERROR] File status isn't active, but %s. Can't continue.", file.Status)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "The file isn't ready to be downloaded yet. Status required: active"}`))
+		return
+	}
+
+	// Fixme: More auth: org and workflow!
+	downloadPath := file.DownloadPath
+	log.Printf("[INFO] Downloadpath: %s", downloadPath)
+	Openfile, err := os.Open(downloadPath)
+	defer Openfile.Close() //Close after function return
+	if err != nil {
+		//File not found, send 404
+		http.Error(resp, "File not found.", 404)
+		return
+	}
+
+	//File is found, create and send the correct headers
+	//Get the Content-Type of the file
+	//Create a buffer to store the header of the file in
+	FileHeader := make([]byte, 512)
+	//Copy the headers into the FileHeader buffer
+	Openfile.Read(FileHeader)
+	//Get content type of file
+	FileContentType := http.DetectContentType(FileHeader)
+
+	//Get the file size
+	FileStat, _ := Openfile.Stat()                     //Get info from file
+	FileSize := strconv.FormatInt(FileStat.Size(), 10) //Get file size as a string
+
+	//Send the headers
+	resp.Header().Set("Content-Disposition", "attachment; filename="+fileId)
+	resp.Header().Set("Content-Type", FileContentType)
+	resp.Header().Set("Content-Length", FileSize)
+
+	//Send the file
+	//We read 512 bytes from the file already, so we reset the offset back to 0
+	Openfile.Seek(0, 0)
+	io.Copy(resp, Openfile) //'Copy' the file to the client
+	return
+
+	//log.Printf("Should download file %s", downloadPath)
+}
+func handleUploadFile(resp http.ResponseWriter, request *http.Request) {
+	cors := handleCors(resp, request)
+	if cors {
+		return
+	}
+
+	var fileId string
+	location := strings.Split(request.URL.String(), "/")
+	if location[1] == "api" {
+		if len(location) <= 4 {
+			log.Printf("Path too short: %d", len(location))
+			resp.WriteHeader(401)
+			resp.Write([]byte(`{"success": false}`))
+			return
+		}
+
+		fileId = location[4]
+	}
+
+	if len(fileId) != 36 {
+		log.Printf("Bad format for fileId %s", fileId)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
+		return
+	}
+
+	// 1. Check user directly
+	// 2. Check workflow execution authorization
+	user, err := handleApiAuthentication(resp, request)
+	if err != nil {
+		log.Printf("INITIAL Api authentication failed in file upload: %s", err)
+
+		orgId, err := fileAuthentication(request)
+		if err != nil {
+			log.Printf("Bad file authentication in create file: %s", err)
+			resp.WriteHeader(401)
+			resp.Write([]byte(`{"success": false}`))
+			return
+		}
+
+		user.ActiveOrg.Id = orgId
+		user.Username = "Execution File API"
+	}
+
+	log.Printf("[INFO] Should UPLOAD file %s if user has access", fileId)
+	ctx := context.Background()
+	file, err := getFile(ctx, fileId)
+	if err != nil {
+		log.Printf("File %s not found: %s", fileId, err)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false}`))
+		return
+	}
+
+	found := false
+	if file.OrgId == user.ActiveOrg.Id {
+		found = true
+	} else {
+		for _, item := range user.Orgs {
+			if item == file.OrgId {
+				found = true
+				break
+			}
+		}
+	}
+
+	if !found {
+		log.Printf("User %s doesn't have access to %s", user.Username, fileId)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false}`))
+		return
+	}
+
+	log.Printf("[INFO] STATUS: %s", file.Status)
+	if file.Status != "created" {
+		log.Printf("File status isn't created. Can't upload.")
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "This file already has data."}`))
+		return
+	}
+
+	request.ParseMultipartForm(32 << 20)
+	parsedFile, _, err := request.FormFile("shuffle_file")
+	if err != nil {
+		log.Printf("[ERROR] Couldn't upload file: %s", err)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "Failed uploading file"}`))
+		return
+	}
+	defer parsedFile.Close()
+
+	file.Status = "uploading"
+	err = setFile(ctx, *file)
+	if err != nil {
+		log.Printf("Failed setting file to uploading")
+		resp.WriteHeader(500)
+		resp.Write([]byte(`{"success": false, "reason": "Failed setting file to uploading"}`))
+		return
+	}
+
+	// Can be used for validation files for change
+	var buf bytes.Buffer
+	io.Copy(&buf, parsedFile)
+	contents := buf.Bytes()
+	md5 := md5sum(contents)
+	buf.Reset()
+
+	sha256Sum := sha256.Sum256(contents)
+	//parsedFile.Reset()
+
+	f, err := os.OpenFile(file.DownloadPath, os.O_WRONLY|os.O_CREATE, os.ModePerm)
+	if err != nil {
+		// Rolling back file
+		file.Status = "created"
+		setFile(ctx, *file)
+
+		log.Printf("[ERROR] Failed uploading and creating file: %s", err)
+		resp.WriteHeader(500)
+		resp.Write([]byte(`{"success": false}`))
+		return
+	}
+
+	defer f.Close()
+	parsedFile.Seek(0, io.SeekStart)
+	io.Copy(f, parsedFile)
+
+	// FIXME: Set this one to 200 anyway? Can't download file then tho..
+	file.Status = "active"
+	file.Md5sum = md5
+	file.Sha256sum = fmt.Sprintf("%x", sha256Sum)
+	log.Printf("[INFO] MD5 for file %s (%s) is %s and SHA256 is %s", file.Filename, file.Id, file.Md5sum, file.Sha256sum)
+
+	err = setFile(ctx, *file)
+	if err != nil {
+		log.Printf("[ERROR] Failed setting file back to active")
+		resp.WriteHeader(500)
+		resp.Write([]byte(`{"success": false, "reason": "Failed setting file to active"}`))
+		return
+	}
+
+	log.Printf("[INFO] Successfully uploaded file ID %s", file.Id)
+	resp.WriteHeader(200)
+	resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
+}
+
+func handleCreateFile(resp http.ResponseWriter, request *http.Request) {
+	cors := handleCors(resp, request)
+	if cors {
+		return
+	}
+
+	// 1. Check user directly
+	// 2. Check workflow execution authorization
+	user, err := handleApiAuthentication(resp, request)
+	if err != nil {
+		log.Printf("[INFO] INITIAL Api authentication failed in file creation: %s", err)
+
+		orgId, err := fileAuthentication(request)
+		if err != nil {
+			log.Printf("[ERROR] Bad file authentication in create file: %s", err)
+			resp.WriteHeader(401)
+			resp.Write([]byte(`{"success": false}`))
+			return
+		}
+
+		user.ActiveOrg.Id = orgId
+		user.Username = "Execution File API"
+	}
+
+	body, err := ioutil.ReadAll(request.Body)
+	if err != nil {
+		log.Println("Failed reading body")
+		resp.WriteHeader(401)
+		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to read data"}`)))
+		return
+	}
+
+	type FileStructure struct {
+		Filename   string `json:"filename"`
+		OrgId      string `json:"org_id"`
+		WorkflowId string `json:"workflow_id"`
+	}
+
+	var curfile FileStructure
+	err = json.Unmarshal(body, &curfile)
+	if err != nil {
+		log.Printf("[ERROR] Failed unmarshaling: %s", err)
+		resp.WriteHeader(401)
+		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to unmarshal data"}`)))
+		return
+	}
+
+	// Loads of validation below
+	if len(curfile.Filename) == 0 || len(curfile.OrgId) == 0 || len(curfile.WorkflowId) == 0 {
+		log.Printf("[ERROR] Missing field during upload.")
+		resp.WriteHeader(401)
+		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Missing field. Required: filename, org_id, workflow_id"}`)))
+		return
+	}
+
+	ctx := context.Background()
+	if user.ActiveOrg.Id != curfile.OrgId {
+		log.Printf("[ERROR] User can't access org %s", curfile.OrgId)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "Error with organization"}`))
+		return
+	}
+
+	// Try to get the org and workflow in case they don't exist
+	workflow, err := getWorkflow(ctx, curfile.WorkflowId)
+	if err != nil {
+		log.Printf("[ERROR] Workflow %s doesn't exist.", curfile.WorkflowId)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`))
+		return
+	}
+
+	_, err = getOrg(ctx, curfile.OrgId)
+	if err != nil {
+		log.Printf("[ERROR] Org %s doesn't exist.", curfile.OrgId)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`))
+		return
+	}
+
+	if workflow.ExecutingOrg.Id != curfile.OrgId {
+		found := false
+		for _, curorg := range workflow.Org {
+			if curorg.Id == curfile.OrgId {
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			log.Printf("[ERROR] Org %s doesn't have access to %s.", curfile.OrgId, curfile.WorkflowId)
+			resp.WriteHeader(401)
+			resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`))
+			return
+		}
+	}
+
+	if strings.Contains(curfile.Filename, "/") || strings.Contains(curfile.Filename, `"`) || strings.Contains(curfile.Filename, "..") || strings.Contains(curfile.Filename, "~") {
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "Invalid characters in filename"}`))
+		return
+	}
+
+	// 1. Create the file object.
+	if len(basepath) == 0 {
+		basepath = "shuffle-files"
+	}
+	folderPath := fmt.Sprintf("%s/%s/%s", basepath, curfile.OrgId, curfile.WorkflowId)
+
+	// Try to make the full file location
+	err = os.MkdirAll(folderPath, os.ModePerm)
+	if err != nil {
+		log.Printf("[ERROR] Writing issue for file location creation: %s", err)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "Failed creating upload location"}`))
+		return
+	}
+
+	filename := curfile.Filename
+	fileId := uuid.NewV4().String()
+	downloadPath := fmt.Sprintf("%s/%s", folderPath, fileId)
+
+	timeNow := time.Now().Unix()
+	newFile := File{
+		Id:           fileId,
+		CreatedAt:    timeNow,
+		UpdatedAt:    timeNow,
+		Description:  "",
+		Status:       "created",
+		Filename:     filename,
+		OrgId:        curfile.OrgId,
+		WorkflowId:   curfile.WorkflowId,
+		DownloadPath: downloadPath,
+	}
+
+	err = setFile(ctx, newFile)
+	if err != nil {
+		log.Printf("[ERROR] Failed setting file: %s", err)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "Failed setting file reference"}`))
+		return
+	} else {
+		log.Printf("[INFO] Created file %s", newFile.DownloadPath)
+	}
+
+	resp.WriteHeader(200)
+	resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, fileId)))
+}
+
+func getFile(ctx context.Context, id string) (*File, error) {
+	key := datastore.NameKey("Files", id, nil)
+	curFile := &File{}
+	if err := dbclient.Get(ctx, key, curFile); err != nil {
+		return &File{}, err
+	}
+
+	return curFile, nil
+}
+
+func setFile(ctx context.Context, file File) error {
+	// clear session_token and API_token for user
+	k := datastore.NameKey("Files", file.Id, nil)
+	if _, err := dbclient.Put(ctx, k, &file); err != nil {
+		log.Println(err)
+		return err
+	}
+
+	return nil
+}

backend/go-app/main.go

@@ -2422,7 +2422,7 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	if count == 0 {
-		log.Printf("No users - redirecting for management user")
+		log.Printf("[WARNING] No users - redirecting for management user")
 		resp.WriteHeader(200)
 		resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "stay"}`)))
 		return
@@ -2555,12 +2555,12 @@ func getApikey(ctx context.Context, apikey string) (User, error) {
 	var users []User
 	_, err := dbclient.GetAll(ctx, q, &users)
 	if err != nil {
-		log.Printf("Error getting users apikey (getapikey): %s", err)
+		log.Printf("[ERROR] Error getting users apikey (getapikey): %s", err)
 		return User{}, err
 	}
 
 	if len(users) == 0 {
-		log.Printf("No users found for apikey %s", apikey)
+		log.Printf("[WARNING] No users found for apikey %s", apikey)
 		return User{}, err
 	}
 
@@ -2577,28 +2577,6 @@ func getSession(ctx context.Context, thissession string) (*session, error) {
 	return curUser, nil
 }
 
-// ListBooks returns a list of books, ordered by title.
-func getFile(ctx context.Context, id string) (*File, error) {
-	key := datastore.NameKey("Files", id, nil)
-	curFile := &File{}
-	if err := dbclient.Get(ctx, key, curFile); err != nil {
-		return &File{}, err
-	}
-
-	return curFile, nil
-}
-
-func setFile(ctx context.Context, file File) error {
-	// clear session_token and API_token for user
-	k := datastore.NameKey("Files", file.Id, nil)
-	if _, err := dbclient.Put(ctx, k, &file); err != nil {
-		log.Println(err)
-		return err
-	}
-
-	return nil
-}
-
 // ListBooks returns a list of books, ordered by title.
 func getOrg(ctx context.Context, id string) (*Org, error) {
 	key := datastore.NameKey("Organizations", id, nil)
@@ -6552,7 +6530,7 @@ func handleAppHotload(location string, forceUpdate bool) error {
 	}
 
 	//log.Printf("Reading app folder: %#v", dir)
-	err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate)
+	_, _, err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate)
 	if err != nil {
 		log.Printf("Err: %s", err)
 		return err
@@ -6835,7 +6813,7 @@ func remoteOrgJobHandler(org Org, interval int) error {
 
 	respBody, err := ioutil.ReadAll(newresp.Body)
 	if err != nil {
-		log.Printf("Failed body read in job sync: %s", err)
+		log.Printf("[ERROR] Failed body read in job sync: %s", err)
 		return err
 	}
 
@@ -6843,7 +6821,7 @@ func remoteOrgJobHandler(org Org, interval int) error {
 
 	err = remoteOrgJobController(org, respBody)
 	if err != nil {
-		log.Printf("Failed job controller run: %s", err)
+		log.Printf("[ERROR] Failed job controller run for %s: %s", respBody, err)
 		return err
 	}
 	return nil
@@ -7141,37 +7119,39 @@ func runInit(ctx context.Context) {
 			}
 		}
 
-		fileq := datastore.NewQuery("Files").Limit(1)
+		/*
-		count, err := dbclient.Count(ctx, fileq)
+			fileq := datastore.NewQuery("Files").Limit(1)
-		log.Printf("FILECOUNT: %d", count)
+			count, err := dbclient.Count(ctx, fileq)
-		if err == nil && count < 10 {
+			log.Printf("FILECOUNT: %d", count)
-			basepath := "."
+			if err == nil && count < 10 {
-			filename := "testfile.txt"
+				basepath := "."
-			fileId := uuid.NewV4().String()
+				filename := "testfile.txt"
-			log.Printf("Creating new file reference %s because none exist!", fileId)
+				fileId := uuid.NewV4().String()
-			workflowId := "2cf1169d-b460-41de-8c36-28b2092866f8"
+				log.Printf("Creating new file reference %s because none exist!", fileId)
-			downloadPath := fmt.Sprintf("%s/%s/%s/%s", basepath, activeOrgs[0].Id, workflowId, fileId)
+				workflowId := "2cf1169d-b460-41de-8c36-28b2092866f8"
+				downloadPath := fmt.Sprintf("%s/%s/%s/%s", basepath, activeOrgs[0].Id, workflowId, fileId)
 
-			timeNow := time.Now().Unix()
+				timeNow := time.Now().Unix()
-			newFile := File{
+				newFile := File{
-				Id:           fileId,
+					Id:           fileId,
-				CreatedAt:    timeNow,
+					CreatedAt:    timeNow,
-				UpdatedAt:    timeNow,
+					UpdatedAt:    timeNow,
-				Description:  "Created by system for testing",
+					Description:  "Created by system for testing",
-				Status:       "active",
+					Status:       "active",
-				Filename:     filename,
+					Filename:     filename,
-				OrgId:        activeOrgs[0].Id,
+					OrgId:        activeOrgs[0].Id,
-				WorkflowId:   workflowId,
+					WorkflowId:   workflowId,
-				DownloadPath: downloadPath,
+					DownloadPath: downloadPath,
-			}
+				}
 
-			err = setFile(ctx, newFile)
+				err = setFile(ctx, newFile)
-			if err != nil {
+				if err != nil {
-				log.Printf("Failed setting file: %s", err)
+					log.Printf("Failed setting file: %s", err)
-			} else {
+				} else {
-				log.Printf("Created file %s in init", newFile.DownloadPath)
+					log.Printf("Created file %s in init", newFile.DownloadPath)
+				}
 			}
-		}
+		*/
 
 		var allworkflowapps []AppAuthenticationStorage
 		q = datastore.NewQuery("workflowappauth")
@@ -7934,172 +7914,6 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) {
 	resp.Write(respBody)
 }
 
-type File struct {
-	Id           string   `json:"id" datastore:"id"`
-	Type         string   `json:"type" datastore:"type"`
-	CreatedAt    int64    `json:"created_at" datastore:"created_at"`
-	UpdatedAt    int64    `json:"updated_at" datastore:"updated_at"`
-	Description  string   `json:"description" datastore:"description"`
-	ExpiresAt    string   `json:"expires_at" datastore:"expires_at"`
-	Status       string   `json:"status" datastore:"status"`
-	Filename     string   `json:"filename" datastore:"filename"`
-	URL          string   `json:"url" datastore:"org"`
-	OrgId        string   `json:"org_id" datastore:"org_id"`
-	WorkflowId   string   `json:"workflow_id" datastore:"workflow_id"`
-	Workflows    []string `json:"workflows" datastore:"workflows"`
-	DownloadPath string   `json:"download_path" datastore:"download_path"`
-}
-
-func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
-	cors := handleCors(resp, request)
-	if cors {
-		return
-	}
-
-	var fileId string
-	location := strings.Split(request.URL.String(), "/")
-	if location[1] == "api" {
-		if len(location) <= 4 {
-			log.Printf("Path too short: %d", len(location))
-			resp.WriteHeader(401)
-			resp.Write([]byte(`{"success": false}`))
-			return
-		}
-
-		fileId = location[4]
-	}
-
-	log.Printf("\n\nUser is trying to get file %s\n\n", fileId)
-
-	// 1. Check user directly
-	// 2. Check workflow execution authorization
-	setOrgId := false
-	user, err := handleApiAuthentication(resp, request)
-	if err != nil {
-		// r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS")
-		log.Printf("INITIAL Api authentication failed in file download: %s", err)
-		executionId, ok := request.URL.Query()["execution_id"]
-		if ok && len(executionId) > 0 {
-			ctx := context.Background()
-			workflowExecution, err := getWorkflowExecution(ctx, executionId[0])
-			if err != nil {
-				log.Printf("Couldn't find execution ID %s", executionId[0])
-				resp.WriteHeader(401)
-				resp.Write([]byte(`{"success": false}`))
-				return
-			}
-
-			apikey := request.Header.Get("Authorization")
-			if !strings.HasPrefix(apikey, "Bearer ") {
-				log.Printf("Apikey doesn't start with bearer (2)")
-				resp.WriteHeader(401)
-				resp.Write([]byte(`{"success": false}`))
-				return
-			}
-
-			apikeyCheck := strings.Split(apikey, " ")
-			if len(apikeyCheck) != 2 {
-				log.Printf("Invalid format for apikey (2)")
-				resp.WriteHeader(401)
-				resp.Write([]byte(`{"success": false}`))
-				return
-			}
-
-			// This is annoying af and is done because of maxlength lol
-			newApikey := apikeyCheck[1]
-			if newApikey != workflowExecution.Authorization {
-				log.Printf("Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization)
-				resp.WriteHeader(401)
-				resp.Write([]byte(`{"success": false}`))
-				return
-			}
-
-			log.Printf("Authorization is correct for execution %s! %s vs %s", executionId, apikey, workflowExecution.Authorization)
-			setOrgId = true
-		} else {
-
-			resp.WriteHeader(401)
-			resp.Write([]byte(`{"success": false}`))
-			return
-		}
-	}
-
-	// 1. Verify if the user has access to the file: org_id and workflow
-
-	log.Printf("Should get file %s", fileId)
-	ctx := context.Background()
-	file, err := getFile(ctx, fileId)
-	if err != nil {
-		log.Printf("File %s not found: %s", fileId, err)
-		resp.WriteHeader(401)
-		resp.Write([]byte(`{"success": false}`))
-		return
-	}
-
-	// This is a workaround for file grabs from an app
-	if setOrgId == true {
-		user.ActiveOrg.Id = file.OrgId
-	}
-
-	found := false
-	if file.OrgId == user.ActiveOrg.Id {
-		found = true
-	} else {
-		for _, item := range user.Orgs {
-			if item == file.OrgId {
-				found = true
-				break
-			}
-		}
-	}
-
-	if !found {
-		log.Printf("User %s doesn't have access to %s", user.Username, fileId)
-		resp.WriteHeader(401)
-		resp.Write([]byte(`{"success": false}`))
-		return
-	}
-
-	// Fixme: More auth: org and workflow!
-	downloadPath := file.DownloadPath
-	log.Printf("Downloadpath: %s", downloadPath)
-	Openfile, err := os.Open(downloadPath)
-	defer Openfile.Close() //Close after function return
-	if err != nil {
-		//File not found, send 404
-		http.Error(resp, "File not found.", 404)
-		return
-	}
-
-	//File is found, create and send the correct headers
-	//Get the Content-Type of the file
-	//Create a buffer to store the header of the file in
-	FileHeader := make([]byte, 512)
-	//Copy the headers into the FileHeader buffer
-	Openfile.Read(FileHeader)
-	//Get content type of file
-	FileContentType := http.DetectContentType(FileHeader)
-
-	//Get the file size
-	FileStat, _ := Openfile.Stat()                     //Get info from file
-	FileSize := strconv.FormatInt(FileStat.Size(), 10) //Get file size as a string
-
-	//Send the headers
-	resp.Header().Set("Content-Disposition", "attachment; filename="+fileId)
-	resp.Header().Set("Content-Type", FileContentType)
-	resp.Header().Set("Content-Length", FileSize)
-
-	//Send the file
-	//We read 512 bytes from the file already, so we reset the offset back to 0
-	Openfile.Seek(0, 0)
-	io.Copy(resp, Openfile) //'Copy' the file to the client
-	return
-
-	//log.Printf("Should download file %s", downloadPath)
-	//resp.WriteHeader(200)
-	//resp.Write([]byte("OK"))
-}
-
 func initHandlers() {
 	var err error
 	ctx := context.Background()
@@ -8233,10 +8047,10 @@ func initHandlers() {
 	// https://developer.box.com/reference/get-files-id-content/
 	// 1. Creating the "get file" option. Make it possible to run this in the frontend.
 	r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS")
-	//r.HandleFunc("/api/v1/files/create", handleGetFile).Methods("POST", "OPTIONS")
+	r.HandleFunc("/api/v1/files/create", handleCreateFile).Methods("POST", "OPTIONS")
-	//r.HandleFunc("/api/v1/files/upload", handleGetFile).Methods("POST", "OPTIONS")
+	r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS")
-	//r.HandleFunc("/api/v1/files/{fileId}", handleGetFile).Methods("GET", "OPTIONS")
+	r.HandleFunc("/api/v1/files/{fileId}", handleGetFileMeta).Methods("GET", "OPTIONS")
-	//r.HandleFunc("/api/v1/files/{fileId}", handleGetFile).Methods("DELETE", "OPTIONS")
+	r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS")
 
 	http.Handle("/", r)
 }

backend/go-app/walkoff.go

@@ -954,6 +954,30 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
 		return
 	}
 
+	runWorkflowExecutionTransaction(ctx, 0, workflowExecution.ExecutionId, actionResult, resp)
+}
+
+// Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times
+func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult ActionResult, resp http.ResponseWriter) {
+	// Should start a tx for the execution here
+	tx, err := dbclient.NewTransaction(ctx)
+	if err != nil {
+		log.Printf("client.NewTransaction: %v", err)
+		resp.WriteHeader(401)
+		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed creating transaction"}`)))
+		return
+	}
+
+	key := datastore.NameKey("workflowexecution", workflowExecutionId, nil)
+	workflowExecution := &WorkflowExecution{}
+	if err := tx.Get(key, workflowExecution); err != nil {
+		log.Printf("tx.Get bug: %v", err)
+		tx.Rollback()
+		resp.WriteHeader(401)
+		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting the workflow key"}`)))
+		return
+	}
+
 	if actionResult.Status == "ABORTED" || actionResult.Status == "FAILURE" {
 		log.Printf("Actionresult is %s. Should set workflowExecution and exit all running functions", actionResult.Status)
 
@@ -1228,18 +1252,37 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
 		}
 	}
 
-	err = setWorkflowExecution(ctx, *workflowExecution)
+	// Transactions: https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-go
-	if err != nil {
+	// Prevents timing issues
-		//workflowExecution.Result = "Error setting workflow: result too large"
+	//ExecutionId
-		//workflowExecution.Status = "FINISHED"
+	if _, err := tx.Put(key, workflowExecution); err != nil {
-		//workflowExecution.CompletedAt = int64(time.Now().Unix())
+		tx.Rollback()
+		log.Printf("[ERROR] tx.Put bug: %v", err)
 
-		log.Printf("Error saving workflow execution actionresult setting: %s", err)
 		resp.WriteHeader(401)
 		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution actionresult: %s"}`, err)))
 		return
 	}
 
+	if _, err = tx.Commit(); err != nil {
+		if attempts >= 5 {
+			log.Printf("[ERROR] QUITTING: tx.Commit %d: %v", attempts, err)
+			tx.Rollback()
+			workflowExecution.Status = "ABORTED"
+			setWorkflowExecution(ctx, *workflowExecution)
+
+			resp.WriteHeader(401)
+			resp.Write([]byte(`{"success": false}`))
+			return
+		}
+
+		log.Printf("[WARNING] tx.Commit %d: %v", attempts, err)
+
+		attempts += 1
+		runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp)
+		return
+	}
+
 	resp.WriteHeader(200)
 	resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
 }
@@ -1360,6 +1403,7 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) {
 	q := datastore.NewQuery("workflow").Filter("owner =", user.Id)
 	if user.Role == "admin" {
 		q = datastore.NewQuery("workflow").Filter("org_id =", user.ActiveOrg.Id)
+		log.Printf("[INFO] Getting workflows (ADMIN) for organization %s", user.ActiveOrg.Id)
 	}
 
 	var workflows []Workflow
@@ -1830,7 +1874,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
 	for _, action := range workflow.Actions {
 		allNodes = append(allNodes, action.ID)
 
-		if len(action.Errors) > 0 {
+		if len(action.Errors) > 0 || !action.IsValid {
 			action.IsValid = true
 			action.Errors = []string{}
 		}
@@ -2192,9 +2236,11 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
 					// Handles check for required params
 					if !found && param.Required {
 						log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name)
-						resp.WriteHeader(401)
+						action.Errors = append(action.Errors, "Parameter %s is required", param.Name)
-						resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name)))
+						//newActions = append(newActions, action)
-						return
+						//resp.WriteHeader(401)
+						//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name)))
+						//return
 					}
 
 				}
@@ -2209,6 +2255,15 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
 	workflow.IsValid = true
 	log.Printf("Tags: %#v", workflow.Tags)
 
+	// FIXME: Is this too drastic? May lead to issues in the future.
+	// Should maybe make a copy for the old org.
+	if workflow.OrgId != user.ActiveOrg.Id {
+		log.Printf("[WARNING] Editing workflow to be owned by %s", user.ActiveOrg.Id)
+		workflow.OrgId = user.ActiveOrg.Id
+		workflow.ExecutingOrg = user.ActiveOrg
+		workflow.Org = append(workflow.Org, user.ActiveOrg)
+	}
+
 	err = setWorkflow(ctx, workflow, fileId)
 	if err != nil {
 		log.Printf("Failed saving workflow to database: %s", err)
@@ -2234,7 +2289,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
 		Errors:  workflow.Errors,
 	}
 
-	log.Printf("Saved new version of workflow %s (%s)", workflow.Name, fileId)
+	log.Printf("Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId)
 	resp.WriteHeader(200)
 	newBody, err := json.Marshal(returndata)
 	if err != nil {
@@ -2795,7 +2850,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
 					continue
 				}
 
-				log.Printf("Should set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID)
+				log.Printf("[WARNING] Set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID)
 				defaultResults = append(defaultResults, ActionResult{
 					Action:        action,
 					ExecutionId:   workflowExecution.ExecutionId,
@@ -2836,7 +2891,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
 
 	var allEnvs []Environment
 	if len(workflowExecution.ExecutionOrg) > 0 {
-		log.Printf("Executing ORG: %s", workflowExecution.ExecutionOrg)
+		log.Printf("[INFO] Executing ORG: %s", workflowExecution.ExecutionOrg)
 
 		allEnvironments, err := getEnvironments(ctx, workflowExecution.ExecutionOrg)
 		if err != nil {
@@ -5336,11 +5391,24 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra
 	return err
 }
 
+type buildLaterStruct struct {
+	Tags  []string
+	Extra string
+	Id    string
+}
+
 // Onlyname is used to
-func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) error {
+func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) ([]buildLaterStruct, []buildLaterStruct, error) {
 	var err error
 
 	allapps := []WorkflowApp{}
+	reservedNames := []string{
+		"OWA",
+		"NLP",
+	}
+
+	buildLaterFirst := []buildLaterStruct{}
+	buildLaterList := []buildLaterStruct{}
 
 	// It's here to prevent getting them in every iteration
 	ctx := context.Background()
@@ -5360,11 +5428,20 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
 			}
 
 			// Go routine? Hmm, this can be super quick I guess
-			err = iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate)
+			buildFirst, buildLast, err := iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate)
 			if err != nil {
 				log.Printf("Error reading folder: %s", err)
 				continue
 			}
+
+			for _, item := range buildFirst {
+				buildLaterFirst = append(buildLaterFirst, item)
+			}
+
+			for _, item := range buildLast {
+				buildLaterList = append(buildLaterList, item)
+			}
+
 		case mode.IsRegular():
 			// Check the file
 			filename := file.Name()
@@ -5562,22 +5639,69 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
 
 				//log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion)
 
-				/// Only upload if successful and no errors
+				// ID  can be used to e.g. set a build status.
-				err = buildImageMemory(fs, tags, extra)
+				buildLater := buildLaterStruct{
-				if err != nil {
+					Tags:  tags,
-					log.Printf("Failed image build memory: %s", err)
+					Extra: extra,
-				} else {
+					Id:    workflowapp.ID,
-					if len(tags) > 0 {
+				}
-						log.Printf("Successfully built image %s", tags[0])
+
-					} else {
+				reservedFound := false
-						log.Printf("Successfully built Docker image")
+				for _, appname := range reservedNames {
+					if strings.ToUpper(workflowapp.Name) == strings.ToUpper(appname) {
+						buildLaterList = append(buildLaterList, buildLater)
+
+						reservedFound = true
+						break
 					}
 				}
+
+				/// Only upload if successful and no errors
+				if !reservedFound {
+					buildLaterFirst = append(buildLaterFirst, buildLater)
+				} else {
+					log.Printf("\n\n[WARNING] Skipping build of %s to later\n\n", workflowapp.Name)
+				}
 			}
 		}
 	}
 
-	return err
+	if len(buildLaterFirst) == 0 && len(buildLaterList) == 0 {
+		return buildLaterFirst, buildLaterList, err
+	}
+
+	//log.Printf("BUILDLATERFIRST: %d, BUILDLATERLIST: %d", len(buildLaterFirst), len(buildLaterList))
+	if len(extra) == 0 {
+		log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst))
+		for _, item := range buildLaterFirst {
+			err = buildImageMemory(fs, item.Tags, item.Extra)
+			if err != nil {
+				log.Printf("Failed image build memory: %s", err)
+			} else {
+				if len(item.Tags) > 0 {
+					log.Printf("Successfully built image %s", item.Tags[0])
+				} else {
+					log.Printf("Successfully built Docker image")
+				}
+			}
+		}
+
+		log.Printf("Starting build of %d skipped docker images", len(buildLaterList))
+		for _, item := range buildLaterList {
+			err = buildImageMemory(fs, item.Tags, item.Extra)
+			if err != nil {
+				log.Printf("Failed image build memory: %s", err)
+			} else {
+				if len(item.Tags) > 0 {
+					log.Printf("Successfully built image %s", item.Tags[0])
+				} else {
+					log.Printf("Successfully built Docker image")
+				}
+			}
+		}
+	}
+
+	return buildLaterFirst, buildLaterList, err
 }
 
 func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) {
@@ -5722,7 +5846,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	// Query for the specifci workflowId
-	q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(20)
+	q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(30)
 	var workflowExecutions []WorkflowExecution
 	_, err = dbclient.GetAll(ctx, q, &workflowExecutions)
 	if err != nil {

docker-compose.yml

@@ -2,7 +2,7 @@ version: '3'
 services:
   frontend:
     #build: ./frontend
-    image: ghcr.io/frikky/shuffle-frontend:0.8.0
+    image: ghcr.io/frikky/shuffle-frontend:0.8.3
     container_name: shuffle-frontend
     hostname: shuffle-frontend
     ports:
@@ -17,7 +17,7 @@ services:
       - backend
   backend:
     #build: ./backend
-    image: ghcr.io/frikky/shuffle-backend:0.8.0
+    image: ghcr.io/frikky/shuffle-backend:0.8.3
     container_name: shuffle-backend
     hostname: ${BACKEND_HOSTNAME}
     # Here for debugging:
@@ -28,9 +28,11 @@ services:
     volumes: 
       - /var/run/docker.sock:/var/run/docker.sock 
       - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps     
+      - ${SHUFFLE_FILE_LOCATION}:/shuffle-files
     environment:
       - DATASTORE_EMULATOR_HOST=shuffle-database:8000
       - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps
+      - SHUFFLE_FILE_LOCATION=/shuffle-files
       - ORG_ID=${ORG_ID}
       - SHUFFLE_APP_DOWNLOAD_LOCATION=${SHUFFLE_APP_DOWNLOAD_LOCATION}
       - SHUFFLE_DEFAULT_USERNAME=${SHUFFLE_DEFAULT_USERNAME}

frontend/src/views/AngularWorkflow.jsx

@@ -899,6 +899,41 @@ const AngularWorkflow = (props) => {
 	const onUnselect = (event) => {
 		console.time("UNSELECT")
 
+		// Attempt at rewrite of name in other actions in following nodes.
+		// Should probably be done in the onBlur for the textfield instead
+		/*
+		if (event.target.data().type === "ACTION") {
+			const nodeaction = event.target.data()
+			const curaction = workflow.actions.find(a => a.id === nodeaction.id)
+			console.log("workflowaction: ", curaction)
+			console.log("nodeaction: ", nodeaction)
+			if (nodeaction.label !== curaction.label) {
+				console.log("BEACH!")
+
+				var params = []
+				const fixedName = "$"+curaction.label.toLowerCase().replace(" ", "_")
+				for (var actionkey in workflow.actions) {
+					if (workflow.actions[actionkey].id === curaction.id) {
+						continue
+					}
+
+					for (var paramkey in workflow.actions[actionkey].parameters) {
+						const param = workflow.actions[actionkey].parameters[paramkey]
+						if (param.value === null || param.value === undefined || !param.value.includes("$")) {
+							continue
+						}
+
+						const innername = param.value.toLowerCase().replace(" ", "_")
+						if (innername.includes(fixedName)) {
+							//workflow.actions[actionkey].parameters[paramkey].replace(
+							//console.log("FOUND!: ", innername)
+						}
+					}
+				}
+			}
+		}
+		*/
+
 		// FIXME - check if they have value before overriding like this for no reason.
 		// Would save a lot of time (400~ ms -> 30ms)
 		//console.log("ACTION: ", selectedAction)
@@ -1009,8 +1044,29 @@ const AngularWorkflow = (props) => {
 			}
 
 			setSelectedActionName(curaction.name)	
-
 			setSelectedAction(curaction)
+
+			/*
+			var params = []
+			const fixedName = "$"+curaction.label.toLowerCase().replace(" ", "_")
+			for (var actionkey in workflow.actions) {
+				if (workflow.actions[actionkey].id === curaction.id) {
+					continue
+				}
+
+				for (var paramkey in workflow.actions[actionkey].parameters) {
+					const param = workflow.actions[actionkey].parameters[paramkey]
+					if (param.value === null || param.value === undefined || !param.value.includes("$")) {
+						continue
+					}
+
+					const innername = param.value.toLowerCase().replace(" ", "_")
+					if (innername.includes(fixedName)) {
+						console.log("FOUND!: ", innername)
+					}
+				}
+			}
+			*/
 		} else if (data.type === "TRIGGER") {
 			//console.log("Should handle trigger "+data.triggertype)
 			//console.log(data)
@@ -1278,6 +1334,7 @@ const AngularWorkflow = (props) => {
 				} else {
 					setEnvironments({"name": "Onprem", "type": "onprem"})
 				}
+
 				return
 			}
 
@@ -2419,14 +2476,43 @@ const AngularWorkflow = (props) => {
 	// ACTION select
 	//
 	const selectedNameChange = (event) => {
+		console.log("OLDNAME: ", selectedActionName)
 		event.target.value = event.target.value.replace("(", "")
 		event.target.value = event.target.value.replace(")", "")
 		event.target.value = event.target.value.replace("$", "")
 		event.target.value = event.target.value.replace("#", "")
 		event.target.value = event.target.value.replace(".", "")
 		event.target.value = event.target.value.replace(",", "")
+		event.target.value = event.target.value.replace(" ", "_")
 		selectedAction.label = event.target.value
 		setSelectedAction(selectedAction)
+
+		/*
+		if (nodeaction.label !== curaction.label) {
+			console.log("BEACH!")
+
+			var params = []
+			const fixedName = "$"+curaction.label.toLowerCase().replace(" ", "_")
+			for (var actionkey in workflow.actions) {
+				if (workflow.actions[actionkey].id === curaction.id) {
+					continue
+				}
+
+				for (var paramkey in workflow.actions[actionkey].parameters) {
+					const param = workflow.actions[actionkey].parameters[paramkey]
+					if (param.value === null || param.value === undefined || !param.value.includes("$")) {
+						continue
+					}
+
+					const innername = param.value.toLowerCase().replace(" ", "_")
+					if (innername.includes(fixedName)) {
+						//workflow.actions[actionkey].parameters[paramkey].replace(
+						//console.log("FOUND!: ", innername)
+					}
+				}
+			}
+		}
+		*/
 	}
 
 	const selectedTriggerChange = (event) => {
@@ -2836,15 +2922,56 @@ const AngularWorkflow = (props) => {
 								}}
 							/>
 
-						console.log(selectedActionParameters[count])
 						if (selectedActionParameters[count].schema !== undefined && selectedActionParameters[count].schema !== null && selectedActionParameters[count].schema.type === "file") {
-							const fileId = "6daabec1-892b-469c-b603-c902e47223a9"
+							datafield = 
-							datafield = `SHOW FILES FROM OTHER NODES? Filename: ${selectedActionParameters[count].value}`	
+								<TextField
+									style={{backgroundColor: inputColor, borderRadius: borderRadius,}} 
+									InputProps={{
+										style:{
+											color: "white",
+											minHeight: "50px", 
+											marginLeft: "5px",
+											maxWidth: "95%",
+											fontSize: "1em",
+										},
+										endAdornment: (
+											<InputAdornment position="end">
+												<Tooltip title="Autocomplete text" placement="top">
+													<AddCircleOutlineIcon style={{cursor: "pointer"}} onClick={(event) => {
+														setMenuPosition({
+															top: event.pageY,
+															left: event.pageX,
+														})
+														setShowDropdownNumber(count)
+														setShowDropdown(true)
+														setShowAutocomplete(true)
+													}}/>
+												</Tooltip>
+											</InputAdornment>
+										)
+									}}
+									fullWidth
+									multiline={multiline}
+									rows="5"
+									color="primary"
+									defaultValue={data.value}
+									type={"text"}
+									placeholder={"The file ID to get"}
+									onChange={(event) => {
+										changeActionParameter(event, count)
+									}}
+									onBlur={(event) => {
+									}}
+								/>
+							//const fileId = "6daabec1-892b-469c-b603-c902e47223a9"
+							//datafield = `SHOW FILES FROM OTHER NODES? Filename: ${selectedActionParameters[count].value}`	
+							/*
 							if (selectedActionParameters[count].value != fileId) {
 								changeActionParameter(fileId, count)
 								setUpdate(Math.random())
 
 							}
+							*/
 						} else if (selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0) {
 							if (selectedActionParameters[count].value === "" && selectedActionParameters[count].required) {
 								// Rofl, dirty workaround :)
@@ -4551,7 +4678,6 @@ const AngularWorkflow = (props) => {
 						placeholder={selectedTrigger.label}
 						onChange={selectedTriggerChange}
 					/>
-					{showEnvironment ? 
 						<div style={{marginTop: "20px"}}>
 							<Typography>
 								Environment
@@ -4599,7 +4725,6 @@ const AngularWorkflow = (props) => {
 								})}
 							</Select>
 						</div>
-						: null}
 					<Divider style={{marginTop: "20px", height: "1px", width: "100%", backgroundColor: "rgb(91, 96, 100)"}}/>
 					<div style={{flex: "6", marginTop: "20px"}}>
 						<div>
@@ -5095,59 +5220,57 @@ const AngularWorkflow = (props) => {
 						placeholder={selectedTrigger.label}
 						onChange={selectedTriggerChange}
 					/>
-					{showEnvironment ? 
+					<div style={{marginTop: "20px"}}>
-						<div style={{marginTop: "20px"}}>
+						<Typography>
-							<Typography>
+							Environment
-								Environment
+						</Typography>
-							</Typography>
+						<Select
-							<Select
+							value={selectedTrigger.environment}
-								value={selectedTrigger.environment}
+							disabled={selectedTrigger.status === "running"}
-								disabled={selectedTrigger.status === "running"}
+							SelectDisplayProps={{
-								SelectDisplayProps={{
+								style: {
-									style: {
+									marginLeft: 10,
-										marginLeft: 10,
 
-									}
+								}
-								}}
+							}}
-								fullWidth
+							fullWidth
-								onChange={(e) => {
+							onChange={(e) => {
-									selectedTrigger.environment = e.target.value
+								selectedTrigger.environment = e.target.value
-									setSelectedTrigger(selectedTrigger)
+								setSelectedTrigger(selectedTrigger)
-									if (e.target.value === "cloud") {
+								if (e.target.value === "cloud") {
-										console.log("Set cloud config")
+									console.log("Set cloud config")
-										workflow.triggers[selectedTriggerIndex].parameters[0].value = "*/2 * * * *" 
+									workflow.triggers[selectedTriggerIndex].parameters[0].value = "*/2 * * * *" 
 
-										//var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/")
+									//var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/")
-										//const urlpath = tmpvalue.slice(3, tmpvalue.length)
+									//const urlpath = tmpvalue.slice(3, tmpvalue.length)
-										//const newurl = "https://shuffler.io/"+urlpath.join("/")
+									//const newurl = "https://shuffler.io/"+urlpath.join("/")
-										//workflow.triggers[selectedTriggerIndex].parameters[0].value = newurl
+									//workflow.triggers[selectedTriggerIndex].parameters[0].value = newurl
-									} else {
+								} else {
-										console.log("Set cloud config")
+									console.log("Set cloud config")
-										//var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/")
+									//var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/")
-										//const urlpath = tmpvalue.slice(3, tmpvalue.length)
+									//const urlpath = tmpvalue.slice(3, tmpvalue.length)
-										//const newurl = window.location.origin+"/"+urlpath.join("/")
+									//const newurl = window.location.origin+"/"+urlpath.join("/")
-										workflow.triggers[selectedTriggerIndex].parameters[0].value = "120" 
+									workflow.triggers[selectedTriggerIndex].parameters[0].value = "120" 
-									}
+								}
 
-									setWorkflow(workflow)
+								setWorkflow(workflow)
-									setUpdate(Math.random())
+								setUpdate(Math.random())
-								}}
+							}}
-								style={{backgroundColor: inputColor, color: "white", height: "50px"}}
+							style={{backgroundColor: inputColor, color: "white", height: "50px"}}
-							>
+						>
-								{triggerEnvironments.map(data => {
+							{triggerEnvironments.map(data => {
-									if (data.archived) {
+								if (data.archived) {
-										return null
+									return null
-									}
+								}
 								
-									return (
+								return (
-										<MenuItem key={data} style={{backgroundColor: inputColor, color: "white"}} value={data}>
+									<MenuItem key={data} style={{backgroundColor: inputColor, color: "white"}} value={data}>
-											{data}
+										{data}
-										</MenuItem>
+									</MenuItem>
-									)
+								)
-								})}
+							})}
-							</Select>
+						</Select>
-						</div>
+					</div>
-						: null}
 					<Divider style={{marginTop: "20px", height: "1px", width: "100%", backgroundColor: "rgb(91, 96, 100)"}}/>
 					<div style={{flex: "6", marginTop: "20px"}}>
 						<div>
@@ -5228,7 +5351,7 @@ const AngularWorkflow = (props) => {
 		return null 
 	}
 
-	const cytoscapeViewWidths = 700
+	const cytoscapeViewWidths = 750
 	const bottomBarStyle = {
 		position: "fixed", 
 		right: 20, 
@@ -5286,6 +5409,62 @@ const AngularWorkflow = (props) => {
 		return null
 	}
 
+	const FileMenu = () => {
+		const [newAnchor, setNewAnchor] = React.useState(null);
+		const [showShuffleMenu, setShowShuffleMenu] = React.useState(false)
+
+		{ /*const [showShuffleMenu, setShowShuffleMenu] = React.useState(true) */}
+		return (
+			<div style={{"display": "inline-block"}}>
+				<Menu
+					id="long-menu"
+					anchorEl={newAnchor}
+					open={showShuffleMenu}
+					onClose={() => {
+						setShowShuffleMenu(false)
+					}}
+				>
+					<div style={{margin: 15, color: "white", maxWidth: 250, minWidth: 250, }}>
+						<h4>This menu is used to control the workflow itself.</h4>
+						<Divider style={{backgroundColor: "white", marginTop: 10, marginBottom: 10,}}/>
+						<FormControlLabel
+							style={{marginBottom: 15, color: "white",}}
+							label={<div style={{color: "white"}}>Exit on Error</div>}
+							control={
+								<Switch checked={workflow.configuration.exit_on_error} onChange={() => {
+									workflow.configuration.exit_on_error = !workflow.configuration.exit_on_error
+									setWorkflow(workflow)
+									setUpdate("exit_on_error_"+workflow.configuration.exit_on_error ? "true" : "false")
+									setShowShuffleMenu(false)
+								}} />
+							}
+						/>
+						<FormControlLabel
+							style={{marginBottom: 15, color: "white",}}
+							label={<div style={{color: "white"}}>Start from top</div>}
+							control={
+								<Switch checked={workflow.configuration.start_from_top} onChange={() => {
+									workflow.configuration.start_from_top = !workflow.configuration.start_from_top
+									setWorkflow(workflow)
+									setUpdate("start_from_top_"+workflow.configuration.start_from_top ? "true" : "false")
+									setShowShuffleMenu(false)
+								}} />
+							}
+						/>
+					</div>
+				</Menu>
+				<Tooltip color="secondary" title="Workflow settings" placement="top-start">
+					<Button color="primary" style={{height: 50, marginLeft: 10, }} variant="outlined" onClick={(event) => {
+						setShowShuffleMenu(!showShuffleMenu)
+						setNewAnchor(event.currentTarget)
+					}}>
+						<SettingsIcon />
+					</Button>
+				</Tooltip>
+			</div>
+		)
+	}
+
 	const WorkflowMenu = () => {
 		const [newAnchor, setNewAnchor] = React.useState(null);
 		const [showShuffleMenu, setShowShuffleMenu] = React.useState(false)
@@ -5413,6 +5592,7 @@ const AngularWorkflow = (props) => {
 							<DirectionsRunIcon />
 						</Button>
 					</Tooltip>
+					{/* <FileMenu />	*/}
 					<WorkflowMenu />	
 				</div>
 			</div>
@@ -5719,7 +5899,7 @@ const AngularWorkflow = (props) => {
 							try {
 								const tmp = String(JSON.parse(showResult))
 								if (!showResult.includes("{") && !showResult.includes("[")) {
-									console.log("IN HERE: ", tmp)
+									//console.log("IN HERE: ", tmp)
 									jsonvalid = false
 								}
 							} catch (e) {

frontend/src/views/Apps.jsx

@@ -116,7 +116,7 @@ const Apps = (props) => {
 	const [apps, setApps] = React.useState([])
 	const [filteredApps, setFilteredApps] = React.useState([])
 	const [validation, setValidation] = React.useState(false)
-	const [isLoading, setIsLoading] = React.useState(false)
+	const [isLoading, setIsLoading] = React.useState(true)
 	const [appSearchLoading, setAppSearchLoading] = React.useState(false)
 	const [selectedAction, setSelectedAction] = React.useState({})
 	const [searchBackend, setSearchBackend] = React.useState(false)
@@ -207,6 +207,7 @@ const Apps = (props) => {
 	  			credentials: "include",
     		})
 		.then((response) => {
+			setIsLoading(false)
 			if (response.status !== 200) {
 				console.log("Status not 200 for apps :O!")
 			}
@@ -231,6 +232,7 @@ const Apps = (props) => {
     })
 		.catch(error => {
 			alert.error(error.toString())
+			setIsLoading(false)
 		});
 	}
 
@@ -742,7 +744,7 @@ const Apps = (props) => {
 						<Divider style={{height: 1, backgroundColor: dividerColor, marginTop: 20, marginBottom: 20}} />
 						<div style={{}}>
 							<Button
-								variant="text"
+								variant="contained"
 								component="label"
 								color="primary"
 								style={{marginRight: 10, }}
@@ -932,11 +934,14 @@ const Apps = (props) => {
 								}
 							</Paper>
 						: 
-						<Paper square style={uploadViewPaperStyle}>
+						isLoading ? 
-							<h4 style={{margin: 10}}>
+							<CircularProgress style={{width: 40, height: 40, margin: "auto"}}/>
-								No apps have been created, uploaded or downloaded yet. Click "Load existing apps" above to get the baseline. This may take a while as its building docker images.
+							:
-							</h4>
+							<Paper square style={uploadViewPaperStyle}>
-						</Paper>
+								<h4 style={{margin: 10}}>
+									No apps have been created, uploaded or downloaded yet. Click "Load existing apps" above to get the baseline. This may take a while as its building docker images.
+								</h4>
+							</Paper>
 						}
 					</div>
 				</div>

frontend/src/views/Workflows.jsx

@@ -15,6 +15,7 @@ import FormControlLabel from '@material-ui/core/FormControlLabel';
 import Chip from '@material-ui/core/Chip';
 import Switch from '@material-ui/core/Switch';
 import Typography from '@material-ui/core/Typography';
+import Zoom from '@material-ui/core/Zoom';
 
 import CircularProgress from '@material-ui/core/CircularProgress';
 import CachedIcon from '@material-ui/icons/Cached';
@@ -74,6 +75,7 @@ const Workflows = (props) => {
 	const [update, setUpdate] = React.useState("test");
 	const [deleteModalOpen, setDeleteModalOpen] = React.useState(false);
 	const [editingWorkflow, setEditingWorkflow] = React.useState({})
+	const [executionLoading, setExecutionLoading] = React.useState(false)
 	const { start, stop } = useInterval({
 	  	duration: 5000,
 	  	startImmediate: false,
@@ -245,6 +247,7 @@ const Workflows = (props) => {
 	}
 
 	const getWorkflowExecution = (id) => {
+		setExecutionLoading(true)
 		fetch(globalUrl+"/api/v1/workflows/"+id+"/executions", {
     	  method: 'GET',
 				headers: {
@@ -254,6 +257,7 @@ const Workflows = (props) => {
 	  			credentials: "include",
     		})
 		.then((response) => {
+			setExecutionLoading(false)
 			if (response.status !== 200) {
 				console.log("Status not 200 for WORKFLOW EXECUTION :O!")
 			}
@@ -275,6 +279,7 @@ const Workflows = (props) => {
 			}
 		})
 		.catch(error => {
+			setExecutionLoading(false)
 			alert.error(error.toString())
 		});
 	}
@@ -360,7 +365,8 @@ const Workflows = (props) => {
 				data.triggers[key].status = "stopped"
 			}
 		}
-
+		data["org"] = []
+		data["org_id"] = ""
 		data.execution_org = {"id": ""}
 		console.log(data)
 
@@ -858,10 +864,17 @@ const Workflows = (props) => {
 				</div> 
 			)
 		}
+
 		return (
-			<h4>
+			executionLoading ?
-				There are no executiondetails yet. Click "execute" to run your first one.
+				<div style={{marginTop: 25, textAlign: "center"}}>
-			</h4>
+					<CircularProgress />
+				</div>
+				: 
+				<h4>
+					There are no executiondetails yet. Click "execute" to run your first one.
+				</h4>
+			
 		)
 	}
 
@@ -880,9 +893,14 @@ const Workflows = (props) => {
 			)
 		} 
 		return (
-			<h4>
+			executionLoading ?
-				There are no executions for this workflow yet
+				<div style={{marginTop: 25, textAlign: "center"}}>
-			</h4>
+					<CircularProgress />
+				</div>
+				:
+				<h4>
+					There are no executions for this workflow yet
+				</h4>
 		)
 	}
 
@@ -1012,8 +1030,19 @@ const Workflows = (props) => {
 				},
 			}}
 		>
+			<DialogTitle>
+				<div style={{color: "rgba(255,255,255,0.9)"}}>
+					{editingWorkflow.id !== undefined ? "Editing" : "New"} workflow
+					<div style={{float: "right"}}>
+						<Tooltip color="primary" title={"Import manually"} placement="top">
+							<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
+								<PublishIcon />
+							</Button> 				
+						</Tooltip>
+					</div>
+				</div>
+			</DialogTitle>
 			<FormControl>
-			<DialogTitle><div style={{color: "white"}}>{editingWorkflow.id !== undefined ? "Editing" : "New"} workflow</div></DialogTitle>
 				<DialogContent>
 					<TextField
 						onBlur={(event) => setNewWorkflowName(event.target.value)}
@@ -1111,107 +1140,126 @@ const Workflows = (props) => {
 			workflowViewStyle.display = "none"
 		}
 
-		const workflowView = workflows.length > 0 ? 
+		const workflowButtons = 
-		<div style={viewStyle}>	
+			<span>
-			<div style={workflowViewStyle}>
+				{workflows.length > 0 ?
-				<div style={{display: "flex"}}>
+					<Tooltip color="primary" title={"Create new workflow"} placement="top">
-					<div style={{flex: "4"}}>
+						<Button color="primary" style={{}} variant="text" onClick={() => setModalOpen(true)}><AddIcon /></Button> 				
-						<h2>Workflows</h2> 
+					</Tooltip>
-					</div>
+				: null}
-					<div style={{marginTop: 20}}>
+				<Tooltip color="primary" title={"Import workflows"} placement="top">
-					 	<Tooltip color="primary" title={"Create new workflow"} placement="top">
+					<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
-							<Button color="primary" style={{}} variant="text" onClick={() => setModalOpen(true)}><AddIcon /></Button> 				
+						<PublishIcon />
-						</Tooltip>
+					</Button> 				
-						{/*
+				</Tooltip>
-					 	<Tooltip color="primary" title={"Import workflows"} placement="top">
+				<input hidden type="file" multiple="multiple" ref={(ref) => upload = ref} onChange={importFiles} />
-							<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
+				{workflows.length > 0 ? 
-								<PublishIcon />
+					<Tooltip color="primary" title={`Download ALL workflows (${workflows.length})`} placement="top">
-							</Button> 				
+						<Button color="primary" style={{}} variant="text" onClick={() => {
-						</Tooltip>
+							exportAllWorkflows()
-						*/}
+						}}>
-						<Tooltip color="primary" title={`Download ALL workflows (${workflows.length})`} placement="top">
+							<GetAppIcon />
-							<Button color="primary" style={{}} variant="text" onClick={() => {
-								exportAllWorkflows()
-							}}>
-								<GetAppIcon />
-							</Button> 				
-						</Tooltip>
-					 	<Tooltip color="primary" title={"Import workflows"} placement="top">
-							<Button color="primary" style={{}} variant="text" onClick={() => setLoadWorkflowsModalOpen(true)}>
-								<CloudDownloadIcon />
-							</Button> 				
-						</Tooltip>
-						<input hidden type="file" multiple="multiple" ref={(ref) => upload = ref} onChange={importFiles} />
-					</div>
-				</div>
-				<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
-
-				<div style={scrollStyle}>
-					{workflows.map((data, index) => {
-						return (
-							<WorkflowPaper key={index} data={data} />
-						)
-					})}
-				</div>
-			</div>
-			<div style={{flex: viewSize.executionsView, marginLeft: "10px", marginRight: "10px"}}>
-				<div style={{display: "flex"}}>
-					<div style={{flex: "10"}}>
-						<h2>Executions: {selectedWorkflow.name}</h2> 
-					</div>
-					<div style={{flex: "1"}}>
-						<Button color="primary" style={{marginTop: "20px"}} variant="text" onClick={() => {
-								alert.info("Refreshing executions"); 
-								getWorkflowExecution(selectedWorkflow.id)
-							}}>
-							<CachedIcon />
 						</Button> 				
-					</div>
+					</Tooltip>
-				</div>
+				: null}
-				<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
+				<Tooltip color="primary" title={"Download workflows"} placement="top">
-				<div style={scrollStyle}>
+					<Button color="primary" style={{}} variant="text" onClick={() => setLoadWorkflowsModalOpen(true)}>
-					<ExecutionsView />
+						<CloudDownloadIcon />
-				</div>
+					</Button> 				
-			</div>
+				</Tooltip>
-			<div style={{flex: viewSize.executionResults, marginLeft: "10px", marginRight: "10px", minWidth: "33%"}}>
+			</span>
-				<div style={{display: "flex"}}>
-					<div style={{flex: "1"}}>
-						<h2>Execution Timeline</h2>
-					</div>
-					<div style={{flex: 1}}>
-				    	<FormControlLabel
-							style={{color: "white", marginBottom: "0px", marginTop: "10px"}}
-							label={<div style={{color: "white"}}>Collapse results</div>}
-							control={<Switch checked={collapseJson} onChange={() => {setCollapseJson(!collapseJson)}} />}
-						/>
-					</div>
-				</div>
-				<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
-				<div style={scrollStyle}>
-					<ExecutionDetails />
-				</div>
-			</div>
-		</div>
-		: 
-		<div style={emptyWorkflowStyle}>	
-			<Paper style={boxStyle}>
-				<div>
-					<h2>Welcome to Shuffle</h2>
-				</div>
-				<div>
-					<p>
-						<b>Shuffle</b> is a flexible, easy to use, automation platform allowing users to integrate their services and devices freely. It's made to significantly reduce the amount of manual labor, and is focused on security applications. <a href="/docs/about" style={{textDecoration: "none", color: "#f85a3e"}}>Click here to learn more.</a>
-					</p>
-				</div>
-				<div>
-					If you want to jump straight into it, click here to create your first workflow: 
-				</div>
-				<div>
-					<Button color="primary" style={{marginTop: "20px",}} variant="outlined" onClick={() => setModalOpen(true)}>New workflow</Button> 				
-				</div>
-			</Paper>
-		</div>
 
+		const WorkflowView = () => {
+			if (workflows.length === 0) {
+				return (
+					<div style={emptyWorkflowStyle}>	
+						<Paper style={boxStyle}>
+							<div>
+								<h2>Welcome to Shuffle</h2>
+							</div>
+							<div>
+								<p>
+									<b>Shuffle</b> is a flexible, easy to use, automation platform allowing users to integrate their services and devices freely. It's made to significantly reduce the amount of manual labor, and is focused on security applications. <a href="/docs/about" style={{textDecoration: "none", color: "#f85a3e"}}>Click here to learn more.</a>
+								</p>
+							</div>
+							<div>
+								If you want to jump straight into it, click here to create your first workflow: 
+							</div>
+							<div style={{display: "flex"}}>
+								<Button color="primary" style={{marginTop: "20px",}} variant="outlined" onClick={() => setModalOpen(true)}>New workflow</Button> 				
+								<span style={{paddingTop: 20, display: "flex",}}>
+									<Typography style={{marginTop: 5, marginLeft: 30, marginRight: 15}}>
+										..OR
+									</Typography>
+									{workflowButtons}
+								</span>
+							</div>
+						</Paper>
+					</div>
+				)
+			}
+
+		return (
+			<div style={viewStyle}>	
+				<div style={workflowViewStyle}>
+					<div style={{display: "flex"}}>
+						<div style={{flex: "4"}}>
+							<h2>Workflows</h2> 
+						</div>
+						<div style={{marginTop: 20}}>
+							{workflowButtons}
+						</div>
+					</div>
+					<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
+
+					<div style={scrollStyle}>
+						{workflows.map((data, index) => {
+							return (
+									<WorkflowPaper key={index} data={data} />
+							)
+						})}
+					</div>
+				</div>
+				<div style={{flex: viewSize.executionsView, marginLeft: "10px", marginRight: "10px"}}>
+					<div style={{display: "flex"}}>
+						<div style={{flex: "10"}}>
+							<h2>Executions: {selectedWorkflow.name}</h2> 
+						</div>
+						<div style={{flex: "1"}}>
+							<Button color="primary" style={{marginTop: "20px"}} variant="text" onClick={() => {
+									alert.info("Refreshing executions"); 
+									getWorkflowExecution(selectedWorkflow.id)
+								}}>
+								<CachedIcon />
+							</Button> 				
+						</div>
+					</div>
+					<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
+					<div style={scrollStyle}>
+						<ExecutionsView />
+					</div>
+				</div>
+				<div style={{flex: viewSize.executionResults, marginLeft: "10px", marginRight: "10px", minWidth: "33%"}}>
+					<div style={{display: "flex"}}>
+						<div style={{flex: "1"}}>
+							<h2>Execution Timeline</h2>
+						</div>
+						<div style={{flex: 1}}>
+					    	<FormControlLabel
+								style={{color: "white", marginBottom: "0px", marginTop: "10px"}}
+								label={<div style={{color: "white"}}>Collapse results</div>}
+								control={<Switch checked={collapseJson} onChange={() => {setCollapseJson(!collapseJson)}} />}
+							/>
+						</div>
+					</div>
+					<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
+					<div style={scrollStyle}>
+						<ExecutionDetails />
+					</div>
+				</div>
+			</div>
+		)
+	}
 
 	const importWorkflowsFromUrl = (url) => {
 		console.log("IMPORT WORKFLOWS FROM ", downloadUrl)
@@ -1384,13 +1432,17 @@ const Workflows = (props) => {
 
 	const loadedCheck = isLoaded && isLoggedIn && workflowDone ? 
 		<div>
-			{workflowView}
+			<WorkflowView />
 			{modalView}
 			{deleteModal}
 			{workflowDownloadModalOpen}
 		</div>
 		:
-		<div>
+		<div style={{paddingTop: 250, width: 250, margin: "auto", textAlign: "center"}}>
+			<CircularProgress />
+			<Typography>
+				Loading Workflows
+			</Typography>
 		</div>