Merge pull request #234 from frikky/launch

Shuffle v0.56 - Scalability and file fixing

.env

@@ -39,4 +39,7 @@ SHUFFLE_PASS_WORKER_PROXY=TRUE
 
 SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io
 SHUFFLE_BASE_IMAGE_NAME=frikky
-SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.0"
+SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.3"
+
+# Used for auto-cleanup of containers. REALLY important at scale.
+SHUFFLE_CONTAINER_AUTO_CLEANUP=false

.github/FUNDING.yml

@@ -2,8 +2,8 @@
 
 github: frikky
 patreon: # Replace with a single Patreon username
-open_collective: # Replace with a single Open Collective username
+open_collective: shuffle
-ko_fi: # Replace with a single Ko-fi username
+ko_fi: frikky
 tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
 community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
 liberapay: # Replace with a single Liberapay username

README.md

@@ -43,6 +43,19 @@ Please consider [sponsoring](https://github.com/sponsors/frikky) the project if
 ## Website
 https://shuffler.io
 
+## Contributors 
+![ICPL logo](https://github.com/frikky/Shuffle/blob/launch/frontend/src/assets/img/icpl_logo.png)
+
+**Shuffle**
+<a href="https://github.com/frikky/shuffle/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=frikky/shuffle" />
+</a>
+
+[**App magicians**](https://github.com/frikky/shuffle-apps)
+<a href="https://github.com/frikky/shuffle-apps/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=frikky/shuffle-apps" />
+</a>
+
 ## License
 All modular information related to Shuffle will be under MIT (anyone can use it for whatever purpose), with Shuffle itself using AGPLv3. 
 

backend/app_sdk/Dockerfile

@@ -7,7 +7,7 @@ RUN mkdir /install
 WORKDIR /install
 
 COPY requirements.txt /requirements.txt
-RUN pip install --prefix="/install" -r /requirements.txt
+RUN pip3 install -r /requirements.txt
 
 FROM base
 

backend/app_sdk/build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 NAME=shuffle-app_sdk
-VERSION=0.8.2
+VERSION=0.8.54
 
 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
 docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
@@ -8,6 +8,7 @@ docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.
 #docker push frikky/$NAME:$VERSION
 #docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION
 #docker push ghcr.io/frikky/$NAME:$VERSION
+#docker tag ghcr.io/frikky/$NAME:$VERSION frikky/shuffle:app_sdk
 
 docker push frikky/shuffle:app_sdk
 docker push ghcr.io/frikky/$NAME:$VERSION

backend/app_sdk/requirements.txt

@@ -1,2 +1,2 @@
-requests
 urllib3
+requests

backend/go-app/codegen.go

@@ -244,7 +244,7 @@ func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) {
 
 // This function generates the python code that's being used.
 // This is really meta when you program it. Handling parameters is hard here.
-func makePythoncode(swagger *openapi3.Swagger, name, url, method string, parameters, optionalQueries, headers []string) (string, string) {
+func makePythoncode(swagger *openapi3.Swagger, name, url, method string, parameters, optionalQueries, headers []string, fileField string) (string, string) {
 	method = strings.ToLower(method)
 	queryString := ""
 	queryData := ""
@@ -365,21 +365,39 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet
 		preparedHeaders += "}"
 	}
 
+	fileBalance := ""
+	fileAdder := ``
+	fileGrabber := ``
+	fileParameter := ``
+	if method == "post" && len(fileField) > 0 {
+		fileParameter = ", file_id"
+		fileGrabber = `filedata = self.get_file(file_id)`
+
+		// This indentation is confusing (but correct) ROFL
+		fileAdder = fmt.Sprintf(`if not filedata["success"]:
+            return file_id+" is not a valid File ID"
+        files = {"%s": (filedata["filename"], filedata["data"])}`, fileField)
+		fileBalance = ", files=files"
+	}
+
 	// Extra param for url if it's changeable
 	// Extra param for authentication scheme(s)
 	// The last weird one is the body.. Tabs & spaces sucks.
-	data := fmt.Sprintf(`    async def %s(self%s%s%s%s%s%s):
+	data := fmt.Sprintf(`    async def %s(self%s%s%s%s%s%s%s):
         %s
         url=f"%s%s"
         %s
         %s
         %s
         %s
-        return requests.%s(url, headers=headers%s%s%s).text
+        %s
+        %s
+        return requests.%s(url, headers=headers%s%s%s%s).text
 		`,
 		functionname,
 		authenticationParameter,
 		urlParameter,
+		fileParameter,
 		parameterData,
 		queryString,
 		bodyParameter,
@@ -391,18 +409,20 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet
 		authenticationSetup,
 		queryData,
 		bodyFormatter,
+		fileGrabber,
+		fileAdder,
 		method,
 		authenticationAddin,
 		bodyAddin,
 		verifyAddin,
+		fileBalance,
 	)
 
-	/*
+	if strings.Contains(functionname, "filescan") {
-		if strings.Contains(functionname, "search") {
+		//log.Printf("FUNCTION: %s", data)
-			log.Println(data)
+		log.Println(data)
-			log.Printf("Queries: %s", queryString)
+		log.Printf("Queries: %s", queryString)
-		}
+	}
-	*/
 
 	//log.Printf(data)
 	return functionname, data
@@ -446,6 +466,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger,
 	api.Sharing = false
 	api.Verified = false
 	api.Tested = false
+	api.Invalid = false
 	api.PrivateID = newmd5
 	api.Generated = true
 	api.Activated = true
@@ -638,10 +659,15 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger,
 	// This is the python code to be generated
 	// Could just as well be go at this point lol
 	pythonFunctions := []string{}
+	//Verified      bool   `json:"verified" yaml:"verified" required:false datastore:"verified"`
 	for actualPath, path := range swagger.Paths {
 		actualPath = strings.Replace(actualPath, " ", "_", -1)
 		//actualPath = strings.Replace(actualPath, ".", "", -1)
 		actualPath = strings.Replace(actualPath, "\\", "", -1)
+		if !api.Invalid && strings.HasPrefix(actualPath, "tmp") {
+			log.Printf("[WARNING] Set api %s to invalid because of path %s", swagger.Info.Title, actualPath)
+			api.Invalid = true
+		}
 
 		// FIXME: Handle everything behind questionmark (?) with dots as well.
 		// https://godoc.org/github.com/getkin/kin-openapi/openapi3#PathItem
@@ -846,10 +872,10 @@ def run(request):
 func deployAppToDatastore(ctx context.Context, workflowapp WorkflowApp) error {
 	err := setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID)
 	if err != nil {
-		log.Printf("Failed setting workflowapp: %s", err)
+		log.Printf("[ERROR] Failed setting workflowapp: %s", err)
 		return err
 	} else {
-		log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion)
+		log.Printf("[INFO] Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion)
 	}
 
 	return nil
@@ -948,6 +974,12 @@ func validateParameterName(name string) string {
 		}
 	}
 
+	newname = strings.ReplaceAll(newname, " ", "_")
+	newname = strings.ReplaceAll(newname, ",", "_")
+	newname = strings.ReplaceAll(newname, ".", "_")
+	newname = strings.ReplaceAll(newname, "|", "_")
+	newname = strings.ReplaceAll(newname, "-", "_")
+
 	return newname
 }
 
@@ -1001,6 +1033,7 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [
 			parsedName = strings.ReplaceAll(parsedName, ",", "_")
 			parsedName = strings.ReplaceAll(parsedName, ".", "_")
 			parsedName = strings.ReplaceAll(parsedName, "|", "_")
+			parsedName = strings.ReplaceAll(parsedName, "-", "_")
 			parsedName = validateParameterName(parsedName)
 			param.Value.Name = parsedName
 			path.Connect.Parameters[counter].Value.Name = parsedName
@@ -1076,7 +1109,7 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [
 		action.Parameters = append(action.Parameters, optionalParam)
 	}
 
-	functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "connect", parameters, optionalQueries, headersFound)
+	functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "connect", parameters, optionalQueries, headersFound, "")
 
 	if len(functionname) > 0 {
 		action.Name = functionname
@@ -1211,7 +1244,7 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
 		action.Parameters = append(action.Parameters, optionalParam)
 	}
 
-	functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "get", parameters, optionalQueries, headersFound)
+	functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "get", parameters, optionalQueries, headersFound, "")
 
 	if len(functionname) > 0 {
 		action.Name = functionname
@@ -1344,7 +1377,7 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
 		action.Parameters = append(action.Parameters, optionalParam)
 	}
 
-	functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "head", parameters, optionalQueries, headersFound)
+	functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "head", parameters, optionalQueries, headersFound, "")
 
 	if len(functionname) > 0 {
 		action.Name = functionname
@@ -1478,7 +1511,7 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []
 		action.Parameters = append(action.Parameters, optionalParam)
 	}
 
-	functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound)
+	functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound, "")
 
 	if len(functionname) > 0 {
 		action.Name = functionname
@@ -1521,6 +1554,40 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
 		},
 	})
 
+	fileField := ""
+	if path.Post.RequestBody != nil {
+		//log.Printf("DATA: %#v",
+		value := path.Post.RequestBody.Value
+		//log.Printf("VAL: %#v", value.Content)
+		if val, ok := value.Content["multipart/form-data"]; ok {
+			if val.Schema.Value != nil {
+				if innerval, ok := val.Schema.Value.Properties["fieldname"]; ok {
+					if extensionvalue, ok := innerval.Value.ExtensionProps.Extensions["value"]; ok {
+						fieldname := extensionvalue.(json.RawMessage)
+						newName := string(fmt.Sprintf("%s", string(fieldname)))
+						if newName[0] == 0x22 && newName[len(newName)-1] == 0x22 {
+							parsedName := newName[1 : len(newName)-1]
+							log.Printf("Parse name: %s", parsedName)
+							fileField = parsedName
+
+							curParam := WorkflowAppActionParameter{
+								Name:        "file_id",
+								Description: "Files to be uploaded",
+								Multiline:   false,
+								Required:    true,
+								Schema: SchemaDefinition{
+									Type: "string",
+								},
+							}
+
+							action.Parameters = append(action.Parameters, curParam)
+						}
+					}
+				}
+			}
+		}
+	}
+
 	headersFound := []string{}
 	if len(path.Post.Parameters) > 0 {
 		for counter, param := range path.Post.Parameters {
@@ -1610,12 +1677,17 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
 		action.Parameters = append(action.Parameters, optionalParam)
 	}
 
-	functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "post", parameters, optionalQueries, headersFound)
+	functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "post", parameters, optionalQueries, headersFound, fileField)
 
 	if len(functionname) > 0 {
 		action.Name = functionname
 	}
 
+	//log.Printf("PARAMS: %d", len(action.Parameters))
+	//for _, param := range action.Parameters {
+	//	log.Printf("%#v", param)
+	//}
+
 	return action, curCode
 }
 
@@ -1743,7 +1815,7 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W
 		action.Parameters = append(action.Parameters, optionalParam)
 	}
 
-	functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "patch", parameters, optionalQueries, headersFound)
+	functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "patch", parameters, optionalQueries, headersFound, "")
 
 	if len(functionname) > 0 {
 		action.Name = functionname
@@ -1877,7 +1949,7 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
 		action.Parameters = append(action.Parameters, optionalParam)
 	}
 
-	functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "put", parameters, optionalQueries, headersFound)
+	functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "put", parameters, optionalQueries, headersFound, "")
 
 	if len(functionname) > 0 {
 		action.Name = functionname

backend/go-app/docker.go

@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
@@ -125,12 +126,29 @@ func getParsedTarMemory(fs billy.Filesystem, tw *tar.Writer, baseDir, extra stri
 				return err
 			}
 
+			//log.Printf("FILENAME: %s", filename)
 			readFile, err := ioutil.ReadAll(fileReader)
 			if err != nil {
 				log.Printf("Not file: %s", err)
 				return err
 			}
 
+			// Fixes issues with older versions of Docker and reference formats
+			// Specific to Shuffle rn. Could expand.
+			// FIXME: Seems like the issue was with multi-stage builds
+			/*
+				if filename == "Dockerfile" {
+					log.Printf("Should search and replace in readfile.")
+
+					referenceCheck := "FROM frikky/shuffle:"
+					if strings.Contains(string(readFile), referenceCheck) {
+						log.Printf("SHOULD SEARCH & REPLACE!")
+						newReference := fmt.Sprintf("FROM registry.hub.docker.com/frikky/shuffle:")
+						readFile = []byte(strings.Replace(string(readFile), referenceCheck, newReference, -1))
+					}
+				}
+			*/
+
 			//log.Printf("Filename: %s", filename)
 			// FIXME - might need the folder from EXTRA here
 			// Name has to be e.g. just "requirements.txt"
@@ -156,8 +174,23 @@ func getParsedTarMemory(fs billy.Filesystem, tw *tar.Writer, baseDir, extra stri
 	return nil
 }
 
+/*
+// Fixes App SDK issues.. meh
+func fixTags(tags []string) []string {
+	checkTag := "frikky/shuffle"
+	newTags := []string{}
+	for _, tag := range tags {
+		if strings.HasPrefix(tag, checkTags) {
+			newTags.append(newTags, fmt.Sprintf("registry.hub.docker.com/%s", tag))
+		}
+
+		newTags.append(tag)
+	}
+}
+*/
+
 // Custom Docker image builder wrapper in memory
-func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string) error {
+func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string, downloadIfFail bool) error {
 	ctx := context.Background()
 	client, err := client.NewEnvClient()
 	if err != nil {
@@ -169,7 +202,7 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
 	tw := tar.NewWriter(buf)
 	defer tw.Close()
 
-	log.Printf("Setting up memory build structure for folder: %s", dockerfileFolder)
+	log.Printf("[INFO] Setting up memory build structure for folder: %s", dockerfileFolder)
 	err = getParsedTarMemory(fs, tw, dockerfileFolder, "")
 	if err != nil {
 		log.Printf("Tar issue: %s", err)
@@ -197,17 +230,56 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
 	}
 
 	// Build the actual image
+	log.Printf("[INFO] Building %s. This may take up to a few minutes.", dockerfileFolder)
 	imageBuildResponse, err := client.ImageBuild(
 		ctx,
 		dockerFileTarReader,
 		buildOptions,
 	)
+
+	//log.Printf("Response: %#v", imageBuildResponse.Body)
 	//log.Printf("IMAGERESPONSE: %#v", imageBuildResponse.Body)
 
 	defer imageBuildResponse.Body.Close()
-	_, newerr := io.Copy(os.Stdout, imageBuildResponse.Body)
+	buildBuf := new(strings.Builder)
+	_, newerr := io.Copy(buildBuf, imageBuildResponse.Body)
 	if newerr != nil {
 		log.Printf("Failed reading Docker build STDOUT: %s", newerr)
+	} else {
+		log.Printf("STRING: %s", buildBuf.String())
+		if strings.Contains(buildBuf.String(), "errorDetail") {
+			log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), strings.Join(tags, "\n"))
+
+			// Handles pulling of the same image if applicable
+			// This fixes some issues with older versions of Docker which can't build
+			// on their own ( <17.05 )
+			pullOptions := types.ImagePullOptions{}
+			downloaded := false
+			for _, image := range tags {
+				// Is this ok? Not sure. Tags shouldn't be controlled here prolly.
+				image = strings.ToLower(image)
+
+				newImage := fmt.Sprintf("%s/%s", registryName, image)
+				log.Printf("[INFO] Pulling image %s", newImage)
+				reader, err := client.ImagePull(ctx, newImage, pullOptions)
+				if err != nil {
+					log.Printf("[ERROR] Failed getting image %s: %s", newImage, err)
+					continue
+				}
+
+				// Attempt to retag the image to not contain registry...
+
+				//newBuf := buildBuf
+				downloaded = true
+				io.Copy(os.Stdout, reader)
+				log.Printf("[INFO] Successfully downloaded and built %s", newImage)
+			}
+
+			if !downloaded {
+				return errors.New(fmt.Sprintf("Failed to build / download images %s", strings.Join(tags, ",")))
+			}
+			//baseDockerName
+		}
 	}
 
 	if err != nil {
@@ -272,9 +344,15 @@ func buildImage(tags []string, dockerfileFolder string) error {
 
 	// Read the STDOUT from the build process
 	defer imageBuildResponse.Body.Close()
-	_, err = io.Copy(os.Stdout, imageBuildResponse.Body)
+	buildBuf := new(strings.Builder)
+	_, err = io.Copy(buildBuf, imageBuildResponse.Body)
 	if err != nil {
 		return err
+	} else {
+		if strings.Contains(buildBuf.String(), "errorDetail") {
+			log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), strings.Join(tags, "\n"))
+			return errors.New(fmt.Sprintf("Failed building %s. Check backend logs for details. Most likely means you have an old version of Docker.", strings.Join(tags, ",")))
+		}
 	}
 
 	return nil
@@ -424,7 +502,7 @@ func handleStopHookDocker(resp http.ResponseWriter, request *http.Request) {
 	ctx := context.Background()
 	hook, err := getHook(ctx, fileId)
 	if err != nil {
-		log.Printf("Failed getting hook: %s", err)
+		log.Printf("Failed getting hook %s (stop docker): %s", fileId, err)
 		resp.WriteHeader(401)
 		resp.Write([]byte(`{"success": false}`))
 		return
@@ -556,7 +634,7 @@ func handleStartHookDocker(resp http.ResponseWriter, request *http.Request) {
 	ctx := context.Background()
 	hook, err := getHook(ctx, fileId)
 	if err != nil {
-		log.Printf("Failed getting hook: %s", err)
+		log.Printf("Failed getting hook %s (start docker): %s", fileId, err)
 		resp.WriteHeader(401)
 		resp.Write([]byte(`{"success": false}`))
 		return
@@ -636,7 +714,7 @@ func handleStartHookDocker(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	// FIXME - get some real data?
-	log.Printf("Successfully started %s-%s on port %s with filepath %s", image, fileId, port, filepath)
+	log.Printf("[INFO] Successfully started %s-%s on port %s with filepath %s", image, fileId, port, filepath)
 	resp.WriteHeader(200)
 	resp.Write([]byte(`{"success": true, "message": "Started webhook"}`))
 	return
@@ -644,7 +722,7 @@ func handleStartHookDocker(resp http.ResponseWriter, request *http.Request) {
 
 // Checks if an image exists
 func imageCheckBuilder(images []string) error {
-	log.Printf("[FIXME] ImageNames to check: %#v", images)
+	//log.Printf("[FIXME] ImageNames to check: %#v", images)
 	return nil
 
 	ctx := context.Background()
@@ -704,10 +782,115 @@ func hookTest() {
 
 	returnHook, err := getHook(ctx, hook.Id)
 	if err != nil {
-		log.Printf("Failed getting hook: %s", err)
+		log.Printf("Failed getting hook %s (test): %s", hook.Id, err)
 	}
 
 	if len(returnHook.Id) > 0 {
 		log.Printf("Success! - %s", returnHook.Id)
 	}
 }
+
+//https://stackoverflow.com/questions/23935141/how-to-copy-docker-images-from-one-host-to-another-without-using-a-repository
+func getDockerImage(resp http.ResponseWriter, request *http.Request) {
+	cors := handleCors(resp, request)
+	if cors {
+		return
+	}
+
+	// Just here to verify that the user is logged in
+	_, err := handleApiAuthentication(resp, request)
+	if err != nil {
+		log.Printf("Api authentication failed in validate swagger: %s", err)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false}`))
+		return
+	}
+
+	body, err := ioutil.ReadAll(request.Body)
+	if err != nil {
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
+		return
+	}
+
+	type requestCheck struct {
+		Name string `datastore:"name" json:"name" yaml:"name"`
+	}
+
+	//body = []byte(`swagger: "2.0"`)
+	//body = []byte(`swagger: '1.0'`)
+	//newbody := string(body)
+	//newbody = strings.TrimSpace(newbody)
+	//body = []byte(newbody)
+	//log.Println(string(body))
+	//tmpbody, err := yaml.YAMLToJSON(body)
+	//log.Println(err)
+	//log.Println(string(tmpbody))
+
+	// This has to be done in a weird way because Datastore doesn't
+	// support map[string]interface and similar (openapi3.Swagger)
+	var version requestCheck
+
+	err = json.Unmarshal(body, &version)
+	if err != nil {
+		resp.WriteHeader(422)
+		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed JSON marshalling: %s"}`, err)))
+		return
+	}
+
+	log.Printf("Image to load: %s", version.Name)
+	//cli, err := client.NewEnvClient()
+	//if err != nil {
+	//	log.Println("Unable to create docker client")
+	//	return err
+	//}
+
+	dockercli, err := client.NewEnvClient()
+	if err != nil {
+		log.Printf("Unable to create docker client: %s", err)
+		resp.WriteHeader(422)
+		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed JSON marshalling: %s"}`, err)))
+		return
+	}
+
+	ctx := context.Background()
+	images, err := dockercli.ImageList(ctx, types.ImageListOptions{
+		All: true,
+	})
+
+	img := types.ImageSummary{}
+	tagFound := ""
+	for _, image := range images {
+		for _, tag := range image.RepoTags {
+			log.Printf("Image: %s", tag)
+
+			if strings.ToLower(tag) == strings.ToLower(version.Name) {
+				img = image
+				tagFound = tag
+				break
+			}
+		}
+	}
+
+	if len(img.ID) == 0 {
+		resp.WriteHeader(401)
+		resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't find image %s"}`, version.Name)))
+		return
+	}
+	_ = tagFound
+
+	/*
+		log.Printf("IMg: %#v", img)
+		pullOptions := types.ImagePullOptions{}
+		log.Printf("[INFO] Pulling image %s", image)
+		reader, err := dockercli.ImagePull(ctx, tag, pullOptions)
+		if err != nil {
+			log.Printf("[ERROR] Failed getting image %s: %s", image, err)
+		}
+
+		io.Copy(os.Stdout, r)
+	*/
+
+	resp.WriteHeader(200)
+	resp.Write([]byte(fmt.Sprintf(`{"success": true, "message": "Downloading image %s"}`, version.Name)))
+}

backend/go-app/files.go

@@ -42,6 +42,9 @@ type File struct {
 	DownloadPath string   `json:"download_path" datastore:"download_path"`
 	Md5sum       string   `json:"md5_sum" datastore:"md5_sum"`
 	Sha256sum    string   `json:"sha256_sum" datastore:"sha256_sum"`
+	FileSize     int64    `json:"filesize" datastore:"filesize"`
+	Duplicate    bool     `json:"duplicate" datastore:"duplicate"`
+	Subflows     []string `json:"subflows" datastore:"subflows"`
 }
 
 var basepath = os.Getenv("SHUFFLE_FILE_LOCATION")
@@ -100,6 +103,51 @@ func fileExists(filename string) bool {
 	return !info.IsDir()
 }
 
+func handleGetFiles(resp http.ResponseWriter, request *http.Request) {
+	cors := handleCors(resp, request)
+	if cors {
+		return
+	}
+
+	// 1. Check user directly
+	// 2. Check workflow execution authorization
+	user, err := handleApiAuthentication(resp, request)
+	if err != nil {
+		log.Printf("[INFO] INITIAL Api authentication failed in file LIST: %s", err)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false}`))
+		return
+	}
+
+	if user.Role != "admin" {
+		log.Printf("[AUTH] User isn't admin")
+		resp.WriteHeader(401)
+		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Need to be admin"}`)))
+		return
+	}
+
+	ctx := context.Background()
+	files, err := getAllFiles(ctx, user.ActiveOrg.Id)
+	if err != nil {
+		log.Printf("[ERROR] Failed to get files: %s", err)
+		resp.WriteHeader(500)
+		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error getting files."}`)))
+		return
+	}
+
+	log.Printf("Got %d files for org %s", len(files), user.ActiveOrg.Id)
+	newBody, err := json.Marshal(files)
+	if err != nil {
+		log.Printf("[ERROR] Failed marshaling files: %s", err)
+		resp.WriteHeader(500)
+		resp.Write([]byte(`{"success": false, "reason": "Failed to marshal files"}`))
+		return
+	}
+
+	resp.WriteHeader(200)
+	resp.Write([]byte(newBody))
+}
+
 func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) {
 	cors := handleCors(resp, request)
 	if cors {
@@ -347,7 +395,7 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
 		return
 	}
 
-	log.Printf("\n\nUser is trying to download file %s\n\n", fileId)
+	log.Printf("\n\n[INFO] User is trying to download file %s\n\n", fileId)
 
 	// 1. Check user directly
 	// 2. Check workflow execution authorization
@@ -417,8 +465,18 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
 	Openfile, err := os.Open(downloadPath)
 	defer Openfile.Close() //Close after function return
 	if err != nil {
+		file.Status = "deleted"
+		err = setFile(ctx, *file)
+		if err != nil {
+			log.Printf("Failed setting file to uploading")
+			resp.WriteHeader(500)
+			resp.Write([]byte(`{"success": false, "reason": "Failed setting file to uploading"}`))
+			return
+		}
+
 		//File not found, send 404
-		http.Error(resp, "File not found.", 404)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "File doesn't exist locally"}`))
 		return
 	}
 
@@ -552,6 +610,7 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) {
 	var buf bytes.Buffer
 	io.Copy(&buf, parsedFile)
 	contents := buf.Bytes()
+	file.FileSize = int64(len(contents))
 	md5 := md5sum(contents)
 	buf.Reset()
 
@@ -642,7 +701,8 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) {
 
 	// Loads of validation below
 	if len(curfile.Filename) == 0 || len(curfile.OrgId) == 0 || len(curfile.WorkflowId) == 0 {
-		log.Printf("[ERROR] Missing field during upload.")
+		log.Printf("[ERROR] Missing field during fileupload. Required: filename, org_id, workflow_id")
+		log.Printf("INPUT: %s", string(body))
 		resp.WriteHeader(401)
 		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Missing field. Required: filename, org_id, workflow_id"}`)))
 		return
@@ -715,6 +775,30 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) {
 	fileId := uuid.NewV4().String()
 	downloadPath := fmt.Sprintf("%s/%s", folderPath, fileId)
 
+	duplicateWorkflows := []string{}
+	for _, trigger := range workflow.Triggers {
+		if trigger.AppName == "Shuffle Workflow" && trigger.TriggerType == "SUBFLOW" {
+			for _, parameter := range trigger.Parameters {
+				if parameter.Name == "workflow" && len(parameter.Value) > 0 {
+
+					found := false
+					for _, workflow := range duplicateWorkflows {
+						if workflow == parameter.Value {
+							found = true
+							break
+						}
+					}
+
+					if !found {
+						duplicateWorkflows = append(duplicateWorkflows, parameter.Value)
+					}
+
+					break
+				}
+			}
+		}
+	}
+
 	timeNow := time.Now().Unix()
 	newFile := File{
 		Id:           fileId,
@@ -726,6 +810,7 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) {
 		OrgId:        curfile.OrgId,
 		WorkflowId:   curfile.WorkflowId,
 		DownloadPath: downloadPath,
+		Subflows:     duplicateWorkflows,
 	}
 
 	err = setFile(ctx, newFile)
@@ -740,6 +825,7 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) {
 
 	resp.WriteHeader(200)
 	resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, fileId)))
+
 }
 
 func getFile(ctx context.Context, id string) (*File, error) {
@@ -754,6 +840,9 @@ func getFile(ctx context.Context, id string) (*File, error) {
 
 func setFile(ctx context.Context, file File) error {
 	// clear session_token and API_token for user
+	timeNow := time.Now().Unix()
+	file.UpdatedAt = timeNow
+
 	k := datastore.NameKey("Files", file.Id, nil)
 	if _, err := dbclient.Put(ctx, k, &file); err != nil {
 		log.Println(err)
@@ -762,3 +851,23 @@ func setFile(ctx context.Context, file File) error {
 
 	return nil
 }
+
+func getAllFiles(ctx context.Context, orgId string) ([]File, error) {
+	var files []File
+	q := datastore.NewQuery("Files").Filter("org_id =", orgId).Order("-updated_at").Limit(100)
+
+	_, err := dbclient.GetAll(ctx, q, &files)
+	if err != nil {
+		if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") {
+			q = q.Limit(50)
+			_, err := dbclient.GetAll(ctx, q, &files)
+			if err != nil {
+				return []File{}, err
+			}
+		} else {
+			return []File{}, err
+		}
+	}
+
+	return files, nil
+}

backend/go-app/go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/gorilla/mux v1.7.4
 	github.com/h2non/filetype v1.0.12
 	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/satori/go.uuid v1.2.0
 	golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d

backend/go-app/go.sum

@@ -160,6 +160,8 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

backend/tests/execute.sh

@@ -1,5 +1,26 @@
 #!/bin/sh
-curl -XPOST http://localhost:5001/api/v1/workflows/1d9d8ce2-566e-4c3f-8a37-5d6c7d2000b5/execute -d '{"execution_argument":""}' -H "Authorization: Bearer 144308d0-6aab-4d4f-8bb2-75189281ee26"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
-
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
-
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
-curl -XPOST http://localhost:5001/api/v1/workflows/1d9d8ce2-566e-4c3f-8a37-5d6c7d2000b5/execute -d '{"execution_argument":""}' -H "Authorization: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjYwZjQwNjBlNThkNzVmZDNmNzBiZWZmODhjNzk0YTc3NTMyN2FhMzEiLCJ0eXAiOiJKV1QifQ.eyJhdWQiOiJodHRwczovL3NodWZmbGVyLmlvL2FwaS92MS93b3JrZmxvd3MvMWQ5ZDhjZTItNTY2ZS00YzNmLThhMzctNWQ2YzdkMjAwMGI1L2V4ZWN1dGUiLCJhenAiOiIxMDMwNzY3ODIwNjE0MjQ2MTg0MjIiLCJlbWFpbCI6InNjaGVkdWxlckBzaHVmZmxlLTI0MTUxNy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJleHAiOjE1NjU1Mjc1NTEsImlhdCI6MTU2NTUyMzk1MSwiaXNzIjoiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tIiwic3ViIjoiMTAzMDc2NzgyMDYxNDI0NjE4NDIyIn0.r0EDq9fjhf_5CPTiltyfk_L3uYJp577Uy0yYPcCAl2nv50_z_oUtbWGBpQLL8gcj-NGd3g4E52Qur8k6hCMIQweLS6WAb1279vGffEoCNDfkWb3Oy-yJGP1kzwLvqFJqnHLkSWYXNWvSyWnEimW8Rryx_m1BXS5wcA8l4NIr83kS7fPZrTwjnwFSeGSThwk91DVARzapQb8r0GEgOUyHZ1aBXnV98mikzSUt-5xFKe9eMdD22YJAj0Ru-DxAxs5nOqghX4PMRysWjshjOMrlR1piPWxqAmewp8YKZDCQ5gXskpeAFBDoULT971Wsx_NCohnJsFqx1JfPS9ZYMTW2oQ"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"

docker-compose.yml

@@ -2,7 +2,7 @@ version: '3'
 services:
   frontend:
     #build: ./frontend
-    image: ghcr.io/frikky/shuffle-frontend:0.8.3
+    image: ghcr.io/frikky/shuffle-frontend:0.8.56
     container_name: shuffle-frontend
     hostname: shuffle-frontend
     ports:
@@ -17,7 +17,7 @@ services:
       - backend
   backend:
     #build: ./backend
-    image: ghcr.io/frikky/shuffle-backend:0.8.3
+    image: ghcr.io/frikky/shuffle-backend:0.8.56
     container_name: shuffle-backend
     hostname: ${BACKEND_HOSTNAME}
     # Here for debugging:
@@ -45,7 +45,7 @@ services:
       - database
   orborus:
     #build: ./functions/onprem/orborus
-    image: ghcr.io/frikky/shuffle-orborus:0.8.0
+    image: ghcr.io/frikky/shuffle-orborus:0.8.5
     container_name: shuffle-orborus
     hostname: shuffle-orborus
     networks:
@@ -53,8 +53,8 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     environment:
-      - SHUFFLE_APP_SDK_VERSION=0.8.0
+      - SHUFFLE_APP_SDK_VERSION=0.8.51
-      - SHUFFLE_WORKER_VERSION=0.8.0
+      - SHUFFLE_WORKER_VERSION=0.8.54
       - ORG_ID=${ORG_ID}
       - ENVIRONMENT_NAME=${ENVIRONMENT_NAME}
       - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT}
@@ -66,6 +66,7 @@ services:
       - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME}
       - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY}
       - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX}
+      - CLEANUP=${SHUFFLE_CONTAINER_AUTO_CLEANUP}
     restart: unless-stopped
   database:
     #build: ./backend/database

frontend/Dockerfile

@@ -8,7 +8,8 @@ ENV PATH /usr/src/app/node_modules/.bin:$PATH
 
 COPY package.json /usr/src/app/package.json
 
-RUN npm install --verbose
+#RUN npm install --verbose
+RUN yarn install 
 
 # copy only required files to not trigger rebuilding every time
 COPY ./certs /usr/src/app/certs/
@@ -19,9 +20,9 @@ COPY ./*.json /usr/src/app/
 
 # There were issues with the webpack installer from package.json
 RUN rm -rf /usr/src/app/node_modules/webpack
-RUN npm install webpack@4.42.0
+#RUN yarn add webpack@4.42.0
 
-RUN npm run-script build
+RUN yarn build
 
 # Production environment
 FROM nginx:latest

frontend/package.json

@@ -1,13 +1,13 @@
 {
   "name": "shuffler",
   "homepage": "https://shuffler.io",
-  "version": "0.6.0",
+  "version": "0.8.53",
   "private": true,
   "dependencies": {
     "@material-ui/core": "^4.5.2",
     "@material-ui/icons": "^4.5.1",
     "@material-ui/styles": "^4.5.2",
-    "@use-it/interval": "^0.1.3",
+    "@use-it/interval": "^1.0.0",
     "babel-eslint": "^10.1.0",
     "class-transformer": "^0.3.1",
     "create-react-app": "^2.0.3",
@@ -32,7 +32,7 @@
     "md5-file": "^4.0.0",
     "mdbreact": "^4.21.1",
     "moment": "~2.20.1",
-    "react": "^16.10.2",
+    "react": "^16.14.0",
     "react-alert": "^5.5.0",
     "react-alert-template-basic": "^1.0.0",
     "react-beforeunload": "^2.2.1",
@@ -40,7 +40,7 @@
     "react-cookie": "^4.0.1",
     "react-cytoscapejs": "^1.2.0",
     "react-device-detect": "^1.9.10",
-    "react-dom": "^16.10.2",
+    "react-dom": "^16.14.0",
     "react-draggable": "^3.3.2",
     "react-dropzone": "^10.1.10",
     "react-ga": "^2.7.0",

frontend/src/App.jsx

@@ -142,7 +142,7 @@ const App = (message, props) => {
 			<Route exact path="/apps/new" render={props => <AppCreator isLoaded={isLoaded} isLoggedIn={isLoggedIn} globalUrl={globalUrl} {...props} />} />
 			<Route exact path="/apps/edit/:appid" render={props => <AppCreator isLoaded={isLoaded} isLoggedIn={isLoggedIn} globalUrl={globalUrl} {...props} />} />
 			<Route exact path="/schedules/:key" render={props => <EditSchedule globalUrl={globalUrl} {...props} />} />
-			<Route exact path="/workflows" render={props => <Workflows cookies={cookies} removeCookie={removeCookie} isLoaded={isLoaded} isLoggedIn={isLoggedIn} globalUrl={globalUrl} cookies={cookies} {...props} />} />
+			<Route exact path="/workflows" render={props => <Workflows cookies={cookies} removeCookie={removeCookie} isLoaded={isLoaded} isLoggedIn={isLoggedIn} globalUrl={globalUrl} cookies={cookies} userdata={userdata} {...props} />} />
 			<Route exact path="/workflows/:key" render={props => <AngularWorkflow userdata={userdata} globalUrl={globalUrl} isLoaded={isLoaded} isLoggedIn={isLoggedIn} {...props} />} />
 			<Route exact path="/docs/:key" render={props => <Docs isLoaded={isLoaded} globalUrl={globalUrl} {...props} />} />
 			<Route exact path="/docs" render={props => { window.location.pathname = "/docs/about" }} />

frontend/src/components/OrgHeader.js

@@ -3,9 +3,15 @@ import { makeStyles } from '@material-ui/styles';
 import { useTheme } from '@material-ui/core/styles';
 
 import Tooltip from '@material-ui/core/Tooltip';
+import Grid from '@material-ui/core/Grid';
 import Button from '@material-ui/core/Button';
 import TextField from '@material-ui/core/TextField';
+import Typography from '@material-ui/core/Typography';
 import { useAlert } from "react-alert";
+import IconButton from '@material-ui/core/IconButton';
+import ExpandLessIcon from '@material-ui/icons/ExpandLess';
+import ExpandMoreIcon from '@material-ui/icons/ExpandMore';
+import SaveIcon from '@material-ui/icons/Save';
 
 const useStyles = makeStyles({
 	notchedOutline: {
@@ -23,11 +29,17 @@ const OrgHeader = (props) => {
 	const classes = useStyles()
 
 	var upload = ""
+	const defaultBranch = "master"
 	const [orgName, setOrgName] = React.useState(selectedOrganization.name)
 	const [orgDescription, setOrgDescription] = React.useState(selectedOrganization.description)
+	const [appDownloadUrl, setAppDownloadUrl] = React.useState(selectedOrganization.defaults === undefined ? "https://github.com/frikky/shuffle-apps" : selectedOrganization.defaults.app_download_repo === undefined || selectedOrganization.defaults.app_download_repo.length === 0 ? "https://github.com/frikky/shuffle-apps" : selectedOrganization.defaults.app_download_repo)
+	const [appDownloadBranch, setAppDownloadBranch] = React.useState(selectedOrganization.defaults === undefined ? defaultBranch : selectedOrganization.defaults.app_download_branch === undefined || selectedOrganization.defaults.app_download_branch.length === 0 ? defaultBranch : selectedOrganization.defaults.app_download_branch)
+	const [workflowDownloadUrl, setWorkflowDownloadUrl] = React.useState(selectedOrganization.defaults === undefined ? "https://github.com/frikky/shuffle-apps" : selectedOrganization.defaults.workflow_download_repo === undefined || selectedOrganization.defaults.workflow_download_repo.length === 0 ? "https://github.com/frikky/shuffle-workflows" : selectedOrganization.defaults.workflow_download_repo)
+	const [workflowDownloadBranch, setWorkflowDownloadBranch] = React.useState(selectedOrganization.defaults === undefined ? defaultBranch : selectedOrganization.defaults.workflow_download_branch === undefined || selectedOrganization.defaults.workflow_download_branch.length === 0 ? defaultBranch : selectedOrganization.defaults.workflow_download_branch)
 
 	const [file, setFile] = React.useState("")
 	const [fileBase64, setFileBase64] = React.useState(selectedOrganization.image)
+	const [expanded, setExpanded] = React.useState(false)
 
 	if (file !== "") {
 		const img = document.getElementById('logo')
@@ -55,12 +67,13 @@ const OrgHeader = (props) => {
 		}
 	}
 
-	const handleEditOrg = (name, description, orgId, image) => {
+	const handleEditOrg = (name, description, orgId, image, defaults) => {
 		const data = { 
 			"name": name, 
 			"description": description,
 			"org_id": orgId,
 			"image": image,
+			"defaults": defaults,
 		}
 
 		const url = globalUrl + `/api/v1/orgs/${selectedOrganization.id}`;
@@ -102,9 +115,14 @@ const OrgHeader = (props) => {
 			style={{ width: 150, height: 55, flex: 1 }}
 			variant="contained"
 			color="primary"
-			onClick={() => handleEditOrg(orgName, orgDescription, selectedOrganization.id, selectedOrganization.image)}
+			onClick={() => handleEditOrg(orgName, orgDescription, selectedOrganization.id, selectedOrganization.image, {
+				"app_download_repo": appDownloadUrl, 
+				"app_download_branch": appDownloadBranch,
+				"workflow_download_repo": workflowDownloadUrl,
+				"workflow_download_branch": workflowDownloadBranch,
+			})}
 		>
-			Save Changes	
+			<SaveIcon />
 		</Button>
 
 	var imageData = file.length > 0 ? file : fileBase64 
@@ -113,7 +131,7 @@ const OrgHeader = (props) => {
 	return (
 		<div>
 			<div style={{color: "white", flex: "1", display: "flex", flexDirection: "row"}}>
-				<Tooltip title="Click to edit the app's image - 174x174" placement="bottom">
+				<Tooltip title="Click to edit the organizations's image (174x174)" placement="bottom">
 					<div style={{flex: "1", margin: "10px 25px 10px 0px", border: imageData !== undefined && imageData.length > 0 ? null : "1px solid #f85a3e", cursor: "pointer", backgroundColor: imageData !== undefined && imageData.length > 0 ? null : theme.palette.inputColor, maxWidth: 174, maxHeight: 174}} onClick={() => {upload.click()}}>
 						<input hidden type="file" ref={(ref) => upload = ref} onChange={editHeaderImage} />
 						{imageInfo}
@@ -188,9 +206,151 @@ const OrgHeader = (props) => {
 						<div style={{margin: "auto", textalign: "center",}}>
 							{orgSaveButton}
 						</div>
-								</div>
-							</div>
 						</div>
+					</div>
+				</div>
+				<div style={{textAlign: "center",}}>
+					<IconButton style={{color: "white", marginTop: 10, }} onClick={() => {
+						setExpanded(!expanded)	
+					}}>
+						{expanded ? 
+							<ExpandLessIcon />
+							:
+							<ExpandMoreIcon />
+						}
+					</IconButton>
+					{expanded ? 
+						<Grid container spacing={3} style={{textAlign: "left"}}>
+							<Grid item xs={6} style={{}}>
+								<span>
+									<Typography>
+										App Download URL
+									</Typography>
+									<TextField
+										required
+										style={{flex: "1", marginTop: "5px", marginRight: "15px", backgroundColor: theme.palette.inputColor}}
+										fullWidth={true}
+										type="name"
+										id="outlined-with-placeholder"
+										margin="normal"
+										variant="outlined"
+										placeholder="A description for the organization"
+										value={appDownloadUrl}
+										onChange={e => {
+											setAppDownloadUrl(e.target.value)
+										}}
+										InputProps={{
+											classes: {
+												notchedOutline: classes.notchedOutline,
+											},
+											style:{
+												color: "white",
+											},
+										}}
+										/>
+								</span>
+							</Grid>
+							<Grid item xs={6} style={{}}>
+								<span>
+									<Typography>
+										App Download Branch 
+									</Typography>
+									<TextField
+										required
+										style={{flex: "1", marginTop: "5px", marginRight: "15px", backgroundColor: theme.palette.inputColor}}
+										fullWidth={true}
+										type="name"
+										id="outlined-with-placeholder"
+										margin="normal"
+										variant="outlined"
+										placeholder="A description for the organization"
+										value={appDownloadBranch}
+										onChange={e => {
+											setAppDownloadBranch(e.target.value)
+										}}
+										InputProps={{
+											classes: {
+												notchedOutline: classes.notchedOutline,
+											},
+											style:{
+												color: "white",
+											},
+										}}
+										/>
+								</span>
+							</Grid>
+							<Grid item xs={6} style={{}}>
+								<span>
+									<Typography>
+										Workflow Download URL
+									</Typography>
+									<TextField
+										required
+										style={{flex: "1", marginTop: "5px", marginRight: "15px", backgroundColor: theme.palette.inputColor}}
+										fullWidth={true}
+										type="name"
+										id="outlined-with-placeholder"
+										margin="normal"
+										variant="outlined"
+										placeholder="A description for the organization"
+										value={workflowDownloadUrl}
+										onChange={e => {
+											setWorkflowDownloadUrl(e.target.value)
+										}}
+										InputProps={{
+											classes: {
+												notchedOutline: classes.notchedOutline,
+											},
+											style:{
+												color: "white",
+											},
+										}}
+										/>
+								</span>
+							</Grid>
+							<Grid item xs={6} style={{}}>
+								<span>
+									<Typography>
+										Workflow Download Branch 
+									</Typography>
+									<TextField
+										required
+										style={{flex: "1", marginTop: "5px", marginRight: "15px", backgroundColor: theme.palette.inputColor}}
+										fullWidth={true}
+										type="name"
+										id="outlined-with-placeholder"
+										margin="normal"
+										variant="outlined"
+										placeholder="A description for the organization"
+										value={workflowDownloadBranch}
+										onChange={e => {
+											setWorkflowDownloadBranch(e.target.value)
+										}}
+										InputProps={{
+											classes: {
+												notchedOutline: classes.notchedOutline,
+											},
+											style:{
+												color: "white",
+											},
+										}}
+										/>
+								</span>
+							</Grid>
+							{/*
+							<span style={{textAlign: "center"}}>
+								{expanded ? 
+									<ExpandLessIcon />
+									:
+									<ExpandMoreIcon />
+								}
+							</span>
+							*/}
+						</Grid>
+					: 
+					null
+					}
+				</div>
 			</div>
 	)
 }

frontend/src/defaultCytoscapeStyle.js

@@ -74,6 +74,8 @@ const data = [{
 				'shape': 'octagon',
 				'border-color': 'orange',
 				'background-color': '#213243',
+				'background-width': '100%',
+				'background-height': '100%',
 			},
 		},
 		{

frontend/src/views/Admin.jsx

@@ -1,4 +1,4 @@
-import React, { useEffect} from 'react';
+import React, { useState } from 'react';
 
 import { makeStyles } from '@material-ui/styles';
 import {Link} from 'react-router-dom';
@@ -31,6 +31,11 @@ import { useTheme } from '@material-ui/core/styles';
 import HandlePayment from './HandlePayment'
 import OrgHeader from '../components/OrgHeader'
 
+import EditIcon from '@material-ui/icons/Edit';
+import SelectAllIcon from '@material-ui/icons/SelectAll';
+import OpenInNewIcon from '@material-ui/icons/OpenInNew';
+import CloudDownloadIcon from '@material-ui/icons/CloudDownload';
+import DescriptionIcon from '@material-ui/icons/Description';
 import PolymerIcon from '@material-ui/icons/Polymer';
 import CheckCircleIcon from '@material-ui/icons/CheckCircle';
 import CloseIcon from '@material-ui/icons/Close';
@@ -59,6 +64,7 @@ const Admin = (props) => {
 	const theme = useTheme();
 	const classes = useStyles();
 	const [firstRequest, setFirstRequest] = React.useState(true);
+	const [orgRequest, setOrgRequest] = React.useState(true);
 	const [modalUser, setModalUser] = React.useState({});
 	const [modalOpen, setModalOpen] = React.useState(false);
 
@@ -78,11 +84,13 @@ const Admin = (props) => {
 	const [environments, setEnvironments] = React.useState([]);
 	const [authentication, setAuthentication] = React.useState([]);
 	const [schedules, setSchedules] = React.useState([])
+	const [files, setFiles] = React.useState([])
 	const [selectedUser, setSelectedUser] = React.useState({})
 	const [newPassword, setNewPassword] = React.useState("");
 	const [selectedUserModalOpen, setSelectedUserModalOpen] = React.useState(false)
 	const [selectedAuthentication, setSelectedAuthentication] = React.useState({})
 	const [selectedAuthenticationModalOpen, setSelectedAuthenticationModalOpen] = React.useState(false)
+	const [authenticationFields, setAuthenticationFields] = React.useState([])
 	const [showArchived, setShowArchived] = React.useState(false)
 
 	const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" 
@@ -272,9 +280,70 @@ const Admin = (props) => {
 		})
 	}
 
-	
+	const saveAuthentication = (authentication) => {
-	
+		const data = authentication
-	
+		const url = globalUrl + '/api/v1/apps/authentication';
+
+		fetch(url, {
+			mode: 'cors',
+			method: 'PUT',
+			body: JSON.stringify(data),
+			credentials: 'include',
+			crossDomain: true,
+			withCredentials: true,
+			headers: {
+				'Content-Type': 'application/json; charset=utf-8',
+			},
+		})
+		.then(response =>
+			response.json().then(responseJson => {
+				if (responseJson["success"] === false) {
+					alert.error("Failed changing authentication")
+				} else {
+					//alert.success("Successfully password!")
+					setSelectedUserModalOpen(false)
+					getAppAuthentication() 
+				}
+			}),
+		)
+		.catch(error => {
+			alert.error("Err: " + error.toString())
+		});
+	}
+
+	const editAuthenticationConfig = (id) => {
+		const data = { 
+			"id": id, 
+			"action": "assign_everywhere",
+		}
+		const url = globalUrl + '/api/v1/apps/authentication/'+id+"/config";
+
+		fetch(url, {
+			mode: 'cors',
+			method: 'POST',
+			body: JSON.stringify(data),
+			credentials: 'include',
+			crossDomain: true,
+			withCredentials: true,
+			headers: {
+				'Content-Type': 'application/json; charset=utf-8',
+			},
+		})
+			.then(response =>
+				response.json().then(responseJson => {
+					if (responseJson["success"] === false) {
+						alert.error("Failed overwriting appauth in workflows")
+					} else {
+						alert.success("Successfully updated auth everywhere!")
+						setSelectedUserModalOpen(false)
+						getAppAuthentication() 
+					}
+				}),
+			)
+			.catch(error => {
+				alert.error("Err: " + error.toString())
+			});
+	}
 
 	const onPasswordChange = () => {
 		const data = { "username": selectedUser.username, "newpassword": newPassword }
@@ -296,7 +365,7 @@ const Admin = (props) => {
 					if (responseJson["success"] === false) {
 						alert.error("Failed setting new password")
 					} else {
-						alert.success("Successfully password!")
+						alert.success("Successfully updated password!")
 						setSelectedUserModalOpen(false)
 					}
 				}),
@@ -466,6 +535,33 @@ const Admin = (props) => {
 		})
 	}
 
+	const flushQueue = (name) => {
+		// Just use this one?
+		const url = globalUrl + '/api/v1/flush_queue';
+		fetch(url, {
+			method: 'DELETE',
+			credentials: "include",
+			headers: {
+				'Content-Type': 'application/json',
+			},
+		})
+			.then(response =>
+				response.json().then(responseJson => {
+					if (responseJson["success"] === false) {
+						alert.error(responseJson.reason)
+						getEnvironments()
+					} else {
+						setLoginInfo("")
+						setModalOpen(false)
+						getEnvironments()
+					}
+				}),
+			)
+		.catch(error => {
+			console.log("Error when deleting: ", error)
+		})
+	}
+
 	const deleteEnvironment = (name) => {
 		// FIXME - add some check here ROFL
 		alert.info("Deleting environment " + name)
@@ -550,6 +646,78 @@ const Admin = (props) => {
 			});
 	}
 
+	const getFiles = () => {
+		fetch(globalUrl + "/api/v1/files", {
+			method: 'GET',
+			headers: {
+				'Content-Type': 'application/json',
+				'Accept': 'application/json',
+			},
+			credentials: "include",
+		})
+			.then((response) => {
+				if (response.status !== 200) {
+					console.log("Status not 200 for apps :O!")
+					return
+				}
+
+				return response.json()
+			})
+			.then((responseJson) => {
+				//console.log(responseJson)
+				setFiles(responseJson)
+			})
+			.catch(error => {
+				alert.error(error.toString())
+			});
+	}
+
+	const downloadFile = (file) => {
+		fetch(globalUrl + "/api/v1/files/"+file.id+"/content", {
+			method: 'GET',
+			headers: {
+				'Content-Type': 'application/json',
+				'Accept': 'application/json',
+			},
+			credentials: "include",
+		})
+			.then((response) => {
+				if (response.status !== 200) {
+					console.log("Status not 200 for apps :O!")
+					return ""
+				}
+
+				return response.text()
+			})
+			.then((respdata) => {
+				if (respdata.length === 0) {
+					alert.error("Failed getting file")
+					return
+				}
+
+				var blob = new Blob( [ respdata ], {
+					type: 'application/octet-stream'
+				})
+
+				var url = URL.createObjectURL( blob )
+				var link = document.createElement( 'a' )
+				link.setAttribute( 'href', url )
+				link.setAttribute( 'download', `${file.filename}` )
+				var event = document.createEvent( 'MouseEvents' )
+				event.initMouseEvent( 'click', true, true, window, 1, 0, 0, 0, 0, false, false, false, false, 0, null)
+				link.dispatchEvent( event )
+
+				//return response.json()
+			})
+			.then((responseJson) => {
+				//console.log(responseJson)
+				//setSchedules(responseJson)
+			})
+			.catch(error => {
+				alert.error(error.toString())
+			});
+	}
+
 	const getSchedules = () => {
 		fetch(globalUrl + "/api/v1/workflows/schedules", {
 			method: 'GET',
@@ -596,6 +764,7 @@ const Admin = (props) => {
 			.then((responseJson) => {
 				if (responseJson.success) {
 					//console.log(responseJson.data)
+					console.log(responseJson)
 					setAuthentication(responseJson.data)
 				} else {
 					alert.error("Failed getting authentications")
@@ -705,6 +874,48 @@ const Admin = (props) => {
 		});
 	}
 
+	const setConfig = (event, newValue) => {
+		if (newValue === 1) {
+			getUsers()
+		} else if (newValue === 2) {
+			getAppAuthentication()
+		} else if (newValue === 3) {
+			getEnvironments()
+		} else if (newValue === 4) {
+			getSchedules()
+		} else if (newValue === 5) {
+			getFiles()
+		} else if (newValue === 6) {
+			getOrgs() 
+		}
+
+		if (newValue === 6) {
+			console.log("Should get apps for categories.")
+		}
+
+		const views = {
+			0: "organization",
+			1: "users",
+			2: "app_auth",
+			3: "environments",
+			4: "schedules",
+			5: "files",
+			6: "categories",
+		}
+
+		//var theURL = window.location.pathname
+		//FIXME: Add url edits
+		//var theURL = window.location
+		//theURL.replace(`/${views[curTab]}`, `/${views[newValue]}`)
+		//window.history.pushState({"html":response.html,"pageTitle":response.pageTitle},"", urlPath);
+
+		//console.log(newpath)
+		//window.location.pathame = newpath
+
+		setModalUser({})
+		setCurTab(newValue)
+	}
+
 
 	if (firstRequest) {
 		setFirstRequest(false)
@@ -720,19 +931,20 @@ const Admin = (props) => {
 			"app_auth": 2,
 			"environments": 3,
 			"schedules": 4,
-			"categories": 5,
+			"files": 5,
 		}
 
 		if (props.match.params.key !== undefined) {
 			const tmpitem = views[props.match.params.key]
 			if (tmpitem !== undefined) {
-				setCurTab(tmpitem)
+				//setCurTab(tmpitem)
+				setConfig("", tmpitem)
 			}
 		}
 	}
 
-	if (selectedOrganization.id === undefined && userdata !== undefined && userdata.active_org !== undefined) {
+	if (selectedOrganization.id === undefined && userdata !== undefined && userdata.active_org !== undefined && orgRequest) {
-		//setSelectedOrganization(userdata.active_org)
+		setOrgRequest(false)
 		handleGetOrg(userdata.active_org.id)
 	}
 
@@ -820,8 +1032,8 @@ const Admin = (props) => {
 			});
 	}
 
-	const editAuthenticationModal =
+	const editAuthenticationModal = selectedAuthenticationModalOpen ? 
-		<Dialog modal
+		<Dialog 
 			open={selectedAuthenticationModalOpen}
 			onClose={() => { setSelectedAuthenticationModalOpen(false) }}
 			PaperProps={{
@@ -833,56 +1045,71 @@ const Admin = (props) => {
 				},
 			}}
 		>
-			<DialogTitle><span style={{ color: "white" }}>Edit authentication</span></DialogTitle>
+			<DialogTitle><span style={{ color: "white" }}>Edit authentication for {selectedAuthentication.app.name} ({selectedAuthentication.label})</span></DialogTitle>
 			<DialogContent>
-				<div style={{ display: "flex" }}>
+				{selectedAuthentication.fields.map((data, index) => {
-					<TextField
+					return (
-						style={{ backgroundColor: theme.palette.inputColor, flex: 3 }}
+						<div key={index}>
-						InputProps={{
+							<Typography style={{marginBottom: 0, marginTop: 10}}>{data.key}</Typography>
-							style: {
+							<TextField
-								height: 50,
+								style={{ backgroundColor: theme.palette.inputColor, marginTop: 0, }}
-								color: "white",
+								InputProps={{
-							},
+									style: {
-						}}
+										height: 50,
-						color="primary"
+										color: "white",
-						required
+									},
-						fullWidth={true}
+								}}
-						placeholder="New password"
+								color="primary"
-						type="password"
+								required
-						id="standard-required"
+								fullWidth={true}
-						autoComplete="password"
+								placeholder={data.key}
-						margin="normal"
+								type="text"
-						variant="outlined"
+								id={`authentication-${index}`}
-						onChange={e => setNewPassword(e.target.value)}
+								margin="normal"
-					/>
+								variant="outlined"
-					<Button
+								onChange={e => {
-						style={{ maxHeight: 50, flex: 1 }}
+									authenticationFields[index].value = e.target.value
-						variant="outlined"
+									setAuthenticationFields(authenticationFields)
-						color="primary"
+								}}
-						onClick={() => onPasswordChange()}
+							/>
-					>
+						</div>
-						Submit
+					)
-					</Button>
+				})}
-				</div>
-				<Divider style={{ marginTop: 20, marginBottom: 20, backgroundColor: theme.palette.inputColor }} />
-				<Button
-					style={{}}
-					variant="outlined"
-					color="primary"
-					onClick={() => deleteUser(selectedUser)}
-				>
-					{selectedUser.active ? "Deactivate" : "Activate"}
-				</Button>
-				<Button
-					style={{}}
-					variant="outlined"
-					color="primary"
-					onClick={() => generateApikey(selectedUser.id)}
-				>
-					Get new API key
-				</Button>
 			</DialogContent>
+			<DialogActions>
+				<Button style={{ borderRadius: "0px" }} onClick={() => setSelectedAuthenticationModalOpen(false)} color="primary">
+					Cancel
+				</Button>
+				<Button variant="contained" style={{ borderRadius: "0px" }} onClick={() => {
+					var error = false
+					for (var key in authenticationFields) {
+						const item = authenticationFields[key]
+						if (item.value.length === 0) {
+							console.log("ITEM: ", item)
+							//var currentnode = cy.getElementById(data.id)
+							var textfield = document.getElementById(`authentication-${key}`)
+							if (textfield !== null && textfield !== undefined) {
+								console.log("HANDLE ERROR FOR KEY ", key)
+							}
+							error = true
+						}
+					}
+
+					if (error) {
+						alert.error("All fields must have a new value")
+					} else {
+						alert.success("Saving new version of this authentication")
+						selectedAuthentication.fields = authenticationFields
+						saveAuthentication(selectedAuthentication)
+						setSelectedAuthentication({})
+						setSelectedAuthenticationModalOpen(false)
+					}
+				}} color="primary">
+					Submit
+				</Button>
+			</DialogActions>
 		</Dialog>
+	 : null
 
 	const editUserModal =
 		<Dialog modal
@@ -897,7 +1124,7 @@ const Admin = (props) => {
 				},
 			}}
 		>
-			<DialogTitle><span style={{ color: "white" }}>Edit user</span></DialogTitle>
+			<DialogTitle><span style={{ color: "white" }}><EditIcon /></span></DialogTitle>
 			<DialogContent>
 				<div style={{ display: "flex" }}>
 					<TextField
@@ -1270,7 +1497,7 @@ const Admin = (props) => {
 						})}
 					</Grid>
 					<Divider style={{ marginTop: 20, marginBottom: 20, backgroundColor: theme.palette.inputColor }} />
-					{isCloud && selectedOrganization.subscriptions !== null && selectedOrganization.subscriptions.length > 0 ? 
+					{isCloud && selectedOrganization.subscriptions !== undefined && selectedOrganization.subscriptions !== null && selectedOrganization.subscriptions.length > 0 ? 
 						<div style={{marginTop: 30, marginBottom: 20}}>
 							<Typography style={{marginTop: 40, marginLeft: 10, marginBottom: 5,}}>
 								Your subscription{selectedOrganization.subscriptions.length > 1 ? "s" : ""}
@@ -1429,6 +1656,7 @@ const Admin = (props) => {
 			</div>
 			<div />
 			<Button
+				disabled={isCloud}
 				style={{}}
 				variant="contained"
 				color="primary"
@@ -1463,8 +1691,13 @@ const Admin = (props) => {
 					/>
 				</ListItem>
 				{users === undefined ? null : users.map((data, index) => {
+					var bgColor = "#27292d"
+					if (index % 2 === 0) {
+						bgColor = "#1f2023"
+					}
+
 					return (
-						<ListItem key={index}>
+						<ListItem key={index} style={{backgroundColor: bgColor}}>
 							<ListItemText
 								primary={data.username}
 								style={{ minWidth: 200, maxWidth: 200 }}
@@ -1532,6 +1765,113 @@ const Admin = (props) => {
 		</div>
 		: null
 
+	const filesView = curTab === 5 ?
+		<div>
+			<div style={{marginTop: 20, marginBottom: 20,}}>
+				<h2 style={{display: "inline",}}>Files</h2>
+				<span style={{marginLeft: 25}}>Files from Workflows. <a target="_blank" href="https://shuffler.io/docs/organizations#files" style={{textDecoration: "none", color: "#f85a3e"}}>Learn more</a></span>
+			</div>
+			<Divider style={{marginTop: 20, marginBottom: 20, backgroundColor: theme.palette.inputColor}}/>
+			<List>
+				<ListItem>
+					<ListItemText
+						primary="Created"
+						style={{maxWidth: 225, minWidth: 225}}
+					/>
+					<ListItemText
+						primary="Name"
+						style={{maxWidth: 150, minWidth: 150, overflow: "hidden",}}
+					/>
+					<ListItemText
+						primary="Workflow"
+						style={{maxWidth: 100, minWidth: 100, overflow: "hidden",}}
+					/>
+					<ListItemText
+						primary="Md5"
+						style={{minWidth: 300, maxWidth: 300, overflow: "hidden"}}
+					/>
+					<ListItemText
+						primary="Status"
+						style={{minWidth: 75, maxWidth: 75}}
+					/>
+					<ListItemText
+						primary="Filesize"
+						style={{minWidth: 125, maxWidth: 125}}
+					/>
+					<ListItemText
+						primary="Actions"
+					/>
+				</ListItem>
+				{files === undefined || files === null ? null : files.map((file, index) => {
+					var bgColor = "#27292d"
+					if (index % 2 === 0) {
+						bgColor = "#1f2023"
+					}
+
+					return (
+						<ListItem key={index} style={{backgroundColor: bgColor}} >
+							<ListItemText
+								style={{maxWidth: 225, minWidth: 225}}
+								primary={new Date(file.created_at*1000).toISOString()}
+							/>
+							<ListItemText
+								style={{maxWidth: 150, minWidth: 150}}
+								primary={file.filename}
+							/>
+							<ListItemText
+								primary=
+									<Tooltip title={"Go to workflow"} style={{}} aria-label={"Download"}>
+										<a style={{textDecoration: "none", color: "#f85a3e"}} href={`/workflows/${file.workflow_id}`} target="_blank">
+											<IconButton>
+												<OpenInNewIcon style={{color: "white"}} />
+											</IconButton>
+										</a>
+									</Tooltip>
+								style={{minWidth: 100, maxWidth: 100, overflow: "hidden"}}
+							/>
+							<ListItemText
+								primary={file.md5_sum}
+								style={{minWidth: 300, maxWidth: 300, overflow: "hidden"}}
+							/>
+							<ListItemText
+								primary={file.status}
+								style={{minWidth: 75, maxWidth: 75, overflow: "hidden"}}
+							/>
+							<ListItemText
+								primary={file.filesize}
+								style={{minWidth: 125, maxWidth: 125, overflow: "hidden"}}
+							/>
+							<ListItemText
+								primary=
+									<Tooltip title={"Download file"} style={{}} aria-label={"Download"}>
+										<IconButton onClick={() => {
+											downloadFile(file)
+										}}>
+											<CloudDownloadIcon style={{color: "white"}} />
+										</IconButton>
+									</Tooltip>
+								style={{minWidth: 75, maxWidth: 75, overflow: "hidden"}}
+							/>
+							{/*
+							<ListItemText>
+								<Button 
+									style={{}} 
+									variant="contained"
+									color="primary"
+									disabled
+									onClick={() => deleteSchedule(file)}
+								>
+									Stop schedule	
+								</Button>
+							</ListItemText>
+							*/}
+						</ListItem>
+					)
+				})}
+			</List>
+		</div>
+		: null
+
 	const schedulesView = curTab === 4 ?
 		<div>
 			<div style={{marginTop: 20, marginBottom: 20,}}>
@@ -1562,8 +1902,13 @@ const Admin = (props) => {
 					/>
 				</ListItem>
 				{schedules === undefined || schedules === null ? null : schedules.map((schedule, index) => {
+					var bgColor = "#27292d"
+					if (index % 2 === 0) {
+						bgColor = "#1f2023"
+					}
+
 					return (
-						<ListItem key={index}>
+						<ListItem key={index} style={{backgroundColor: bgColor}}>
 							<ListItemText
 								style={{maxWidth: 200, minWidth: 200}}
 								primary={schedule.environment === "cloud" ? schedule.frequency : <span>{schedule.seconds} seconds</span>}
@@ -1673,35 +2018,53 @@ const Admin = (props) => {
 		</div>
 		: null
 
+	const updateAppAuthentication = (field) => {
+		setSelectedAuthenticationModalOpen(true)
+		setSelectedAuthentication(field)
+		//{selectedAuthentication.fields.map((data, index) => {
+		var newfields = []
+		for (var key in field.fields) {
+			newfields.push({
+				"key": field.fields[key].key,
+				"value": "",
+			})
+		}
+		setAuthenticationFields(newfields)
+	}
+
 	const authenticationView = curTab === 2 ?
 		<div>
 			<div style={{marginTop: 20, marginBottom: 20,}}>
 				<h2 style={{display: "inline",}}>App Authentication</h2>
 				<span style={{marginLeft: 25}}>Control the authentication options for individual apps. <b>Actions can be destructive!</b></span>
-				 .&nbsp;<a target="_blank" href="https://shuffler.io/docs/organizations#app_authentication" style={{textDecoration: "none", color: "#f85a3e"}}>Learn more</a>
+				 &nbsp;<a target="_blank" href="https://shuffler.io/docs/organizations#app_authentication" style={{textDecoration: "none", color: "#f85a3e"}}>Learn more</a>
 			</div>
 			<Divider style={{marginTop: 20, marginBottom: 20, backgroundColor: theme.palette.inputColor}}/>
 			<List>
 				<ListItem>
 					<ListItemText
 						primary="Icon"
-						style={{minWidth: 150, maxWidth: 150}}
+						style={{minWidth: 75, maxWidth: 75}}
 					/>
 					<ListItemText
 						primary="Label"
-						style={{minWidth: 250, maxWidth: 250}}
+						style={{minWidth: 225, maxWidth: 225}}
 					/>
 					<ListItemText
 						primary="App Name"
 						style={{minWidth: 150, maxWidth: 150}}
 					/>
 					<ListItemText
-						primary="Workflows"
+						primary="Ready"
-						style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
+						style={{minWidth: 100, maxWidth: 100}}
 					/>
 					<ListItemText
-						primary="Action amount"
+						primary="Workflows"
-						style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
+						style={{minWidth: 110, maxWidth: 110, overflow: "hidden"}}
+					/>
+					<ListItemText
+						primary="Actions"
+						style={{minWidth: 110, maxWidth: 110, overflow: "hidden"}}
 					/>
 					<ListItemText
 						primary="Fields"
@@ -1712,27 +2075,36 @@ const Admin = (props) => {
 					/>
 				</ListItem>
 				{authentication === undefined ? null : authentication.map((data, index) => {
+					var bgColor = "#27292d"
+					if (index % 2 === 0) {
+						bgColor = "#1f2023"
+					}
+
 					return (
-						<ListItem key={index}>
+						<ListItem key={index} style={{backgroundColor: bgColor}}>
 							<ListItemText
 								primary=<img alt="" src={data.app.large_image} style={{maxWidth: 50,}} />
-								style={{minWidth: 150, maxWidth: 150}}
+								style={{minWidth: 75, maxWidth: 75}}
 							/>
 							<ListItemText
 								primary={data.label}
-								style={{minWidth: 250, maxWidth: 250}}
+								style={{minWidth: 225, maxWidth: 225}}
 							/>
 							<ListItemText
 								primary={data.app.name}
 								style={{minWidth: 150, maxWidth: 150}}
 							/>
 							<ListItemText
-								primary={data.usage === null ? 0 : data.usage.length}
+								primary={data.defined === false ? "No" : "Yes"}
-								style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
+								style={{minWidth: 100, maxWidth: 100}}
+							/>
+							<ListItemText
+								primary={data.workflow_count === null ? 0 : data.workflow_count}
+								style={{minWidth: 110, maxWidth: 110, overflow: "hidden"}}
 							/>
 							<ListItemText
 								primary={data.node_count}
-								style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
+								style={{minWidth: 110, maxWidth: 110, overflow: "hidden"}}
 							/>
 							<ListItemText
 								primary={data.fields.map(data => {
@@ -1741,16 +2113,43 @@ const Admin = (props) => {
 								style={{minWidth: 200, maxWidth: 200, overflow: "hidden"}}
 							/>
 							<ListItemText>
-								<Button 
+								<IconButton
-									style={{}} 
+									onClick={() => {
-									variant="outlined"
+										updateAppAuthentication(data)
-									color="primary"
+									}}
+								>
+									<EditIcon color="primary"/>
+								</IconButton>
+								{data.defined ? 
+									<Tooltip color="primary" title="Set in EVERY workflow" placement="top">
+										<IconButton 
+											style={{marginRight: 10}}
+											disabled={data.defined === false}
+											onClick={() => {
+												editAuthenticationConfig(data.id)
+											}}
+										>
+											<SelectAllIcon color={data.defined ? "primary" : "secondary"} />
+										</IconButton>
+									</Tooltip>
+								: 
+									<Tooltip color="primary" title="Must edit before you can set in all workflows" placement="top">
+										<IconButton 
+											style={{marginRight: 10}}
+											onClick={() => {
+											}}
+										>
+											<SelectAllIcon color={data.defined ? "primary" : "secondary"} />
+										</IconButton>
+									</Tooltip>
+								}
+								<IconButton 
 									onClick={() => {
 										deleteAuthentication(data)
 									}}
 								>
-									Delete	
+									<DeleteIcon color="primary"/> 
-								</Button>
+								</IconButton>
 							</ListItemText>
 						</ListItem>
 					)
@@ -1820,6 +2219,11 @@ const Admin = (props) => {
 						return null
 					}
 
+					//var bgColor = "#27292d"
+					//if (index % 2 === 0) {
+					//	bgColor = "#1f2023"
+					//}
+
 					return (
 						<ListItem key={index}>
 							<ListItemText
@@ -1848,6 +2252,7 @@ const Admin = (props) => {
 								style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
 							>
 								<Button disabled={environment.archived} variant="outlined" style={{borderRadius: "0px"}} onClick={() => deleteEnvironment(environment.Name)} color="primary">Archive</Button>
+								{/*<Button disabled={environment.archived} variant="outlined" style={{borderRadius: "0px"}} onClick={() => flushQueue(environment.Name)} color="primary">Flush Queue</Button>*/}
 							</ListItemText>
 							<ListItemText
 								style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
@@ -1860,7 +2265,7 @@ const Admin = (props) => {
 		</div>
 		: null
 
-	const organizationsTab = curTab === 6 ?
+	const organizationsTab = curTab === 7 ?
 		<div>
 			<div style={{marginTop: 20, marginBottom: 20,}}>
 				<h2 style={{display: "inline",}}>Organizations</h2>
@@ -1941,7 +2346,7 @@ const Admin = (props) => {
 		</div>
 		: null
 
-	const hybridTab = curTab === 5 ?
+	const hybridTab = curTab === 6 ?
 		<div>
 			<div style={{marginTop: 20, marginBottom: 20,}}>
 				<h2 style={{display: "inline",}}>Hybrid</h2>
@@ -1983,48 +2388,11 @@ const Admin = (props) => {
 
 		// primary={environment.Registered ? "true" : "false"}
 
-	const setConfig = (event, newValue) => {
+	
-		if (newValue === 1) {
-			getUsers()
-		} else if (newValue === 2) {
-			getAppAuthentication()
-		} else if (newValue === 3) {
-			getEnvironments()
-		} else if (newValue === 4) {
-			getSchedules()
-		} else if (newValue === 6) {
-			getOrgs() 
-		}
-
-		if (newValue === 6) {
-			console.log("Should get apps for categories.")
-		}
-
-		const views = {
-			0: "organization",
-			1: "users",
-			2: "app_auth",
-			3: "environments",
-			4: "schedules",
-			5: "categories",
-		}
-
-		//var theURL = window.location.pathname
-		//FIXME: Add url edits
-		//var theURL = window.location
-		//theURL.replace(`/${views[curTab]}`, `/${views[newValue]}`)
-		//window.history.pushState({"html":response.html,"pageTitle":response.pageTitle},"", urlPath);
-
-		//console.log(newpath)
-		//window.location.pathame = newpath
-
-		setModalUser({})
-		setCurTab(newValue)
-	}
 
 	const iconStyle = {marginRight: 10}
 	const data = 
-		<div style={{minWidth: 1366, margin: "auto"}}>
+		<div style={{width: 1366, margin: "auto", overflowX: "hidden",}}>
 			<Paper style={paperStyle}>
 				<Tabs
 					value={curTab}
@@ -2033,10 +2401,11 @@ const Admin = (props) => {
 					aria-label="disabled tabs example"
 				>
 					<Tab label=<span><BusinessIcon style={iconStyle} /> Organization</span>/>
-					{isCloud ? null : <Tab label=<span><AccessibilityNewIcon style={iconStyle} />Users</span> />}
+					<Tab label=<span><AccessibilityNewIcon style={iconStyle} />Users</span> />
 					{isCloud ? null : <Tab label=<span><LockIcon style={iconStyle} />App Authentication</span>/>}
 					{isCloud ? null : <Tab label=<span><EcoIcon style={iconStyle} />Environments</span>/>}
 					{isCloud ? null : <Tab label=<span><ScheduleIcon style={iconStyle} />Schedules</span> />}
+					{isCloud ? null : <Tab label=<span><DescriptionIcon style={iconStyle} />Files</span> />}
 					{window.location.protocol == "http:" && window.location.port === "3000" ? <Tab label=<span><CloudIcon style={iconStyle} /> Hybrid</span>/> : null}
 					{window.location.protocol == "http:" && window.location.port === "3000" ? <Tab label=<span><BusinessIcon style={iconStyle} /> Organizations</span>/> : null}
 					{window.location.protocol === "http:" && window.location.port === "3000" ? <Tab label=<span><LockIcon style={iconStyle} />Categories</span>/> : null}
@@ -2049,6 +2418,7 @@ const Admin = (props) => {
 					{usersView}	
 					{environmentView}
 					{schedulesView}
+					{filesView}
 					{hybridTab}
 					{organizationsTab}
 				</div>

frontend/src/views/AdminSetup.jsx

@@ -6,11 +6,6 @@ import TextField from '@material-ui/core/TextField';
 import Button from '@material-ui/core/Button';
 import Paper from '@material-ui/core/Paper';
 
-const hrefStyle = {
-	color: "white", 
-	textDecoration: "none"
-}
-
 const bodyDivStyle = {
 	margin: "auto",
 	marginTop: "100px",
@@ -35,7 +30,7 @@ const useStyles = makeStyles({
 });
 
 const AdminAccount = props => {
-  const { globalUrl, isLoaded, isLoggedIn, setIsLoggedIn, setCookie, } = props;
+  const { globalUrl, isLoaded, isLoggedIn, } = props;
 
   const [username, setUsername] = useState("");
   const [password, setPassword] = useState("");

frontend/src/views/AppCreator.jsx

@@ -4,6 +4,7 @@ import {BrowserView, MobileView} from "react-device-detect";
 
 import {Link} from 'react-router-dom';
 import Paper from '@material-ui/core/Paper';
+import Typography from '@material-ui/core/Typography';
 import FormControlLabel from '@material-ui/core/FormControlLabel';
 import Button from '@material-ui/core/Button';
 import Divider from '@material-ui/core/Divider';
@@ -18,6 +19,7 @@ import DialogActions from '@material-ui/core/DialogActions';
 import TextField from '@material-ui/core/TextField';
 import Tooltip from '@material-ui/core/Tooltip';
 import CheckCircleIcon from '@material-ui/icons/CheckCircle';
+import AttachFileIcon from '@material-ui/icons/AttachFile';
 import Breadcrumbs from '@material-ui/core/Breadcrumbs';
 import AppsIcon from '@material-ui/icons/Apps';
 import CircularProgress from '@material-ui/core/CircularProgress';
@@ -25,6 +27,7 @@ import CircularProgress from '@material-ui/core/CircularProgress';
 import Chip from '@material-ui/core/Chip';
 import ChipInput from 'material-ui-chip-input'
 
+import YAML from 'yaml'
 import ErrorOutline from '@material-ui/icons/ErrorOutline';
 import { useAlert } from "react-alert";
 import words from "shellwords"
@@ -99,7 +102,12 @@ const parseCurl = (s) => {
 		return ""
 	}
 
-  var args = rewrite(words.split(s))
+	try {
+  	var args = rewrite(words.split(s))
+	} catch (e) {
+		return s
+	}
+	
   var out = { method: 'GET', header: {} }
   var state = ''
 
@@ -187,6 +195,7 @@ const AppCreator = (props) => {
 	const alert = useAlert()
 
 	var upload = ""
+	const increaseAmount = 30
 	const actionNonBodyRequest = ["GET", "HEAD", "DELETE", "CONNECT"]
 	const actionBodyRequest = ["POST", "PUT", "PATCH",]
 	const authenticationOptions = ["No authentication", "API key", "Bearer auth", "Basic auth", ]
@@ -218,6 +227,9 @@ const AppCreator = (props) => {
 	const [actions, setActions] = useState([])
 	const [errorCode, setErrorCode] = useState("")
 	const [appBuilding, setAppBuilding] = useState(false)
+	const [extraBodyFields, setExtraBodyFields] = useState([])
+	const [fileUploadEnabled, setFileUploadEnabled] = useState(false)
+	const [actionAmount, setActionAmount] = useState(increaseAmount)
 
 	//const [actions, setActions] = useState([{
 	//	"name": "Get workflows",
@@ -246,6 +258,7 @@ const AppCreator = (props) => {
   	const [currentActionMethod, setCurrentActionMethod] = useState(actionNonBodyRequest[0])
   	const [currentAction, setCurrentAction] = useState({
 			"name": "",
+			"file_field": "",
 			"description": "",
 			"url": "",
 			"headers": "",
@@ -253,6 +266,7 @@ const AppCreator = (props) => {
 			"queries": [],
 			"body": "",
 			"errors": [],
+			"example_response": "",
 			"method": actionNonBodyRequest[0],
 		});
 
@@ -262,7 +276,7 @@ const AppCreator = (props) => {
 		if (firstrequest) {
 			setFirstrequest(false)
 			if (window.location.pathname.includes("apps/edit")) {
-  				setIsEditing(true)
+  			setIsEditing(true)
 				handleEditApp()
 			} else {
 				checkQuery()
@@ -322,15 +336,37 @@ const AppCreator = (props) => {
 				throw new Error("NOT 200 :O")
 			}
 
+			//console.log("DATA: ", response.text())
 			return response.json()
 		})
 		.then((responseJson) => {
+			console.log("THE BODY IS HERE")
   		setIsAppLoaded(true)
 			if (!responseJson.success) {
 				alert.error("Failed to verify")
-			} else {
+			} else{
-				const data = JSON.parse(responseJson.body)
+				console.log("HMM 2")
-				parseIncomingOpenapiData(data)
+				var jsonvalid = false 
+				var tmpvalue = ""
+				try {
+					tmpvalue = JSON.parse(responseJson.body)
+					jsonvalid = true
+				} catch (e) {
+					console.log("Error JSON: ", e)
+				}
+
+				if (!jsonvalid) {
+					try {
+						tmpvalue = YAML.parse(responseJson.body, )
+						jsonvalid = true
+					} catch(e) {
+						console.log("Error YAML: ", e)
+					}
+				}
+
+				if (jsonvalid) {
+					parseIncomingOpenapiData(tmpvalue)
+				}
 			}
 		})
 		.catch(error => {
@@ -353,16 +389,52 @@ const AppCreator = (props) => {
 		//}
 	}
 
+
+	const handleGetRef = (parameter, data) => {
+		if (parameter["$ref"] === undefined) {
+			return parameter
+		}
+
+		const paramsplit = parameter["$ref"].split("/")
+		if (paramsplit[0] !== "#") {
+			console.log("Bad param: ", paramsplit)
+			return parameter
+		}
+
+		var newitem = data
+		for (var key in paramsplit) {
+			var tmpparam = paramsplit[key]
+			if (tmpparam === "#") {
+				continue
+			}
+
+			if (newitem[tmpparam] === undefined) {
+				return parameter
+			}
+
+			newitem = newitem[tmpparam]
+		}
+
+		return newitem 
+
+		//console.log("Should get ", parameter["$ref"])
+		//const subkeys = parameter["$ref"].split("/")
+		// setBasedata(data)
+
+		//	handleGetReference(parameter["$ref"])
+	}
+
 	// Sets the data up as it should be at later points
 	// This is the data FROM the database, not what's being saved
 	const parseIncomingOpenapiData = (data) => {
+		//console.log("DATA: ", data.info)
 		setBasedata(data)
 
-		setName(data.info.title)
-		setDescription(data.info.description)
-		document.title = "Apps - "+data.info.title
-
 		if (data.info !== null && data.info !== undefined)  {
+			setName(data.info.title)
+			setDescription(data.info.description)
+			document.title = "Apps - "+data.info.title
+
 			if (data.info["x-logo"] !== undefined) {
 				setFileBase64(data.info["x-logo"])
 			}
@@ -377,11 +449,21 @@ const AppCreator = (props) => {
 		}
 
 		if (data.tags !== undefined && data.tags.length > 0) {
+			var newtags = []
 			for (var key in data.tags) {
-				newWorkflowTags.push(data.tags[key].name)
+				if (data.tags[key].name.length > 50) {
+					console.log("Skipping tag cus it's too long: ", data.tags[key].name.length)
+					continue
+				}
+
+				newtags.push(data.tags[key].name)
 			}
 
-			setNewWorkflowTags(newWorkflowTags)
+			if (newtags.length > 10) {
+				newtags = newtags.slice(0,9)
+			}
+
+			setNewWorkflowTags(newtags)
 		}
 
 		// This is annoying (:
@@ -410,16 +492,17 @@ const AppCreator = (props) => {
 		for (let [path, pathvalue] of Object.entries(data.paths)) {
 			for (let [method, methodvalue] of Object.entries(pathvalue)) {
 				if (methodvalue === null) {
-					alert.info("Skipped method "+method)
+					alert.info("Skipped method (null)"+method)
 					continue
 				}
 
 				if (!allowedfunctions.includes(method.toUpperCase())) {
+					alert.info("Skipped method (not allowed) "+method)
 					continue
 				}
 
 				var tmpname = methodvalue.summary
-				if (methodvalue.operationId !== undefined && methodvalue.operationId !== null && methodvalue.operationId.length > 0) {
+				if (methodvalue.operationId !== undefined && methodvalue.operationId !== null && methodvalue.operationId.length > 0 && (tmpname === undefined || tmpname.length === 0)) {
 					tmpname = methodvalue.operationId
 				}
 
@@ -427,16 +510,94 @@ const AppCreator = (props) => {
 					"name": tmpname,
 					"description": methodvalue.description,
 					"url": path,
+					"file_field": "",
 					"method": method.toUpperCase(),
 					"headers": "",
 					"queries": [],
 					"paths": [],
 					"body": "",
 					"errors": [],
+					"example_response": "",
+				}
+
+				if (methodvalue["requestBody"] !== undefined) {
+					//console.log("Handle requestbody: ", methodvalue["requestBody"])
+					if (methodvalue["requestBody"]["content"] !== undefined) {
+						if (methodvalue["requestBody"]["content"]["application/json"] !== undefined) {
+							if (methodvalue["requestBody"]["content"]["application/json"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["application/json"]["schema"] !== null) {
+								if (methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"] !== undefined) {
+									var tmpobject = {}
+									for (let [prop, propvalue] of Object.entries(methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"])) {
+										tmpobject[prop] = `\$\{${prop}\}`
+									}
+
+									for (var subkey in methodvalue["requestBody"]["content"]["application/json"]["schema"]["required"]) {
+										const tmpitem = methodvalue["requestBody"]["content"]["application/json"]["schema"]["required"][subkey]
+										tmpobject[tmpitem] = `\$\{${tmpitem}\}`
+									}
+
+									newaction["body"] = JSON.stringify(tmpobject, null, 2)
+								}
+							}
+						} else if (methodvalue["requestBody"]["content"]["application/xml"] !== undefined) {
+							console.log("METHOD XML: ", methodvalue)
+							if (methodvalue["requestBody"]["content"]["application/xml"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["application/xml"]["schema"] !== null) {
+								if (methodvalue["requestBody"]["content"]["application/xml"]["schema"]["properties"] !== undefined) {
+									var tmpobject = {}
+									for (let [prop, propvalue] of Object.entries(methodvalue["requestBody"]["content"]["application/xml"]["schema"]["properties"])) {
+										tmpobject[prop] = `\$\{${prop}\}`
+									}
+
+									for (var subkey in methodvalue["requestBody"]["content"]["application/xml"]["schema"]["required"]) {
+										const tmpitem = methodvalue["requestBody"]["content"]["application/xml"]["schema"]["required"][subkey]
+										tmpobject[tmpitem] = `\$\{${tmpitem}\}`
+									}
+
+									//console.log("OBJ XML: ", tmpobject)
+									//newaction["body"] = XML.stringify(tmpobject, null, 2)
+								}
+							}
+						} else {
+							if (methodvalue["requestBody"]["content"]["example"] !== undefined) {
+								if (methodvalue["requestBody"]["content"]["example"]["example"] !== undefined) {
+									newaction["body"] = methodvalue["requestBody"]["content"]["example"]["example"] 
+									//JSON.stringify(tmpobject, null, 2)
+								}
+							}
+
+							console.log(methodvalue["requestBody"]["content"])
+							if (methodvalue["requestBody"]["content"]["multipart/form-data"] !== undefined) {
+								if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"] !== null) {
+									if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["type"] === "object") {
+										const fieldname = methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["properties"]["fieldname"]
+										if (fieldname !== undefined) {
+											console.log("FIELDNAME: ", fieldname)
+											newaction.file_field = fieldname["value"]
+										}
+									}
+								}
+							}
+						}
+					}
+				}
+
+				// HAHAHA wtf is this.
+				if (methodvalue.responses !== undefined && methodvalue.responses !== null) {
+					if (methodvalue.responses.default !== undefined) {
+						if (methodvalue.responses.default.content !== undefined) {
+							if (methodvalue.responses.default.content["text/plain"] !== undefined) {
+								if (methodvalue.responses.default.content["text/plain"]["schema"] !== undefined) {
+									if (methodvalue.responses.default.content["text/plain"]["schema"]["example"] !== undefined) {
+										newaction.example_response = methodvalue.responses.default.content["text/plain"]["schema"]["example"] 
+									}
+								}
+							}
+						}
+					}
 				}
 
 				for (var key in methodvalue.parameters) {
-					const parameter = methodvalue.parameters[key]
+					const parameter = handleGetRef(methodvalue.parameters[key], data)
 					if (parameter.in === "query") {
 						var tmpaction = {
 							"description": parameter.description,
@@ -466,9 +627,12 @@ const AppCreator = (props) => {
 						}
 					} else if (parameter.in === "header") {
 						newaction.headers += `${parameter.name}=${parameter.example}\n`	
+					} else {
+						console.log("WARNING: don't know how to handle this param: ", parameter)
 					}
 				}
 
+
 				if (newaction.name === "" || newaction.name === undefined) {
 					// Find a unique part of the string
 					// FIXME: Looks for length between /, find the one where they differ
@@ -582,6 +746,17 @@ const AppCreator = (props) => {
 			}
 		}
 
+		if (newActions.length > increaseAmount-1) {
+			setActionAmount(increaseAmount)
+		} else {
+			setActionAmount(newActions.length)
+		}
+
+		if (newActions.length > 1000) {
+			alert.error("Cut down actions from "+newActions.length+" to 999 because of limit")
+			newActions = newActions.slice(0,999)
+		}
+
 		setActions(newActions)
   	setIsAppLoaded(true)
 	}
@@ -659,6 +834,11 @@ const AppCreator = (props) => {
 			}
 
 			const regex = /[A-Za-z0-9 _]/g;
+			if (item.name === undefined) {
+				console.log("Skipping action ", item)
+				continue
+			}
+
 			const found = item.name.match(regex);
 			if (found !== null) {
 				item.name = found.join("")
@@ -668,17 +848,58 @@ const AppCreator = (props) => {
 				"responses": {
 					"default": {
 					  "description": "default",
-					  "schema": {}
+						"content": {
+							"text/plain": {
+					  		"schema": {
+									"type": "string",
+									"example": "",
+								},
+							},
+						},
 					}
 				},
 				"summary": item.name,
 				"operationId": item.name.split(" ").join("_"),
 				"description": item.description,
-				"parameters": []
+				"parameters": [],
+				"requestBody": {
+					"content": {
+
+					}
+				},
 			}
 
 			//console.log("ACTION: ", item)
 
+			if (item.example_response !== undefined && item.example_response.length > 0) {
+				// FIXME: Shallow copy of the string
+				var showResult = Object.assign("", item.example_response).trim()
+				showResult = showResult.split(" None").join(" \"None\"")
+				showResult = showResult.split("\'").join("\"")
+				showResult = showResult.split(" False").join(" false")
+				showResult = showResult.split(" True").join(" true")
+
+				var jsonvalid = true
+				try {
+					const tmp = String(JSON.parse(showResult))
+					if (!showResult.includes("{") && !showResult.includes("[")) {
+						jsonvalid = false
+					}
+				} catch (e) {
+					jsonvalid = false
+				}
+
+				data.paths[item.url][item.method.toLowerCase()].responses["default"]["content"]["text/plain"].schema.type = "string"
+				if (jsonvalid) {
+					// FIXME: Add a JSON parser here - don't run it as a string.
+					data.paths[item.url][item.method.toLowerCase()].responses["default"]["content"]["text/plain"].schema.example = showResult
+
+				} else {
+					data.paths[item.url][item.method.toLowerCase()].responses["default"]["content"]["text/plain"].schema.example = item.example_response
+
+				}
+			}
+
 			if (item.queries.length > 0) {
 				for (var querykey in item.queries) {
 					const queryitem = item.queries[querykey]
@@ -761,7 +982,7 @@ const AppCreator = (props) => {
 						"type": "string",
 					},
 				}
-
+				
 				// FIXME - add application/json if JSON example?
 				data.paths[item.url][item.method.toLowerCase()]["requestBody"] = {
 					"description": "Generated by Shuffler.io",
@@ -773,16 +994,46 @@ const AppCreator = (props) => {
 					},
 				}
 
+				/*
+				data.paths[item.url][item.method.toLowerCase()]["requestBody"] = {
+					"description": "Generated by Shuffler.io",
+					"required": required,
+					"content": {
+						"example": {
+							"example": item.body,
+						},
+					},
+				}
+				*/
+
 				data.paths[item.url][item.method.toLowerCase()].parameters.push(newitem)
 			}
 
+			// https://swagger.io/docs/specification/describing-request-body/file-upload/
+			if (item.file_field !== undefined && item.file_field !== null && item.file_field.length > 0) {
+				console.log("HANDLE FILEFIELD SAVE: ", item.file_field)
+				data.paths[item.url][item.method.toLowerCase()]["requestBody"]["content"]["multipart/form-data"] = {
+
+					"schema": {
+						"type": "object",
+						"properties": {
+							"fieldname": {
+								"type": "string",
+								"value": item.file_field,
+							},
+						},
+					},
+				}
+
+				console.log(data.paths[item.url][item.method.toLowerCase()]["requestBody"]["content"]["multipart/form-data"])
+			}
+
 			if (item.headers.length > 0) {
 				const required = false
 
 				const headersSplit = item.headers.split("\n")
 				for (var key in headersSplit) {
 					const header = headersSplit[key]
-					console.log("HEADER: ", header)
 					var key = ""
 					var value = ""
 					if (header.length > 0 && header.includes("= ")) {
@@ -989,6 +1240,7 @@ const AppCreator = (props) => {
 			"name": "",
 			"description": "",
 			"url": "",
+			"file_field": "",
 			"headers": "",
 			"paths": [],
 			"queries": [],
@@ -1102,14 +1354,14 @@ const AppCreator = (props) => {
 		null
 		:
 		<div>
-			{actions.map((data, index) => {
+			{actions.slice(0,actionAmount).map((data, index) => {
 				var error = data.errors.length > 0 ? 
 					<Tooltip color="primary" title={data.errors.join("\n")} placement="bottom">
 						<ErrorOutline />
 					</Tooltip>
 					:
 					<Tooltip color="secondary" title={data.errors.join("\n")} placement="bottom">
-						<CheckCircleIcon />
+						<CheckCircleIcon style={{marginTop: 6}}/>
 					</Tooltip>
 				
 
@@ -1127,6 +1379,7 @@ const AppCreator = (props) => {
 				}
 
 				const url = data.url
+				const hasFile = data["file_field"] !== undefined && data["file_field"] !== null && data["file_field"].length > 0
 				return (
 					<Paper style={actionListStyle}>
 						{error} 
@@ -1137,15 +1390,23 @@ const AppCreator = (props) => {
 								setUrlPathQueries(data.queries)
 								setUrlPath(data.url)
 								setActionsModalOpen(true)
+
+								if (data["body"] !== undefined && data["body"] !== null && data["body"].length > 0) {
+									findBodyParams(data["body"])
+								}
+
+								if (hasFile) {
+									setFileUploadEnabled(true)
+								}
 							}}>
 							<div style={{display: "flex"}}>
 								<Chip
 									style={{backgroundColor: bgColor, color: "white", borderRadius: 5, minWidth: 80, marginRight: 10, marginTop: 2, cursor: "pointer", fontSize: 14,}}
 									label={data.method}
-									variant="contained"
 								/>
 								<span style={{fontSize: 16, marginTop: "auto", marginBottom: "auto",}}>
-									{url} - {data.name}
+								{hasFile ? <AttachFileIcon style={{height: 20, width: 20}} /> : null} {url} - {data.name}
+						
 								</span>
 							</div>
 							</div>
@@ -1177,24 +1438,51 @@ const AppCreator = (props) => {
 	const setActionField = (field, value) => {
 		currentAction[field] = value
 		setCurrentAction(currentAction)	
+
 		//setUrlPathQueries(currentAction.queries)
 	}
 
+	const findBodyParams = (body) => {
+		const regex = /\${(\w+)}/g
+		const found = body.match(regex)
+		if (found === null) {
+			setExtraBodyFields([])
+		} else {
+			setExtraBodyFields(found)
+		}
+	}
+
 	const bodyInfo = actionBodyRequest.includes(currentActionMethod) ?
-		<div>
+		<div style={{marginTop: 10}}>
-			Body - used as example in action argument
+			<b>Request Body</b>: {extraBodyFields.length > 0 ? 
+				<Typography style={{display: "inline-block"}}>
+					Variables: {extraBodyFields.join(", ")}
+				</Typography>
+			: 
+			<Typography style={{display: "inline-block"}}>
+				{`Add variables with \$\{ variable_name }`}
+			</Typography>
+			}
 			<TextField
 				required
 				style={{flex: "1", marginRight: "15px", backgroundColor: inputColor}}
 				fullWidth={true}
-				placeholder={'{\n\t"username": "testing@test.com"\n\t"name": "test testington"\n}'}
+				placeholder={'{\n\t"username": "${username}",\n\t"apikey": "${apikey}",\n\t"search": "1.2.3.5"}'}
 				margin="normal"
 				variant="outlined"
 				multiline
 				rows="5"
 				defaultValue={currentAction["body"]}
-				onChange={e => setActionField("body", e.target.value)}
+				onChange={e => {
+					setActionField("body", e.target.value)
+					findBodyParams(e.target.value)
+				}}
 				key={currentAction}
+				helperText={
+					<span style={{color:"white", marginBottom: "2px",}}>
+						Shows an example body to the user. ${} creates variables.
+					</span>
+				}
 				InputProps={{
 					classes: {
 						notchedOutline: classes.notchedOutline,
@@ -1205,9 +1493,40 @@ const AppCreator = (props) => {
 				}}
 			/>
 
+			<div>
+			</div>
 		</div>
 		: null
 
+	const exampleResponse = 
+		<div style={{}}>
+			<b>Example success response</b> 
+			<TextField
+				required
+				style={{flex: "1", marginRight: "15px", backgroundColor: inputColor}}
+				fullWidth={true}
+				placeholder={'{\n\t"email": "testing@test.com",\n\t"firstname": "testing"\n}'}
+				margin="normal"
+				variant="outlined"
+				multiline
+				rows="2"
+				defaultValue={currentAction["example_response"]}
+				onChange={e => setActionField("example_response", e.target.value)}
+				helperText={<span style={{color:"white", marginBottom: "2px",}}>
+					Helps with autocompletion and understanding of the endpoint 
+				</span>}
+				key={currentAction}
+				InputProps={{
+					classes: {
+						notchedOutline: classes.notchedOutline,
+					},
+					style:{
+						color: "white",
+					},
+				}}
+			/>
+		</div>
+
 	const addActionToView = (errors) => {
 		currentAction.errors = errors
 		currentAction.queries = urlPathQueries
@@ -1329,7 +1648,7 @@ const AppCreator = (props) => {
 		const queries = values[1]
 
 		if (currentAction.paths !== paths && urlPath.length > 0) {
-			console.log("IN PATHS SETTER: !", paths)
+			//console.log("IN PATHS SETTER: !", paths)
 			setActionField("paths", paths)
 		} 
 
@@ -1361,12 +1680,12 @@ const AppCreator = (props) => {
 			open={actionsModalOpen} 
 			fullWidth
 			onClose={() => {
-				console.log("CLOSED?")
 				setUrlPath("")
 				setCurrentAction({
 					"name": "",
 					"description": "",
 					"url": "",
+					"file_field": "",
 					"headers": "",
 					"paths": [],
 					"queries": [],
@@ -1377,6 +1696,7 @@ const AppCreator = (props) => {
 				setCurrentActionMethod(apikeySelection[0])
 				setUrlPathQueries([])
 				setActionsModalOpen(false)
+				setFileUploadEnabled(false)
 			}}
 		>
 			<FormControl style={{backgroundColor: surfaceColor, color: "white",}}>
@@ -1453,11 +1773,13 @@ const AppCreator = (props) => {
 							id: 'method-option',
 						}}
 						>
-						{actionNonBodyRequest.map(data => (
+						{actionNonBodyRequest.map((data, index) => {
-							<MenuItem style={{backgroundColor: inputColor, color: "white"}} value={data}>
+							return (
+							<MenuItem key={index} style={{backgroundColor: inputColor, color: "white"}} value={data}>
 								{data}
 							</MenuItem>
-						))}
+							)
+						})}
 						{actionBodyRequest.map(data => (
 							<MenuItem style={{backgroundColor: inputColor, color: "white"}} value={data}>
 								{data}
@@ -1492,66 +1814,81 @@ const AppCreator = (props) => {
 						}}
 						onBlur={event => {
 							var parsedurl = event.target.value
-							if (parsedurl.startsWith("curl")) {
+							if (parsedurl.startsWith("PUT ") || parsedurl.startsWith("GET ") ||parsedurl.startsWith("POST ") || parsedurl.startsWith("DELETE ") ||parsedurl.startsWith("PATCH ") || parsedurl.startsWith("CONNECT ")) {
-								const request = parseCurl(event.target.value)
+								const tmp = parsedurl.split(" ")
-								console.log(request)
-								if (request.method.toUpperCase() !== currentAction.Method) {
-									setCurrentActionMethod(request.method.toUpperCase())
-									setActionField("method", request.method.toUpperCase())
-								}
 
-								if (request.header !== undefined && request.header !== null) {
+								if (tmp.length > 1) {
-									var headers = []
+									parsedurl = tmp[1]
-									for (let [key, value] of Object.entries(request.header)) {
-										if (parameterName !== undefined && key.toLowerCase() === parameterName.toLowerCase()) {
-											continue
-										}
-
-										if (key === "Authorization" && authenticationOption === "Bearer auth") {
-											continue
-										} 
-
-										headers += key+"="+value+"\n"
-									}
-
-									setActionField("headers", headers)
-								}
-
-								if (request.body !== undefined && request.body !== null) {
-									setActionField("body", request.body)
-								}
-
-								// Parse URL
-								if (request.url !== undefined) {
-									parsedurl = request.url
-								}
-							}
-
-							if (parsedurl !== undefined) {
-								if (parsedurl.includes("<") && parsedurl.includes(">")) {
-									parsedurl = parsedurl.split("<").join("{")
-									parsedurl = parsedurl.split(">").join("}")
-								}
-
-								if (parsedurl.startsWith("http") || parsedurl.startsWith("ftp")) {
-									if (parsedurl !== undefined && parsedurl.includes(parameterName)) {
-										// Remove <> etc.
-										// 
-										
-										console.log("IT HAS THE PARAM NAME!")
-										const newurl = new URL(encodeURI(parsedurl))
-										newurl.searchParams.delete(parameterName)
-										parsedurl = decodeURI(newurl.href)
-									}
-
-									// Remove the base URL itself
-									if (parsedurl !== undefined && baseUrl !== undefined && baseUrl.length > 0 && parsedurl.includes(baseUrl)) {
-										parsedurl = parsedurl.replace(baseUrl, "")
-									}
-
-									// Check URL query && headers 
 									setActionField("url", parsedurl)
 									setUrlPath(parsedurl)
+
+									setCurrentActionMethod(tmp[0].toUpperCase())
+									setActionField("method", tmp[0].toUpperCase())
+								}
+
+								setUpdate(Math.random())
+							} else if (parsedurl.startsWith("curl")) {
+								const request = parseCurl(event.target.value)
+								if (request !== event.target.value) {
+									if (request.method.toUpperCase() !== currentAction.Method) {
+										setCurrentActionMethod(request.method.toUpperCase())
+										setActionField("method", request.method.toUpperCase())
+									}
+
+									if (request.header !== undefined && request.header !== null) {
+										var headers = []
+										for (let [key, value] of Object.entries(request.header)) {
+											if (parameterName !== undefined && key.toLowerCase() === parameterName.toLowerCase()) {
+												continue
+											}
+
+											if (key === "Authorization" && authenticationOption === "Bearer auth") {
+												continue
+											} 
+
+											headers += key+"="+value+"\n"
+										}
+
+										setActionField("headers", headers)
+									}
+
+									if (request.body !== undefined && request.body !== null) {
+										setActionField("body", request.body)
+									}
+
+									// Parse URL
+									if (request.url !== undefined) {
+										parsedurl = request.url
+									}
+							}
+
+							console.log("PARSED: ", parsedurl)
+							if (parsedurl !== undefined) {
+									if (parsedurl.includes("<") && parsedurl.includes(">")) {
+										parsedurl = parsedurl.split("<").join("{")
+										parsedurl = parsedurl.split(">").join("}")
+									}
+
+									if (parsedurl.startsWith("http") || parsedurl.startsWith("ftp")) {
+										if (parsedurl !== undefined && parsedurl.includes(parameterName)) {
+											// Remove <> etc.
+											// 
+											
+											console.log("IT HAS THE PARAM NAME!")
+											const newurl = new URL(encodeURI(parsedurl))
+											newurl.searchParams.delete(parameterName)
+											parsedurl = decodeURI(newurl.href)
+										}
+
+										// Remove the base URL itself
+										if (parsedurl !== undefined && baseUrl !== undefined && baseUrl.length > 0 && parsedurl.includes(baseUrl)) {
+											parsedurl = parsedurl.replace(baseUrl, "")
+										}
+
+										// Check URL query && headers 
+										setActionField("url", parsedurl)
+										setUrlPath(parsedurl)
+									}
 								}
 							}
 
@@ -1563,8 +1900,38 @@ const AppCreator = (props) => {
 					<Button color="primary" style={{marginTop: "5px", marginBottom: "10px", borderRadius: "0px"}} variant="outlined" onClick={() => {
 						addPathQuery()
 					}}>New query</Button> 				
+					{currentActionMethod === "POST" ?
+						<Button color="primary" variant={fileUploadEnabled ? "contained" : "outlined"} style={{marginLeft: 10, marginTop: "5px", marginBottom: "10px", borderRadius: "0px"}} onClick={() => {
+							setFileUploadEnabled(!fileUploadEnabled)
+							if (fileUploadEnabled && currentAction["file_field"].length > 0) {
+								setActionField("file_field", "")
+							}
+							setUpdate(Math.random())
+						}}>Enable Fileupload</Button> 				
+					: null}
+					{fileUploadEnabled ? 
+						<TextField
+							required
+							style={{backgroundColor: inputColor, display: "inline-block",}}
+							placeholder={"file"}
+							margin="normal"
+							variant="outlined"
+							id="standard-required"
+							defaultValue={currentAction["file_field"]}
+							onChange={e => setActionField("file_field", e.target.value)}
+							helperText={<span style={{color:"white", marginBottom: "2px",}}>The File field to interact with</span>}
+							InputProps={{
+								classes: {
+									notchedOutline: classes.notchedOutline,
+								},
+								style:{
+									color: "white",
+								},
+							}}
+						/>
+						: null}
 					<div/>
-					Headers - static for the action
+					<b>Headers</b>: static for the action
 					<TextField
 						required
 						style={{flex: "1", marginRight: "15px", marginTop: "5px", backgroundColor: inputColor}}
@@ -1575,7 +1942,7 @@ const AppCreator = (props) => {
 						id="standard-required"
 						defaultValue={currentAction["headers"]}
 						multiline
-						rows="5"
+						rows="2"
 						onChange={e => setActionField("headers", e.target.value)}
 						helperText={<span style={{color:"white", marginBottom: "2px",}}>Headers that are part of the request. Default: EMPTY</span>}
 						InputProps={{
@@ -1588,6 +1955,8 @@ const AppCreator = (props) => {
 						}}
 					/>
 					{bodyInfo}
+					<Divider style={{backgroundColor: "rgba(255,255,255,0.5)", marginTop: 15, marginBottom: 15}} />
+					{exampleResponse}
 				</DialogContent>
 				<DialogActions>
 	      <Button style={{borderRadius: "0px"}} onClick={() => {
@@ -1595,14 +1964,15 @@ const AppCreator = (props) => {
 					Cancel	
 				</Button>
 	      <Button color="primary" variant="outlined" style={{borderRadius: "0px"}} onClick={() => {
-						console.log(urlPathQueries)
+						//console.log(urlPathQueries)
-						console.log(urlPath)
+						//console.log(urlPath)
-						// value={urlPath}
+						//console.log(currentAction)
 						const errors = getActionErrors()		
 						addActionToView(errors)
 						setActionsModalOpen(false)
 						setUrlPathQueries([]) 
 						setUrlPath("")
+						setFileUploadEnabled(false)
 					}}>
 						Submit	
 					</Button>
@@ -1661,26 +2031,40 @@ const AppCreator = (props) => {
 
 	const actionView = 
 		<div style={{color: "white"}}>
-			<h2>Actions</h2>
+			<h2>Actions ({actions.length})</h2>
 			Actions are the tasks performed by an app. Read more about actions and apps
 			<Link target="_blank" to="https://shuffler.io/docs/apps#actions" style={{textDecoration: "none", color: "#f85a3e"}}> here</Link>.
 			<div>
 				{loopActions}
-				<Button color="primary" style={{marginTop: "20px", borderRadius: "0px"}} variant="outlined" onClick={() => {
+				<div style={{display: "flex"}}>
-					setCurrentAction({
+					<Button color="primary" style={{marginTop: "20px", borderRadius: "0px"}} variant="outlined" onClick={() => {
-						"name": "",
+						setCurrentAction({
-						"description": "",
+							"name": "",
-						"url": "",
+							"description": "",
-						"headers": "",
+							"url": "",
-						"queries": [],
+							"file_field": "",
-						"paths": [],
+							"headers": "",
-						"body": "",
+							"queries": [],
-						"errors": [],
+							"paths": [],
-						"method": actionNonBodyRequest[0],
+							"body": "",
-					})
+							"errors": [],
-  				setCurrentActionMethod(actionNonBodyRequest[0])
+							"method": actionNonBodyRequest[0],
-					setActionsModalOpen(true)
+						})
-				}}>New action</Button> 				
+						setCurrentActionMethod(actionNonBodyRequest[0])
+						setActionsModalOpen(true)
+					}}>New action</Button> 				
+					{actionAmount > 0 && actionAmount < actions.length ? null :  
+						<Button color="primary" style={{marginTop: "20px", borderRadius: "0px", textAlign: "center"}} variant="outlined" onClick={() => {
+							if (actionAmount+increaseAmount > actions.length) {
+								setActionAmount(actions.length)
+							} else {
+								setActionAmount(actionAmount+increaseAmount)
+							}
+						}}>
+							See more actions	
+						</Button>
+					}
+				</div>
 			</div>
 		</div>
 

frontend/src/views/Apps.jsx

@@ -21,6 +21,7 @@ import {Link} from 'react-router-dom';
 import Breadcrumbs from '@material-ui/core/Breadcrumbs';
 import ReactJson from 'react-json-view'
 import Chip from '@material-ui/core/Chip';
+import { useTheme } from '@material-ui/core/styles';
 
 import CachedIcon from '@material-ui/icons/Cached';
 import CloudDownloadIcon from '@material-ui/icons/CloudDownload';
@@ -46,6 +47,10 @@ const inputColor = "#383B40"
 export const GetParsedPaths = (inputdata, basekey) => {
 	const splitkey = " > "
 	var parsedValues = []
+	if (inputdata === undefined || inputdata === null) {
+		return parsedValues
+	}
+
 	if (typeof(inputdata) !== "object") {
 		return parsedValues
 	}
@@ -106,9 +111,10 @@ export const GetParsedPaths = (inputdata, basekey) => {
 
 
 const Apps = (props) => {
-  const { globalUrl, isLoggedIn, isLoaded } = props;
+  const { globalUrl, isLoggedIn, isLoaded, userdata } = props;
 
 	//const [workflows, setWorkflows] = React.useState([]);
+	const theme = useTheme();
 	const baseRepository = "https://github.com/frikky/shuffle-apps"
 	const alert = useAlert()
 	const [selectedApp, setSelectedApp] = React.useState({});
@@ -134,6 +140,7 @@ const Apps = (props) => {
 	const [field2, setField2] = React.useState("")
 	const [cursearch, setCursearch] = React.useState("")
 	const [sharingConfiguration, setSharingConfiguration] = React.useState("you")
+	const [downloadBranch, setDownloadBranch] = React.useState("master")
 
 	const [isDropzone, setIsDropzone] = React.useState(false);
 	const upload = React.useRef(null);
@@ -311,10 +318,18 @@ const Apps = (props) => {
 			boxColor = "orange"
 		}
 
+		if (data.invalid) {
+			boxColor = "red"
+		}
+
+		//<div style={{backgroundColor: theme.palette.inputColor, height: 100, width: 100, borderRadius: 3, verticalAlign: "middle", textAlign: "center", display: "table-cell"}}>
+		// <div style={{width: "100px", height: "100px", border: "1px solid black", verticalAlign: "middle", textAlign: "center", display: "table-cell"}}>
 		var imageline = data.large_image.length === 0 ?
-			<img alt={data.title} style={{width: 100, height: 100}} />
+			<img alt={data.title} style={{width: 100, height: 100, backgroundColor: theme.palette.inputColor,}} />
 			: 
-			<img alt={data.title} src={data.large_image} style={{width: 100, height: 100, maxWidth: "100%"}} />
+			<img alt={data.title} src={data.large_image} style={{maxWidth: 100, maxHeight: "100%", display: "block", margin: "0 auto"}} onLoad={(event) => {
+				//console.log("IMG LOADED!: ", event.target)
+			}} />
 
 		// FIXME - add label to apps, as this might be slow with A LOT of apps
 		var newAppname = data.name
@@ -336,7 +351,7 @@ const Apps = (props) => {
 		}
 
 		var description = data.description
-		const maxDescLen = 60
+		const maxDescLen = 51
 		if (description.length > maxDescLen) {
 			description = data.description.slice(0, maxDescLen)+"..."
 		}
@@ -359,8 +374,8 @@ const Apps = (props) => {
 					} 
 				}
 			}}>
-				<Grid container style={{margin: 10, flex: "10"}}>
+				<Grid container style={{margin: 10, flex: "10", maxHeight: 110, overflow: "hidden",}}>
-					<ButtonBase>
+					<ButtonBase style={{backgroundColor: theme.palette.inputColor, border: 3}}>
 						{imageline}
 					</ButtonBase>
 					<div style={{marginLeft: "10px", marginTop: "5px", marginBottom: "5px", width: boxWidth, backgroundColor: boxColor}}>
@@ -515,9 +530,9 @@ const Apps = (props) => {
 			: null
 
 		var imageline = selectedApp.large_image === undefined || selectedApp.large_image.length === 0 ?
-			<img alt={selectedApp.title} style={{width: 100, height: 100}} />
+			<img alt={selectedApp.title} style={{width: 100, height: 100, backgroundColor: theme.palette.inputColor,}} />
 			: 
-			<img alt={selectedApp.title} src={selectedApp.large_image} style={{width: 100, height: 100, maxWidth: "100%"}} />
+			<img alt={selectedApp.title} src={selectedApp.large_image} style={{maxHeight: 100, maxWidth: 100, backgroundColor: theme.palette.inputColor}} />
 
 		const GetAppExample = () => {
 			if (selectedAction.returns === undefined) {
@@ -588,10 +603,10 @@ const Apps = (props) => {
 					<div style={{marginRight: 15, marginTop: 10}}>
 						{imageline}
 					</div>
-					<div style={{maxWidth: "75%", overflow: "hidden"}}>
+					<div style={{maxWidth: "85%", overflow: "hidden"}}>
 						<h2 style={{marginTop: 20, marginBottom: 0, }}>{newAppname}</h2>
 						<p style={{marginTop: 5, marginBottom: 0,}}>Version {selectedApp.app_version}</p>	
-						<p style={{marginTop: 5, marginBottom: 0}}>{description}</p>	
+						<p style={{marginTop: 5, marginBottom: 0, maxHeight: 150, overflowY: "auto", overflowX: "hidden",}}>{description}</p>	
 					</div>
 				</div>
 				{activateButton}
@@ -634,7 +649,7 @@ const Apps = (props) => {
 								updateAppField(selectedApp.id, "sharing", !selectedApp.sharing)
 								//setSelectedAction(event.target.value)
 							}}
-							style={{width: 150, backgroundColor: inputColor, color: "white", height: 35, marginleft: 10,}}
+							style={{width: 150, backgroundColor: theme.palette.surfaceColor, backgroundColor: inputColor, color: "white", height: 35, marginleft: 10,}}
 							SelectDisplayProps={{
 								style: {
 									marginLeft: 10,
@@ -699,7 +714,7 @@ const Apps = (props) => {
 
 					{selectedAction.parameters !== undefined && selectedAction.parameters !== null ? 
 						<div style={{marginTop: 15, marginBottom: 15}}>
-							<b>Arguments</b>
+							<b>Parameters</b>
 							{selectedAction.parameters.map(data => {
 									var itemColor = "#f85a3e"
 									if (!data.required) {
@@ -802,12 +817,16 @@ const Apps = (props) => {
 		
     const reader = new FileReader();
 
-    reader.addEventListener('load', (e) => {
+		try {
-      const content = e.target.result;
+			reader.addEventListener('load', (e) => {
-			setOpenApiData(content);
+				const content = e.target.result;
-			setIsDropzone(isDropzone);
+				setOpenApiData(content);
-			setOpenApiModal(true)
+				setIsDropzone(isDropzone);
-    })
+				setOpenApiModal(true)
+			})
+		} catch (e) {
+			console.log("Error in dropzone: ", e)
+		}
 
 		reader.readAsText(files[0]);
   };
@@ -907,7 +926,7 @@ const Apps = (props) => {
 					<div style={{marginTop: 15}}>
 						{apps.length > 0 ? 
 							filteredApps.length > 0 ? 
-								<div style={{height: "75vh", overflowY: "scroll"}}>
+								<div style={{height: "75vh", overflowY: "auto"}}>
 									{filteredApps.map(app => {
 										return (
 											appPaper(app)
@@ -959,6 +978,7 @@ const Apps = (props) => {
 
 		const parsedData = {
 			"url": url,
+			"branch": downloadBranch || 'master'
 		}
 
 		if (field1.length > 0) {
@@ -988,18 +1008,23 @@ const Apps = (props) => {
 			}
 			setIsLoading(false)
 			stop()
+			setValidation(false)
 
 			return response.json()
 		})
     .then((responseJson) => {
-				console.log("DATA: ", responseJson)
+			console.log("DATA: ", responseJson)
-				if (responseJson.reason !== undefined) {
+			if (responseJson.reason !== undefined) {
-					alert.error("Failed loading: "+responseJson.reason)
+				alert.error("Failed loading: "+responseJson.reason)
-				}
+			}
 		})
 		.catch(error => {
 			console.log("ERROR: ", error.toString())
 			alert.error(error.toString())
+
+			stop()
+			setIsLoading(false)
+			setValidation(false)
 		})
 	}
 
@@ -1167,6 +1192,7 @@ const Apps = (props) => {
 				return
 			}
 
+			console.log("Validating response!")
 			validateOpenApi(responseJson)
     })
 		.catch(error => {
@@ -1185,10 +1211,12 @@ const Apps = (props) => {
 
 
 		try {
-			return JSON.stringify(YAML.parse(apidata))
+			const parsed = YAML.parse(YAML.stringify(apidata))
+			//const parsed = YAML.parse(apidata))
+			return YAML.stringify(parsed)
 		} catch(error) {
 			console.log("YAML DECODE ERROR - TRY SOMETHING ELSE?: "+error)
-			setOpenApiError(error.toString())
+			setOpenApiError("Local error: "+ error.toString())
 		}
 
 		return ""
@@ -1197,19 +1225,23 @@ const Apps = (props) => {
 	// Sends the data to backend, which should return a version 3 of the same API
 	// If 200 - continue, otherwise, there's some issue somewhere
 	const validateOpenApi = (openApidata) => {
-		const newApidata = escapeApiData(openApidata)
+		var newApidata = escapeApiData(openApidata)
 		if (newApidata === "") {
+			// Used to return here
+			newApidata = openApidata
 			return
 		}
 
+		//console.log(newApidata)
+
 		setValidation(true)
 		fetch(globalUrl+"/api/v1/validate_openapi", {
-    	  	method: 'POST',
+    	method: 'POST',
 			headers: {
 				'Accept': 'application/json',
 			},
-			body: newApidata,
+			body: openApidata,
-	  		credentials: "include",
+	  	credentials: "include",
 		})
 		.then((response) => {
 			setValidation(false)
@@ -1302,7 +1334,7 @@ const Apps = (props) => {
 					style={{backgroundColor: inputColor}}
 					variant="outlined"
 					margin="normal"
-					defaultValue="https://github.com/frikky/shuffle-apps"
+					defaultValue={userdata.active_org.defaults.app_download_repo !== undefined && userdata.active_org.defaults.app_download_repo.length > 0 ? userdata.active_org.defaults.app_download_repo : "https://github.com/frikky/shuffle-apps"}
 					InputProps={{
 						style:{
 							color: "white",
@@ -1314,6 +1346,25 @@ const Apps = (props) => {
 					placeholder="https://github.com/frikky/shuffle-apps"
 					fullWidth
 					/>
+				<span style={{marginTop: 10}}>Branch (default value is "master"):</span>
+					<div style={{display: "flex"}}>
+						<TextField
+							style={{backgroundColor: inputColor}}
+							variant="outlined"
+							margin="normal"
+							defaultValue={userdata.active_org.defaults.app_download_branch !== undefined && userdata.active_org.defaults.app_download_branch.length > 0 ? userdata.active_org.defaults.app_download_branch : downloadBranch}
+							InputProps={{
+								style:{
+									color: "white",
+									height: "50px",
+									fontSize: "1em",
+								},
+							}}
+							onChange={e => setDownloadBranch(e.target.value)}
+							placeholder="master"
+							fullWidth
+							/>
+					</div>
 
 				<span style={{marginTop: 10}}>Authentication (optional - private repos etc):</span>
 				<div style={{display: "flex"}}>

frontend/src/views/LoginPage.jsx

@@ -69,7 +69,7 @@ const LoginDialog = props => {
 				}),
 			)
 			.catch(error => {
-				setLoginInfo("Error in userdata: ", error)
+				setLoginInfo("Error logging in: ", error)
 			})
 	}
 
@@ -80,6 +80,7 @@ const LoginDialog = props => {
 
 	const onSubmit = (e) => {
 		e.preventDefault()
+		setLoginInfo("")
 		// FIXME - add some check here ROFL
 
 		// Just use this one?
@@ -114,7 +115,7 @@ const LoginDialog = props => {
 					}),
 				)
 				.catch(error => {
-					setLoginInfo("Error in userdata: " + error)
+					setLoginInfo("Error logging in: " + error)
 				});
 		} else {
 			url = baseurl + '/api/v1/users/register';
@@ -130,7 +131,7 @@ const LoginDialog = props => {
 						if (responseJson["success"] === false) {
 							setLoginInfo(responseJson["reason"])
 						} else {
-							setLoginInfo("Successful register :)")
+							setLoginInfo("Successful register!")
 						}
 					}),
 				)

frontend/src/views/SettingsPage.jsx

@@ -85,7 +85,7 @@ const Settings = (props) => {
 
 	const generateApikey = () => {
 		fetch(globalUrl+"/api/v1/generateapikey", {
-    	  	method: 'GET',
+    	  method: 'GET',
 				headers: {
 					'Content-Type': 'application/json',
 					'Accept': 'application/json',
@@ -99,7 +99,7 @@ const Settings = (props) => {
 
 			return response.json()
 		})
-    	.then((responseJson) => {
+    .then((responseJson) => {
 			setUserSettings(responseJson)
     	})
 		.catch(error => {

frontend/src/views/Workflows.jsx

@@ -43,8 +43,40 @@ import CloudDownloadIcon from '@material-ui/icons/CloudDownload';
 const inputColor = "#383B40"
 const surfaceColor = "#27292D"
 
+export const validateJson = (showResult) => {
+	//showResult = showResult.split(" None").join(" \"None\"")
+	showResult = showResult.split(" False").join(" false")
+	showResult = showResult.split(" True").join(" true")
+
+	var jsonvalid = true
+	try {
+		const tmp = String(JSON.parse(showResult))
+		if (!showResult.includes("{") && !showResult.includes("[")) {
+			jsonvalid = false
+		}
+	} catch (e) {
+		showResult = showResult.split("\'").join("\"")
+
+		try {
+			const tmp = String(JSON.parse(showResult))
+			if (!showResult.includes("{") && !showResult.includes("[")) {
+				jsonvalid = false
+			}
+		} catch (e) {
+			jsonvalid = false
+		}
+	}
+
+	const result = jsonvalid ? JSON.parse(showResult) : showResult
+	//console.log("VALID: ", jsonvalid, result)
+	return {
+		"valid": jsonvalid, 
+		"result": result,
+	}
+}
+
 const Workflows = (props) => {
-  const { globalUrl, isLoggedIn, isLoaded, removeCookie, cookies} = props;
+  const { globalUrl, isLoggedIn, isLoaded, removeCookie, cookies, userdata} = props;
 	document.title = "Shuffle - Workflows"
 
 	const alert = useAlert()
@@ -80,7 +112,7 @@ const Workflows = (props) => {
 	  	duration: 5000,
 	  	startImmediate: false,
 	  	callback: () => {
-				getWorkflowExecution(selectedWorkflow.id) 
+				//getWorkflowExecution(selectedWorkflow.id) 
 	  	}
 	})
 
@@ -183,7 +215,7 @@ const Workflows = (props) => {
 
 			if (responseJson.length > 0){
 				setSelectedWorkflow(responseJson[0])
-				getWorkflowExecution(responseJson[0].id)
+				//getWorkflowExecution(responseJson[0].id)
 			}
     	})
 		.catch(error => {
@@ -202,8 +234,8 @@ const Workflows = (props) => {
 		color: "#ffffff",
 		width: "100%",
 		display: "flex",
-		minWidth: 1366,
+		minWidth: 1024,
-		maxWidth: 1766,
+		maxWidth: 1024,
 		margin: "auto",
 		maxHeight: "90vh",
 	}
@@ -272,7 +304,7 @@ const Workflows = (props) => {
 					setSelectedExecution(responseJson[0])
 					setWorkflowExecutions(responseJson)
 				} else {
-					alert.info("Couldn't find executions for the workflow")
+					//alert.info("Couldn't find executions for the workflow")
 					setSelectedExecution({})
 					setWorkflowExecutions([])
 				}
@@ -298,7 +330,7 @@ const Workflows = (props) => {
 			if (response.status !== 200) {
 				console.log("Status not 200 for WORKFLOW EXECUTION :O!")
 			}
-			getWorkflowExecution(workflowid) 
+			//getWorkflowExecution(workflowid) 
 
 			return response.json()
 		})
@@ -361,10 +393,24 @@ const Workflows = (props) => {
 
 		data["owner"] = ""
 		for (var key in data.triggers) {
-			if (data.triggers[key].status == "running") {
+			const trigger = data.triggers[key]
-				data.triggers[key].status = "stopped"
+			if (trigger.app_name === "Shuffle Workflow") {
+				if (trigger.parameters.length > 2) {
+					trigger.parameters[2].value = ""
+				}
+			} 
+			
+			if (trigger.status == "running") {
+				trigger.status = "stopped"
 			}
 		}
+
+		for (var key in data.actions) {
+			data.actions[key].authentication_id = ""
+		}
+
+		//return
+
 		data["org"] = []
 		data["org_id"] = ""
 		data.execution_org = {"id": ""}
@@ -377,12 +423,15 @@ const Workflows = (props) => {
 	}
 
 	const copyWorkflow = (data) => {
+		data = JSON.parse(JSON.stringify(data))
 		alert.success("Copying workflow "+data.name)
+		console.log("data: ", data)
 		data.id = ""
 		data.name = data.name+"_copy"
+		//return
 
 		fetch(globalUrl+"/api/v1/workflows", {
-    	  	method: 'POST',
+    	  method: 'POST',
 				headers: {
 					'Content-Type': 'application/json',
 					'Accept': 'application/json',
@@ -397,9 +446,9 @@ const Workflows = (props) => {
 			}
 			return response.json()
 		})
-    	.then((responseJson) => {
+    .then((responseJson) => {
 			getAvailableWorkflows()
-    	})
+    })
 		.catch(error => {
 			alert.error(error.toString())
 		});
@@ -480,7 +529,7 @@ const Workflows = (props) => {
 							<div style={{flex: "10",}} onClick={() => {
 								if (selectedWorkflow.id !== data.id) {
 									setSelectedWorkflow(data)
-									getWorkflowExecution(data.id)
+									//getWorkflowExecution(data.id)
 								}
 							}}>
 								<Typography variant="h6" style={{marginTop: 10, marginBottom: 0, }}>
@@ -537,7 +586,7 @@ const Workflows = (props) => {
 						<div style={{display: "flex", flex: 1}} onClick={() => {
 							if (selectedWorkflow.id !== data.id) {
 								setSelectedWorkflow(data)
-								getWorkflowExecution(data.id)
+								//getWorkflowExecution(data.id)
 							}
 						}}>
 							<Grid item style={{flex: "1", justifyContent: "center", overflow: "hidden", float: "bottom",}}>
@@ -684,22 +733,12 @@ const Workflows = (props) => {
 		}
 
 		var t = new Date(data.started_at*1000)
-		var jsonvalid = true
 		var showResult = data.result.trim()
-		showResult = replaceAll(showResult, " None", " \"None\"");
+		const validate = validateJson(showResult)
-		try {
-			const tmp = String(JSON.parse(showResult))
-			if (!tmp.includes("{") && !tmp.includes("[")) {
-				jsonvalid = false
-			}
-		} catch (e) {
-			jsonvalid = false
-		}
 
-		//console.log("VALID: ", jsonvalid)
+		if (validate.valid) {
-		if (jsonvalid) {
 			showResult = <ReactJson 
-				src={JSON.parse(showResult)} 
+				src={validate.result} 
 				theme="solarized" 
 				collapsed={collapseJson}
 				displayDataTypes={false}
@@ -707,7 +746,7 @@ const Workflows = (props) => {
 			/>
 		} else {
 			// FIXME - have everything parsed as json, either just for frontend
-			// or in the backend
+			// or in the backend?
 			/*	
 			const newdata = {"result": data.result}
 			showResult = <ReactJson 
@@ -767,7 +806,7 @@ const Workflows = (props) => {
 			No results yet 
 		</div>
 
-	const resultsLength = Object.getOwnPropertyNames(selectedExecution).length > 0 && selectedExecution.results !== null ? selectedExecution.results.length : 0 
+	const resultsLength = Object.getOwnPropertyNames(selectedExecution).length > 0 && selectedExecution.results !== null ? selectedExecution.results.length : 0 	
 
 	const ExecutionDetails = () => {
 		var starttime = new Date(selectedExecution.started_at*1000)
@@ -780,23 +819,12 @@ const Workflows = (props) => {
 
 		var arg = null
 		if (selectedExecution.execution_argument !== undefined && selectedExecution.execution_argument.length > 0) {
-			var jsonvalid = true
-
 			var showResult = selectedExecution.execution_argument.trim()
-			showResult = replaceAll(showResult, " None", " \"None\"");
+			const validate = validateJson(showResult)
 
-			try {
+			arg = validate.valid ? 
-				const tmp = String(JSON.parse(showResult))
-				if (!tmp.includes("{") && !tmp.includes("[")) {
-					jsonvalid = false
-				}
-			} catch (e) {
-				jsonvalid = false
-			}
-
-			arg = jsonvalid ? 
 				<ReactJson 
-					src={JSON.parse(showResult)} 
+					src={validate.result} 
 					theme="solarized" 
 					collapsed={true}
 					displayDataTypes={false}
@@ -807,22 +835,11 @@ const Workflows = (props) => {
 
 		var lastresult = null
 		if (selectedExecution.result !== undefined && selectedExecution.result.length > 0) {
-			var jsonvalid = true
 			var showResult = selectedExecution.result.trim()
-			showResult = replaceAll(showResult, " None", " \"None\"");
+			const validate = validateJson(showResult)
-
+			lastresult = validate.valid ? 
-			try {
-				const tmp = JSON.parse(showResult)
-				if (!tmp.includes("{") && !tmp.includes("[")) {
-					jsonvalid = false
-				}
-			} catch (e) {
-				jsonvalid = false
-			}
-
-			lastresult = jsonvalid ? 
 				<ReactJson 
-					src={JSON.parse(showResult)} 
+					src={validate.result} 
 					theme="solarized" 
 					collapsed={true}
 					displayDataTypes={false}
@@ -899,7 +916,7 @@ const Workflows = (props) => {
 				</div>
 				:
 				<h4>
-					There are no executions for this workflow yet
+					Executions have been moved to the Workflow itself. <div/><Link to={`/workflows/${selectedWorkflow.id}?view=executions`} style={{textDecoration: "none", color: "#f85a3e"}}>Click here to see them</Link>
 				</h4>
 		)
 	}
@@ -1204,7 +1221,7 @@ const Workflows = (props) => {
 				<div style={workflowViewStyle}>
 					<div style={{display: "flex"}}>
 						<div style={{flex: "4"}}>
-							<h2>Workflows</h2> 
+							<h2>Workflows ({workflows.length})</h2> 
 						</div>
 						<div style={{marginTop: 20}}>
 							{workflowButtons}
@@ -1222,24 +1239,27 @@ const Workflows = (props) => {
 				</div>
 				<div style={{flex: viewSize.executionsView, marginLeft: "10px", marginRight: "10px"}}>
 					<div style={{display: "flex"}}>
-						<div style={{flex: "10"}}>
+						<div style={{flex: 10}}>
 							<h2>Executions: {selectedWorkflow.name}</h2> 
 						</div>
-						<div style={{flex: "1"}}>
+						{/*
+						<div style={{flex: 1}}>
 							<Button color="primary" style={{marginTop: "20px"}} variant="text" onClick={() => {
 									alert.info("Refreshing executions"); 
-									getWorkflowExecution(selectedWorkflow.id)
+									//getWorkflowExecution(selectedWorkflow.id)
 								}}>
 								<CachedIcon />
 							</Button> 				
 						</div>
+						*/}
 					</div>
 					<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
 					<div style={scrollStyle}>
 						<ExecutionsView />
 					</div>
 				</div>
-				<div style={{flex: viewSize.executionResults, marginLeft: "10px", marginRight: "10px", minWidth: "33%"}}>
+				{/*
+				<div style={{flex: viewSize.executionResults, marginLeft: "10px", marginRight: "10px", minWidth: "40%"}}>
 					<div style={{display: "flex"}}>
 						<div style={{flex: "1"}}>
 							<h2>Execution Timeline</h2>
@@ -1257,6 +1277,7 @@ const Workflows = (props) => {
 						<ExecutionDetails />
 					</div>
 				</div>
+				*/}
 			</div>
 		)
 	}
@@ -1348,7 +1369,7 @@ const Workflows = (props) => {
 					style={{backgroundColor: inputColor}}
 					variant="outlined"
 					margin="normal"
-					value={downloadUrl}
+					defaultValue={userdata.active_org.defaults.workflow_download_repo !== undefined && userdata.active_org.defaults.workflow_download_repo.length > 0 ? userdata.active_org.defaults.workflow_download_repo : downloadUrl}
 					InputProps={{
 						style:{
 							color: "white",
@@ -1367,7 +1388,7 @@ const Workflows = (props) => {
 							style={{backgroundColor: inputColor}}
 							variant="outlined"
 							margin="normal"
-							value={downloadBranch}
+							defaultValue={userdata.active_org.defaults.workflow_download_branch !== undefined && userdata.active_org.defaults.workflow_download_branch.length > 0 ? userdata.active_org.defaults.workflow_download_branch : downloadBranch}
 							InputProps={{
 								style:{
 									color: "white",

functions/extensions/aws-lambda/deploy.sh

@@ -0,0 +1,9 @@
+GOOS=linux go build main.go
+zip function.zip main
+
+aws lambda update-function-code \
+	--function-name shuffler-forwarder \
+	--runtime go1.* \
+  --zip-file fileb://function.zip \
+	--handler main \
+	--role arn:aws:iam::123456789012:role/execution_role

functions/extensions/cortex-responders/Shuffle/shuffle.json

@@ -0,0 +1,35 @@
+{
+  "name": "Shuffle",
+  "version": "1.0",
+  "author": "@frikkylikeme",
+  "url": "https://github.com/frikky/shuffle",
+  "license": "AGPL-V3",
+  "description": "Execute a workflow in Shuffle",
+  "dataTypeList": ["thehive:case", "thehive:alert"],
+  "command": "Shuffle/shuffle.py",
+  "baseConfig": "Shuffle",
+  "configurationItems": [
+    {
+      "name": "url",
+      "description": "The URL to your shuffle instance",
+      "type": "string",
+      "multi": false,
+      "required": true,
+      "defaultValue": "https://shuffler.io"
+    },
+		{
+      "name": "api_key",
+      "description": "The API key to your Shuffle user",
+      "type": "string",
+      "multi": false,
+      "required": true
+    },
+    {
+      "name": "workflow_id",
+      "description": "The ID of the workflow to execute",
+      "type": "string",
+      "multi": false,
+      "required": true
+    }
+  ]
+}

functions/extensions/cortex-responders/Shuffle/shuffle.py

@@ -0,0 +1,28 @@
+
+#!/usr/bin/env python
+# encoding: utf-8
+
+from cortexutils.responder import Responder
+import requests
+
+class Shuffle(Responder):
+    def __init__(self):
+        Responder.__init__(self)
+        self.api_key = self.get_param("config.api_key", "")
+        self.url = self.get_param("config.url", "")
+        self.workflow_id = self.get_param("config.workflow_id", "")
+
+    def run(self):
+        Responder.run(self)
+
+        parsed_url = "%s/api/v1/workflows/%s/execute" % (self.url, self.workflow_id)
+        headers = {
+            "Authorization": "Bearer %s"  % self.api_key
+        }
+        requests.post(parsed_url, headers=headers)
+
+        self.report({'message': 'message sent'})
+
+if __name__ == '__main__':
+    Shuffle().run()
+

functions/extensions/wazuh/custom-shuffle

@@ -0,0 +1,36 @@
+#!/bin/sh
+# Created by Shuffle, AS. <frikky@shuffler.io>.
+
+WPYTHON_BIN="framework/python/bin/python3"
+
+SCRIPT_PATH_NAME="$0"
+
+DIR_NAME="$(cd $(dirname ${SCRIPT_PATH_NAME}); pwd -P)"
+SCRIPT_NAME="$(basename ${SCRIPT_PATH_NAME})"
+
+case ${DIR_NAME} in
+    */active-response/bin | */wodles*)
+        if [ -z "${WAZUH_PATH}" ]; then
+            WAZUH_PATH="$(cd ${DIR_NAME}/../..; pwd)"
+        fi
+
+        PYTHON_SCRIPT="${DIR_NAME}/${SCRIPT_NAME}.py"
+    ;;
+    */bin)
+        if [ -z "${WAZUH_PATH}" ]; then
+            WAZUH_PATH="$(cd ${DIR_NAME}/..; pwd)"
+        fi
+
+        PYTHON_SCRIPT="${WAZUH_PATH}/framework/scripts/${SCRIPT_NAME}.py"
+    ;;
+     */integrations)
+        if [ -z "${WAZUH_PATH}" ]; then
+            WAZUH_PATH="$(cd ${DIR_NAME}/..; pwd)"
+        fi
+
+        PYTHON_SCRIPT="${DIR_NAME}/${SCRIPT_NAME}.py"
+    ;;
+esac
+
+
+${WAZUH_PATH}/${WPYTHON_BIN} ${PYTHON_SCRIPT} "$@"

functions/extensions/wazuh/custom-shuffle.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python
+# Created by Shuffle, AS. <frikky@shuffler.io>.
+# Based on the Slack integration using Webhooks
+
+import json
+import sys
+import time
+import os
+
+try:
+    import requests
+    from requests.auth import HTTPBasicAuth
+except Exception as e:
+    print("No module 'requests' found. Install: pip install requests")
+    sys.exit(1)
+
+# ADD THIS TO ossec.conf configuration:
+#  <integration>
+#      <name>custom-shuffle</name>
+#      <hook_url>http://<IP>:3001/api/v1/hooks/<HOOK_ID></hook_url>
+#      <level>3</level>
+#      <alert_format>json</alert_format>
+#  </integration>
+
+# Global vars
+
+debug_enabled = False
+pwd = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+json_alert = {}
+now = time.strftime("%a %b %d %H:%M:%S %Z %Y")
+
+# Set paths
+log_file = '{0}/logs/integrations.log'.format(pwd)
+
+
+def main(args):
+    debug("# Starting")
+
+    # Read args
+    alert_file_location = args[1]
+    webhook = args[3]
+
+    debug("# Webhook")
+    debug(webhook)
+
+    debug("# File location")
+    debug(alert_file_location)
+
+    # Load alert. Parse JSON object.
+    with open(alert_file_location) as alert_file:
+        json_alert = json.load(alert_file)
+    debug("# Processing alert")
+    debug(json_alert)
+
+    debug("# Generating message")
+    msg = generate_msg(json_alert)
+    if isinstance(msg, str):
+        if len(msg) == 0:
+            return
+    debug(msg)
+
+    debug("# Sending message")
+    send_msg(msg, webhook)
+
+
+def debug(msg):
+    if debug_enabled:
+        msg = "{0}: {1}\n".format(now, msg)
+        print(msg)
+        f = open(log_file, "a")
+        f.write(msg)
+        f.close()
+
+# Skips container kills to stop self-recursion
+def filter_msg(alert):
+    # These are things that recursively happen because Shuffle starts Docker containers
+    # Docker integration rules: https://github.com/wazuh/wazuh-ruleset/blob/ae36745db1d3f312db0392f5925c2f2b0ec009a9/rules/0560-docker_integration_rules.xml
+    skip = ["87924", "87900", "87901", "87902", "87903", "87904", "86001", "86002", "86003", "87932", "80710", "87929", "87928",]
+    if alert["rule"]["id"] in skip:
+        return False
+
+    #try:
+    #    if "docker" in alert["rule"]["description"].lower() and "
+    #msg['text'] = alert.get('full_log')
+    #except:
+    #    pass
+    #msg['title'] = alert['rule']['description'] if 'description' in alert['rule'] else "N/A"
+
+    return True
+
+def generate_msg(alert):
+    if not filter_msg(alert):
+        print("Skipping rule %s" % alert["rule"]["id"])
+        return ""
+
+    level = alert['rule']['level']
+
+    if (level <= 4):
+        color = "good"
+    elif (level >= 5 and level <= 7):
+        color = "warning"
+    else:
+        color = "danger"
+
+    msg = {}
+    msg['color'] = color
+    msg['pretext'] = "WAZUH Alert"
+    msg['title'] = alert['rule']['description'] if 'description' in alert['rule'] else "N/A"
+    msg['text'] = alert.get('full_log')
+    msg['rule_id'] = alert["rule"]["id"]
+    msg['timestamp'] = alert["timestamp"]
+    msg['id'] = alert['id']
+    msg["all_fields"] = alert
+
+    #msg['fields'] = []
+    #    msg['fields'].append({
+    #        "title": "Agent",
+    #        "value": "({0}) - {1}".format(
+    #            alert['agent']['id'],
+    #            alert['agent']['name']
+    #        ),
+    #    })
+    #if 'agentless' in alert:
+    #    msg['fields'].append({
+    #        "title": "Agentless Host",
+    #        "value": alert['agentless']['host'],
+    #    })
+
+    #msg['fields'].append({"title": "Location", "value": alert['location']})
+    #msg['fields'].append({
+    #    "title": "Rule ID",
+    #    "value": "{0} _(Level {1})_".format(alert['rule']['id'], level),
+    #})
+
+    #attach = {'attachments': [msg]}
+
+    return json.dumps(msg)
+
+
+def send_msg(msg, url):
+    headers = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'}
+    res = requests.post(url, data=msg, headers=headers)
+    debug(res)
+
+
+if __name__ == "__main__":
+    try:
+        # Read arguments
+        bad_arguments = False
+        if len(sys.argv) >= 4:
+            msg = '{0} {1} {2} {3} {4}'.format(
+                now,
+                sys.argv[1],
+                sys.argv[2],
+                sys.argv[3],
+                sys.argv[4] if len(sys.argv) > 4 else '',
+            )
+            debug_enabled = (len(sys.argv) > 4 and sys.argv[4] == 'debug')
+        else:
+            msg = '{0} Wrong arguments'.format(now)
+            bad_arguments = True
+
+        # Logging the call
+        f = open(log_file, 'a')
+        f.write(msg + '\n')
+        f.close()
+
+        if bad_arguments:
+            debug("# Exiting: Bad arguments.")
+            sys.exit(1)
+
+        # Main function
+        main(sys.argv)
+
+    except Exception as e:
+        debug(str(e))
+        raise

functions/extensions/wazuh/ossec.conf

@@ -0,0 +1,5 @@
+<integration>
+  <name>custom-shuffle</name>
+  <hook_url>http://<IP>:3001/api/v1/hooks/webhook_<HOOK_ID></hook_url>
+  <alert_format>json</alert_format>
+</integration>

functions/onprem/orborus/build.sh

@@ -1,11 +1,11 @@
 NAME=shuffle-orborus
-VERSION=0.8.0
+VERSION=0.8.54
 
 echo "Running docker build with $NAME:$VERSION"
 #docker rmi frikky/shuffle:$NAME --force
 docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
 
 #docker push frikky/$NAME:$VERSION
-#docker push frikky/shuffle:$NAME
 # docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION
+docker push frikky/shuffle:$NAME
 docker push ghcr.io/frikky/$NAME:$VERSION

functions/onprem/orborus/orborus.go

@@ -21,17 +21,22 @@ import (
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
+	//"github.com/docker/docker/api/types/filters"
 	dockerclient "github.com/docker/docker/client"
 	"github.com/satori/go.uuid"
 	//network "github.com/docker/docker/api/types/network"
 	//natting "github.com/docker/go-connections/nat"
+	"github.com/mackerelio/go-osstat/cpu"
+	"github.com/mackerelio/go-osstat/memory"
 )
 
 // Starts jobs in bulk, so this could be increased
 var sleepTime = 3
+var maxConcurrency = 50
 
 // Timeout if something rashes
 var workerTimeoutEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_TIMEOUT")
+var concurrencyEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY")
 var appSdkVersion = os.Getenv("SHUFFLE_APP_SDK_VERSION")
 var workerVersion = os.Getenv("SHUFFLE_WORKER_VERSION")
 
@@ -47,6 +52,8 @@ var baseUrl = os.Getenv("BASE_URL")
 var environment = os.Getenv("ENVIRONMENT_NAME")
 var dockerApiVersion = os.Getenv("DOCKER_API_VERSION")
 var runningMode = strings.ToLower(os.Getenv("RUNNING_MODE"))
+var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP"))
+var workerIds = []string{}
 
 type ExecutionRequestWrapper struct {
 	Data []ExecutionRequest `json:"data"`
@@ -103,13 +110,25 @@ func getThisContainerId() {
 		out, err := exec.Command("bash", "-c", cmd).Output()
 		if err == nil {
 			containerId = strings.TrimSpace(string(out))
+
+			// cgroup error. Hardcoding this.
+			// https://github.com/moby/moby/issues/7015
+			//log.Printf("Checking if %s is in %s", ".scope", string(out))
+			if strings.Contains(string(out), ".scope") {
+				containerId = "shuffle-orborus"
+				//docker-76c537e9a4b7c7233011f5d70e6b7f2d600b6413ac58a96519b8dca7a3f7117a.scope
+			}
 		} else {
-			log.Printf("Failed getting container ID: %s", err)
+			containerId = "shuffle-orborus"
+			log.Printf("[WARNING] Failed getting container ID: %s", err)
 		}
 	}
+
+	log.Printf("Started with containerId %s", containerId)
 }
 
 // Deploys the internal worker whenever something happens
+// https://docs.docker.com/engine/api/sdk/examples/
 func deployWorker(image string, identifier string, env []string) {
 	// Binds is the actual "-v" volume.
 	hostConfig := &container.HostConfig{
@@ -124,23 +143,28 @@ func deployWorker(image string, identifier string, env []string) {
 
 	// form container id and use it as network source if it's not empty
 	if containerId != "" {
-		log.Printf("[INFO] Found container ID %s", containerId)
+		//log.Printf("[INFO] Found container ID %s", containerId)
 		hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId))
 	} else {
 		//log.Printf("[INFO] Empty self container id, continue without NetworkMode")
 	}
 
+	if cleanupEnv == "true" {
+		hostConfig.AutoRemove = true
+	}
+
 	config := &container.Config{
 		Image: image,
 		Env:   env,
 	}
 
-	log.Printf("Identifier: %s", identifier)
+	//log.Printf("[INFO] Identifier: %s", identifier)
 	cont, err := dockercli.ContainerCreate(
 		context.Background(),
 		config,
 		hostConfig,
 		nil,
+		nil,
 		identifier,
 	)
 
@@ -154,6 +178,7 @@ func deployWorker(image string, identifier string, env []string) {
 				config,
 				hostConfig,
 				nil,
+				nil,
 				identifier,
 			)
 
@@ -167,7 +192,8 @@ func deployWorker(image string, identifier string, env []string) {
 		}
 	}
 
-	err = dockercli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{})
+	containerStartOptions := types.ContainerStartOptions{}
+	err = dockercli.ContainerStart(context.Background(), cont.ID, containerStartOptions)
 	if err != nil {
 		log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err)
 		return
@@ -195,6 +221,7 @@ func deployWorker(image string, identifier string, env []string) {
 		//}
 	} else {
 		log.Printf("[INFO] Container %s was created under environment %s", cont.ID, environment)
+		//workerIds = append(workerIds, cont.ID)
 	}
 
 	return
@@ -227,11 +254,11 @@ func initializeImages() {
 	ctx := context.Background()
 
 	if appSdkVersion == "" {
-		appSdkVersion = "0.8.0"
+		appSdkVersion = "0.8.5"
 		log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion)
 	}
 	if workerVersion == "" {
-		workerVersion = "0.8.0"
+		workerVersion = "0.8.54"
 		log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion)
 	}
 
@@ -248,9 +275,7 @@ func initializeImages() {
 
 	// check whether they are the same first
 	images := []string{
-		//fmt.Sprintf("%s/%s:app_sdk%s", baseimageregistry, baseimagename, baseimagetagsuffix),
+		fmt.Sprintf("frikky/shuffle:app_sdk"),
-		//fmt.Sprintf("%s/%s:worker%s", baseimageregistry, baseimagename, baseimagetagsuffix),
-
 		fmt.Sprintf("%s/%s/shuffle-app_sdk:%s", baseimageregistry, baseimagename, appSdkVersion),
 		fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion),
 		// fmt.Sprintf("docker.io/%s:app_sdk", baseimagename),
@@ -275,6 +300,40 @@ func initializeImages() {
 	}
 }
 
+// Will be used for checking if there's enough to deploy based on a threshold
+// E.g. having maximum CPU and maxmimum RAM
+// Does this work containerized?
+func getStats() {
+	fmt.Printf("\n")
+
+	memory, err := memory.Get()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+		return
+	}
+
+	before, err := cpu.Get()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+		return
+	}
+	time.Sleep(time.Duration(250) * time.Millisecond)
+	after, err := cpu.Get()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+		return
+	}
+	total := float64(after.Total - before.Total)
+
+	fmt.Printf("[INFO] memory total: %d bytes\n", memory.Total)
+	fmt.Printf("[INFO] memory used: %d bytes\n", memory.Used)
+	fmt.Printf("[INFO] cpu used  : %f%%\n", float64(after.User-before.User)/total*100)
+	fmt.Printf("[INFO] cpu system: %f%%\n", float64(after.System-before.System)/total*100)
+	fmt.Printf("[INFO] cpu idle  : %f%%\n", float64(after.Idle-before.Idle)/total*100)
+
+	fmt.Printf("\n")
+}
+
 // Initial loop etc
 func main() {
 	log.Println("[INFO] Setting up execution environment")
@@ -302,7 +361,19 @@ func main() {
 		log.Printf("[INFO] Cleanup process running every %d seconds", workerTimeout)
 	}
 
-	go zombiecheck(workerTimeout)
+	if concurrencyEnv != "" {
+		//var concurrencyEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY")
+		tmpInt, err := strconv.Atoi(concurrencyEnv)
+		if err == nil {
+			maxConcurrency = tmpInt
+			log.Printf("[INFO] Max workflow execution concurrency set to %d", maxConcurrency)
+		} else {
+			log.Printf("[WARNING] Env SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY must be a number, not %s. Defaulted to %d", workerTimeoutEnv, maxConcurrency)
+		}
+	}
+
+	ctx := context.Background()
+	go zombiecheck(ctx, workerTimeout)
 
 	log.Printf("[INFO] Running towards %s with Org %s", baseUrl, orgId)
 	httpProxy := os.Getenv("HTTP_PROXY")
@@ -337,6 +408,8 @@ func main() {
 		},
 	}
 
+	//getStats()
+
 	if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" {
 		client = &http.Client{}
 	} else {
@@ -367,13 +440,15 @@ func main() {
 	hasStarted := false
 	for {
 		//log.Printf("Prerequest")
+		//go getStats()
 		newresp, err := client.Do(req)
+		executionCount := getRunningWorkers(ctx, workerTimeout)
 		//log.Printf("Postrequest")
 		if err != nil {
 			log.Printf("[WARNING] Failed making request: %s", err)
 			zombiecounter += 1
 			if zombiecounter*sleepTime > workerTimeout {
-				go zombiecheck(workerTimeout)
+				go zombiecheck(ctx, workerTimeout)
 				zombiecounter = 0
 			}
 			time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -394,7 +469,7 @@ func main() {
 			log.Printf("[ERROR] Failed reading body: %s", err)
 			zombiecounter += 1
 			if zombiecounter*sleepTime > workerTimeout {
-				go zombiecheck(workerTimeout)
+				go zombiecheck(ctx, workerTimeout)
 				zombiecounter = 0
 			}
 			time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -408,7 +483,7 @@ func main() {
 			sleepTime = 10
 			zombiecounter += 1
 			if zombiecounter*sleepTime > workerTimeout {
-				go zombiecheck(workerTimeout)
+				go zombiecheck(ctx, workerTimeout)
 				zombiecounter = 0
 			}
 			time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -423,13 +498,31 @@ func main() {
 		if len(executionRequests.Data) == 0 {
 			zombiecounter += 1
 			if zombiecounter*sleepTime > workerTimeout {
-				go zombiecheck(workerTimeout)
+				go zombiecheck(ctx, workerTimeout)
 				zombiecounter = 0
 			}
 			time.Sleep(time.Duration(sleepTime) * time.Second)
 			continue
 		}
 
+		// Anything below here verifies concurrency virification
+		if executionCount >= maxConcurrency {
+			if zombiecounter*sleepTime > workerTimeout {
+				go zombiecheck(ctx, workerTimeout)
+				zombiecounter = 0
+			}
+			time.Sleep(time.Duration(sleepTime) * time.Second)
+			continue
+		}
+
+		//log.Printf("[INFO] Got %d new requests. Executing: %d. Max: %d", len(executionRequests.Data), executionCount, maxConcurrency)
+
+		allowed := maxConcurrency - executionCount
+		if len(executionRequests.Data) > allowed {
+			log.Printf("[WARNING] Throttle - Cutting down requests from %d to %d (MAX: %d, CUR: %d)", len(executionRequests.Data), allowed, maxConcurrency, executionCount)
+			executionRequests.Data = executionRequests.Data[0:allowed]
+		}
+
 		// New, abortable version. Should check executionid and remove everything else
 		var toBeRemoved ExecutionRequestWrapper
 		for _, execution := range executionRequests.Data {
@@ -453,6 +546,7 @@ func main() {
 				fmt.Sprintf("EXECUTIONID=%s", execution.ExecutionId),
 				fmt.Sprintf("ENVIRONMENT_NAME=%s", environment),
 				fmt.Sprintf("BASE_URL=%s", baseUrl),
+				fmt.Sprintf("CLEANUP=%s", cleanupEnv),
 			}
 
 			if strings.ToLower(os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) != "false" {
@@ -466,7 +560,7 @@ func main() {
 
 			go deployWorker(workerImage, containerName, env)
 
-			log.Printf("[INFO] %s is deployed and to be removed from queue.", execution.ExecutionId)
+			log.Printf("[INFO] ExecutionID %s was deployed and to be removed from queue.", execution.ExecutionId)
 			zombiecounter += 1
 			toBeRemoved.Data = append(toBeRemoved.Data, execution)
 		}
@@ -528,25 +622,22 @@ func main() {
 	}
 }
 
-// FIXME - add this to remove exited workers
+// Is this ok to do with Docker? idk :)
-// Should it check what happened to the execution? idk
+func getRunningWorkers(ctx context.Context, workerTimeout int) int {
-func zombiecheck(workerTimeout int) error {
-	log.Println("[INFO] Looking for old containers")
-	ctx := context.Background()
-
 	containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
 		All: true,
 	})
+	//Filters: filters.Args{
+	//	map[string][]string{"ancestor": {"<imagename>:<version>"}},
+	//},
 
 	if err != nil {
-		log.Printf("[ERROR] Failed creating Containerlist: %s", err)
+		log.Printf("[ERROR] Error getting containers: %s", err)
-		return err
+		return maxConcurrency
 	}
 
-	containerNames := map[string]string{}
+	currenttime := time.Now().Unix()
-
+	counter := 0
-	stopContainers := []string{}
-	removeContainers := []string{}
 	for _, container := range containers {
 		// Skip random containers. Only handle things related to Shuffle.
 		if !strings.Contains(container.Image, baseimagename) {
@@ -568,14 +659,75 @@ func zombiecheck(workerTimeout int) error {
 
 		for _, name := range container.Names {
 			// FIXME - add name_version_uid_uid regex check as well
-			if strings.HasPrefix(name, "/shuffle") {
+			if !strings.HasPrefix(name, "/worker") {
 				continue
 			}
 
-			log.Printf("[INFO] NAME: %s", name)
+			//log.Printf("Time: %d - %d", currenttime-container.Created, int64(workerTimeout))
+			if container.State == "running" && currenttime-container.Created < int64(workerTimeout) {
+				counter += 1
+				break
+			}
+		}
+	}
+
+	return counter
+}
+
+// FIXME - add this to remove exited workers
+// Should it check what happened to the execution? idk
+func zombiecheck(ctx context.Context, workerTimeout int) error {
+	log.Println("[INFO] Looking for old containers (zombies)")
+	containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
+		All: true,
+	})
+
+	//log.Printf("Len: %d", len(containers))
+
+	if err != nil {
+		log.Printf("[ERROR] Failed creating Containerlist: %s", err)
+		return err
+	}
+
+	containerNames := map[string]string{}
+
+	stopContainers := []string{}
+	removeContainers := []string{}
+	log.Printf("[INFO] Baseimage: %s, Workertimeout: %d", baseimagename, int64(workerTimeout))
+	baseString := `/bin/sh -c 'python app.py --log-level DEBUG'`
+	for _, container := range containers {
+		// Skip random containers. Only handle things related to Shuffle.
+		if !strings.Contains(container.Image, baseimagename) && container.Command != baseString && container.Command != "./worker" {
+			shuffleFound := false
+			for _, item := range container.Labels {
+				if item == "shuffle" {
+					shuffleFound = true
+					break
+				}
+			}
+
+			// Check image name
+			if !shuffleFound {
+				log.Printf("Skipping: %s, %s", container.Labels, container.Image)
+				continue
+			}
+			//} else {
+			//	log.Printf("NAME: %s", container.Image)
+		} else {
+			//log.Printf("Img: %s", container.Image)
+			//log.Printf("Names: %s", container.Names)
+		}
+
+		for _, name := range container.Names {
+			// FIXME - add name_version_uid_uid regex check as well
+			if strings.HasPrefix(name, "/shuffle") && !strings.HasPrefix(name, "/shuffle-subflow") {
+				continue
+			}
+
+			currenttime := time.Now().Unix()
+			//log.Printf("[INFO] (%s) NAME: %s. TIME: %d", container.State, name, currenttime-container.Created)
 
 			// Need to check time here too because a container can be removed the same instant as its created
-			currenttime := time.Now().Unix()
 			if container.State != "running" && currenttime-container.Created > int64(workerTimeout) {
 				removeContainers = append(removeContainers, container.ID)
 				containerNames[container.ID] = name
@@ -591,9 +743,10 @@ func zombiecheck(workerTimeout int) error {
 	}
 
 	// FIXME - add killing of apps with same execution ID too
+	log.Printf("[INFO] Should STOP %d containers.", len(stopContainers))
 	for _, containername := range stopContainers {
 		log.Printf("[INFO] Stopping and removing container %s", containerNames[containername])
-		go dockercli.ContainerStop(ctx, containername, nil)
+		dockercli.ContainerStop(ctx, containername, nil)
 		removeContainers = append(removeContainers, containername)
 	}
 
@@ -602,8 +755,9 @@ func zombiecheck(workerTimeout int) error {
 		Force:         true,
 	}
 
+	log.Printf("[INFO] Should REMOVE %d containers.", len(removeContainers))
 	for _, containername := range removeContainers {
-		go dockercli.ContainerRemove(ctx, containername, removeOptions)
+		dockercli.ContainerRemove(ctx, containername, removeOptions)
 	}
 
 	return nil

functions/onprem/worker/Dockerfile

@@ -5,6 +5,8 @@ WORKDIR /app
 RUN go get -u github.com/docker/docker/api/types 
 RUN go get -u github.com/docker/docker/api/types/container
 RUN go get -u github.com/docker/docker/client
+RUN go get -u github.com/gorilla/mux
+RUN go get -u github.com/patrickmn/go-cache
 
 COPY worker.go /app/worker.go
 RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker .
@@ -13,7 +15,7 @@ FROM alpine:3.12
 
 ENV SHUFFLE_BASE_IMAGE_REGISTRY=docker.io
 ENV SHUFFLE_BASE_IMAGE_NAME=frikky/shuffle
-ENV SHUFFLE_BASE_IMAGE_TAG_SUFFIX=0.6.0
+ENV SHUFFLE_BASE_IMAGE_TAG_SUFFIX=0.8.5
 
 RUN apk add --no-cache bash
 COPY --from=builder /app/ /

functions/onprem/worker/build.sh

@@ -1,12 +1,14 @@
 NAME=shuffle-worker
-VERSION=0.8.0
+VERSION=0.8.56
 
 echo "Running docker build with $NAME:$VERSION"
 #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin .
-docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
+docker build . -t frikky/shuffle:$NAME -t frikky/shuffle:$NAME_$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
 
 # Push both for now..
 #docker push frikky/$NAME:$VERSION 
-docker push frikky/shuffle:$NAME
+#docker push frikky/shuffle:$NAME_$VERSION
 #docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION
+#docker tag frikky/shuffle:0.8.51 ghcr.io/frikky/shuffle-worker:0.8.5
+docker tag frikky/shuffle:$NAME ghcr.io/frikky/shuffle-worker:0.8.52
 docker push ghcr.io/frikky/$NAME:$VERSION