diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 08fca979..00000000 --- a/.dockerignore +++ /dev/null @@ -1,15 +0,0 @@ -**/.git* -**/.gitlab-ci.yml -**/.env -**/.dockerignore -**/Dockerfile* -**/node_modules -buildSrc/libs -.gradle/ -build/ -**build/ -.idea/ -!.idea/codeStyles/codeStyleConfig.xml -.DS_Store -*.log -out/ diff --git a/.env b/.env index 45389f1f..3bb32c35 100644 --- a/.env +++ b/.env @@ -39,7 +39,7 @@ SHUFFLE_PASS_WORKER_PROXY=TRUE SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io SHUFFLE_BASE_IMAGE_NAME=frikky -SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.3" +SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.60" # Used for auto-cleanup of containers. REALLY important at scale. SHUFFLE_CONTAINER_AUTO_CLEANUP=false diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 00000000..d3f66e5b --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,39 @@ +# Contributing to Shuffle + +First off, thank you for contributing to [Shuffle](https://shuffler.io/)! Your talents and your contributions are greatly appreciated. With Shuffle, we aim to make cybersecurity more accessible, and keep that in mind with everything we make. + +## Opening a new issue + +If you find a bug or think of an improvement or fix, please open a [new issue](https://github.com/frikky/Shuffle/issues/new). Outline every step necessary to reproduce the bug. Include screenshots, logs and/or code examples where applicable. The more thorough you are, the better. + +## What you can work on +There are a lot of things to work on in our complex ecosystem. The most pressing issues are documentation, use-cases and content-creation, but any help is appreciated. We'll make sure you get the help you need to get started. Below is an incomplete list of items. If you see an issue, tell us or fix it! :) + +#### App Creation (Python & GUI w/OpenAPI) +As with everything else, app creation for Shuffle is made as accessibl as possible with the app editor. However, there are some instances where it can't do the job, and you'll have to write Python code. The App Editor generates OpenAPI specifications and can be widely shared, while Python apps only work for Shuffle and NSA's WALKOFF (which Shuffle is based on). You can find our [OpenAPI apps here](https://github.com/frikky/security-openapis) and our [Python apps here](https://github.com/frikky/shuffle-apps). Apps in these repositories are automatically available after installation. Shuffle apps are searchable on [https://shuffler.io](https://shuffler.io/search). + +#### Workflow creation (GUI & Conceptualizing) +Workflows are where the magic of Shuffle automation happens. Our current ones [are outlined here](https://github.com/frikky/security-openapis), and will be automatically imported into Shuffle instances in the future. They are split into Prepare and Response, but don't necessarily have to be. If you'd like to talk about workflow creation or use-cases in general, either Open a [new issue](https://github.com/frikky/shuffle-workflows/issues/new) or send us an email at [frikky@shuffler.io](mailto:frikky@shuffler.io) + +#### Documentation (Markdown) +Documentation is essential to any product, and Shuffle is no exception. Documentation in Shuffle uses markdown and is located in the [shuffle-docs](https://github.com/frikky/shuffle-docs/tree/master/docs) repository. These are then loaded into Shuffle when someone visits [https://shuffler/docs/about](https://shuffler/docs/about), then cached for later use. If you make an edit, expect it on our website in about an hour. + +#### Frontend (ReactJS) +The frontend of Shuffle is what everyone sees when they log in. Our goal here is to make it easy to get started and keep going with Shuffle - removing any blockers from the point of accessibility. If you'd like to get started, find [an issue](https://github.com/frikky/Shuffle/issues) and check the [installation guide](https://github.com/frikky/Shuffle/blob/master/install-guide.md#local-development-installation) for setting it up locally without Docker. + +#### Backend (Golang) +The backend of Shuffle is our REST API Server that runs in the background, handling all the API-calls in general, whether from users or apps. If you'd like to get started, find [an issue](https://github.com/frikky/Shuffle/issues) and check the [installation guide](https://github.com/frikky/Shuffle/blob/master/install-guide.md#local-development-installation) for setting it up locally without Docker. + +## Working on an issue + +**Shuffle** uses the [GitHub flow](https://guides.github.com/introduction/flow/index.html). All project changes are made through pull requests. +If you see an issue that you would like to work on, leave a quick comment or just get cracking. + +### License + +All contributions are made under either the **GNU Affero General Public License v3.0** or **MIT** license. See below for further details. + +* [Main project license - AGPLv3](https://github.com/frikky/Shuffle/blob/master/LICENSE) +* [Apps - MIT](https://github.com/frikky/Shuffle-apps/blob/master/LICENSE) +* [Workflows - MIT](https://github.com/frikky/Shuffle-workflows/blob/master/LICENSE) +* [Documentation - MIT](https://github.com/frikky/Shuffle-docs/blob/master/LICENSE) diff --git a/install-guide.md b/.github/install-guide.md similarity index 83% rename from install-guide.md rename to .github/install-guide.md index 5f5fbd22..d1c21169 100644 --- a/install-guide.md +++ b/.github/install-guide.md @@ -82,8 +82,13 @@ docker-compose up Related issue: #47 -## Local development installation -**Frontend - ReactJS /w cytoscape** +# Local development installation +Local development is pretty straight forward with **ReactJS** and **Golang**. This part is intended to help you run the code for development purposes. + +**PS: You have to stop the Backend Docker container to get this one working** +**PPS: Use the "Launch" branch when developing to get it set up easier** + +## Frontend - ReactJS /w cytoscape http://localhost:3000 - Requires [npm](https://nodejs.org/en/download/)/[yarn](https://yarnpkg.com/lang/en/docs/install/#debian-stable)/your preferred manager. Runs independently from backend. ```bash cd frontend @@ -91,35 +96,39 @@ npm i npm start ``` -**Backend - Golang** +## Backend - Golang http://localhost:5001 - REST API - requires [>=go1.13](https://golang.org/dl/) ```bash export DATASTORE_EMULATOR_HOST=0.0.0.0:8000 cd backend/go-app -go build go run *.go ``` -**Database - Datastore** +**WINDOWS USERS:** You'll have to to add the "export" part as an environment variable. + +## Database - Datastore Based on Google datastore ``` docker run -p 8000:8000 google/cloud-sdk gcloud beta emulators datastore start --project=shuffle --host-port 0.0.0.0:8000 --no-store-on-disk ``` -**Orborus** +## Orborus Execution of Workflows: PS: This requires some specific environment variables ``` cd functions/onprem/orborus go run orborus.go ``` -Environments: + + +Environments (modify for Windows): ``` export ORG_ID=Shuffle export ENVIRONMENT_NAME=Shuffle export BASE_URL=http://YOUR-IP:5001 export DOCKER_API_VERSION=1.40 -export SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} ``` +**WINDOWS USERS:** You'll have to to add the "export" part as an environment variable. +AND THAT's it - hopefully it worked. If it didn't please email [frikky@shuffler.io](mailto:frikky@shuffler.io) diff --git a/README.md b/README.md index f9589f40..895bd472 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,22 @@ # Shuffle -[Shuffle](https://shuffler.io) is an automation platform to unify your security services (SOAR). It has thousands of premade integrations and is based on open frameworks like OpenAPI and Mitre Att&ck. The workflow editor is based on a no-code thought process to empower non-developers, and the app creator makes you able to integrate any platform in minutes. +[Shuffle](https://shuffler.io) is an automation platform focused on accessibility. We believe everyone should have access to efficient processes, and are striving to make that a possibility by making integrations for YOUR tools. Security Operations is complex, but it doesn't have to be. [![Discord](https://img.shields.io/discord/463752820026376202.svg?label=&logo=discord&logoColor=ffffff&color=7389D8&labelColor=6A7EC2)](https://discord.gg/B2CBzUm) -![Example Shuffle webhook integration](https://github.com/frikky/Shuffle/blob/master/frontend/src/assets/img/shuffle_webhook.png) +![Example Shuffle webhook integration](https://github.com/frikky/Shuffle/blob/master/frontend/src/assets/img/github_shuffle_img.png) ## Try it -* Self-hosted: Check out the [installation guide](https://github.com/frikky/shuffle/blob/master/install-guide.md) +* Self-hosted: Check out the [installation guide](https://github.com/frikky/shuffle/blob/master/.github/install-guide.md) * Cloud: Register at https://shuffler.io/register and get cooking (missing a lot of features) Please consider [sponsoring](https://github.com/sponsors/frikky) the project if you want to see more rapid development. ## Support * [Discord](https://discord.gg/B2CBzUm) +* [Twitter](https://twitter.com/shuffleio) * [Email](mailto:frikky@shuffler.io) * [Open issue](https://github.com/frikky/Shuffle/issues/new) +* [Shuffler.io](https://shuffler.io/contact) ## Blogposts * [1. Introducing Shuffle](https://medium.com/security-operation-capybara/introducing-shuffle-an-open-source-soar-platform-part-1-58a529de7d12) @@ -23,25 +25,36 @@ Please consider [sponsoring](https://github.com/sponsors/frikky) the project if * [4. Real-time executions with TheHive, Cortex and MISP](https://medium.com/@Frikkylikeme/indicators-and-webhooks-with-thehive-cortex-and-misp-open-source-soar-part-4-f70cde942e59) ## Documentation -[Documentation](https://shuffler.io/docs) can be found on https://shuffler.io/docs and is written in https://github.com/frikky/shuffle-docs. +[Documentation](https://shuffler.io/docs) can be found on [https://shuffler.io/docs](https://shuffler.io/docs) and is written here: [https://github.com/frikky/shuffle-docs](https://github.com/frikky/shuffle-docs). ## Related repositories -* Apps: https://github.com/frikky/shuffle-apps -* Workflows: https://github.com/frikky/shuffle-workflows -* Security OpenAPI apps: https://github.com/frikky/security-openapis -* Documentation: https://github.com/frikky/shuffle-docs +* OpenAPI apps: [https://github.com/frikky/security-openapis](https://github.com/frikky/security-openapis) +* Documentation: [https://github.com/frikky/shuffle-docs](https://github.com/frikky/shuffle-docs) +* Workflows: [https://github.com/frikky/shuffle-workflows](https://github.com/frikky/shuffle-workflows) +* Python apps: [https://github.com/frikky/shuffle-apps](https://github.com/frikky/shuffle-apps) ## Features -* Simple workflow automation editor -* Premade apps for a number of security tools -* App creator for [OpenAPI](https://github.com/frikky/OpenAPI-security-definitions) -* Easy to learn Python library for custom apps - -## Architecture -![Shuffle Architecture](https://github.com/frikky/Shuffle/blob/master/frontend/src/assets/img/shuffle_architecture.png) +* Simple, feature rich [workflow editor](https://shuffler.io/docs/workflows) +* App creator using [OpenAPI](https://github.com/frikky/OpenAPI-security-definitions) +* Premade apps for your security tools +* Organization and sub-organization control +* Hybrid resource sharing with shuffler.io (optional) ## Website -https://shuffler.io +[https://shuffler.io](https://shuffler.io) + +## Contributing +We want to make the world of cybersecurity more accessible and need all the help we can get. Send an email to [frikky@shuffler](mailto:frikky@shuffler.io) and we'll make sure to give you any training you may need. + +These are the main areas to contribute in: +* Frontend (ReactJS) +* Backend (Golang) +* App Creation (Python & GUI w/OpenAPI) +* Documentation (Markdown) +* Workflow creation (GUI & Conceptualizing) +* Content Creation (Blogs, videos etc) + +Contributing guidelines for Github are outlined [here](https://github.com/frikky/Shuffle/blob/master/.github/CONTRIBUTING.md). ## Contributors ![ICPL logo](https://github.com/frikky/Shuffle/blob/launch/frontend/src/assets/img/icpl_logo.png) @@ -59,8 +72,13 @@ https://shuffler.io ## License All modular information related to Shuffle will be under MIT (anyone can use it for whatever purpose), with Shuffle itself using AGPLv3. -Apps & App SDK: MIT +Workflows: MIT +Documentation: MIT Shuffle backend: AGPLv3 +Apps, specification and App SDK: MIT + +## Architecture +![Shuffle Architecture](https://github.com/frikky/Shuffle/blob/master/frontend/src/assets/img/shuffle_architecture.png) ### Repository overview Below is the folder structure with a short explanation @@ -68,17 +86,14 @@ Below is the folder structure with a short explanation ├── README.md # What you're reading right now ├── backend # Contains backend related code. │   ├── go-app # The backend golang webserver -│   ├── app_gen # Code for app generation outside the Shuffle platform │ └── app_sdk # The SDK used for apps -├── frontend # Contains frontend code. ReactJS and cytoscape. Horrible code :) -├── functions # Contains google cloud function code mainly. -│   ├── static_baseline.py # Static code used by stitcher.go to generate code -│   ├── stitcher.go # Attempts to stitch together an app - part of backend now +├── frontend # Contains frontend code. ReactJS, Material UI and cytoscape +├── functions # Has execution and extension resources, such as the Wazuh integration │   ├── onprem # Code for onprem solutions │  │   ├── Orborus # Distributes execution locations │  │   ├── Worker # Runs a workflow └ docker-compose.yml # Used for deployments ``` -**It's in BETA** - [Get in touch](https://shuffler.io/contact), send a mail to [frikky@shuffler.io](mailto:frikky@shuffler.io) or poke me on twitter [@frikkylikeme](https://twitter.com/frikkylikeme) +**It's in BETA (0.8.60)** - [Get in touch](https://shuffler.io/contact), send a mail to [frikky@shuffler.io](mailto:frikky@shuffler.io) or poke me on twitter [@frikkylikeme](https://twitter.com/frikkylikeme) diff --git a/backend/app_sdk/Dockerfile b/backend/app_sdk/Dockerfile index 370c31f8..75ec12c4 100644 --- a/backend/app_sdk/Dockerfile +++ b/backend/app_sdk/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7-alpine as base +FROM python:3.9.1-alpine as base FROM base as builder RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index e0c9ec6a..e428da9f 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -7,9 +7,10 @@ import json import logging import requests import urllib.parse +import http.client +import urllib3 class AppBase: - """ The base class for Python-based apps in Shuffle, handles logging and callbacks configurations""" __version__ = None app_name = None @@ -21,7 +22,7 @@ class AppBase: # apikey is for the user / org # authorization is for the specific workflow self.url = os.getenv("CALLBACK_URL", "https://shuffler.io") - self.base_url = os.getenv("BASE_URL", "") + self.base_url = os.getenv("BASE_URL", "https://shuffler.io") self.action = os.getenv("ACTION", "") self.authorization = os.getenv("AUTHORIZATION", "") self.current_execution_id = os.getenv("EXECUTIONID", "") @@ -29,7 +30,10 @@ class AppBase: self.result_wrapper_count = 0 if isinstance(self.action, str): - self.action = json.loads(self.action) + try: + self.action = json.loads(self.action) + except: + print("[WARNING] Failed parsing action as JSON") if len(self.base_url) == 0: self.base_url = self.url @@ -40,20 +44,33 @@ class AppBase: if action_result["status"] == "EXECUTING": action_result["status"] = "FAILURE" + # FIXME: Add cleanup of parameters to not send to frontend here + params = {} + #action = action_result["action"] + #try: + # for item in action["authentication"]: + # for action["parameters"] + # print("AUTH: ", key, value) + # params[item["key"]] = item["value"] + #except KeyError: + # print("No authentication specified!") + # pass + # I wonder if this actually works self.logger.info("Before last stream result") url = "%s%s" % (self.base_url, stream_path) - print("URL: %s" % url) + #print("[INFO] URL (URL): %s" % url) try: ret = requests.post(url, headers=headers, json=action_result) self.logger.info("Result: %d" % ret.status_code) if ret.status_code != 200: self.logger.info(ret.text) except requests.exceptions.ConnectionError as e: - self.logger.exception(e) + #self.logger.exception("ConnectionError: %s" % e) + self.logger.info("Expected ConnectionError happened") return except TypeError as e: - self.logger.exception(e) + #self.logger.exception(e) action_result["status"] = "FAILURE" action_result["result"] = "POST error: %s" % e self.logger.info("Before typeerror stream result") @@ -61,6 +78,12 @@ class AppBase: self.logger.info("Result: %d" % ret.status_code) if ret.status_code != 200: self.logger.info(ret.text) + except http.client.RemoteDisconnected as e: + self.logger.info("Expected Remotedisconnect happened") + return + except urllib3.exceptions.ProtocolError as e: + self.logger.info("Expected ProtocolError happened") + return async def cartesian_product(self, L): if L: @@ -108,6 +131,11 @@ class AppBase: # 1. For the first array, take the total amount(y) (2x3=6) and divide it by the current array (x): 2. x/y = 3. This means do 3 of each value # 2. For the second array, take the total amount(y) (2x3=6) and divide it by the current array (x): 3. x/y = 2. # 3. What does the 3rd array do? Same, but ehhh? + # + # Example4: + # What if there are multiple loops inside a single item? + # + # paramlist = [] listitems = [] @@ -130,7 +158,7 @@ class AppBase: octothorpe_count = param["value"].count(".#") if octothorpe_count > self.result_wrapper_count: self.result_wrapper_count = octothorpe_count - print("NEW OCTOTHORPE WRAPPER: %d" % octothorpe_count) + print("[INFO] NEW OCTOTHORPE WRAPPER: %d" % octothorpe_count) # This whole thing is hard. # item = [{"data": "1.2.3.4", "dataType": "ip"}] @@ -270,7 +298,7 @@ class AppBase: newparams[key] = value[0] has_loop = True else: - print("Key %s is NOT a list within a list: %s" % (key, value)) + print("Key %s is NOT a list within a list" % (key)) newparams[key] = value @@ -408,7 +436,7 @@ class AppBase: content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"]) ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers) - print("RET2 (file get): %s" % ret2.text) + print("RET2 (file get) done") if ret2.status_code == 200: tmpdata = ret1.json() returndata = { @@ -513,8 +541,21 @@ class AppBase: "status": "EXECUTING" } + # Simple validation of parameters in general + try: + tmp_parameters = action["parameters"] + except KeyError: + action["parameters"] = [] + except TypeError: + pass + self.action = copy.deepcopy(action) - self.logger.info("ACTION RESULT (start): %s", action_result) + self.logger.info("Sending starting action result (EXECUTING)") + + headers = { + "Content-Type": "application/json", + "Authorization": "Bearer %s" % self.authorization + } if len(self.action) == 0: print("ACTION env not defined") @@ -534,10 +575,6 @@ class AppBase: self.send_result(action_result, headers, stream_path) return - headers = { - "Content-Type": "application/json", - "Authorization": "Bearer %s" % self.authorization - } # Add async logger # self.console_logger.handlers[0].stream.set_execution_id() @@ -720,7 +757,7 @@ class AppBase: else: tmp = json.loads(parsedlist)[lastsplit[0]] - print(tmp) + #print(tmp) return tmp except IndexError as e: return default_error @@ -751,8 +788,8 @@ class AppBase: # Do stuff here. innervalue = parse_nested_param(data, maxDepth(data)-0) outervalue = parse_nested_param(data, maxDepth(data)-1) - print("INNER: ", innervalue) - print("OUTER: ", outervalue) + #print("INNER: ", innervalue) + #print("OUTER: ", outervalue) if outervalue != innervalue: #print("Outer: ", outervalue, " inner: ", innervalue) @@ -769,7 +806,7 @@ class AppBase: print("Parsed value from %s: %s" % (thistype, parsed_value)) return (parsed_value, True) - print("DATA: %s\n" % data) + #print("DATA: %s\n" % data) return (parse_wrapper(data)[0], True) @@ -829,12 +866,12 @@ class AppBase: return data if len(parsedlist) > 0 and not non_string: - print("Returning parsed list: ", parsedlist) + #print("Returning parsed list: ", parsedlist) return " ".join(parsedlist) elif len(parsedlist) == 1 and non_string: return parsedlist[0] else: - print("Casting back to string because multi: ", parsedlist) + #print("Casting back to string because multi: ", parsedlist) newlist = [] for item in parsedlist: try: @@ -848,13 +885,13 @@ class AppBase: # Parses JSON loops and such down to the item you're looking for def recurse_json(basejson, parsersplit): match = "#(\d+):?-?([0-9a-z]+)?#?" - print("Split: %s\n%s" % (parsersplit, basejson)) + #print("Split: %s\n%s" % (parsersplit, basejson)) try: outercnt = 0 # Loops over split values for value in parsersplit: - print("VALUE: %s\n" % value) + #print("VALUE: %s\n" % value) actualitem = re.findall(match, value, re.MULTILINE) if value == "#": newvalue = [] @@ -875,7 +912,7 @@ class AppBase: return newvalue, True elif len(actualitem) > 0: - print("[INFO] In recursion v2: ", actualitem) + #print("[INFO] In recursion v2: ", actualitem) is_loop = True newvalue = [] @@ -884,7 +921,7 @@ class AppBase: # Means it's a single item -> continue if seconditem == "": - print("[INFO] In first - handling %s" % firstitem) + #print("[INFO] In first - handling %s" % firstitem) tmpitem = basejson[int(firstitem)] try: newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:]) @@ -1018,7 +1055,7 @@ class AppBase: except KeyError as error: print(f"KeyError in JSON: {error}") - print(f"[INFO] After first trycatch. Baseresult: ", baseresult) + print(f"[INFO] After first trycatch. Baseresult")#, baseresult) # 2. Find the JSON data if len(baseresult) == 0: @@ -1031,7 +1068,7 @@ class AppBase: baseresult = baseresult.replace(" True,", " true,") baseresult = baseresult.replace(" False", " false,") - print("[INFP] After third parser return - Formatted: ", baseresult) + print("[INFO] After third parser return - Formatted")#, baseresult) basejson = {} try: basejson = json.loads(baseresult) @@ -1346,6 +1383,8 @@ class AppBase: actionname = action["name"] if " " in actionname: actionname.replace(" ", "_", -1) + + #if action.generated: # actionname = actionname.lower() @@ -1384,6 +1423,20 @@ class AppBase: for parameter in action["parameters"]: counter += 1 + # Hack for key:value in options using || + try: + if parameter["options"] != None and len(parameter["options"]) > 0: + #print(f'OPTIONS: {parameter["options"]}') + #print(f'OPTIONS VAL: {parameter}') + if "||" in parameter["value"]: + splitvalue = parameter["value"].split("||") + if len(splitvalue) > 1: + print(f'[INFO] Parsed split || options of actions["parameters"]["name"]') + action["parameters"][counter]["value"] = splitvalue[1] + + except (IndexError, KeyError, TypeError) as e: + print("Options err: {e}") + if parameter["name"] == "body": bodyindex = counter #print("PARAM: %s" % parameter) @@ -1441,10 +1494,10 @@ class AppBase: except KeyError: pass - print("Return value: %s" % value) + #print("Return value: %s" % value) actionname = action["name"] #print("Multicheck ", actualitem) - print("ITEM LENGTH: %d, Actual item: %s" % (len(actualitem), actualitem)) + #print("ITEM LENGTH: %d, Actual item: %s" % (len(actualitem), actualitem)) if len(actualitem) > 0: multiexecution = True @@ -1464,6 +1517,7 @@ class AppBase: #json_replacement = tmpitem.replace(actualitem[0][0], replacement, 1) #print("AFTER POST replacement: %s" % json_replacement) + #json_replacement = replacement try: json_replacement = json.loads(replacement) except json.decoder.JSONDecodeError as e: @@ -1475,11 +1529,13 @@ class AppBase: if len(json_replacement) > minlength: minlength = len(json_replacement) + + print("PRE new_replacement") # FIXME: Only do this IF they want to loop new_replacement = [] for i in range(len(json_replacement)): - if isinstance(json_replacement[i], dict) or isinstance(json_replacement[i], dict): + if isinstance(json_replacement[i], dict) or isinstance(json_replacement[i], list): tmp_replacer = json.dumps(json_replacement[i]) newvalue = tmpitem.replace(actualitem[0][0], tmp_replacer, 1) else: @@ -1531,9 +1587,9 @@ class AppBase: multi_parameters[parameter["name"]] = resultarray multi_execution_lists.append(new_replacement) - print("MULTI finished: %s" % json_replacement) + #print("MULTI finished: %s" % json_replacement) else: - print("(2) Pre replacement: %s" % actualitem) + print("(2) Pre replacement. ") #% actualitem) # This is here to handle for loops within variables.. kindof # 1. Find the length of the longest array # 2. Build an array with the base values based on parameter["value"] @@ -1603,14 +1659,14 @@ class AppBase: # With this parameter ready, add it to... a greater list of parameters. Rofl print("LENGTH OF ARR: %d" % len(resultarray)) - print("RESULTARRAY: %s" % resultarray) + #print("RESULTARRAY: %s" % resultarray) if resultarray not in multi_execution_lists: multi_execution_lists.append(resultarray) multi_parameters[parameter["name"]] = resultarray else: # Parses things like int(value) - print("Normal parsing (not looping) with data %s" % value) + print("Normal parsing (not looping)")#with data %s" % value) value = parse_wrapper_start(value) if parameter["id"] == "body_replacement": @@ -1630,7 +1686,7 @@ class AppBase: # print("PARAM: %s" % parameter) #if param.id == "body_replacement": - print("POST data value: %s" % value) + #print("POST data value: %s" % value) params[parameter["name"]] = value multi_parameters[parameter["name"]] = value @@ -1685,10 +1741,10 @@ class AppBase: # "id": "body_replacement", #}) - print("[INFO] APP_SDK DONE: Starting NORMAL execution of function") - print("[INFO] Running with params (0): %s" % params) + #print("[INFO] APP_SDK DONE: Starting NORMAL execution of function") + print("[INFO] Running normal execution\n") newres = await func(**params) - print("[INFO] Returned from execution:", newres) + print("\n[INFO] Returned from execution with datalength!")#, newres) if isinstance(newres, tuple): print("[INFO] Handling return as tuple") # Handles files. @@ -1714,7 +1770,7 @@ class AppBase: result = json.dumps(tmp_result) elif isinstance(newres, str): - print("[INFO] Handling return as string") + print("[INFO] Handling return as string of length %d" % len(newres)) result += newres else: try: @@ -1723,9 +1779,9 @@ class AppBase: result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) print("Can't handle type %s value from function" % (type(newres))) - print("[INFO] POST NEWRES RESULT: ", result) + print("[INFO] POST NEWRES RESULT!")#, result) else: - print("[INFO] APP_SDK DONE: Starting MULTI execution (length: %d) with values %s" % (minlength, multi_parameters)) + #print("[INFO] APP_SDK DONE: Starting MULTI execution (length: %d) with values %s" % (minlength, multi_parameters)) # 1. Use number of executions based on the arrays being similar # 2. Find the right value from the parsed multi_params @@ -1901,21 +1957,40 @@ class AppBase: self.send_result(action_result, headers, stream_path) return - - #STOPCOPY - # !!! Let the above line stay - its used for some horrible codegeneration / stitching !!! # - @classmethod - async def run(cls): - """ Connect to Redis and HTTP session, await actions """ + async def run(cls, action=""): logging.basicConfig(format="{asctime} - {name} - {levelname}:{message}", style='{') logger = logging.getLogger(f"{cls.__name__}") logger.setLevel(logging.DEBUG) - print("Started execution!!") + + #print("Started execution: %s!!" % cls) + #print("Action: %s" % action) + #if isinstance(cls, object): + # self.action = cls app = cls(redis=None, logger=logger, console_logger=logger) - # Authorization for the app/function to control the workflow - # Function will crash if its wrong, which it probably should. + if isinstance(action, str): + print("Normal execution. Action is a string.") + elif isinstance(action, object): + app.action = action + + try: + app.authorization = action["authorization"] + app.current_execution_id = action["execution_id"] + except: + pass + + try: + app.url = action["url"] + except: + pass + + try: + app.base_url = action["base_url"] + except: + pass + else: + print("ACTION TYPE (unhandled): %s" % type(action)) await app.execute_action(app.action) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 5f09b969..25fa59d6 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.54 +VERSION=0.8.60 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/app.yaml b/backend/go-app/app.yaml deleted file mode 100644 index 59f28320..00000000 --- a/backend/go-app/app.yaml +++ /dev/null @@ -1,98 +0,0 @@ -runtime: go111 - -env_variables: - -automatic_scaling: - max_instances: 1 - min_instances: 1 - -handlers: -- url: /api/(.*) - script: auto - secure: always -- url: /static/js/(.*) - static_files: build/static/js/\1 - upload: build/static/js/(.*) - secure: always -- url: /static/css/(.*) - static_files: build/static/css/\1 - upload: build/static/css/(.*) - secure: always -- url: /images/(.*) - static_files: build/images/\1 - upload: build/images/(.*) - secure: always -- url: /(.*\.(json|ico))$ - static_files: build/\1 - upload: build/.*\.(json|ico)$ - secure: always -- url: /manifest.json - static_files: build/manifest.json - upload: build/manifest.json - secure: always - -# lol.. wildcard doesn't work with /api/(.*) for some reason -- url: / - static_files: build/index.html - upload: build/index.html - secure: always -- url: /home - static_files: build/index.html - upload: build/index.html - secure: always -- url: /passwordreset - static_files: build/index.html - upload: build/index.html - secure: always -- url: /login - static_files: build/index.html - upload: build/index.html - secure: always -- url: /register - static_files: build/index.html - upload: build/index.html - secure: always -- url: /workflows - static_files: build/index.html - upload: build/index.html - secure: always -- url: /workflows/(.*) - static_files: build/index.html - upload: build/index.html - secure: always -- url: /info/(.*) - static_files: build/index.html - upload: build/index.html - secure: always -- url: /docs/(.*) - static_files: build/index.html - upload: build/index.html - secure: always -- url: /docs - static_files: build/index.html - upload: build/index.html - secure: always -- url: /settings - static_files: build/index.html - upload: build/index.html - secure: always -- url: /apps - static_files: build/index.html - upload: build/index.html - secure: always -- url: /contact - static_files: build/index.html - upload: build/index.html - secure: always -- url: /apps/(.*) - static_files: build/index.html - upload: build/index.html - secure: always -- url: /register/(.*) - static_files: build/index.html - upload: build/index.html - secure: always -- url: /passwordreset/(.*) - static_files: build/index.html - upload: build/index.html - secure: always diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index 4b20d2fa..74776e88 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -418,11 +418,14 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet fileBalance, ) - if strings.Contains(functionname, "filescan") { - //log.Printf("FUNCTION: %s", data) - log.Println(data) - log.Printf("Queries: %s", queryString) - } + // Use lowercase when checking + /* + if strings.Contains(functionname, "filter") { + //log.Printf("FUNCTION: %s", data) + log.Println(data) + log.Printf("Queries: %s", queryString) + } + */ //log.Printf(data) return functionname, data diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 0a0b4bae..776dbea0 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -299,7 +299,7 @@ func buildImage(tags []string, dockerfileFolder string) error { return err } - log.Printf("Tags: %s", tags) + log.Printf("[INFO] Docker Tags: %s", tags) dockerfileSplit := strings.Split(dockerfileFolder, "/") // Create a buffer diff --git a/backend/go-app/files.go b/backend/go-app/files.go index 0779a08c..419d834d 100644 --- a/backend/go-app/files.go +++ b/backend/go-app/files.go @@ -135,7 +135,7 @@ func handleGetFiles(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("Got %d files for org %s", len(files), user.ActiveOrg.Id) + log.Printf("[INFO] Got %d files for org %s", len(files), user.ActiveOrg.Id) newBody, err := json.Marshal(files) if err != nil { log.Printf("[ERROR] Failed marshaling files: %s", err) @@ -716,38 +716,45 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) { return } - // Try to get the org and workflow in case they don't exist - workflow, err := getWorkflow(ctx, curfile.WorkflowId) - if err != nil { - log.Printf("[ERROR] Workflow %s doesn't exist.", curfile.WorkflowId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) - return - } - - _, err = getOrg(ctx, curfile.OrgId) - if err != nil { - log.Printf("[ERROR] Org %s doesn't exist.", curfile.OrgId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) - return - } - - if workflow.ExecutingOrg.Id != curfile.OrgId { - found := false - for _, curorg := range workflow.Org { - if curorg.Id == curfile.OrgId { - found = true - break - } - } - - if !found { - log.Printf("[ERROR] Org %s doesn't have access to %s.", curfile.OrgId, curfile.WorkflowId) + var workflow *Workflow + if curfile.WorkflowId == "global" { + // PS: Not a security issue. + // Files are global anyway, but the workflow_id is used to identify origin + log.Printf("[INFO] Uploading filename %s for org %s as global file.", curfile.Filename, curfile.OrgId) + } else { + // Try to get the org and workflow in case they don't exist + workflow, err = getWorkflow(ctx, curfile.WorkflowId) + if err != nil { + log.Printf("[ERROR] Workflow %s doesn't exist.", curfile.WorkflowId) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) return } + + _, err = getOrg(ctx, curfile.OrgId) + if err != nil { + log.Printf("[ERROR] Org %s doesn't exist.", curfile.OrgId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) + return + } + + if workflow.ExecutingOrg.Id != curfile.OrgId { + found := false + for _, curorg := range workflow.Org { + if curorg.Id == curfile.OrgId { + found = true + break + } + } + + if !found { + log.Printf("[ERROR] Org %s doesn't have access to %s.", curfile.OrgId, curfile.WorkflowId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) + return + } + } } if strings.Contains(curfile.Filename, "/") || strings.Contains(curfile.Filename, `"`) || strings.Contains(curfile.Filename, "..") || strings.Contains(curfile.Filename, "~") { @@ -776,24 +783,26 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) { downloadPath := fmt.Sprintf("%s/%s", folderPath, fileId) duplicateWorkflows := []string{} - for _, trigger := range workflow.Triggers { - if trigger.AppName == "Shuffle Workflow" && trigger.TriggerType == "SUBFLOW" { - for _, parameter := range trigger.Parameters { - if parameter.Name == "workflow" && len(parameter.Value) > 0 { + if curfile.WorkflowId != "global" { + for _, trigger := range workflow.Triggers { + if trigger.AppName == "Shuffle Workflow" && trigger.TriggerType == "SUBFLOW" { + for _, parameter := range trigger.Parameters { + if parameter.Name == "workflow" && len(parameter.Value) > 0 { - found := false - for _, workflow := range duplicateWorkflows { - if workflow == parameter.Value { - found = true - break + found := false + for _, workflow := range duplicateWorkflows { + if workflow == parameter.Value { + found = true + break + } } - } - if !found { - duplicateWorkflows = append(duplicateWorkflows, parameter.Value) - } + if !found { + duplicateWorkflows = append(duplicateWorkflows, parameter.Value) + } - break + break + } } } } diff --git a/backend/go-app/main.go b/backend/go-app/main.go index b225de77..3031b0b5 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -17,6 +17,7 @@ import ( "log" "net" "net/http" + "net/url" "os" "os/exec" //"regexp" @@ -2877,8 +2878,8 @@ func fixUserOrg(ctx context.Context, user *User) *User { // Used for testing only. Shouldn't impact production. func handleCors(resp http.ResponseWriter, request *http.Request) bool { - //allowedOrigins := "http://localhost:3000" - allowedOrigins := "http://localhost:3002" + allowedOrigins := "http://localhost:3000" + //allowedOrigins := "http://localhost:3002" resp.Header().Set("Vary", "Origin") resp.Header().Set("Access-Control-Allow-Headers", "Content-Type, Accept, X-Requested-With, remember-me, Authorization") @@ -3558,10 +3559,13 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { // bodyWrapper = string(parsedBody) //} + url := &url.URL{} newRequest := &http.Request{ + URL: url, Method: "POST", Body: ioutil.NopCloser(bytes.NewReader(b)), } + //start, startok := request.URL.Query()["start"] // OrgId: activeOrgs[0].Id, workflowExecution, executionResp, err := handleExecution(item, workflow, newRequest) @@ -4628,7 +4632,7 @@ func findAvailablePorts(startRange int64, endRange int64) string { func handleSendalert(resp http.ResponseWriter, request *http.Request) { user, err := handleApiAuthentication(resp, request) if err != nil { - log.Printf("Api authentication failed in getworkflows: %s", err) + log.Printf("Api authentication failed in sendalert: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -6360,13 +6364,13 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { // FIXME: Check whether it's in use. if user.Id != app.Owner && user.Role != "admin" { - log.Printf("Wrong user (%s) for app %s when verifying swagger", user.Username, app.Name) + log.Printf("[WARNING] Wrong user (%s) for app %s when verifying swagger", user.Username, app.Name) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } - log.Printf("EDITING APP WITH ID %s", app.ID) + log.Printf("[INFO] EDITING APP WITH ID %s", app.ID) newmd5 = app.ID } @@ -6455,7 +6459,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { identifier = strings.Replace(identifier, " ", "-", -1) identifier = strings.Replace(identifier, "_", "-", -1) - log.Printf("Successfully parsed %s. Proceeding to docker container", identifier) + log.Printf("[INFO] Successfully parsed %s. Proceeding to docker container", identifier) // Now that the baseline is setup, we need to make it into a cloud function // 1. Upload the API to datastore for use @@ -6600,7 +6604,9 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { log.Printf("Failed to increase success execution stats: %s", err) } - cacheKey := fmt.Sprintf("workflowapps-sorted") + cacheKey := fmt.Sprintf("workflowapps-sorted-100") + requestCache.Delete(cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") requestCache.Delete(cacheKey) resp.WriteHeader(200) @@ -6613,7 +6619,7 @@ func healthCheckHandler(resp http.ResponseWriter, request *http.Request) { // Creates osfs from folderpath with a basepath as directory base func createFs(basepath, pathname string) (billy.Filesystem, error) { - log.Printf("base: %s, pathname: %s", basepath, pathname) + log.Printf("[INFO] MemFS base: %s, pathname: %s", basepath, pathname) fs := memfs.New() err := filepath.Walk(pathname, @@ -6674,19 +6680,19 @@ func handleAppHotload(location string, forceUpdate bool) error { log.Printf("Failed memfs creation - probably bad path: %s", err) return errors.New(fmt.Sprintf("Failed to find directory %s", location)) } else { - log.Printf("Memfs creation from %s done", location) + log.Printf("[INFO] Memfs creation from %s done", location) } dir, err := fs.ReadDir("") if err != nil { - log.Printf("Failed reading folder: %s", err) + log.Printf("[WARNING] Failed reading folder: %s", err) return err } //log.Printf("Reading app folder: %#v", dir) _, _, err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate) if err != nil { - log.Printf("Err: %s", err) + log.Printf("[WARNING] Githubfolders error: %s", err) return err } @@ -6760,6 +6766,7 @@ func handleCloudExecutionOnprem(workflowId, startNode, executionSource, executio log.Println(string(b)) newRequest := &http.Request{ + URL: &url.URL{}, Method: "POST", Body: ioutil.NopCloser(bytes.NewReader(b)), } @@ -6902,19 +6909,19 @@ func remoteOrgJobController(org Org, body []byte) error { ctx := context.Background() if !responseData.Success { - log.Printf("Should stop org job controller") + log.Printf("Should stop org job controller because no success?") - if strings.Contains(responseData.Reason, "Bad apikey") { - log.Printf("Bad apikey. Stopping sync for org?: %s", responseData.Reason) + if strings.Contains(responseData.Reason, "Bad apikey") || strings.Contains(responseData.Reason, "Error getting the organization") { + log.Printf("[WARNING] Remote error; Bad apikey or org error. Stopping sync for org: %s", responseData.Reason) if value, exists := scheduledOrgs[org.Id]; exists { // Looks like this does the trick? Hurr - log.Printf("STOPPING ORG SCHEDULE for: %s", org.Id) + log.Printf("[WARNING] STOPPING ORG SCHEDULE for: %s", org.Id) value.Lock() org, err := getOrg(ctx, org.Id) if err != nil { - log.Printf("Failed finding org %s: %s", org.Id, err) + log.Printf("[WARNING] Failed finding org %s: %s", org.Id, err) return err } @@ -6923,9 +6930,9 @@ func remoteOrgJobController(org Org, body []byte) error { org.CloudSync = false err = setOrg(ctx, *org, org.Id) if err != nil { - log.Printf("Failed setting organization when stopping sync: %s", err) + log.Printf("[WARNING] Failed setting organization when stopping sync: %s", err) } else { - log.Printf("Successfully updated the org to not sync") + log.Printf("[INFO] Successfully STOPPED org cloud sync for %s", org.Id) } return errors.New("Stopped schedule for org locally because of bad apikey.") @@ -7515,7 +7522,7 @@ func runInit(ctx context.Context) { } } - log.Printf("Downloading OpenAPI data for search - EXTRA APPS") + log.Printf("[INFO] Downloading OpenAPI data for search - EXTRA APPS") apis := "https://github.com/frikky/security-openapis" // THis gets memory problems hahah diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 0dc28d3a..5fbdb3e6 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -190,6 +190,10 @@ type WorkflowApp struct { Name string `json:"name" datastore:"name" yaml:"name"` Url string `json:"url" datastore:"url" yaml:"url"` } `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false` + ReferenceInfo struct { + DocumentationUrl string `json:"documentation_url" datastore:"documentation_url"` + GithubUrl string `json:"github_url" datastore:"github_url"` + } Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"` Tags []string `json:"tags" yaml:"tags" required:false datastore:"activated"` @@ -266,6 +270,7 @@ type WorkflowExecution struct { ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` ExecutionId string `json:"execution_id" datastore:"execution_id"` ExecutionSource string `json:"execution_source" datastore:"execution_source"` + ExecutionParent string `json:"execution_parent" datastore:"execution_parent"` ExecutionOrg string `json:"execution_org" datastore:"execution_org"` WorkflowId string `json:"workflow_id" datastore:"workflow_id"` LastNode string `json:"last_node" datastore:"last_node"` @@ -317,6 +322,7 @@ type Action struct { AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` Example string `json:"example,omitempty" datastore:"example"` AuthNotRequired bool `json:"auth_not_required,omitempty" datastore:"auth_not_required" yaml:"auth_not_required"` + Category string `json:"category" datastore:"category"` } // Added environment for location to execute @@ -406,8 +412,27 @@ type Workflow struct { Name string `json:"name" datastore:"name"` Value string `json:"value" datastore:"value,noindex"` } `json:"execution_variables,omitempty" datastore:"execution_variables"` - ExecutionEnvironment string `json:"execution_environment" datastore:"execution_environment"` - PreviouslySaved bool `json:"first_save" datastore:"first_save"` + ExecutionEnvironment string `json:"execution_environment" datastore:"execution_environment"` + PreviouslySaved bool `json:"first_save" datastore:"first_save"` + Categories Categories `json:"categories" datastore:"categories"` + ExampleArgument string `json:"example_argument" datastore:"example_argument,noindex"` +} + +type Category struct { + Name string `json:"name" datastore:"name"` + Description string `json:"description" datastore:"description"` + Count int64 `json:"count" datastore:"count"` +} + +type Categories struct { + SIEM Category `json:"siem" datastore:"siem"` + Communication Category `json:"communication" datastore:"communication"` + Assets Category `json:"assets" datastore:"assets"` + Cases Category `json:"cases" datastore:"cases"` + Network Category `json:"network" datastore:"network"` + Intel Category `json:"intel" datastore:"intel"` + EDR Category `json:"edr" datastore:"edr"` + Other Category `json:"other" datastore:"other"` } type ActionResult struct { @@ -796,7 +821,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { if len(executionRequests.Data) == 0 { executionRequests.Data = []ExecutionRequest{} } else { - log.Printf("[INFO] Executionrequests: %d", len(executionRequests.Data)) + log.Printf("[INFO] Executionrequests (%s): %d", id, len(executionRequests.Data)) } newjson, err := json.Marshal(executionRequests) @@ -1138,6 +1163,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl log.Printf("[WARNING] Actionresult is %s for node %s in %s. Continuing anyway because of workflow configuration.", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) // Finds ALL childnodes to set them to SKIPPED childNodes = findChildNodes(*workflowExecution, actionResult.Action.ID) + // Remove duplicates //log.Printf("CHILD NODES: %d", len(childNodes)) for _, nodeId := range childNodes { @@ -1198,6 +1224,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl Name: curAction.Name, ID: curAction.ID, } + newResult := ActionResult{ Action: newAction, ExecutionId: actionResult.ExecutionId, @@ -1641,13 +1668,16 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) { q = q.Limit(35) _, err = dbclient.GetAll(ctx, q, &workflows) if err != nil { - log.Printf("Failed getting workflows for user %s: %s", user.Username, err) + log.Printf("Failed getting workflows for user %s: %s (0)", user.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } } else { - log.Printf("Failed getting workflows for user %s: %s", user.Username, err) + log.Printf("Failed getting workflows for user %s: %s (1)", user.Username, err) + //DeleteKey(ctx, "workflow", "5694357e-8063-4580-8529-301cc72df951") + + //log.Printf("Workflows: %#v", workflows) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -1998,11 +2028,10 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { return } - err = increaseStatisticsField(ctx, "total_workflows", fileId, -1, workflow.OrgId) - if err != nil { - log.Printf("Failed to increase total workflows: %s", err) - } - + //err = increaseStatisticsField(ctx, "total_workflows", fileId, -1, workflow.OrgId) + //if err != nil { + // log.Printf("Failed to increase total workflows: %s", err) + //} //memcacheName := fmt.Sprintf("%s_%s", user.Username, fileId) //memcache.Delete(ctx, memcacheName) //memcacheName = fmt.Sprintf("%s_workflows", user.Username) @@ -2036,7 +2065,7 @@ func updateAppAuth(auth AppAuthenticationStorage, workflowId, nodeId string, add // FIXME: Add a way to use !add to remove updateAuth := false if !workflowFound && add { - log.Printf("Adding workflow things to auth!") + log.Printf("[INFO] Adding workflow things to auth!") usageItem := AuthenticationUsage{ WorkflowId: workflowId, Nodes: []string{nodeId}, @@ -2047,14 +2076,14 @@ func updateAppAuth(auth AppAuthenticationStorage, workflowId, nodeId string, add auth.NodeCount += 1 updateAuth = true } else if !nodeFound && add { - log.Printf("Adding node things to auth!") + log.Printf("[INFO] Adding node things to auth!") auth.Usage[workflowIndex].Nodes = append(auth.Usage[workflowIndex].Nodes, nodeId) auth.NodeCount += 1 updateAuth = true } if updateAuth { - log.Printf("Updating auth!") + log.Printf("[INFO] Updating auth!") ctx := context.Background() err := setWorkflowAppAuthDatastore(ctx, auth, auth.Id) if err != nil { @@ -2066,6 +2095,52 @@ func updateAppAuth(auth AppAuthenticationStorage, workflowId, nodeId string, add return nil } +// Identifies what a category defined really is +func handleCategoryIncrease(categories Categories, action Action, workflowapps []WorkflowApp) Categories { + if action.Category == "" { + appName := action.AppName + for _, app := range workflowapps { + if appName != strings.ToLower(app.Name) { + continue + } + + if len(app.Categories) > 0 { + log.Printf("[INFO] Setting category for %s: %s", app.Name, app.Categories) + action.Category = app.Categories[0] + break + } + } + + //log.Printf("Should find app's categories as it's empty during save") + return categories + } + + //log.Printf("Action: %s, category: %s", action.AppName, action.Category) + // FIXME: Make this an "autodiscover" that's controlled by the category itself + // Should just be a list that's looped against :) + newCategory := strings.ToLower(action.Category) + if strings.Contains(newCategory, "case") || strings.Contains(newCategory, "ticket") || strings.Contains(newCategory, "alert") || strings.Contains(newCategory, "mssp") { + categories.Cases.Count += 1 + } else if strings.Contains(newCategory, "siem") || strings.Contains(newCategory, "event") || strings.Contains(newCategory, "log") || strings.Contains(newCategory, "search") { + categories.SIEM.Count += 1 + } else if strings.Contains(newCategory, "sms") || strings.Contains(newCategory, "comm") || strings.Contains(newCategory, "phone") || strings.Contains(newCategory, "call") || strings.Contains(newCategory, "chat") || strings.Contains(newCategory, "mail") || strings.Contains(newCategory, "phish") { + categories.Communication.Count += 1 + } else if strings.Contains(newCategory, "intel") || strings.Contains(newCategory, "crim") || strings.Contains(newCategory, "ti") { + categories.Intel.Count += 1 + } else if strings.Contains(newCategory, "sand") || strings.Contains(newCategory, "virus") || strings.Contains(newCategory, "malware") || strings.Contains(newCategory, "scan") || strings.Contains(newCategory, "edr") || strings.Contains(newCategory, "endpoint detection") { + // Sandbox lol + categories.EDR.Count += 1 + } else if strings.Contains(newCategory, "vuln") || strings.Contains(newCategory, "fim") || strings.Contains(newCategory, "fim") || strings.Contains(newCategory, "integrity") { + categories.Assets.Count += 1 + } else if strings.Contains(newCategory, "network") || strings.Contains(newCategory, "firewall") || strings.Contains(newCategory, "waf") || strings.Contains(newCategory, "switch") { + categories.Network.Count += 1 + } else { + categories.Other.Count += 1 + } + + return categories +} + // Saves a workflow to an ID func saveWorkflow(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) @@ -2166,6 +2241,9 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // FIXME - this shouldn't be necessary with proper API checks newActions := []Action{} allNodes := []string{} + workflow.Categories = Categories{} + + workflowapps, apperr := getAllWorkflowApps(ctx, 500) //log.Printf("Action: %#v", action.Authentication) for _, action := range workflow.Actions { @@ -2193,6 +2271,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { action.Errors = []string{} } + workflow.Categories = handleCategoryIncrease(workflow.Categories, action, workflowapps) newActions = append(newActions, action) } @@ -2200,10 +2279,9 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { log.Printf("[WORKFLOW INIT] NOT PREVIOUSLY SAVED - SET ACTION AUTH!") //AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"` - workflowapps, apperr := getAllWorkflowApps(ctx, 500) allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) if err == nil && len(workflowapps) > 0 && apperr == nil { - log.Printf("Setting actions") + //log.Printf("Setting actions") actionFixing := []Action{} appsAdded := []string{} for _, action := range newActions { @@ -2335,7 +2413,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { workflow.Actions = newActions newTriggers := []Trigger{} for _, trigger := range workflow.Triggers { - log.Printf("Trigger %s: %s", trigger.TriggerType, trigger.Status) + log.Printf("[INFO] Trigger %s: %s", trigger.TriggerType, trigger.Status) // Check if it's actually running // FIXME: Do this for other triggers too @@ -2533,7 +2611,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // FIXME - might be a sploit to run someone elses app if getAllWorkflowApps // doesn't check sharing=true // Have to do it like this to add the user's apps - log.Println("Apps set starting") + //log.Println("Apps set starting") //log.Printf("EXIT ON ERROR: %#v", workflow.Configuration.ExitOnError) workflowApps := []WorkflowApp{} //memcacheName = "all_apps" @@ -2758,8 +2836,11 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { Errors: workflow.Errors, } - cacheKey := fmt.Sprintf("workflowapps-sorted") + cacheKey := fmt.Sprintf("workflowapps-sorted-100") requestCache.Delete(cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") + requestCache.Delete(cacheKey) + log.Printf("[INFO] Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId) resp.WriteHeader(200) newBody, err := json.Marshal(returndata) @@ -3048,6 +3129,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } makeNew := true + start, startok := request.URL.Query()["start"] if request.Method == "POST" { body, err := ioutil.ReadAll(request.Body) if err != nil { @@ -3057,9 +3139,43 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf // This one doesn't really matter. log.Printf("[INFO] Running POST execution with body of length %d", len(string(body))) + + if len(body) >= 4 { + if body[0] == 34 && body[len(body)-1] == 34 { + body = body[1 : len(body)-1] + } + if body[0] == 34 && body[len(body)-1] == 34 { + body = body[1 : len(body)-1] + } + } + + //workflowExecution.ExecutionSource = "default" + sourceWorkflow, sourceWorkflowOk := request.URL.Query()["source_workflow"] + if sourceWorkflowOk { + //log.Printf("Got source workflow %s", sourceWorkflow) + workflowExecution.ExecutionSource = sourceWorkflow[0] + } else { + //log.Printf("Did NOT get source workflow") + + } + + sourceExecution, sourceExecutionOk := request.URL.Query()["source_execution"] + if sourceExecutionOk { + //log.Printf("[INFO] Got source execution%s", sourceExecution) + workflowExecution.ExecutionParent = sourceExecution[0] + } else { + //log.Printf("Did NOT get source execution") + } + if len(string(body)) < 50 { + //log.Println(body) + // String in string + //log.Println(body) + + //if string(body)[0] == "\"" && string(body)[string(body) log.Printf("Body: %s", string(body)) } + var execution ExecutionRequest err = json.Unmarshal(body, &execution) if err != nil { @@ -3113,12 +3229,11 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf // Check for parameters of start and ExecutionId // This is mostly used for user input trigger - start, startok := request.URL.Query()["start"] answer, answerok := request.URL.Query()["answer"] referenceId, referenceok := request.URL.Query()["reference_execution"] if answerok && referenceok { // If answer is false, reference execution with result - log.Printf("Answer is OK AND reference is OK!") + log.Printf("[INFO] Answer is OK AND reference is OK!") if answer[0] == "false" { log.Printf("Should update reference and return, no need for further execution!") @@ -3189,12 +3304,12 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } // Don't override workflow defaults - if startok { - log.Printf("Setting start to %s based on query!", start[0]) - //workflowExecution.Workflow.Start = start[0] - workflowExecution.Start = start[0] - } + } + if startok { + //log.Printf("\n\n[INFO] Setting start to %s based on query!\n\n", start[0]) + //workflowExecution.Workflow.Start = start[0] + workflowExecution.Start = start[0] } // FIXME - regex uuid, and check if already exists? @@ -3382,7 +3497,42 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf break } } + + if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" { + found := false + for _, node := range childNodes { + if node == trigger.ID { + found = true + break + } + } + + if !found { + //log.Printf("SHOULD SET TRIGGER %s TO BE SKIPPED", trigger.ID) + + curaction := Action{ + AppName: trigger.AppName, + AppVersion: trigger.AppVersion, + Label: trigger.Label, + Name: trigger.Name, + ID: trigger.ID, + } + + defaultResults = append(defaultResults, ActionResult{ + Action: curaction, + ExecutionId: workflowExecution.ExecutionId, + Authorization: workflowExecution.Authorization, + Result: "Skipped because it's not under the startnode", + StartedAt: 0, + CompletedAt: 0, + Status: "SKIPPED", + }) + } else { + log.Printf("SHOULD KEEP TRIGGER %s", trigger.ID) + } + } } + //childNodes := findChildNodes(workflowExecution, workflowExecution.Start) if !startFound { log.Printf("Startnode %s doesn't exist!", workflowExecution.Start) @@ -4625,7 +4775,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { user.PrivateApps = privateApps err = setUser(ctx, &user) if err != nil { - log.Printf("[ERROR]Failed removing %s app for user %s: %s", app.Name, user.Username, err) + log.Printf("[ERROR] Failed removing %s app for user %s: %s", app.Name, user.Username, err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": true"}`))) return @@ -4645,7 +4795,9 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { if err != nil { log.Printf("Failed to increase total apps loaded stats: %s", err) } - cacheKey := fmt.Sprintf("workflowapps-sorted") + cacheKey := fmt.Sprintf("workflowapps-sorted-100") + requestCache.Delete(cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") requestCache.Delete(cacheKey) //err = memcache.Delete(request.Context(), sessionToken) @@ -4659,13 +4811,7 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { return } - user, userErr := handleApiAuthentication(resp, request) - if userErr != nil { - log.Printf("Api authentication failed in edit workflow: %s", userErr) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } + ctx := context.Background() location := strings.Split(request.URL.String(), "/") var fileId string @@ -4679,23 +4825,67 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { fileId = location[4] } - ctx := context.Background() app, err := getApp(ctx, fileId) if err != nil { - log.Printf("Error getting app (app config): %s", fileId) + log.Printf("[WARNING] Error getting app %s (app config): %s", fileId, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + return + } + + //if IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"` + // Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` + //log.Printf("Sharing: %s", app.Sharing) + //log.Printf("Generated: %s", app.Generated) + //log.Printf("Downloaded: %s", app.Downloaded) + + // FIXME - Handle sharing and such PROPERLY + if app.Sharing && app.Generated { + log.Printf("CAN SHARE APP!") + parsedApi, err := getOpenApiDatastore(ctx, fileId) + if err != nil { + log.Printf("[WARNING] OpenApi doesn't exist for: %s - err: %s", fileId, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if len(parsedApi.ID) > 0 { + parsedApi.Success = true + } else { + parsedApi.Success = false + } + + //log.Printf("PARSEDAPI: %#v", parsedApi) + data, err := json.Marshal(parsedApi) + if err != nil { + log.Printf("[WARNING] Error parsing api json: %s", err) + resp.WriteHeader(422) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed marshalling new parsed swagger: %s"}`, err))) + return + } + + resp.WriteHeader(200) + resp.Write(data) + return + } + + user, userErr := handleApiAuthentication(resp, request) + if userErr != nil { + log.Printf("[WARNING] Api authentication failed in get app: %s", userErr) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } if user.Id != app.Owner && user.Role != "admin" { - log.Printf("Wrong user (%s) for app %s", user.Username, app.Name) + log.Printf("[WARNING] Wrong user (%s) for app %s", user.Username, app.Name) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } - log.Printf("Getting app %s", fileId) + log.Printf("[INFO] Getting app %s (OpenAPI)", fileId) parsedApi, err := getOpenApiDatastore(ctx, fileId) if err != nil { log.Printf("OpenApi doesn't exist for: %s - err: %s", fileId, err) @@ -5212,7 +5402,9 @@ func updateWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { return } - cacheKey := fmt.Sprintf("workflowapps-sorted") + cacheKey := fmt.Sprintf("workflowapps-sorted-100") + requestCache.Delete(cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") requestCache.Delete(cacheKey) log.Printf("Changed workflow app %s", app.ID) @@ -5825,7 +6017,7 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("Starting app hotloading") + log.Printf("[INFO] Starting app hotloading") // Just need to be logged in // FIXME - should have some permissions? @@ -5850,7 +6042,7 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("Hotloading from %s", location) + log.Printf("[INFO] Hotloading from %s", location) err = handleAppHotload(location, true) if err != nil { log.Printf("Failed app hotload: %s", err) @@ -5971,6 +6163,11 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { return } + cacheKey := fmt.Sprintf("workflowapps-sorted-100") + requestCache.Delete(cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") + requestCache.Delete(cacheKey) + resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } @@ -6105,7 +6302,9 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, continue } - cacheKey := fmt.Sprintf("workflowapps-sorted") + cacheKey := fmt.Sprintf("workflowapps-sorted-100") + requestCache.Delete(cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") requestCache.Delete(cacheKey) } } else { @@ -6489,7 +6688,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin if !reservedFound { buildLaterFirst = append(buildLaterFirst, buildLater) } else { - log.Printf("\n\n[WARNING] Skipping build of %s to later\n\n", workflowapp.Name) + log.Printf("[WARNING] Skipping build of %s to later", workflowapp.Name) } } } @@ -6619,7 +6818,9 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { } //memcache.Delete(ctx, "all_apps") - cacheKey := fmt.Sprintf("workflowapps-sorted") + cacheKey := fmt.Sprintf("workflowapps-sorted-100") + requestCache.Delete(cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") requestCache.Delete(cacheKey) resp.WriteHeader(200) @@ -6677,18 +6878,76 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { } // Query for the specifci workflowId - q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(30) + maxAmount := 30 + q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(maxAmount) var workflowExecutions []WorkflowExecution _, err = dbclient.GetAll(ctx, q, &workflowExecutions) if err != nil { + if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { - q = datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(15) - _, err = dbclient.GetAll(ctx, q, &workflowExecutions) - if err != nil { - log.Printf("Error getting workflowexec (2): %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting all workflowexecutions for %s"}`, fileId))) - return + q = datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(1) + /* + _, err = dbclient.GetAll(ctx, q, &workflowExecutions) + if err != nil { + log.Printf("Error getting workflowexec (2): %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting all workflowexecutions for %s"}`, fileId))) + return + } + */ + + cursorStr := "" + for { + it := dbclient.Run(ctx, q) + //_, err = it.Next(&app) + for { + var workflowExecution WorkflowExecution + _, err := it.Next(&workflowExecution) + if err != nil { + break + } + + workflowExecutions = append(workflowExecutions, workflowExecution) + } + + //log.Printf("Len: %d", len(workflowExecutions)) + if len(workflowExecutions) > maxAmount { + break + } + + nextCursor, err := it.Cursor() + if err != iterator.Done && err != nil { + if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { + //log.Printf("NEXT!") + nextStr := fmt.Sprintf("%s", nextCursor) + if cursorStr == nextStr { + break + } + + cursorStr = nextStr + + continue + } else { + log.Printf("BREAK: %s", err) + break + } + } + + if err != nil { + log.Printf("Cursorerror: %s", err) + break + } else { + //log.Printf("NEXTCURSOR: %s", nextCursor) + nextStr := fmt.Sprintf("%s", nextCursor) + if cursorStr == nextStr { + break + } + + cursorStr = nextStr + q = q.Start(nextCursor) + //cursorStr = nextCursor + //break + } } } else { log.Printf("Error getting workflowexec: %s", err) @@ -6763,6 +7022,10 @@ func getAllWorkflowApps(ctx context.Context, maxLen int) ([]WorkflowApp, error) break } + if app.Name == "Shuffle Subflow" { + continue + } + found := false //log.Printf("ACTIONS: %d - %s", len(app.Actions), app.Name) for _, innerapp := range apps { diff --git a/docker-compose.yml b/docker-compose.yml index addef547..67fe27a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.56 + image: ghcr.io/frikky/shuffle-frontend:0.8.60 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.56 + image: ghcr.io/frikky/shuffle-backend:0.8.60 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -45,7 +45,7 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: ghcr.io/frikky/shuffle-orborus:0.8.5 + image: ghcr.io/frikky/shuffle-orborus:0.8.60 container_name: shuffle-orborus hostname: shuffle-orborus networks: @@ -53,8 +53,8 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock environment: - - SHUFFLE_APP_SDK_VERSION=0.8.51 - - SHUFFLE_WORKER_VERSION=0.8.54 + - SHUFFLE_APP_SDK_VERSION=0.8.60 + - SHUFFLE_WORKER_VERSION=0.8.60 - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} diff --git a/frontend/package.json b/frontend/package.json index bb24de65..0b99f34c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -36,7 +36,7 @@ "react-alert": "^5.5.0", "react-alert-template-basic": "^1.0.0", "react-beforeunload": "^2.2.1", - "react-chartjs-2": "^2.8.0", + "react-chartjs-2": "^2.11.1", "react-cookie": "^4.0.1", "react-cytoscapejs": "^1.2.0", "react-device-detect": "^1.9.10", @@ -52,7 +52,7 @@ "react-powerhooks": "0.0.7", "react-router": "^4.3.1", "react-router-dom": "^4.3.1", - "react-scripts": "^3.4.1", + "react-scripts": "^4.0.1", "reactstrap": "^7.1.0", "shellwords": "^0.1.1", "simplebar": "^4.2.3", diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 0864b984..18456036 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -30,6 +30,7 @@ import SettingsPage from "./views/SettingsPage"; import { createMuiTheme, MuiThemeProvider } from '@material-ui/core/styles'; +import ScrollToTop from "./components/ScrollToTop"; import AlertTemplate from "./components/AlertTemplate"; import { positions, Provider } from "react-alert"; @@ -74,6 +75,7 @@ const App = (message, props) => { const [isLoggedIn, setIsLoggedIn] = useState(false); const [dataset, setDataset] = useState(false); const [isLoaded, setIsLoaded] = useState(false); + const [curpath, setCurpath] = useState(typeof window === 'undefined' || window.location === undefined ? "" : window.location.pathname) useEffect(() => { if (dataset === false) { @@ -126,6 +128,7 @@ const App = (message, props) => { } /> :
+
} /> } /> diff --git a/frontend/src/assets/img/github_shuffle_img.png b/frontend/src/assets/img/github_shuffle_img.png new file mode 100644 index 00000000..56c1a04f Binary files /dev/null and b/frontend/src/assets/img/github_shuffle_img.png differ diff --git a/frontend/src/components/Header.js b/frontend/src/components/Header.js index 11dcf59c..1143a696 100644 --- a/frontend/src/components/Header.js +++ b/frontend/src/components/Header.js @@ -4,10 +4,13 @@ import {BrowserView, MobileView} from "react-device-detect"; import {Link} from 'react-router-dom'; import List from '@material-ui/core/List'; +import Avatar from '@material-ui/core/Avatar'; +import Menu from '@material-ui/core/Menu'; import ListItem from '@material-ui/core/ListItem'; import MenuItem from '@material-ui/core/MenuItem'; import Select from '@material-ui/core/Select'; import Button from '@material-ui/core/Button'; +import IconButton from '@material-ui/core/IconButton'; import HomeIcon from '@material-ui/icons/Home'; import PolymerIcon from '@material-ui/icons/Polymer'; import AppsIcon from '@material-ui/icons/Apps'; @@ -27,6 +30,7 @@ const Header = props => { const [LoginHoverColor, setLoginHoverColor] = useState(hoverOutColor); const [DocsHoverColor, setDocsHoverColor] = useState(hoverOutColor); const [HelpHoverColor, setHelpHoverColor] = useState(hoverOutColor); + const [anchorEl, setAnchorEl] = React.useState(null); const hrefStyle = { color: hoverOutColor, @@ -102,6 +106,17 @@ const Header = props => { setLoginHoverColor(hoverOutColor) } + + const handleClick = (event) => { + setAnchorEl(event.currentTarget); + }; + + const handleClose = () => { + setAnchorEl(null); + }; + + + // Should be based on some path const logoCheck = !homePage ? null : null @@ -182,76 +197,45 @@ const Header = props => {
- - -
- Logout -
- - {logoCheck} - + { + setAnchorEl(event.currentTarget); + }}> + + + { + handleClose() + }} + > + { + event.preventDefault() + handleClose() + }}> - + Settings - - {/* - - - - - - */} - {userdata === undefined || userdata.admin === undefined || userdata.admin === null || !userdata.admin ? null : - - - - - - } - {userdata === undefined || userdata.orgs === undefined || userdata.orgs === null || userdata.orgs.length <= 1 ? null : - - - - } - - + + { + event.preventDefault() + handleClose() + handleClickLogout() + }}> + Logout + + + {userdata === undefined || userdata.admin === undefined || userdata.admin === null || !userdata.admin ? null : + + + + }
- + const loginTextMobile = !isLoggedIn ?
@@ -327,7 +311,7 @@ const Header = props => { // const loadedCheck = -
+
{loginTextBrowser} diff --git a/frontend/src/components/ScrollToTop.jsx b/frontend/src/components/ScrollToTop.jsx new file mode 100644 index 00000000..6b3c9911 --- /dev/null +++ b/frontend/src/components/ScrollToTop.jsx @@ -0,0 +1,32 @@ +import { useEffect } from 'react'; +import { withRouter } from 'react-router-dom'; +import ReactGA from 'react-ga'; + +function ScrollToTop({setCurpath, history }) { + useEffect(() => { + const unlisten = history.listen(() => { + window.scroll({ + top: 0, + left: 0, + behavior: "smooth", + }); + + //ReactGA.event({ + // category: "referral", + // action: "new_user_referral", + // label: "", + //}) + + ReactGA.pageview(window.location.pathname) + setCurpath(window.location.pathname) + }); + return () => { + unlisten(); + } + }, []); + + return (null); +} + +// https://stackoverflow.com/questions/36904185/react-router-scroll-to-top-on-every-transition +export default withRouter(ScrollToTop); diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index f590ad60..62f926a4 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -1,4 +1,4 @@ -import React, { useState } from 'react'; +import React, { useState, useEffect } from 'react'; import { makeStyles } from '@material-ui/styles'; import {Link} from 'react-router-dom'; @@ -25,13 +25,17 @@ import IconButton from '@material-ui/core/IconButton'; import Avatar from '@material-ui/core/Avatar'; import Zoom from '@material-ui/core/Zoom'; import { useAlert } from "react-alert"; +import Dropzone from '../components/Dropzone'; import { Dialog, DialogTitle, DialogActions, DialogContent } from '@material-ui/core'; import { useTheme } from '@material-ui/core/styles'; import HandlePayment from './HandlePayment' import OrgHeader from '../components/OrgHeader' +import CircularProgress from '@material-ui/core/CircularProgress'; import EditIcon from '@material-ui/icons/Edit'; +import FileCopyIcon from '@material-ui/icons/FileCopy'; +import PublishIcon from '@material-ui/icons/Publish'; import SelectAllIcon from '@material-ui/icons/SelectAll'; import OpenInNewIcon from '@material-ui/icons/OpenInNew'; import CloudDownloadIcon from '@material-ui/icons/CloudDownload'; @@ -61,6 +65,7 @@ const Admin = (props) => { const { globalUrl, userdata } = props; var upload = "" + var to_be_copied = "" const theme = useTheme(); const classes = useStyles(); const [firstRequest, setFirstRequest] = React.useState(true); @@ -92,6 +97,14 @@ const Admin = (props) => { const [selectedAuthenticationModalOpen, setSelectedAuthenticationModalOpen] = React.useState(false) const [authenticationFields, setAuthenticationFields] = React.useState([]) const [showArchived, setShowArchived] = React.useState(false) + const [isDropzone, setIsDropzone] = React.useState(false); + + useEffect(() => { + if (isDropzone) { + //redirectOpenApi(); + setIsDropzone(false); + } + }, [isDropzone]); const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" const getApps = () => { @@ -646,6 +659,67 @@ const Admin = (props) => { }); } + const handleFileUpload = (file_id, file) => { + //console.log("FILE: ", file_id, file) + fetch(`${globalUrl}/api/v1/files/${file_id}/upload`, { + method: 'POST', + credentials: "include", + body: file, + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for apps :O!") + return + } + + return response.json() + }) + .then((responseJson) => { + //console.log("RESPONSE: ", responseJson) + //setFiles(responseJson) + }) + .catch(error => { + //alert.error(error.toString()) + }); + } + + const handleCreateFile = (filename, file) => { + const data = { + "filename": filename, + "org_id": selectedOrganization.id, + "workflow_id": "global", + } + + fetch(globalUrl + "/api/v1/files/create", { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + body: JSON.stringify(data), + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for apps :O!") + return + } + + return response.json() + }) + .then((responseJson) => { + //console.log("RESP: ", responseJson) + if (responseJson.success) { + handleFileUpload(responseJson.id, file) + } else { + alert.error("Failed to upload file ", filename) + } + }) + .catch(error => { + alert.error(error.toString()) + }); + } + const getFiles = () => { fetch(globalUrl + "/api/v1/files", { method: 'GET', @@ -1048,6 +1122,7 @@ const Admin = (props) => { Edit authentication for {selectedAuthentication.app.name} ({selectedAuthentication.label}) {selectedAuthentication.fields.map((data, index) => { + console.log("DATA: ", data, selectedAuthentication) return (
{data.key} @@ -1374,12 +1449,24 @@ const Admin = (props) => {
{selectedOrganization.id === undefined ? -
- : +
+ + + Loading Organization + +
+ :
{selectedOrganization.name.length > 0 ? - : null} + : +
+ + + Loading Organization + +
+ } Cloud syncronization What does cloud sync do? Cloud syncronization is a way of getting more out of Shuffle. Shuffle will ALWAYS make every option open source, but features relying on other users can't be done without a collaborative approach. @@ -1690,7 +1777,7 @@ const Admin = (props) => { style={{ minWidth: 180, maxWidth: 180 }} /> - {users === undefined ? null : users.map((data, index) => { + {users === undefined || users === null ? null : users.map((data, index) => { var bgColor = "#27292d" if (index % 2 === 0) { bgColor = "#1f2023" @@ -1765,12 +1852,72 @@ const Admin = (props) => {
: null + const uploadFiles = (files) => { + for (var key in files) { + try { + const filename = files[key].name + var filedata = new FormData() + filedata.append('shuffle_file', files[key]) + + if (typeof(files[key]) === "object") { + handleCreateFile(filename, filedata) + } + + /* + reader.addEventListener('load', (e) => { + var data = e.target.result; + setIsDropzone(false) + console.log(filename) + console.log(data) + console.log(files[key]) + }) + reader.readAsText(files[key]) + */ + } catch (e) { + console.log("Error in dropzone: ", e) + } + } + + getFiles() + } + + const uploadFile = (e) => { + const isDropzone = e.dataTransfer === undefined ? false : e.dataTransfer.files.length > 0; + const files = isDropzone ? e.dataTransfer.files : e.target.files; + + //const reader = new FileReader(); + //alert.info("Starting fileupload") + uploadFiles(files) + } + const filesView = curTab === 5 ? + 1366 ? 1366 : 1200, margin: "auto", padding: 20 }} onDrop={uploadFile}>

Files

Files from Workflows. Learn more
+ + upload = ref} onChange={(event) => { + //const file = event.target.value + //const fileObject = URL.createObjectURL(actualFile) + //setFile(fileObject) + //const files = event.target.files[0] + uploadFiles(event.target.files) + }} /> + @@ -1801,6 +1948,9 @@ const Admin = (props) => { + {files === undefined || files === null ? null : files.map((file, index) => { var bgColor = "#27292d" @@ -1820,13 +1970,19 @@ const Admin = (props) => { /> - - - + {file.workflow_id === "global" ? + + - - + : + + + + + + + + } style={{minWidth: 100, maxWidth: 100, overflow: "hidden"}} /> { - { + { downloadFile(file) }}> - + style={{minWidth: 75, maxWidth: 75, overflow: "hidden"}} @@ -1865,11 +2021,31 @@ const Admin = (props) => { */} + { + const elementName = "copy_element_shuffle" + var copyText = document.getElementById(elementName); + if (copyText !== null && copyText !== undefined) { + navigator.clipboard.writeText(file.id) + copyText.select(); + copyText.setSelectionRange(0, 99999); /* For mobile devices */ + + /* Copy the text inside the text field */ + document.execCommand("copy"); + + alert.info(file.id + "copied to clipboard") + } + }}> + + + /> ) })}
+ : null const schedulesView = curTab === 4 ? @@ -2074,7 +2250,7 @@ const Admin = (props) => { primary="Actions" /> - {authentication === undefined ? null : authentication.map((data, index) => { + {authentication === undefined || authentication === null ? null : authentication.map((data, index) => { var bgColor = "#27292d" if (index % 2 === 0) { bgColor = "#1f2023" @@ -2392,7 +2568,7 @@ const Admin = (props) => { const iconStyle = {marginRight: 10} const data = -
+
{ {editUserModal} {editAuthenticationModal} {data} +
) } diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 7ce3669f..79979409 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -83,8 +83,8 @@ import cxtmenu from 'cytoscape-cxtmenu'; import { w3cwebsocket as W3CWebSocket } from "websocket"; import { useAlert } from "react-alert"; -import { validateJson } from "./Workflows"; -import { GetParsedPaths } from "./Apps"; +import { validateJson } from "./Workflows.jsx"; +import { GetParsedPaths } from "./Apps.jsx"; const surfaceColor = "#27292D" const inputColor = "#383B40" @@ -121,6 +121,8 @@ const AngularWorkflow = (props) => { const [bodyWidth, bodyHeight] = useWindowSize(); const appBarSize = 74 + const headerSize = 60 + var to_be_copied = "" const [cystyle, ] = useState(cytoscapestyle) const [cy, setCy] = React.useState() @@ -135,6 +137,7 @@ const AngularWorkflow = (props) => { const [workflow, setWorkflow] = React.useState({}); const [userSettings, setUserSettings] = React.useState({}); const [subworkflow, setSubworkflow] = React.useState({}); + const [subworkflowStartnode, setSubworkflowStartnode] = React.useState(""); const [leftViewOpen, setLeftViewOpen] = React.useState(true); const [leftBarSize, setLeftBarSize] = React.useState(350) const [executionText, setExecutionText] = React.useState(""); @@ -155,6 +158,9 @@ const AngularWorkflow = (props) => { const [showSkippedActions, setShowSkippedActions] = React.useState(false) const [lastExecution, setLastExecution] = React.useState("") + // 0 = normal, 1 = just done, 2 = normal + const [savingState, setSavingState] = React.useState(0) + const [selectedResult, setSelectedResult] = React.useState({}) const [codeModalOpen, setCodeModalOpen] = React.useState(false); @@ -253,6 +259,7 @@ const AngularWorkflow = (props) => { setWorkflows(responseJson) if (trigger_index > -1) { + var outersub = {} const trigger = workflow.triggers[trigger_index] if (trigger.parameters.length >= 3) { for (var key in trigger.parameters) { @@ -261,8 +268,23 @@ const AngularWorkflow = (props) => { const sub = responseJson.find(data => data.id === param.value) if (sub !== undefined && subworkflow.id !== sub.id) { setSubworkflow(sub) + outersub = sub } } + + if (param.name === "startnode" && outersub.id !== undefined) { + console.log("SHOULD SET STARTNODE: ", outersub) + const innernode = outersub.actions.find(action => action.id === param.value) + console.log("FOUND NODE: ", innernode) + if (innernode !== undefined && subworkflowStartnode.id !== innernode.id) { + setSubworkflowStartnode(innernode) + } + /* + const sub = responseJson.find(data => data.id === param.value) + setSubworkflow(sub) + } + */ + } } } } @@ -338,7 +360,7 @@ const AngularWorkflow = (props) => { .then((response) => { if (response.status !== 200) { console.log("Status not 200 for setting app auth :O!") - } + } return response.json() }) @@ -346,7 +368,10 @@ const AngularWorkflow = (props) => { if (!responseJson.success) { alert.error("Failed to set app auth: "+responseJson.reason) } else { + getAppAuthentication(true) setAuthenticationModalOpen(false) + + // Needs a refresh with the new authentication.. alert.success("Successfully saved new app auth") } }) @@ -375,7 +400,19 @@ const AngularWorkflow = (props) => { if (responseJson.length > 0) { // FIXME: Sort this by time setWorkflowExecutions(responseJson) + + const cursearch = typeof window === 'undefined' || window.location === undefined ? "" : window.location.search + const tmpView = new URLSearchParams(cursearch).get("execution_id") + if (tmpView !== undefined && tmpView !== null && tmpView.length > 0) { + //console.log("SHOW EXECUTION ", tmpView) + const execution = responseJson.find(data => data.execution_id === tmpView) + if (execution !== null && execution !== undefined) { + setExecutionData(execution) + setExecutionModalView(1) + } + } } + //alert.info("Loaded executions") //setWorkflowExecutions(responseJson) }) @@ -425,7 +462,8 @@ const AngularWorkflow = (props) => { handleUpdateResults(responseJson) }) .catch(error => { - alert.error(error.toString()) + console.log("Error: ", error) + //alert.error(error.toString()) stop() }); } @@ -433,7 +471,7 @@ const AngularWorkflow = (props) => { const abortExecution = () => { setExecutionRunning(false) - alert.info("Aborting execution") + //alert.info("Aborting execution") fetch(globalUrl+"/api/v1/workflows/"+props.match.params.key+"/executions/"+executionRequest.execution_id+"/abort", { method: 'GET', headers: { @@ -537,7 +575,7 @@ const AngularWorkflow = (props) => { currentnode.removeClass('awaiting-data-highlight') currentnode.addClass('success-highlight') - if (!visited.includes(item.action.label)) { + if (visited !== undefined && visited !== null && !visited.includes(item.action.label)) { if (executionRunning) { //alert.show("Success in node "+item.action.label) //+" with result "+item.result) @@ -566,6 +604,9 @@ const AngularWorkflow = (props) => { } break case "FAILURE": + //When status comes as failure, allow user to start workflow execution + setExecutionRunning(false) + currentnode.removeClass('not-executing-highlight') currentnode.removeClass('executing-highlight') currentnode.removeClass('success-highlight') @@ -626,6 +667,7 @@ const AngularWorkflow = (props) => { const saveWorkflow = (curworkflow) => { var success = false + setSavingState(2) // This might not be the right course of action, but seems logical, as items could be running already // Makes it possible to update with a version in current render @@ -634,7 +676,7 @@ const AngularWorkflow = (props) => { if (curworkflow !== undefined) { useworkflow = curworkflow } else { - alert.info("Saving workflow") + //alert.info("Saving workflow") } var cyelements = cy.elements() @@ -732,6 +774,7 @@ const AngularWorkflow = (props) => { credentials: "include", }) .then((response) => { + setSavingState(0) if (response.status !== 200) { console.log("Status not 200 for setting workflows :O!") } @@ -753,10 +796,15 @@ const AngularWorkflow = (props) => { setWorkflow(workflow) } - alert.success("Successfully saved workflow") + //alert.success("Successfully saved workflow") + setSavingState(1) + setTimeout(() => { + setSavingState(0) + }, 1500); } }) .catch(error => { + setSavingState(0) alert.error(error.toString()) }); @@ -775,7 +823,7 @@ const AngularWorkflow = (props) => { return true } - const executeWorkflow = () => { + const executeWorkflow = (executionArgument, startNode) => { if (!lastSaved) { //alert.error("You might have forgotten to save before executing.") console.log("FIXME: Might have forgotten to save before executing.") @@ -797,13 +845,13 @@ const AngularWorkflow = (props) => { curelements[i].addClass("not-executing-highlight") } - if (executionText.length > 0) { - alert.success("Starting execution with an execution argument") + if (executionArgument.length > 0) { + //alert.success("Starting execution WITH an execution argument") } else { - alert.success("Starting execution") + //alert.success("Starting execution") } - const data = {"execution_argument": executionText, "start": workflow.start} + const data = {"execution_argument": executionArgument, "start": startNode} fetch(globalUrl+"/api/v1/workflows/"+props.match.params.key+"/execute", { method: 'POST', headers: { @@ -930,7 +978,7 @@ const AngularWorkflow = (props) => { "Http", ] - const getAppAuthentication = () => { + const getAppAuthentication = (reset) => { fetch(globalUrl+"/api/v1/apps/authentication", { method: 'GET', headers: { @@ -958,6 +1006,10 @@ const AngularWorkflow = (props) => { newauth.push(responseJson.data[key]) } + if (reset === true) { + console.log("APP RESET = reset cy") + cy.on('select', 'node', (e) => onNodeSelect(e, newauth)) + } setAppAuthentication(newauth) } else { alert.error("Failed getting authentications") @@ -992,7 +1044,7 @@ const AngularWorkflow = (props) => { //tmpapps = tmpapps.concat(getExtraApps()) //tmpapps = tmpapps.concat(responseJson) setApps(responseJson) - getAppAuthentication() + //getAppAuthentication() setFilteredApps(responseJson.filter(app => !internalIds.includes(app.name))) setPrioritizedApps(responseJson.filter(app => internalIds.includes(app.name))) @@ -1118,7 +1170,7 @@ const AngularWorkflow = (props) => { setSelectedTrigger({}) } - const onNodeSelect = (event) => { + const onNodeSelect = (event, newAppAuth) => { const data = event.target.data() setLastSaved(false) const branch = workflow.branches.filter(branch => branch.source_id === data.id || branch.destination_id === data.id) @@ -1149,7 +1201,8 @@ const AngularWorkflow = (props) => { findAuthId = curaction.authentication_id } - var tmpAuth = JSON.parse(JSON.stringify(appAuthentication)) + var tmpAuth = JSON.parse(JSON.stringify(newAppAuth)) + console.log("Checking authentication: ", tmpAuth) for (var key in tmpAuth) { var item = tmpAuth[key] @@ -1168,6 +1221,7 @@ const AngularWorkflow = (props) => { } curaction.authentication = authenticationOptions + console.log("Authentication: ", authenticationOptions) if (curaction.selectedAuthentication === null || curaction.selectedAuthentication === undefined || curaction.selectedAuthentication.length === "") { curaction.selectedAuthentication = {} } @@ -1373,7 +1427,7 @@ const AngularWorkflow = (props) => { //throw BreakException return false } - }); + }) } @@ -1562,7 +1616,7 @@ const AngularWorkflow = (props) => { cy.fit(null, 200) - cy.on('select', 'node', (e) => onNodeSelect(e)) + cy.on('select', 'node', (e) => onNodeSelect(e, appAuthentication)) cy.on('select', 'edge', (e) => onEdgeSelect(e)) cy.on('unselect', (e) => onUnselect(e)) @@ -1757,13 +1811,13 @@ const AngularWorkflow = (props) => { const stopSchedule = (trigger, triggerindex) => { alert.info("Stopping schedule") fetch(globalUrl+"/api/v1/workflows/"+props.match.params.key+"/schedule/"+trigger.id, { - method: 'DELETE', - headers: { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - }, - credentials: "include", - }) + method: 'DELETE', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) .then((response) => { if (response.status !== 200) { console.log("Status not 200 for stream results :O!") @@ -1774,21 +1828,16 @@ const AngularWorkflow = (props) => { .then((responseJson) => { // No matter what, it's being stopped. if (!responseJson.success) { - alert.error("Failed to stop schedule: " + responseJson.reason) - - workflow.triggers[triggerindex].status = "stopped" - trigger.status = "stopped" - setSelectedTrigger(trigger) - setWorkflow(workflow) - saveWorkflow(workflow) + alert.WARNING("Failed to stop schedule: " + responseJson.reason) } else { alert.success("Successfully stopped schedule") - workflow.triggers[triggerindex].status = "stopped" - trigger.status = "stopped" - setSelectedTrigger(trigger) - setWorkflow(workflow) - saveWorkflow(workflow) } + + workflow.triggers[triggerindex].status = "stopped" + trigger.status = "stopped" + setSelectedTrigger(trigger) + setWorkflow(workflow) + saveWorkflow(workflow) }) .catch(error => { alert.error(error.toString()) @@ -1859,18 +1908,11 @@ const AngularWorkflow = (props) => { height: "100%", } - const scrollStyle = { - marginTop: 10, - overflow: "scroll", - height: "100%", - overflowX: "auto", - overflowY: "auto", - } const paperAppStyle = { borderRadius: borderRadius, - minHeight: "100px", - maxHeight: "100px", + minHeight: 100, + maxHeight: 100, minWidth: "100%", maxWidth: "100%", marginTop: "5px", @@ -1924,7 +1966,7 @@ const AngularWorkflow = (props) => { return (
- What are WORKFLOW variables? + What are WORKFLOW variables? {workflow.workflow_variables === null ? null : workflow.workflow_variables.map(variable=> { return ( @@ -1991,7 +2033,7 @@ const AngularWorkflow = (props) => { }}>New workflow variable
- What are EXECUTION variables? + What are EXECUTION variables? {workflow.execution_variables === null || workflow.execution_variables === undefined ? null : workflow.execution_variables.map(variable=> { return ( @@ -2212,9 +2254,9 @@ const AngularWorkflow = (props) => {
{triggers.map((trigger, index) => { var imageline = trigger.large_image.length === 0 ? - + : - + const color = trigger.is_valid ? "green" : "orange" return( @@ -2429,8 +2471,11 @@ const AngularWorkflow = (props) => { authentication: [], execution_variable: undefined, example: example, + category: app.categories !== null && app.categories !== undefined && app.categories.length > 0 ? app.categories[0] : "" } + // FIXME: overwrite category if the ACTION chosen has a different category + // const image = "url("+app.large_image+")" // FIXME - find the cytoscape offset position @@ -2530,7 +2575,8 @@ const AngularWorkflow = (props) => { newAppname = newAppname.slice(0, maxlen)+".." } - const image = "url("+app.large_image+")" + //const image = "url("+app.large_image+")" + const image = app.large_image const newAppStyle = JSON.parse(JSON.stringify(paperAppStyle)) const pixelSize = !hover ? "2px" : "4px" newAppStyle.borderLeft = app.is_valid ? `${pixelSize} solid green` : `${pixelSize} solid orange` @@ -2550,9 +2596,9 @@ const AngularWorkflow = (props) => { {setHover(true)}} onMouseOut={() => {setHover(false)}}> -
+ {newAppname} - +

{newAppname}

 @@ -2605,7 +2651,6 @@ const AngularWorkflow = (props) => { return null } - console.log("APP: ", app) return( ) @@ -3310,11 +3355,19 @@ const AngularWorkflow = (props) => { }} style={{backgroundColor: surfaceColor, color: "white", height: "50px"}} > - {selectedActionParameters[count].options.map(data => ( - - {data} - - ))} + {selectedActionParameters[count].options.map((data, index) => { + const split_data = data.split("||") + var viewed_data = data + if (split_data.length > 1) { + viewed_data = split_data[0] + } + + return ( + + {viewed_data} + + ) + })} } else if (data.variant === "STATIC_VALUE") { @@ -3637,6 +3690,7 @@ const AngularWorkflow = (props) => { } tmpitem = tmpitem.charAt(0).toUpperCase()+tmpitem.substring(1) + const description = data.description === undefined ? "" : data.description return (
@@ -3650,10 +3704,12 @@ const AngularWorkflow = (props) => { }}/> : -
+
}
- {tmpitem} + + {tmpitem} +
{selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 ? null : @@ -3825,7 +3881,6 @@ const AngularWorkflow = (props) => { }) } - const headerSize = 68 const rightsidebarStyle = { position: "fixed", right: 0, @@ -3841,6 +3896,21 @@ const AngularWorkflow = (props) => { zIndex: 1000, } + const textFieldStyle = { + backgroundColor: inputColor, + borderRadius: borderRadius, + } + + + const innerTextfieldStyle = { + color: "white", + minHeight: 50, + marginLeft: "5px", + maxWidth: "95%", + fontSize: "1em", + borderRadius: borderRadius, + } + const appApiView = Object.getOwnPropertyNames(selectedAction).length > 0 ?
@@ -3873,7 +3943,7 @@ const AngularWorkflow = (props) => { - What are actions? + What are actions? {selectedAction.errors !== null && selectedAction.errors.length > 0 ?
Errors: {selectedAction.errors.join("\n")} @@ -3894,15 +3964,9 @@ const AngularWorkflow = (props) => { Name {
Authenticate {selectedApp.name}: + +
: null} @@ -4048,11 +4114,12 @@ const AngularWorkflow = (props) => { value={selectedActionName} fullWidth onChange={setNewSelectedAction} - style={{backgroundColor: inputColor, color: "white", height: 50}} + style={{backgroundColor: inputColor, color: "white", height: 50, borderRadius: borderRadius,}} SelectDisplayProps={{ style: { marginLeft: 10, maxHeight: 200, + borderRadius: borderRadius, } }} > @@ -4502,7 +4569,7 @@ const AngularWorkflow = (props) => { }} > - Conditions can't be used for loops [ .# ] Learn more + Conditions can't be used for loops [ .# ] Learn more Condition @@ -4511,6 +4578,7 @@ const AngularWorkflow = (props) => {
+ +
@@ -4754,7 +4823,7 @@ const AngularWorkflow = (props) => {

Branch: Conditions - {selectedEdgeIndex}

- What are conditions? + What are conditions?
@@ -4978,7 +5047,7 @@ const AngularWorkflow = (props) => {

{selectedTrigger.app_name}: {selectedTrigger.status}

- What are email triggers? + What are email triggers?
@@ -5056,6 +5125,7 @@ const AngularWorkflow = (props) => { workflow.triggers[selectedTriggerIndex].parameters[0] = {"name": "workflow", "value": ""} workflow.triggers[selectedTriggerIndex].parameters[1] = {"name": "argument", "value": ""} workflow.triggers[selectedTriggerIndex].parameters[2] = {"name": "user_apikey", "value": ""} + workflow.triggers[selectedTriggerIndex].parameters[3] = {"name": "startnode", "value": ""} console.log("SETTINGS: ", userSettings) if (userSettings !== undefined && userSettings !== null && userSettings.apikey !== null && userSettings.apikey !== undefined && userSettings.apikey.length > 0) { @@ -5068,7 +5138,7 @@ const AngularWorkflow = (props) => {

{selectedTrigger.app_name}

- What are subflows? + What are subflows?
@@ -5116,27 +5186,83 @@ const AngularWorkflow = (props) => { setUpdate(Math.random()) workflow.triggers[selectedTriggerIndex].parameters[0].value = e.target.value.id setWorkflow(workflow) + setSubworkflowStartnode(e.target.value.start) + + // Sets the startnode + if (e.target.value.id !== workflow.id) { + const startnode = e.target.value.actions.find(action => action.id === e.target.value.start) + if (startnode !== undefined && startnode !== null) { + setSubworkflowStartnode(startnode) + } + console.log("STARTNODE: ", startnode) + } }} style={{backgroundColor: inputColor, color: "white", height: "50px"}} > {workflows.map((data, index) => { + /* if (data.id === workflow.id) { return null } + */ return ( - + {data.name} ) })} } - {workflow.triggers[selectedTriggerIndex].parameters[0].value.length === 0 ? null : Explore selected workflow} + {workflow.triggers[selectedTriggerIndex].parameters[0].value.length === 0 ? null : Explore selected workflow} +
- Execution Argument: + Select the Startnode +
+
+ {subworkflow === undefined || subworkflow === null || subworkflow.id === undefined || subworkflow.actions === null || subworkflow.actions === undefined || subworkflow.actions.length === 0 ? null : + + } +
+
+
+ Execution Argument
{

{selectedTrigger.app_name}: {selectedTrigger.status}

- What are webhooks? + What are webhooks?
@@ -5526,6 +5652,7 @@ const AngularWorkflow = (props) => { if (trigger.id === undefined) { return } + alert.info("Stopping webhook") fetch(globalUrl+"/api/v1/hooks/"+trigger.id+"/delete", { @@ -5544,18 +5671,23 @@ const AngularWorkflow = (props) => { return response.json() }) .then((responseJson) => { - workflow.triggers[triggerindex].status = "stopped" - trigger.status = "stopped" - setWorkflow(workflow) - setSelectedTrigger(trigger) + if (workflow.triggers[triggerindex] !== undefined) { + workflow.triggers[triggerindex].status = "stopped" + } if (responseJson.success) { //alert.success("Successfully stopped webhook") // Set the status saveWorkflow(workflow) } else { - alert.error("Failed stopping webhook: "+responseJson.reason) + if (responseJson.reason !== undefined) { + alert.error("Failed stopping webhook: "+responseJson.reason) + } } + + trigger.status = "stopped" + setWorkflow(workflow) + setSelectedTrigger(trigger) }) .catch(error => { alert.error(error.toString()) @@ -5584,7 +5716,7 @@ const AngularWorkflow = (props) => {

{selectedTrigger.app_name}: {selectedTrigger.status}

- What is the user input trigger? + What is the user input trigger?
@@ -5762,7 +5894,7 @@ const AngularWorkflow = (props) => {

{selectedTrigger.app_name}: {selectedTrigger.status}

- What are schedules? + What are schedules?
@@ -6019,12 +6151,14 @@ const AngularWorkflow = (props) => {
+ +
) @@ -6075,12 +6209,14 @@ const AngularWorkflow = (props) => {
+ +
) @@ -6092,19 +6228,23 @@ const AngularWorkflow = (props) => { const boxSize = 100 const executionButton = executionRunning ? + + : - + + + return( @@ -6114,15 +6254,9 @@ const AngularWorkflow = (props) => { { /> - + + + - + + + + + + + {/* */} @@ -6323,6 +6465,9 @@ const AngularWorkflow = (props) => { return {"email"} trigger.trigger_type === "EMAIL").large_image} style={{width: size, height: size}} /> } + if (execution.execution_parent !== null && execution.execution_parent !== undefined && execution.execution_parent.length > 0) { + return {"parent trigger.trigger_type === "SUBFLOW").large_image} style={{width: size, height: size}} /> + } return ( {execution.execution_source} @@ -6487,17 +6632,41 @@ const AngularWorkflow = (props) => { - -

Executing Workflow

+ +
+

Executing Workflow

+ + + + + +
{executionData.status !== undefined && executionData.status.length > 0 ?
- Status: {executionData.status} + Status:   {executionData.status} +
+ : null + } + {executionData.execution_source !== undefined && executionData.execution_source !== null && executionData.execution_source.length > 0 && executionData.execution_source !== "default" ? +
+ Source:   {executionData.execution_parent !== null && executionData.execution_parent !== undefined && executionData.execution_parent.length > 0 ? + Parent Workflow + : + executionData.execution_source + }
: null } {executionData.started_at !== undefined ?
- Started: {new Date(executionData.started_at*1000).toISOString()} + Started:  {new Date(executionData.started_at*1000).toISOString()}
: null } @@ -6507,16 +6676,11 @@ const AngularWorkflow = (props) => {
: null } - {executionData.execution_source !== undefined && executionData.execution_source.length > 0 ? -
- Source: {executionData.execution_source} -
- : null - } +
{executionData.execution_argument !== undefined && executionData.execution_argument.length > 0 ? parsedExecutionArgument() : null } - + {executionData.results !== undefined && executionData.results !== null && executionData.results.length > 1 && executionData.results.find(result => result.status === "SKIPPED" || result.status === "FAILURE") ? { /> {data.action.app_name === "shuffle-subflow" ? - TBD: Load subexecution result for + {validate.valid && data.action.parameters !== undefined && data.action.parameters !== null ? + See subflow execution + : + "TBD: Load subexecution result for" + } : null } : -
+
Result  {data.result}
@@ -6851,7 +7019,7 @@ const AngularWorkflow = (props) => { Execution Variable - Execution Variables are TEMPORARY variables that you can ony be set and used during execution. Learn more here + Execution Variables are TEMPORARY variables that you can ony be set and used during execution. Learn more here setNewVariableName(event.target.value)} color="primary" @@ -6933,7 +7101,7 @@ const AngularWorkflow = (props) => { : null const variablesModal = variablesModalOpen ? - { setNewVariableName("") @@ -7078,8 +7246,9 @@ const AngularWorkflow = (props) => { const handleSubmitCheck = () => { console.log("NEW AUTH: ", authenticationOption) if (authenticationOption.label.length === 0) { - alert.info("Label can't be empty") - return + authenticationOption.label = `Auth for ${selectedApp.name}` + //alert.info("Label can't be empty") + //return } for (var key in selectedApp.authentication.parameters) { @@ -7109,7 +7278,6 @@ const AngularWorkflow = (props) => { setNewAppAuth(newAuthOption) //appAuthentication.push(newAuthOption) //setAppAuthentication(appAuthentication) - getAppAuthentication() setUpdate(authenticationOption.id) /* @@ -7126,7 +7294,7 @@ const AngularWorkflow = (props) => { return (
- What is this?
+ What is this?
These are required fields for authenticating with {selectedApp.name}
Name - what is this used for? @@ -7144,6 +7312,7 @@ const AngularWorkflow = (props) => { fullWidth color="primary" placeholder={"Auth july 2020"} + defaultValue={`Auth for ${selectedApp.name}`} onChange={(event) => { authenticationOption.label = event.target.value }} diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 5b4cb70f..645c77d9 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -195,7 +195,7 @@ const AppCreator = (props) => { const alert = useAlert() var upload = "" - const increaseAmount = 30 + const increaseAmount = 50 const actionNonBodyRequest = ["GET", "HEAD", "DELETE", "CONNECT"] const actionBodyRequest = ["POST", "PUT", "PATCH",] const authenticationOptions = ["No authentication", "API key", "Bearer auth", "Basic auth", ] @@ -444,6 +444,10 @@ const AppCreator = (props) => { } if (data.info["x-categories"] !== undefined && data.info["x-categories"].length > 0) { + if (typeof(data.info["x-categories"]) == "array") { + } else { + + } setNewWorkflowCategories(data.info["x-categories"]) } } @@ -922,6 +926,7 @@ const AppCreator = (props) => { //console.log(queryitem) } } + //data.paths[item.url][item.method.toLowerCase()].parameters.push(newitem) if (item.paths.length > 0) { for (querykey in item.paths) { @@ -1230,6 +1235,11 @@ const AppCreator = (props) => { newAction.errors.push("Can't have the same name") actions.push(newAction) + + if (actions.length > actionAmount) { + setActionAmount(actions.length) + } + setActions(actions) setUpdate(Math.random()) } @@ -1539,6 +1549,11 @@ const AppCreator = (props) => { actions[actionIndex] = currentAction } + + if (actions.length > actionAmount) { + setActionAmount(actions.length) + } + setActions(actions) } @@ -1702,7 +1717,7 @@ const AppCreator = (props) => {
New action
 - Learn more about actions + Learn more about actions
Name { }} onBlur={event => { var parsedurl = event.target.value + console.log("URL: ", parsedurl) + if (parsedurl.includes("<") && parsedurl.includes(">")) { + console.log("REPLACE") + parsedurl = parsedurl.replace("<", "{") + parsedurl = parsedurl.replace(">", "}") + } + if (parsedurl.startsWith("PUT ") || parsedurl.startsWith("GET ") ||parsedurl.startsWith("POST ") || parsedurl.startsWith("DELETE ") ||parsedurl.startsWith("PATCH ") || parsedurl.startsWith("CONNECT ")) { const tmp = parsedurl.split(" ") if (tmp.length > 1) { parsedurl = tmp[1] setActionField("url", parsedurl) - setUrlPath(parsedurl) setCurrentActionMethod(tmp[0].toUpperCase()) setActionField("method", tmp[0].toUpperCase()) @@ -1886,12 +1907,15 @@ const AppCreator = (props) => { } // Check URL query && headers - setActionField("url", parsedurl) - setUrlPath(parsedurl) + //setActionField("url", parsedurl) } } } + if (event.target.value !== parsedurl) { + setUrlPath(parsedurl) + setActionField("url", parsedurl) + } //console.log("URL: ", request.url) }} /> @@ -1966,13 +1990,14 @@ const AppCreator = (props) => { @@ -1980,31 +2005,20 @@ const AppCreator = (props) => {
+ + const categories = [ + "Communication", + "Cases", + "EDR", + "Intel", + "SIEM", + "Network", + "Assets", + "Other", + ] const tagView =
-

Tags

- { - newWorkflowTags.push(chip) - setNewWorkflowTags(newWorkflowTags) - setUpdate("added"+chip) - }} - onDelete={(chip, index) => { - newWorkflowTags.splice(index, 1) - setNewWorkflowTags(newWorkflowTags) - setUpdate("delete "+chip) - }} - /> + {/* { setUpdate("delete "+chip) }} /> + */} +

Categories

+ +

Tags

+ { + newWorkflowTags.push(chip) + setNewWorkflowTags(newWorkflowTags) + setUpdate("added"+chip) + }} + onDelete={(chip, index) => { + newWorkflowTags.splice(index, 1) + setNewWorkflowTags(newWorkflowTags) + setUpdate("delete "+chip) + }} + />
const actionView =

Actions ({actions.length})

Actions are the tasks performed by an app. Read more about actions and apps - here. + here.
{loopActions}
@@ -2053,8 +2112,10 @@ const AppCreator = (props) => { setCurrentActionMethod(actionNonBodyRequest[0]) setActionsModalOpen(true) }}>New action + {/* + {actionAmount} {actions.length} {actionAmount > 0 && actionAmount < actions.length ? null : - } + */}
@@ -2136,7 +2198,7 @@ const AppCreator = (props) => {

- {name} + {name} {actions === null || actions === undefined || actions.length === 0 ? null : ({actions.length})}

diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 15289670..2ee8e716 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -145,6 +145,7 @@ const Apps = (props) => { const [isDropzone, setIsDropzone] = React.useState(false); const upload = React.useRef(null); const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" ? true : false + const borderRadius = 3 const { start, stop } = useInterval({ duration: 5000, @@ -168,6 +169,10 @@ const Apps = (props) => { }) function sortByKey(array, key) { + if (array === undefined || array === null) { + return array + } + return array.sort(function(a, b) { var x = a[key]; var y = b[key]; @@ -222,6 +227,7 @@ const Apps = (props) => { return response.json() }) .then((responseJson) => { + //console.log("Apps: ", responseJson) responseJson = sortByKey(responseJson, "large_image") setApps(responseJson) @@ -325,9 +331,9 @@ const Apps = (props) => { //
//
var imageline = data.large_image.length === 0 ? - {data.title} + {data.title} : - {data.title} { + {data.title} { //console.log("IMG LOADED!: ", event.target) }} /> @@ -530,9 +536,9 @@ const Apps = (props) => { : null var imageline = selectedApp.large_image === undefined || selectedApp.large_image.length === 0 ? - {selectedApp.title} + {selectedApp.title} : - {selectedApp.title} + {selectedApp.title} const GetAppExample = () => { if (selectedAction.returns === undefined) { diff --git a/frontend/src/views/Docs.jsx b/frontend/src/views/Docs.jsx index e8c4b8b0..76258eaf 100644 --- a/frontend/src/views/Docs.jsx +++ b/frontend/src/views/Docs.jsx @@ -1,11 +1,16 @@ import React, {useState, useEffect} from 'react'; +import { useTheme } from '@material-ui/core/styles'; import Divider from '@material-ui/core/Divider'; import ReactMarkdown from 'react-markdown'; import {BrowserView, MobileView} from "react-device-detect"; import Button from '@material-ui/core/Button'; import Menu from '@material-ui/core/Menu'; import MenuItem from '@material-ui/core/MenuItem'; +import Typography from '@material-ui/core/Typography'; +import Paper from '@material-ui/core/Paper'; +import List from '@material-ui/core/List'; +import ListItem from '@material-ui/core/ListItem'; import {Link} from 'react-router-dom'; @@ -20,26 +25,21 @@ const Body = { }; const dividerColor = "rgb(225, 228, 232)" - -const SideBar = { - maxWidth: 250, - flex: "1", - position: "fixed", -} - const hrefStyle = { color: "rgba(255, 255, 255, 0.40)", textDecoration: "none" } const Docs = (props) => { - const { isLoaded, globalUrl, inputColor } = props; + const { isLoaded, globalUrl, inputColor, selectedDoc, serverside, isMobile, update} = props; + const theme = useTheme(); const [data, setData] = useState(""); const [firstrequest, setFirstrequest] = useState(true); const [list, setList] = useState([]); const [listLoaded, setListLoaded] = useState(false); const [anchorEl, setAnchorEl] = React.useState(null); + const [baseUrl, setBaseUrl] = React.useState(serverside === true ? "" : window.location.href) function handleClick(event) { setAnchorEl(event.currentTarget); @@ -49,77 +49,21 @@ const Docs = (props) => { setAnchorEl(null); } - useEffect(() => { - if (firstrequest) { - setFirstrequest(false) - fetchDocList() - fetchDocs(props.match.params.key) - return - } + const SidebarPaperStyle = { + backgroundColor: theme.palette.surfaceColor, + overflowX: "hidden", + position: "relative", + padding: 30, + paddingTop: 15, + borderRadius: 5, + } - // Continue this, and find the h2 with the data in it lol - if (window.location.hash.length > 0) { - var parent = document.getElementById("markdown_wrapper") - if (parent !== null) { - var elements = parent.getElementsByTagName('h2') - - const name = window.location.hash.slice(1, window.location.hash.lenth).toLowerCase().split("%20").join(" ").split("_").join(" ").split("-").join(" ") - - console.log(name) - var found = false - for (var key in elements) { - const element = elements[key] - if (element.innerHTML === undefined) { - continue - } - - // Fix location.. - if (element.innerHTML.toLowerCase() === name) { - element.scrollIntoView({behavior: "smooth"}) - found = true - //element.scrollTo({ - // top: element.offsetTop-100, - // behavior: "smooth" - //}) - } - } - - // H# - if (!found) { - var elements = parent.getElementsByTagName('h3') - console.log(name) - var found = false - for (var key in elements) { - const element = elements[key] - if (element.innerHTML === undefined) { - continue - } - - // Fix location.. - if (element.innerHTML.toLowerCase() === name) { - element.scrollIntoView({behavior: "smooth"}) - found = true - //element.scrollTo({ - // top: element.offsetTop-100, - // behavior: "smooth" - //}) - } - } - } - } - //console.log(element) - - //console.log("NAME: ", name) - //console.log(document.body.innerHTML) - // parent = document.getElementById(parent); - - //var descendants = parent.getElementsByTagName(tagname); - - // this.scrollDiv.current.scrollIntoView({ behavior: 'smooth' }); - - //$(".parent").find("h2:contains('Statistics')").parent(); - } - }) + const SideBar = { + maxWidth: 250, + flex: "1", + position: "fixed", + marginTop: 35, + } const fetchDocList = () => { fetch(globalUrl+"/api/v1/docs", { @@ -143,16 +87,17 @@ const Docs = (props) => { const fetchDocs = (docId) => { fetch(globalUrl+"/api/v1/docs/"+docId, { - method: 'GET', + method: 'GET', headers: { 'Content-Type': 'application/json', 'Accept': 'application/json', }, - }) + }) .then((response) => response.json()) - .then((responseJson) => { + .then((responseJson) => { if (responseJson.success) { setData(responseJson.reason) + document.title = "Shuffle "+docId+" documentation" } else { setData("# Error\nThis page doesn't exist.") } @@ -160,13 +105,99 @@ const Docs = (props) => { .catch(error => {}); } + if (firstrequest) { + setFirstrequest(false) + + if (selectedDoc !== undefined) { + setData(selectedDoc.reason) + setList(selectedDoc.list) + setListLoaded(true) + } else { + fetchDocList() + fetchDocs(props.match.params.key) + } + } + + // Handles search-based changes that origin from outside this file + if (serverside !== true && window.location.href !== baseUrl) { + setBaseUrl(window.location.href) + fetchDocs(props.match.params.key) + } + + const parseElementScroll = () => { + var parent = document.getElementById("markdown_wrapper_outer") + if (parent !== null) { + //console.log("IN PARENT") + var elements = parent.getElementsByTagName('h2') + + const name = window.location.hash.slice(1, window.location.hash.lenth).toLowerCase().split("%20").join(" ").split("_").join(" ").split("-").join(" ") + + //console.log(name) + var found = false + for (var key in elements) { + const element = elements[key] + if (element.innerHTML === undefined) { + continue + } + + // Fix location.. + if (element.innerHTML.toLowerCase() === name) { + element.scrollIntoView({behavior: "smooth"}) + found = true + //element.scrollTo({ + // top: element.offsetTop-100, + // behavior: "smooth" + //}) + } + } + + // H# + if (!found) { + var elements = parent.getElementsByTagName('h3') + console.log(name) + var found = false + for (var key in elements) { + const element = elements[key] + if (element.innerHTML === undefined) { + continue + } + + // Fix location.. + if (element.innerHTML.toLowerCase() === name) { + element.scrollIntoView({behavior: "smooth"}) + found = true + //element.scrollTo({ + // top: element.offsetTop-100, + // behavior: "smooth" + //}) + } + } + } + } + //console.log(element) + + //console.log("NAME: ", name) + //console.log(document.body.innerHTML) + // parent = document.getElementById(parent); + + //var descendants = parent.getElementsByTagName(tagname); + + // this.scrollDiv.current.scrollIntoView({ behavior: 'smooth' }); + + //$(".parent").find("h2:contains('Statistics')").parent(); + } + + if (serverside !== true && window.location.hash.length > 0) { + parseElementScroll() + } + const markdownStyle = { color: "rgba(255, 255, 255, 0.65)", flex: "1", - maxWidth: 750, + maxWidth: isMobile ? "100%" : 750, overflow: "hidden", paddingBottom: 200, - marginLeft: 250, + marginLeft: isMobile ? 0 : 275, } function OuterLink(props) { @@ -182,7 +213,7 @@ const Docs = (props) => { function CodeHandler(props) { return ( - 
+			
 				
 					{props.value}
 				
@@ -190,13 +221,22 @@ const Docs = (props) => {
 		)
 	}
 
+	function TextWrapper(props) {
+		console.log(props)
+		return (
+			
+				{props.value}			
+			
+		)
+	}
+
 	function Heading(props) {
 		const element = React.createElement(`h${props.level}`, {style: {marginTop: 40}}, props.children)
 		return (
-			
-				{props.level !== 1 ?  : null}
+			
+				{props.level !== 1 ?  : null}
 				{element}
-			
+			
 		)
 	}
 	//React.createElement("p", {style: {color: "red", backgroundColor: "blue"}}, this.props.paragraph)
@@ -213,26 +253,28 @@ const Docs = (props) => {
   const postDataBrowser = 
 		

 			

-				
    
-					{list.map((item, index) => {
-						const path = "/docs/"+item
-						const newname = item.charAt(0).toUpperCase()+item.substring(1).split("_").join(" ").split("-").join(" ")
-						return (
-							
  • 
-								 {fetchDocs(item)}}>
-									
    {newname}
    
-								
-							
    • 
-						)
-					})}
-				

+				
+					
+						{list.map((item, index) => {
+							const path = "/docs/"+item
+							const newname = item.charAt(0).toUpperCase()+item.substring(1).split("_").join(" ").split("-").join(" ")
+							return (
+								
  • 
+									 {fetchDocs(item)}}>
+										{newname}
+									
+								
    • 
+							)
+						})}
+					    
+				    
 			
    
-			
    
+			
    
 				 {
 
 	const mobileStyle = {
 		color: "white",
-		marginLeft: "15px",
-		marginRight: "15px",
-		paddingBottom: "50px",
+		marginLeft: 15,
+		marginRight: 15,
+		paddingBottom: 50,
 		backgroundColor: "inherit",
+		display: "flex",
+		flexDirection: "column",
 	}
 
 	const postDataMobile = 
 		
    
-			
-			
-			{list.map(item => {
-				const path = "/docs/"+item
-				const newname = item.charAt(0).toUpperCase()+item.substring(1).split("_").join(" ").split("-").join(" ")
-				return (
-					 {window.location.pathname = path}}>{newname}
-				)
-			})}
-			
-			
    
+			
    
+				
+				
+				{list.map((item, index) => {
+					const path = "/docs/"+item
+					const newname = item.charAt(0).toUpperCase()+item.substring(1).split("_").join(" ").split("-").join(" ")
+					return (
+						 {window.location.pathname = path}}>{newname}
+					)
+				})}
+				
+			
    
+			
    
 				
 			
    
 			
-			
 
@@ -297,7 +348,7 @@ const Docs = (props) => {
  	// {imageModal}
 
 
-	const loadedCheck = isLoaded && listLoaded ? 
+	const loadedCheck = 
 		
    
 			
 				{postDataBrowser}	
@@ -306,9 +357,6 @@ const Docs = (props) => {
 				{postDataMobile}	
 			
 		
    
-		:
-		
    
-		
    
 
 	return (
 		
    	
diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx
index 83616e2b..05bc5196 100644
--- a/frontend/src/views/Workflows.jsx
+++ b/frontend/src/views/Workflows.jsx
@@ -29,6 +29,7 @@ import PublishIcon from '@material-ui/icons/Publish';
 //import JSONPretty from 'react-json-pretty';
 //import JSONPrettyMon from 'react-json-pretty/dist/monikai'
 import ReactJson from 'react-json-view'
+import Dropzone from '../components/Dropzone';
 
 import {Link} from 'react-router-dom';
 import { useAlert } from "react-alert";
@@ -108,6 +109,8 @@ const Workflows = (props) => {
 	const [deleteModalOpen, setDeleteModalOpen] = React.useState(false);
 	const [editingWorkflow, setEditingWorkflow] = React.useState({})
 	const [executionLoading, setExecutionLoading] = React.useState(false)
+	const [isDropzone, setIsDropzone] = React.useState(false);
+
 	const { start, stop } = useInterval({
 	  	duration: 5000,
 	  	startImmediate: false,
@@ -180,6 +183,58 @@ const Workflows = (props) => {
 		
 	: null
 
+	const uploadFile = (e) => {
+		const isDropzone = e.dataTransfer === undefined ? false : e.dataTransfer.files.length > 0;
+		const files = isDropzone ? e.dataTransfer.files : e.target.files;
+		
+    const reader = new FileReader();
+		alert.info("Starting upload. Please wait while we validate the workflows")
+
+		try {
+			reader.addEventListener('load', (e) => {
+				var data = e.target.result;
+				setIsDropzone(false)
+				try {
+					data = JSON.parse(reader.result)
+				} catch (e) {
+					alert.error("Invalid JSON: "+e)
+					return
+				}
+
+				// Initialize the workflow itself
+				const ret = setNewWorkflow(data.name, data.description, data.tags, {}, false)
+				.then((response) => {
+					if (response !== undefined) {
+						// SET THE FULL THING
+						data.id = response.id
+
+						// Actually create it
+						const ret = setNewWorkflow(data.name, data.description, data.tags, data, false)
+						.then((response) => {
+							if (response !== undefined) {
+								alert.success("Successfully imported "+data.name)
+							}
+						})
+					}
+				})
+				.catch(error => {
+					alert.error("Import error: "+error.toString())
+				});
+			})
+		} catch (e) {
+			console.log("Error in dropzone: ", e)
+		}
+
+		reader.readAsText(files[0]);
+  }
+
+	useEffect(() => {
+		if (isDropzone) {
+			//redirectOpenApi();
+			setIsDropzone(false);
+		}
+  }, [isDropzone]);
+
 	const getAvailableWorkflows = () => {
 		fetch(globalUrl+"/api/v1/workflows", {
     	  method: 'GET',
@@ -191,18 +246,21 @@ const Workflows = (props) => {
     })
 		.then((response) => {
 			if (response.status !== 200) {
-				console.log("Status not 200 for workflows :O!")
+				console.log("Status not 200 for workflows :O!: ", response.status)
+				alert.info("Failed getting workflows.")
+				setWorkflowDone(true)
+
 				return 
 			}
 			return response.json()
 		})
-    	.then((responseJson) => {
+    .then((responseJson) => {
 			setSelectedExecution({})
 			setWorkflowExecutions([])
 
 			if (responseJson !== undefined) {
 				setWorkflows(responseJson)
-					setWorkflowDone(true)
+				setWorkflowDone(true)
 			} else {
 				if (isLoggedIn) {
 					alert.error("An error occurred while loading workflows")
@@ -392,23 +450,44 @@ const Workflows = (props) => {
 		let exportFileDefaultName = data.name+'.json';
 
 		data["owner"] = ""
-		for (var key in data.triggers) {
-			const trigger = data.triggers[key]
-			if (trigger.app_name === "Shuffle Workflow") {
-				if (trigger.parameters.length > 2) {
-					trigger.parameters[2].value = ""
+		if (data.triggers !== null && data.triggers !== undefined) {
+			for (var key in data.triggers) {
+				const trigger = data.triggers[key]
+				if (trigger.app_name === "Shuffle Workflow") {
+					if (trigger.parameters.length > 2) {
+						trigger.parameters[2].value = ""
+					}
+				} 
+				
+				if (trigger.status == "running") {
+					trigger.status = "stopped"
 				}
-			} 
-			
-			if (trigger.status == "running") {
-				trigger.status = "stopped"
 			}
 		}
 
-		for (var key in data.actions) {
-			data.actions[key].authentication_id = ""
+		if (data.actions !== null && data.actions !== undefined) {
+			for (var key in data.actions) {
+				data.actions[key].authentication_id = ""
+
+				for (var subkey in data.actions[key].parameters) {
+					const param = data.actions[key].parameters[subkey]
+					if (param.name.includes("key") || param.name.includes("user") || param.name.includes("pass") || param.name.includes("api") || param.name.includes("auth") || param.name.includes("secret")) {
+						param.value = ""
+					}
+				}
+			}
 		}
 
+		if (data.workflow_variables !== null && data.workflow_variables !== undefined) {
+			for (var key in data.workflow_variables) {
+				const param = data.workflow_variables[key]
+				if (param.name.includes("key") || param.name.includes("user") || param.name.includes("pass") || param.name.includes("api") || param.name.includes("auth") || param.name.includes("secret")) {
+					param.value = ""
+				}
+			}
+		}
+
+		//console.log(data)
 		//return
 
 		data["org"] = []
@@ -1453,7 +1532,9 @@ const Workflows = (props) => {
 
 	const loadedCheck = isLoaded && isLoggedIn && workflowDone ? 
 		
    
-			
+			 1366 ? 1366 : 1200, margin: "auto", padding: 20 }} onDrop={uploadFile}>
+				
+			
 			{modalView}
 			{deleteModal}
 			{workflowDownloadModalOpen}
diff --git a/functions/extensions/aws-lambda/main.go b/functions/extensions/aws-lambda/main.go
new file mode 100644
index 00000000..c78467d6
--- /dev/null
+++ b/functions/extensions/aws-lambda/main.go
@@ -0,0 +1,66 @@
+package main
+
+import (
+	"context"
+	//"encoding/json"
+	//"fmt"
+	"github.com/aws/aws-lambda-go/lambda"
+	"net/http"
+)
+
+type LambdaPayload struct {
+	RequestContext struct {
+		Elb struct {
+			TargetGroupArn string `json:"targetGroupArn"`
+		} `json:"elb"`
+	} `json:"requestContext"`
+	HTTPMethod            string            `json:"httpMethod"`
+	Path                  string            `json:"path"`
+	Headers               map[string]string `json:"headers"`
+	QueryStringParameters map[string]string `json:"queryStringParameters"`
+	Body                  string            `json:"body"`
+	IsBase64Encoded       bool              `json:"isBase64Encoded"`
+}
+
+type LambdaResponse struct {
+	IsBase64Encoded   bool   `json:"isBase64Encoded"`
+	StatusCode        int    `json:"statusCode"`
+	StatusDescription string `json:"statusDescription"`
+	Headers           struct {
+		SetCookie   string `json:"Set-cookie"`
+		ContentType string `json:"Content-Type"`
+	} `json:"headers"`
+	Body string `json:"body"`
+}
+
+func lambda_handler(ctx context.Context, payload LambdaPayload) (LambdaResponse, error) {
+	response := &LambdaResponse{}
+	response.Headers.ContentType = "text/html"
+	response.StatusCode = http.StatusBadRequest
+	response.StatusDescription = http.StatusText(http.StatusBadRequest)
+	if payload.HTTPMethod == http.MethodGet && payload.Path == "/myfavoritecar" {
+		res := "TEST"
+		//car := &Car{}
+		//car.Model = "Corvette"
+		//car.Color = "Red"
+		//car.Year = 1999
+		//res, err := json.Marshal(car)
+		//if err != nil {
+		//	fmt.Println(err)
+		//	response.StatusCode = http.StatusInternalServerError
+		//	response.StatusDescription = http.StatusText(http.StatusInternalServerError)
+		//	return *response, err
+		//}
+		response.Headers.ContentType = "application/json"
+		response.Body = string(res)
+		response.StatusCode = http.StatusOK
+		response.StatusDescription = http.StatusText(http.StatusOK)
+		return *response, nil
+	} else {
+		return *response, nil
+	}
+}
+
+func main() {
+	lambda.Start(lambda_handler)
+}
diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh
index 1aa329af..a0fcec2c 100644
--- a/functions/onprem/orborus/build.sh
+++ b/functions/onprem/orborus/build.sh
@@ -1,5 +1,5 @@
 NAME=shuffle-orborus
-VERSION=0.8.54
+VERSION=0.8.60
 
 echo "Running docker build with $NAME:$VERSION"
 #docker rmi frikky/shuffle:$NAME --force
diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod
new file mode 100644
index 00000000..47154cda
--- /dev/null
+++ b/functions/onprem/orborus/go.mod
@@ -0,0 +1,19 @@
+module orborus
+
+go 1.13
+
+require (
+	github.com/containerd/containerd v1.4.3 // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v20.10.1+incompatible
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/gogo/protobuf v1.3.1 // indirect
+	github.com/mackerelio/go-osstat v0.1.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/satori/go.uuid v1.2.0
+	github.com/sirupsen/logrus v1.7.0 // indirect
+	google.golang.org/grpc v1.34.0 // indirect
+)
diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum
new file mode 100644
index 00000000..84c5d0ec
--- /dev/null
+++ b/functions/onprem/orborus/go.sum
@@ -0,0 +1,112 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/containerd/containerd v1.4.3 h1:ijQT13JedHSHrQGWFcGEwzcNKrAGIiZ+jSD5QQG07SY=
+github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
+github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v20.10.1+incompatible h1:u0HIBLwOJdemyBdTCkoBX34u3lb5KyBo0rQE3a5Yg+E=
+github.com/docker/docker v20.10.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
+github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/mackerelio/go-osstat v0.1.0 h1:e57QHeHob8kKJ5FhcXGdzx5O6Ktuc5RHMDIkeqhgkFA=
+github.com/mackerelio/go-osstat v0.1.0/go.mod h1:1K3NeYLhMHPvzUu+ePYXtoB58wkaRpxZsGClZBJyIFw=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
+github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190410235845-0ad05ae3009d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.34.0 h1:raiipEjMOIC/TO2AvyTxP25XFdLxNIBwzDh3FM3XztI=
+google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go
index f4f7f50a..bf7f2919 100644
--- a/functions/onprem/orborus/orborus.go
+++ b/functions/onprem/orborus/orborus.go
@@ -53,7 +53,7 @@ var environment = os.Getenv("ENVIRONMENT_NAME")
 var dockerApiVersion = os.Getenv("DOCKER_API_VERSION")
 var runningMode = strings.ToLower(os.Getenv("RUNNING_MODE"))
 var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP"))
-var workerIds = []string{}
+var executionIds = []string{}
 
 type ExecutionRequestWrapper struct {
 	Data []ExecutionRequest `json:"data"`
@@ -172,7 +172,7 @@ func deployWorker(image string, identifier string, env []string) {
 		if strings.Contains(fmt.Sprintf("%s", err), "Conflict. The container name ") {
 			uuid := uuid.NewV4()
 			identifier = fmt.Sprintf("%s-%s", identifier, uuid)
-			log.Printf("2 - Identifier: %s", identifier)
+			log.Printf("[INFO] 2 - Identifier: %s", identifier)
 			cont, err = dockercli.ContainerCreate(
 				context.Background(),
 				config,
@@ -221,7 +221,6 @@ func deployWorker(image string, identifier string, env []string) {
 		//}
 	} else {
 		log.Printf("[INFO] Container %s was created under environment %s", cont.ID, environment)
-		//workerIds = append(workerIds, cont.ID)
 	}
 
 	return
@@ -254,11 +253,11 @@ func initializeImages() {
 	ctx := context.Background()
 
 	if appSdkVersion == "" {
-		appSdkVersion = "0.8.5"
+		appSdkVersion = "0.8.60"
 		log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion)
 	}
 	if workerVersion == "" {
-		workerVersion = "0.8.54"
+		workerVersion = "0.8.60"
 		log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion)
 	}
 
@@ -515,8 +514,6 @@ func main() {
 			continue
 		}
 
-		//log.Printf("[INFO] Got %d new requests. Executing: %d. Max: %d", len(executionRequests.Data), executionCount, maxConcurrency)
-
 		allowed := maxConcurrency - executionCount
 		if len(executionRequests.Data) > allowed {
 			log.Printf("[WARNING] Throttle - Cutting down requests from %d to %d (MAX: %d, CUR: %d)", len(executionRequests.Data), allowed, maxConcurrency, executionCount)
@@ -538,6 +535,24 @@ func main() {
 			if execution.Status == "ABORT" || execution.Status == "FAILED" {
 				log.Printf("[INFO] Executionstatus issue: ", execution.Status)
 			}
+
+			found := false
+			for _, executionId := range executionIds {
+				if execution.ExecutionId == executionId {
+					found = true
+					break
+				}
+			}
+
+			// Doesn't work because of USER INPUT
+			if found {
+				//log.Printf("[INFO] Skipping duplicate %s", execution.ExecutionId)
+				//continue
+			} else {
+				//log.Printf("[INFO] Adding to be ran %s", execution.ExecutionId)
+				executionIds = append(executionIds, execution.ExecutionId)
+			}
+
 			// Now, how do I execute this one?
 			// FIXME - if error, check the status of the running one. If it's bad, send data back.
 			containerName := fmt.Sprintf("worker-%s", execution.ExecutionId)
@@ -677,6 +692,7 @@ func getRunningWorkers(ctx context.Context, workerTimeout int) int {
 // FIXME - add this to remove exited workers
 // Should it check what happened to the execution? idk
 func zombiecheck(ctx context.Context, workerTimeout int) error {
+	executionIds = []string{}
 	log.Println("[INFO] Looking for old containers (zombies)")
 	containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
 		All: true,
diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh
index f0592678..ba1b2e38 100644
--- a/functions/onprem/worker/build.sh
+++ b/functions/onprem/worker/build.sh
@@ -1,5 +1,5 @@
 NAME=shuffle-worker
-VERSION=0.8.56
+VERSION=0.8.60
 
 echo "Running docker build with $NAME:$VERSION"
 #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin .
diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go
index 1c36bdb2..dcda8b3e 100644
--- a/functions/onprem/worker/worker.go
+++ b/functions/onprem/worker/worker.go
@@ -571,8 +571,6 @@ type WorkflowAppAction struct {
 	AuthNotRequired  bool   `json:"auth_not_required" datastore:"auth_not_required" yaml:"auth_not_required"`
 }
 
-// FIXME: Generate a callback authentication ID?
-// FIXME: Add org check ..
 type WorkflowExecution struct {
 	Type               string         `json:"type" datastore:"type"`
 	Status             string         `json:"status" datastore:"status"`
@@ -580,6 +578,7 @@ type WorkflowExecution struct {
 	ExecutionArgument  string         `json:"execution_argument" datastore:"execution_argument,noindex"`
 	ExecutionId        string         `json:"execution_id" datastore:"execution_id"`
 	ExecutionSource    string         `json:"execution_source" datastore:"execution_source"`
+	ExecutionParent    string         `json:"execution_parent" datastore:"execution_parent"`
 	ExecutionOrg       string         `json:"execution_org" datastore:"execution_org"`
 	WorkflowId         string         `json:"workflow_id" datastore:"workflow_id"`
 	LastNode           string         `json:"last_node" datastore:"last_node"`
@@ -599,6 +598,7 @@ type WorkflowExecution struct {
 	} `json:"execution_variables,omitempty" datastore:"execution_variables,omitempty"`
 	OrgId string `json:"org_id" datastore:"org_id"`
 }
+
 type Action struct {
 	AppName           string                       `json:"app_name,omitempty" datastore:"app_name"`
 	AppVersion        string                       `json:"app_version,omitempty" datastore:"app_version"`
@@ -839,7 +839,7 @@ func shutdown(executionId, workflowId string) {
 		log.Printf("[INFO] Failed abort request: %s", err)
 	}
 
-	sleepDuration := 0
+	sleepDuration := 1
 	log.Printf("[INFO] Finished shutdown (after %d seconds).", sleepDuration)
 	// Allows everything to finish in subprocesses
 	time.Sleep(time.Duration(sleepDuration) * time.Second)
@@ -1114,16 +1114,16 @@ func handleExecutionResult(workflowExecution WorkflowExecution) {
 				if isSkipped {
 					//log.Printf("Skipping %s as all parents are done", item.Action.Label)
 					if !arrayContains(visited, item.Action.ID) {
-						log.Printf("Adding visited (1): %s", item.Action.Label)
+						log.Printf("[INFO] Adding visited (1): %s", item.Action.Label)
 						visited = append(visited, item.Action.ID)
 					}
 				} else {
-					log.Printf("Continuing %s as all parents are NOT done", item.Action.Label)
+					log.Printf("[INFO] Continuing %s as all parents are NOT done", item.Action.Label)
 					appendActions = append(appendActions, item.Action.ID)
 				}
 			} else {
 				if item.Status == "FINISHED" {
-					log.Printf("Adding visited (2): %s", item.Action.Label)
+					log.Printf("[INFO] Adding visited (2): %s", item.Action.Label)
 					visited = append(visited, item.Action.ID)
 				}
 			}
@@ -1149,7 +1149,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) {
 	// care if it gets stuck in a loop.
 	// FIXME: Force killing a worker should result in a notification somewhere
 	if len(nextActions) == 0 {
-		log.Printf("No next action. Finished? Result vs Actions: %d - %d", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions))
+		log.Printf("[INFO] No next action. Finished? Result vs Actions: %d - %d", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions))
 		exit := true
 		for _, item := range workflowExecution.Results {
 			if item.Status == "EXECUTING" {
@@ -1235,6 +1235,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) {
 	// IF NOT VISITED && IN toExecuteOnPrem
 	// SKIP if it's not onprem
 	toRemove := []int{}
+	//log.Printf("\n\nNEXTACTIONS: %#v\n\n", nextActions)
 	for index, nextAction := range nextActions {
 		action := getAction(workflowExecution, nextAction, environment)
 		// check visited and onprem
@@ -1273,12 +1274,23 @@ func handleExecutionResult(workflowExecution WorkflowExecution) {
 				}
 			}
 
+			// FIXME: Add startnode from frontend
 			action.Parameters = []WorkflowAppActionParameter{}
 			for _, parameter := range trigger.Parameters {
 				parameter.Variant = "STATIC_VALUE"
 				action.Parameters = append(action.Parameters, parameter)
 			}
 
+			action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
+				Name:  "source_workflow",
+				Value: workflowExecution.Workflow.ID,
+			})
+
+			action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
+				Name:  "source_execution",
+				Value: workflowExecution.ExecutionId,
+			})
+
 			//trigger.LargeImage = ""
 			//err = handleSubworkflowExecution(client, workflowExecution, trigger, action)
 			//if err != nil {
@@ -1366,7 +1378,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) {
 		}
 
 		if continueOuter {
-			log.Printf("Parents of %s aren't finished: %s", nextAction, strings.Join(parents[nextAction], ", "))
+			log.Printf("[INFO] Parents of %s aren't finished: %s", nextAction, strings.Join(parents[nextAction], ", "))
 			//for _, tmpaction := range parents[nextAction] {
 			//	action := getAction(workflowExecution, tmpaction)
 			//	_ = action
@@ -1379,10 +1391,10 @@ func handleExecutionResult(workflowExecution WorkflowExecution) {
 		// get action status
 		actionResult := getResult(workflowExecution, nextAction)
 		if actionResult.Action.ID == action.ID {
-			log.Printf("%s already has status %s.", action.ID, actionResult.Status)
+			log.Printf("[INFO] %s already has status %s.", action.ID, actionResult.Status)
 			continue
 		} else {
-			log.Printf("%s:%s has no status result yet. Should execute.", action.Name, action.ID)
+			log.Printf("[INFO] %s:%s has no status result yet. Should execute.", action.Name, action.ID)
 		}
 
 		appname := action.AppName
@@ -1434,7 +1446,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) {
 		}
 
 		// marshal action and put it in there rofl
-		log.Printf("Time to execute %s (%s) with app %s:%s, function %s, env %s with %d parameters.", action.ID, action.Label, action.AppName, action.AppVersion, action.Name, action.Environment, len(action.Parameters))
+		log.Printf("[INFO] Time to execute %s (%s) with app %s:%s, function %s, env %s with %d parameters.", action.ID, action.Label, action.AppName, action.AppVersion, action.Name, action.Environment, len(action.Parameters))
 
 		actionData, err := json.Marshal(action)
 		if err != nil {
@@ -1457,7 +1469,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) {
 		// Sending full execution so that it won't have to load in every app
 		// This might be an issue if they can read environments, but that's alright
 		// if everything is generated during execution
-		log.Printf("Deployed with CALLBACK_URL %s and BASE_URL %s", appCallbackUrl, baseUrl)
+		log.Printf("[INFO] Deployed with CALLBACK_URL %s and BASE_URL %s", appCallbackUrl, baseUrl)
 		env := []string{
 			fmt.Sprintf("ACTION=%s", string(actionData)),
 			fmt.Sprintf("EXECUTIONID=%s", workflowExecution.ExecutionId),
@@ -1582,7 +1594,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) {
 			}
 		}
 
-		log.Printf("Adding visited (3): %s", action.Label)
+		log.Printf("[INFO] Adding visited (3): %s", action.Label)
 
 		visited = append(visited, action.ID)
 		executed = append(executed, action.ID)
@@ -1612,7 +1624,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) {
 		}
 
 		if shutdownCheck {
-			log.Println("BREAKING BECAUSE RESULTS IS SAME LENGTH AS ACTIONS. SHOULD CHECK ALL RESULTS FOR WHETHER THEY'RE DONE")
+			log.Println("[INFO] BREAKING BECAUSE RESULTS IS SAME LENGTH AS ACTIONS. SHOULD CHECK ALL RESULTS FOR WHETHER THEY'RE DONE")
 			validateFinished(workflowExecution)
 			shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID)
 		}
@@ -1625,16 +1637,23 @@ func handleExecutionResult(workflowExecution WorkflowExecution) {
 func executionInit(workflowExecution WorkflowExecution) error {
 	parents = map[string][]string{}
 	children = map[string][]string{}
-	triggersHandled := []string{}
 
 	startAction = workflowExecution.Start
+	log.Printf("[INFO] STARTACTION: %s", startAction)
 	if len(startAction) == 0 {
-		log.Printf("Didn't find execution start action. Setting it to workflow start action.")
+		log.Printf("[INFO] Didn't find execution start action. Setting it to workflow start action.")
 		startAction = workflowExecution.Workflow.Start
 	}
 
-	nextActions = append(nextActions, startAction)
+	// Setting up extra counter
+	for _, trigger := range workflowExecution.Workflow.Triggers {
+		//log.Printf("Appname trigger (0): %s", trigger.AppName)
+		if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" {
+			extra += 1
+		}
+	}
 
+	nextActions = append(nextActions, startAction)
 	for _, branch := range workflowExecution.Workflow.Branches {
 		// Check what the parent is first. If it's trigger - skip
 		sourceFound := false
@@ -1652,27 +1671,15 @@ func executionInit(workflowExecution WorkflowExecution) error {
 		for _, trigger := range workflowExecution.Workflow.Triggers {
 			//log.Printf("Appname trigger (0): %s", trigger.AppName)
 			if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" {
-				//log.Printf("%s is a special trigger. Checking where.", trigger.AppName)
-
-				found := false
-				for _, check := range triggersHandled {
-					if check == trigger.ID {
-						found = true
-						break
-					}
-				}
-
-				if !found {
-					extra += 1
-				} else {
-					triggersHandled = append(triggersHandled, trigger.ID)
+				if branch.SourceID == "c9560766-3f85-4589-8324-311acd6be820" {
+					log.Printf("BRANCH: %#v", branch)
 				}
 
 				if trigger.ID == branch.SourceID {
-					log.Printf("Trigger %s is the source!", trigger.AppName)
+					log.Printf("[INFO] Trigger %s is the source!", trigger.AppName)
 					sourceFound = true
 				} else if trigger.ID == branch.DestinationID {
-					log.Printf("Trigger %s is the destination!", trigger.AppName)
+					log.Printf("[INFO] Trigger %s is the destination!", trigger.AppName)
 					destinationFound = true
 				}
 			}
@@ -1681,17 +1688,23 @@ func executionInit(workflowExecution WorkflowExecution) error {
 		if sourceFound {
 			parents[branch.DestinationID] = append(parents[branch.DestinationID], branch.SourceID)
 		} else {
-			log.Printf("ID %s was not found in actions! Skipping parent. (TRIGGER?)", branch.SourceID)
+			log.Printf("[INFO] ID %s was not found in actions! Skipping parent. (TRIGGER?)", branch.SourceID)
 		}
 
 		if destinationFound {
 			children[branch.SourceID] = append(children[branch.SourceID], branch.DestinationID)
 		} else {
-			log.Printf("ID %s was not found in actions! Skipping child. (TRIGGER?)", branch.SourceID)
+			log.Printf("[INFO] ID %s was not found in actions! Skipping child. (TRIGGER?)", branch.SourceID)
 		}
 	}
 
-	log.Printf("Actions: %d + Special Triggers: %d", len(workflowExecution.Workflow.Actions), extra)
+	/*
+		log.Printf("\n\n\n[INFO] CHILDREN FOUND: %#v", children)
+		log.Printf("[INFO] PARENTS FOUND: %#v", parents)
+		log.Printf("[INFO] NEXT ACTIONS: %#v\n\n", nextActions)
+	*/
+
+	log.Printf("[INFO] Actions: %d + Special Triggers: %d", len(workflowExecution.Workflow.Actions), extra)
 	onpremApps := []string{}
 	toExecuteOnprem := []string{}
 	for _, action := range workflowExecution.Workflow.Actions {
@@ -1720,7 +1733,7 @@ func executionInit(workflowExecution WorkflowExecution) error {
 	pullOptions := types.ImagePullOptions{}
 	_ = pullOptions
 	for _, image := range onpremApps {
-		log.Printf("Image: %s", image)
+		log.Printf("[INFO] Image: %s", image)
 		// Kind of gambling that the image exists.
 		if strings.Contains(image, " ") {
 			image = strings.ReplaceAll(image, " ", "-")
@@ -2053,7 +2066,10 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
 	//	return
 	//}
 
+	resp.WriteHeader(200)
+	resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
 	runWorkflowExecutionTransaction(ctx, 0, workflowExecution.ExecutionId, actionResult, resp)
+
 }
 
 func findChildNodes(workflowExecution WorkflowExecution, nodeId string) []string {
@@ -2462,15 +2478,20 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl
 			return
 		}
 	} else {
-		log.Printf("Skipping setexec with status %s", workflowExecution.Status)
+		log.Printf("[INFO] Skipping setexec with status %s", workflowExecution.Status)
+
+		// Just in case. Should MAYBE validate finishing another time as well.
+		// This fixes issues with e.g. Action -> Trigger -> Action.
+		handleExecutionResult(*workflowExecution)
+		//validateFinished(workflowExecution)
 	}
 
 	//if newExecutions && len(nextActions) > 0 {
 	//	handleExecutionResult(*workflowExecution)
 	//}
 
-	resp.WriteHeader(200)
-	resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
+	//resp.WriteHeader(200)
+	//resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
 }
 
 func getWorkflowExecution(ctx context.Context, id string) (*WorkflowExecution, error) {
@@ -2488,7 +2509,7 @@ func getWorkflowExecution(ctx context.Context, id string) (*WorkflowExecution, e
 }
 
 func validateFinished(workflowExecution WorkflowExecution) {
-	log.Printf("Status: %s, Actions: %d, Extra: %d, Results: %d\n", workflowExecution.Status, len(workflowExecution.Workflow.Actions), extra, len(workflowExecution.Results))
+	log.Printf("[INFO] Status: %s, Actions: %d, Extra: %d, Results: %d\n", workflowExecution.Status, len(workflowExecution.Workflow.Actions), extra, len(workflowExecution.Results))
 
 	//if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extra {
 	if (len(environments) == 1 && requestsSent == 0 && len(workflowExecution.Results) >= 1) || (len(workflowExecution.Results) >= len(workflowExecution.Workflow.Actions) && len(workflowExecution.Workflow.Actions) > 0) {
@@ -2521,7 +2542,7 @@ func validateFinished(workflowExecution WorkflowExecution) {
 		}
 
 		body, err := ioutil.ReadAll(newresp.Body)
-		log.Printf("BACKEND STATUS: %d", newresp.StatusCode)
+		log.Printf("[INFO] BACKEND STATUS: %d", newresp.StatusCode)
 		if err != nil {
 			log.Printf("[ERROR] Failed reading body: %s", err)
 		} else {
@@ -2693,7 +2714,7 @@ func main() {
 	} else {
 		authorization = os.Getenv("AUTHORIZATION")
 		executionId = os.Getenv("EXECUTIONID")
-		log.Printf("Running normal execution with auth %s and ID %s", authorization, executionId)
+		log.Printf("[INFO] Running normal execution with auth %s and ID %s", authorization, executionId)
 	}
 
 	if len(authorization) == 0 {
@@ -2754,7 +2775,7 @@ func main() {
 
 		if firstRequest {
 			firstRequest = false
-			workflowExecution.StartedAt = int64(time.Now().Unix())
+			//workflowExecution.StartedAt = int64(time.Now().Unix())
 
 			cacheKey := fmt.Sprintf("workflowexecution-%s", workflowExecution.ExecutionId)
 			requestCache = cache.New(5*time.Minute, 10*time.Minute)
diff --git a/functions/stitcher.go b/functions/stitcher.go
deleted file mode 100644
index aae060f3..00000000
--- a/functions/stitcher.go
+++ /dev/null
@@ -1,703 +0,0 @@
-package main
-
-import (
-	"archive/zip"
-	"bytes"
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"archive/tar"
-	"cloud.google.com/go/storage"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/client"
-	"google.golang.org/api/cloudfunctions/v1"
-	"gopkg.in/yaml.v2"
-)
-
-var gceProject = "shuffler"
-var bucketName = "shuffler.appspot.com"
-
-type WorkflowAppActionParameter struct {
-	Description string `json:"description" datastore:"description"`
-	ID          string `json:"id" datastore:"id"`
-	Name        string `json:"name" datastore:"name"`
-	Example     string `json:"example" datastore:"example"`
-	Value       string `json:"value" datastore:"value"`
-	Multiline   bool   `json:"multiline" datastore:"multiline"`
-	ActionField string `json:"action_field" datastore:"action_field"`
-	Variant     string `json:"variant", datastore:"variant"`
-	Required    bool   `json:"required" datastore:"required"`
-	Schema      struct {
-		Type string `json:"type" datastore:"type"`
-	} `json:"schema"`
-}
-
-type Authentication struct {
-	Required   bool                   `json:"required" datastore:"required" yaml:"required" `
-	Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"`
-}
-
-type AuthenticationParams struct {
-	Description string `json:"description" datastore:"description" yaml:"description"`
-	ID          string `json:"id" datastore:"id" yaml:"id"`
-	Name        string `json:"name" datastore:"name" yaml:"name"`
-	Example     string `json:"example" datastore:"example" yaml:"example"`
-	Value       string `json:"value" datastore:"value" yaml:"value"`
-	Multiline   bool   `json:"multiline" datastore:"multiline" yaml:"multiline"`
-	Required    bool   `json:"required" datastore:"required" yaml:"required"`
-}
-
-type WorkflowApp struct {
-	Name        string `json:"name" yaml:"name" required:true datastore:"name"`
-	IsValid     bool   `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"`
-	ID          string `json:"id" yaml:"id" required:false datastore:"id"`
-	Link        string `json:"link" yaml:"link" required:false datastore:"link"`
-	AppVersion  string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"`
-	Description string `json:"description" datastore:"description" required:false yaml:"description"`
-	Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"`
-	Sharing     bool   `json:"sharing" datastore:"sharing" yaml:"sharing"`
-	SmallImage  string `json:"small_image" datastore:"small_image" required:false yaml:"small_image"`
-	LargeImage  string `json:"large_image" datastore:"large_image" yaml:"large_image" requred:false`
-	ContactInfo struct {
-		Name string `json:"name" datastore:"name" yaml:"name"`
-		Url  string `json:"url" datastore:"url" yaml:"url"`
-	} `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false`
-	Actions        []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions"`
-	Authentication Authentication      `json:"authentication" yaml:"authentication" required:false datastore:"authentication"`
-}
-
-type AuthenticationStore struct {
-	Key   string `json:"key" datastore:"key"`
-	Value string `json:"value" datastore:"value"`
-}
-
-type WorkflowAppAction struct {
-	Description    string                       `json:"description" datastore:"description"`
-	ID             string                       `json:"id" datastore:"id"`
-	Name           string                       `json:"name" datastore:"name"`
-	NodeType       string                       `json:"node_type" datastore:"node_type"`
-	Environment    string                       `json:"environment" datastore:"environment"`
-	Parameters     []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"`
-	Authentication []AuthenticationStore        `json:"authentication" datastore:"authentication"`
-	Returns        struct {
-		Description string `json:"description" datastore:"returns"`
-		ID          string `json:"id" datastore:"id"`
-		Schema      struct {
-			Type string `json:"type" datastore:"type"`
-		} `json:"schema" datastore:"schema"`
-	} `json:"returns" datastore:"returns"`
-}
-
-func getRunner(classname string) string {
-	return fmt.Sprintf(`
-# Run the actual thing after we've checked params
-def run(request):
-    action = request.get_json()
-    print(action)
-    print(type(action))
-    authorization_key = action.get("authorization")
-    current_execution_id = action.get("execution_id")
-
-    if action and "name" in action and "app_name" in action:
-        asyncio.run(%s.run(action), debug=True)
-        return f'Attempting to execute function {action["name"]} in app {action["app_name"]}'
-    else:
-        return f'Invalid action'
-
-	`, classname)
-}
-
-// Could use some kind of linting system too for this, but meh
-func formatAppfile(filedata []byte) (string, []byte) {
-	lines := strings.Split(string(filedata), "\n")
-
-	newfile := []string{}
-	classname := ""
-	for _, line := range lines {
-		if strings.Contains(line, "walkoff_app_sdk") {
-			continue
-		}
-
-		// Remap logging. CBA this right now
-		// This issue also persists in onprem apps because of await thingies.. :(
-		// FIXME
-		if strings.Contains(line, "console_logger") && strings.Contains(line, "await") {
-			continue
-			//line = strings.Replace(line, "console_logger", "logger", -1)
-			//log.Println(line)
-		}
-
-		// Might not work with different import names
-		// Could be fucked up with spaces everywhere? Idk
-		if strings.Contains(line, "class") && strings.Contains(line, "(AppBase)") {
-			items := strings.Split(line, " ")
-			if len(items) > 0 && strings.Contains(items[1], "(AppBase)") {
-				classname = strings.Split(items[1], "(")[0]
-			} else {
-				log.Println("Something wrong :( (horrible programming right here)")
-				os.Exit(3)
-			}
-		}
-
-		if strings.Contains(line, "if __name__ ==") {
-			break
-		}
-
-		// asyncio.run(HelloWorld.run(), debug=True)
-
-		newfile = append(newfile, line)
-	}
-
-	filedata = []byte(strings.Join(newfile, "\n"))
-	return classname, filedata
-}
-
-// https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
-func Copy(src, dst string) error {
-	in, err := os.Open(src)
-	if err != nil {
-		return err
-	}
-	defer in.Close()
-
-	out, err := os.Create(dst)
-	if err != nil {
-		return err
-	}
-	defer out.Close()
-
-	_, err = io.Copy(out, in)
-	if err != nil {
-		return err
-	}
-	return out.Close()
-}
-func ZipFiles(filename string, files []string) error {
-	newZipFile, err := os.Create(filename)
-	if err != nil {
-		return err
-	}
-	defer newZipFile.Close()
-
-	zipWriter := zip.NewWriter(newZipFile)
-	defer zipWriter.Close()
-
-	// Add files to zip
-	for _, file := range files {
-		zipfile, err := os.Open(file)
-		if err != nil {
-			return err
-		}
-		defer zipfile.Close()
-
-		// Get the file information
-		info, err := zipfile.Stat()
-		if err != nil {
-			return err
-		}
-
-		header, err := zip.FileInfoHeader(info)
-		if err != nil {
-			return err
-		}
-
-		// Using FileInfoHeader() above only uses the basename of the file. If we want
-		// to preserve the folder structure we can overwrite this with the full path.
-		filesplit := strings.Split(file, "/")
-		if len(filesplit) > 1 {
-			header.Name = filesplit[len(filesplit)-1]
-		} else {
-			header.Name = file
-		}
-
-		// Change to deflate to gain better compression
-		// see http://golang.org/pkg/archive/zip/#pkg-constants
-		header.Method = zip.Deflate
-
-		writer, err := zipWriter.CreateHeader(header)
-		if err != nil {
-			return err
-		}
-		if _, err = io.Copy(writer, zipfile); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func getAppbase(filepath string) []string {
-	appBase, err := ioutil.ReadFile(filepath)
-	if err != nil {
-		log.Printf("Readerror: %s", err)
-		os.Exit(1)
-	}
-
-	record := false
-	validLines := []string{}
-	for _, line := range strings.Split(string(appBase), "\n") {
-		if strings.Contains(line, "#STOPCOPY") {
-			log.Println("Stopping copy")
-			break
-		}
-
-		if record {
-			validLines = append(validLines, line)
-		}
-
-		if strings.Contains(line, "#STARTCOPY") {
-			log.Println("Starting copy")
-			record = true
-		}
-	}
-
-	return validLines
-}
-
-// Puts together ./static_baseline.py, onprem/app_sdk_app_base.py and the
-// appcode in a generated_app folder based on appname+version
-func stitcher(appname string, appversion string) string {
-	baselinefile := "static_baseline.py"
-	appfolder := "apps"
-	appbasefile := "onprem/app_sdk/app_base.py"
-
-	baseline, err := ioutil.ReadFile(baselinefile)
-	if err != nil {
-		log.Printf("Readerror: %s", err)
-		os.Exit(1)
-	}
-
-	sourceappfile := fmt.Sprintf("%s/%s/%s/src/app.py", appfolder, appname, appversion)
-	appfile, err := ioutil.ReadFile(sourceappfile)
-	if err != nil {
-		log.Printf("App readerror: %s", err)
-		os.Exit(1)
-	}
-
-	classname, appfile := formatAppfile(appfile)
-	if len(classname) == 0 {
-		log.Println("Failed finding classname in file.")
-		os.Exit(3)
-	}
-
-	runner := getRunner(classname)
-	appBase := getAppbase(appbasefile)
-
-	foldername := fmt.Sprintf("generated_apps/%s_%s", appname, appversion)
-	err = os.Mkdir(foldername, os.ModePerm)
-	if err != nil {
-		log.Println("Failed making temporary app folder. Probably already exists. Remaking")
-		os.RemoveAll(foldername)
-		os.MkdirAll(foldername, os.ModePerm)
-	}
-
-	stitched := []byte(string(baseline) + strings.Join(appBase, "\n") + string(appfile) + string(runner))
-	err = ioutil.WriteFile(fmt.Sprintf("%s/main.py", foldername), stitched, os.ModePerm)
-	if err != nil {
-		log.Println("Failed writing to stitched: %s", err)
-		os.Exit(3)
-	}
-
-	err = Copy(fmt.Sprintf("%s/%s/%s/requirements.txt", appfolder, appname, appversion), fmt.Sprintf("%s/requirements.txt", foldername))
-	if err != nil {
-		log.Println("Failed writing to requirement: %s", err)
-		os.Exit(3)
-	}
-
-	log.Printf("Successfully stitched files in %s/main.py", foldername)
-	// Zip the folder
-	files := []string{
-		fmt.Sprintf("%s/main.py", foldername),
-		fmt.Sprintf("%s/requirements.txt", foldername),
-	}
-	outputfile := fmt.Sprintf("%s.zip", foldername)
-
-	err = ZipFiles(outputfile, files)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	ctx := context.Background()
-
-	// Creates a client.
-	client, err := storage.NewClient(ctx)
-	if err != nil {
-		log.Printf("Failed to create client: %v", err)
-		os.Exit(3)
-	}
-
-	// Create bucket handle
-	bucket := client.Bucket(bucketName)
-
-	remotePath := fmt.Sprintf("apps/%s_%s.zip", appname, appversion)
-	err = createFileFromFile(bucket, remotePath, outputfile)
-	if err != nil {
-		log.Printf("Failed to upload to bucket: %v", err)
-		os.Exit(3)
-	}
-
-	os.Remove(outputfile)
-	return fmt.Sprintf("gs://%s/apps/%s_%s.zip", bucketName, appname, appversion)
-}
-
-func createFileFromFile(bucket *storage.BucketHandle, remotePath, localPath string) error {
-	ctx := context.Background()
-	// [START upload_file]
-	f, err := os.Open(localPath)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	wc := bucket.Object(remotePath).NewWriter(ctx)
-	if _, err = io.Copy(wc, f); err != nil {
-		return err
-	}
-	if err := wc.Close(); err != nil {
-		return err
-	}
-	// [END upload_file]
-	return nil
-}
-
-// Deploy to google cloud function :)
-func deployFunction(appname, localization, applocation string, environmentVariables map[string]string) error {
-	ctx := context.Background()
-	service, err := cloudfunctions.NewService(ctx)
-	if err != nil {
-		return err
-	}
-
-	// ProjectsLocationsListCall
-	projectsLocationsFunctionsService := cloudfunctions.NewProjectsLocationsFunctionsService(service)
-	location := fmt.Sprintf("projects/%s/locations/%s", gceProject, localization)
-	functionName := fmt.Sprintf("%s/functions/%s", location, appname)
-
-	cloudFunction := &cloudfunctions.CloudFunction{
-		AvailableMemoryMb:    128,
-		EntryPoint:           "authorization",
-		EnvironmentVariables: environmentVariables,
-		HttpsTrigger:         &cloudfunctions.HttpsTrigger{},
-		MaxInstances:         0,
-		Name:                 functionName,
-		Runtime:              "python37",
-		SourceArchiveUrl:     applocation,
-	}
-
-	//getCall := projectsLocationsFunctionsService.Get(fmt.Sprintf("%s/functions/function-5", location))
-	//resp, err := getCall.Do()
-
-	createCall := projectsLocationsFunctionsService.Create(location, cloudFunction)
-	_, err = createCall.Do()
-	if err != nil {
-		log.Println("Failed creating new function. Attempting patch, as it might exist already")
-
-		createCall := projectsLocationsFunctionsService.Patch(fmt.Sprintf("%s/functions/%s", location, appname), cloudFunction)
-		_, err = createCall.Do()
-		if err != nil {
-			log.Println("Failed patching function")
-			return err
-		}
-
-		log.Printf("Successfully patched %s to %s", appname, localization)
-	} else {
-		log.Printf("Successfully deployed %s to %s", appname, localization)
-	}
-
-	// FIXME - use response to define the HTTPS entrypoint. It's default to an easy one tho
-
-	return nil
-}
-
-func deployAppCloudFunc(appname string, appversion string) {
-	_ = os.Mkdir("generated_apps", os.ModePerm)
-
-	apikey := "eyJhbGciOiJSUzI1NiIsImtpZCI6IjYwZjQwNjBlNThkNzVmZDNmNzBiZWZmODhjNzk0YTc3NTMyN2FhMzEiLCJ0eXAiOiJKV1QifQ.eyJhdWQiOiJodHRwczovL3NodWZmbGVyLmlvL2FwaS92MS93b3JrZmxvd3MvMWQ5ZDhjZTItNTY2ZS00YzNmLThhMzctNWQ2YzdkMjAwMGI1L2V4ZWN1dGUiLCJhenAiOiIxMDMwNzY3ODIwNjE0MjQ2MTg0MjIiLCJlbWFpbCI6InNjaGVkdWxlckBzaHVmZmxlLTI0MTUxNy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJleHAiOjE1NjU1Mjc1NTEsImlhdCI6MTU2NTUyMzk1MSwiaXNzIjoiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tIiwic3ViIjoiMTAzMDc2NzgyMDYxNDI0NjE4NDIyIn0.r0EDq9fjhf_5CPTiltyfk_L3uYJp577Uy0yYPcCAl2nv50_z_oUtbWGBpQLL8gcj-NGd3g4E52Qur8k6hCMIQweLS6WAb1279vGffEoCNDfkWb3Oy-yJGP1kzwLvqFJqnHLkSWYXNWvSyWnEimW8Rryx_m1BXS5wcA8l4NIr83kS7fPZrTwjnwFSeGSThwk91DVARzapQb8r0GEgOUyHZ1aBXnV98mikzSUt-5xFKe9eMdD22YJAj0Ru-DxAxs5nOqghX4PMRysWjshjOMrlR1piPWxqAmewp8YKZDCQ5gXskpeAFBDoULT971Wsx_NCohnJsFqx1JfPS9ZYMTW2oQ"
-	fullAppname := fmt.Sprintf("%s-%s", strings.Replace(appname, "_", "-", -1), strings.Replace(appversion, ".", "-", -1))
-	locations := []string{"europe-west2"}
-
-	// Deploys the app to all locations
-	bucketname := stitcher(appname, appversion)
-	environmentVariables := map[string]string{
-		"FUNCTION_APIKEY": apikey,
-	}
-
-	for _, location := range locations {
-		err := deployFunction(fullAppname, location, bucketname, environmentVariables)
-		if err != nil {
-			log.Printf("Failed to deploy: %s", err)
-			os.Exit(3)
-		}
-	}
-}
-
-func loadYaml(fileLocation string) (WorkflowApp, error) {
-	action := WorkflowApp{}
-
-	yamlFile, err := ioutil.ReadFile(fileLocation)
-	if err != nil {
-		log.Printf("yamlFile.Get err: %s", err)
-		return WorkflowApp{}, err
-	}
-
-	//log.Printf(string(yamlFile))
-	err = yaml.Unmarshal([]byte(yamlFile), &action)
-	if err != nil {
-		return WorkflowApp{}, err
-	}
-
-	return action, nil
-}
-
-// FIXME - deploy to backend (YAML config)
-func deployConfigToBackend(appname string, appversion string) error {
-	// FIXME - no static path pls
-	action, err := loadYaml(fmt.Sprintf("apps/%s/%s/api.yaml", appname, appversion))
-	if err != nil {
-		log.Println(err)
-		return err
-	}
-
-	action.Sharing = true
-
-	data, err := json.Marshal(action)
-	if err != nil {
-		return err
-	}
-
-	url := "http://localhost:5001/api/v1/workflows/apps"
-	client := &http.Client{}
-	req, err := http.NewRequest(http.MethodPut, url, bytes.NewReader(data))
-	if err != nil {
-		return err
-	}
-
-	req.Header.Set("Authorization", "Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjYwZjQwNjBlNThkNzVmZDNmNzBiZWZmODhjNzk0YTc3NTMyN2FhMzEiLCJ0eXAiOiJKV1QifQ.eyJhdWQiOiJodHRwczovL3NodWZmbGVyLmlvL2FwaS92MS93b3JrZmxvd3MvMWQ5ZDhjZTItNTY2ZS00YzNmLThhMzctNWQ2YzdkMjAwMGI1L2V4ZWN1dGUiLCJhenAiOiIxMDMwNzY3ODIwNjE0MjQ2MTg0MjIiLCJlbWFpbCI6InNjaGVkdWxlckBzaHVmZmxlLTI0MTUxNy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJleHAiOjE1NjU1Mjc1NTEsImlhdCI6MTU2NTUyMzk1MSwiaXNzIjoiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tIiwic3ViIjoiMTAzMDc2NzgyMDYxNDI0NjE4NDIyIn0.r0EDq9fjhf_5CPTiltyfk_L3uYJp577Uy0yYPcCAl2nv50_z_oUtbWGBpQLL8gcj-NGd3g4E52Qur8k6hCMIQweLS6WAb1279vGffEoCNDfkWb3Oy-yJGP1kzwLvqFJqnHLkSWYXNWvSyWnEimW8Rryx_m1BXS5wcA8l4NIr83kS7fPZrTwjnwFSeGSThwk91DVARzapQb8r0GEgOUyHZ1aBXnV98mikzSUt-5xFKe9eMdD22YJAj0Ru-DxAxs5nOqghX4PMRysWjshjOMrlR1piPWxqAmewp8YKZDCQ5gXskpeAFBDoULT971Wsx_NCohnJsFqx1JfPS9ZYMTW2oQ")
-
-	ret, err := client.Do(req)
-	if err != nil {
-		return err
-	}
-
-	log.Printf("Status: %s", ret.Status)
-	body, err := ioutil.ReadAll(ret.Body)
-	if err != nil {
-		return err
-	}
-
-	if ret.StatusCode != 200 {
-		return errors.New(fmt.Sprintf("Status %s. App probably already exists. Raw:\n%s", ret.Status, string(body)))
-	}
-
-	log.Println(string(body))
-	return nil
-}
-
-func tarDirectory(filecontext string) (io.Reader, error) {
-
-	// Create a filereader
-	//dockerFileReader, err := os.Open(dockerfile)
-	//if err != nil {
-	//	return err
-	//}
-
-	//// Read the actual Dockerfile
-	//readDockerFile, err := ioutil.ReadAll(dockerFileReader)
-	//if err != nil {
-	//	return err
-	//}
-
-	// Make a TAR header for the file
-	tarHeader := &tar.Header{
-		Name:     filecontext,
-		Typeflag: tar.TypeDir,
-	}
-
-	// Writes the header described for the TAR file
-	buf := new(bytes.Buffer)
-	tw := tar.NewWriter(buf)
-	defer tw.Close()
-	err := tw.WriteHeader(tarHeader)
-	if err != nil {
-		return nil, err
-	}
-
-	dockerFileTarReader := bytes.NewReader(buf.Bytes())
-	return dockerFileTarReader, nil
-}
-
-func tarDir(source string, target string) (*bytes.Reader, error) {
-	filename := filepath.Base(source)
-	target = filepath.Join(target, fmt.Sprintf("%s.tar", filename))
-	tarfile, err := os.Create(target)
-	if err != nil {
-		return nil, err
-	}
-
-	defer tarfile.Close()
-
-	buf := new(bytes.Buffer)
-	_ = buf
-	tarball := tar.NewWriter(tarfile)
-	defer tarball.Close()
-
-	info, err := os.Stat(source)
-	if err != nil {
-		return nil, err
-	}
-
-	var baseDir string
-	if info.IsDir() {
-		baseDir = filepath.Base(source)
-	}
-
-	_ = filepath.Walk(source,
-		func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-			header, err := tar.FileInfoHeader(info, info.Name())
-			if err != nil {
-				return err
-			}
-
-			if baseDir != "" {
-				header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source))
-			}
-
-			if err := tarball.WriteHeader(header); err != nil {
-				return err
-			}
-
-			if info.IsDir() {
-				return nil
-			}
-
-			file, err := os.Open(path)
-			if err != nil {
-				return err
-			}
-			defer file.Close()
-			_, err = io.Copy(tarball, file)
-			return nil
-		})
-
-	dockerFileTarReader := bytes.NewReader(buf.Bytes())
-	return dockerFileTarReader, nil
-}
-
-func buildImage(client *client.Client, tags []string, dockerBuildCtxDir string) error {
-	dockerBuildContext, err := tarDir(dockerBuildCtxDir, ".")
-	if err != nil {
-		log.Printf("Error in taring the docker root folder - %s", err.Error())
-		return err
-	}
-
-	imageBuildResponse, err := client.ImageBuild(
-		context.Background(),
-		dockerBuildContext,
-		types.ImageBuildOptions{
-			Dockerfile: "Dockerfile",
-			PullParent: true,
-			Remove:     true,
-			Tags:       tags,
-			NetworkMode: "host",
-		},
-	)
-
-	if err != nil {
-		return err
-	}
-
-	// Read the STDOUT from the build process
-	defer imageBuildResponse.Body.Close()
-	_, err = io.Copy(os.Stdout, imageBuildResponse.Body)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// FIXME - deploy to dockerhub
-func deployWorker(appname, appversion string) error {
-	// Get dockerfile from ./apps/appname/appversion/Dockerfile
-	client, err := client.NewEnvClient()
-	if err != nil {
-		return err
-	}
-
-	tags := []string{fmt.Sprintf("%s-%s", appname, appversion)}
-	err = buildImage(client, tags, fmt.Sprintf("./apps/%s/%s", appname, appversion))
-	if err != nil {
-		log.Printf("Build error: %s", err)
-		return err
-	}
-
-	return nil
-}
-
-// Deploys all cloud functions. Onprem thooo :(
-func deployAll() {
-	allapps := []string{
-		"hoxhunt",
-		"secureworks",
-		"servicenow",
-		"lastline",
-		"netcraft",
-		"misp",
-		"email",
-		"testing",
-		"http",
-		"recordedfuture",
-		"passivetotal",
-		"carbon_black",
-		"thehive",
-		"cortex",
-		"splunk",
-	}
-
-	for _, appname := range allapps {
-		appversion := "1.0.0"
-
-		err := deployConfigToBackend(appname, appversion)
-		if err != nil {
-			log.Printf("Failed uploading config: %s", err)
-			continue
-		}
-
-		deployAppCloudFunc(appname, appversion)
-	}
-}
-
-func main() {
-	deployAll()
-	return
-
-	appname := "testing"
-	appversion := "1.0.0"
-
-	err := deployConfigToBackend(appname, appversion)
-	if err != nil {
-		log.Printf("Failed uploading config: %s", err)
-		os.Exit(1)
-	}
-
-	deployAppCloudFunc(appname, appversion)
-
-	// FIXME - build and deploy to dockerhub as well :)
-	// Not able to work in remote directory propely... Even tried making an actual tar and checking it rofl
-	//err := deployWorker(appname, appversion)
-	//if err != nil {
-	//	log.Printf("Failed to deploy docker worker: %s", err)
-	//}
-}