diff --git a/.env b/.env index ac324e6a..ff6b1b54 100644 --- a/.env +++ b/.env @@ -2,6 +2,7 @@ ORG_ID=Shuffle ENVIRONMENT_NAME=Shuffle + # Remote github config for first load SHUFFLE_DOWNLOAD_WORKFLOW_LOCATION= SHUFFLE_DOWNLOAD_WORKFLOW_USERNAME= @@ -60,10 +61,10 @@ SHUFFLE_ELASTIC=true # DATABASE CONFIGURATIONS DATASTORE_EMULATOR_HOST=shuffle-database:8000 -#SHUFFLE_OPENSEARCH_URL=https://shuffle-opensearch:9200 -SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200 -SHUFFLE_OPENSEARCH_USERNAME= -SHUFFLE_OPENSEARCH_PASSWORD= +#SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200 +SHUFFLE_OPENSEARCH_URL=https://shuffle-opensearch:9200 +SHUFFLE_OPENSEARCH_USERNAME=admin +SHUFFLE_OPENSEARCH_PASSWORD=admin SHUFFLE_OPENSEARCH_CERTIFICATE_FILE= SHUFFLE_OPENSEARCH_APIKEY= SHUFFLE_OPENSEARCH_CLOUDID= diff --git a/.github/install-guide.md b/.github/install-guide.md index 0864a172..360bc6a8 100644 --- a/.github/install-guide.md +++ b/.github/install-guide.md @@ -10,18 +10,19 @@ The Docker setup is done with docker-compose 1. Make sure you have [Docker](https://docs.docker.com/get-docker/) and [docker-compose](https://docs.docker.com/compose/install/) installed. 2. Download Shuffle -``` +```bash git clone https://github.com/frikky/Shuffle cd Shuffle ``` 3. Fix prerequisites for the Opensearch database (Elasticsearch): -``` -sudo chown -R 1000:1000 shuffle-database # Required for Opensearch +```bash +mkdir shuffle-database +sudo chown -R 1000:1000 shuffle-database ``` 4. Run docker-compose. -``` +```bash docker-compose up -d ``` @@ -38,20 +39,20 @@ This step is for setting up with Docker on windows from scratch. 4. Open the .env file and change the line with "OUTER_HOSTNAME" to contain your IP: -``` +```bash OUTER_HOSTNAME=YOUR.IP.HERE ``` 6. Run docker-compose -``` -docker compose up -d +```bash +docker-compose up -d ``` ### Configurations (proxies, default users etc.) https://shuffler.io/docs/configuration ### After installation -1. After installation, go to http://localhost:3001/adminsetup (or your servername - https is on port 3443) +1. After installation, go to http://localhost:3001 (or your servername - https is on port 3443) 2. Now set up your admin account (username & password). Shuffle doesn't have a default username and password. 3. Sign in with the same Username & Password! Go to /apps and see if you have any apps yet. If not - you may need to [configure proxies](https://shuffler.io/docs/configuration#production_readiness) 4. Check out https://shuffler.io/docs/configuration as it has a lot of useful information to get started diff --git a/README.md b/README.md index 01fd94b1..d4a13732 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Shuffle Automation

-![Shuffle](https://shuffler.io) is an automation platform for and by the community, focusing on accessibility for anyone to automate. Security operations is complex, but it doesn't have to be. +[Shuffle](https://shuffler.io) is an automation platform for and by the community, focusing on accessibility for anyone to automate. Security operations is complex, but it doesn't have to be. [_Key Features_](https://shuffler.io/docs/features) — [_Community & Support_](https://discord.gg/B2CBzUm) — diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index f6a5a83e..75cceffb 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -3,6 +3,7 @@ import copy import sys import re import time +import base64 import json import liquid import logging @@ -14,17 +15,122 @@ import requests import http.client import urllib.parse import jinja2 +from io import StringIO as StringBuffer from io import BytesIO -from liquid import Liquid +from liquid import Liquid, defaults runtime = os.getenv("SHUFFLE_SWARM_CONFIG", "") +### +### +### +#### Filters for liquidpy +### +### +### + +defaults.MODE = 'wild' +defaults.FROM_FILE = False +from liquid.filters.manager import FilterManager +from liquid.filters.standard import standard_filter_manager + +shuffle_filters = FilterManager() +for key, value in standard_filter_manager.filters.items(): + shuffle_filters.filters[key] = value + +#@shuffle_filters.register +#def plus(a, b): +# try: +# a = int(a) +# except: +# a = 0 +# +# try: +# b = int(b) +# except: +# b = 0 +# +# return standard_filter_manager.filters["plus"](a, b) +# +#@shuffle_filters.register +#def minus(a, b): +# a = int(a) +# b = int(b) +# return standard_filter_manager.filters["minus"](a, b) +# +#@shuffle_filters.register +#def multiply(a, b): +# a = int(a) +# b = int(b) +# return standard_filter_manager.filters["multiply"](a, b) +# +#@shuffle_filters.register +#def divide(a, b): +# a = int(a) +# b = int(b) +# return standard_filter_manager.filters["divide"](a, b) + +@shuffle_filters.register +def md5(a): + a = str(a) + return hashlib.md5(a.encode('utf-8')).hexdigest() + +@shuffle_filters.register +def sha256(a): + a = str(a) + return hashlib.sha256(str(a).encode("utf-8")).hexdigest() + +@shuffle_filters.register +def md5_base64(a): + a = str(a) + foundhash = hashlib.md5(a.encode('utf-8')).hexdigest() + return base64.b64encode(foundhash.encode('utf-8')) + +@shuffle_filters.register +def base64_encode(a): + a = str(a) + try: + return base64.b64encode(a.encode('utf-8')).decode() + except: + return base64.b64encode(a).decode() + +@shuffle_filters.register +def base64_decode(a): + a = str(a) + try: + return base64.b64decode(a).decode() + except: + return base64.b64decode(a) + +#print(standard_filter_manager.filters) +#print(shuffle_filters.filters) +#print(Liquid("{{ '10' | plus: 1}}", filters=shuffle_filters.filters).render()) +#print(Liquid("{{ '10' | minus: 1}}", filters=shuffle_filters.filters).render()) +#print(Liquid("{{ asd | size }}", filters=shuffle_filters.filters).render()) +#print(Liquid("{{ 'asd' | md5 }}", filters=shuffle_filters.filters).render()) +#print(Liquid("{{ 'asd' | sha256 }}", filters=shuffle_filters.filters).render()) +#print(Liquid("{{ 'asd' | md5_base64 | base64_decode }}", filters=shuffle_filters.filters).render()) + +### +### +### +### +### +### +### + class AppBase: __version__ = None app_name = None def __init__(self, redis=None, logger=None, console_logger=None):#, docker_client=None): self.logger = logger if logger is not None else logging.getLogger("AppBaseLogger") + self.log_capture_string = StringBuffer() + ch = logging.StreamHandler(self.log_capture_string) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + ch.setFormatter(formatter) + logger.addHandler(ch) + self.redis=redis self.console_logger = logger if logger is not None else logging.getLogger("AppBaseLogger") @@ -139,15 +245,15 @@ class AppBase: try: new_input = input_data.split() except Exception as e: - self.logger.info(f"[ERROR] Failed to run magic parser (1): {e}") + self.logger.info(f"[ERROR] Failed to run magic parser during split (1): {e}") return input_data # Won't ever touch this one? - if isinstance(input_data, list) or isinstance(input_data, object): + if isinstance(new_input, list) or isinstance(new_input, object): try: return json.dumps(new_input) except Exception as e: - self.logger.info(f"[ERROR] Failed to run magic parser: {e}") + self.logger.info(f"[ERROR] Failed to run magic parser (3): {e}") return new_input @@ -165,12 +271,13 @@ class AppBase: else: self.logger.warning(f"[ERROR] Magic output not defined.") except Exception as e: - self.logger.warning(f"[ERROR] Failed to run magic autoparser: {e}") + self.logger.warning(f"[ERROR] Failed to run magic autoparser (send result): {e}") pass # Try it with some magic self.logger.info(f"""[DEBUG] Inside Send result with status {action_result["status"]}""") + #if isinstance(action_result, # FIXME: Add cleanup of parameters to not send to frontend here params = {} @@ -188,11 +295,64 @@ class AppBase: self.logger.info(f"[DEBUG] Before last stream result") url = "%s%s" % (self.base_url, stream_path) self.logger.info("[INFO] URL FOR RESULT (URL): %s" % url) + try: - ret = requests.post(url, headers=headers, json=action_result) - #self.logger.info(f"[DEBUG] Result: {ret.status_code}") - #if ret.status_code != 200: - # self.logger.info(f"[DEBUG] Shuffle Response: {ret.text}") + log_contents = self.log_capture_string.getvalue() + #print("RESULTS: %s" % log_contents) + self.logger.info("[WARNING] Got logs of length {len(log_contents)}") + if len(action_result["action"]["parameters"]) == 0: + action_result["action"]["parameters"] = [] + + param_found = False + for param in action_result["action"]["parameters"]: + if param["name"] == "shuffle_action_logs": + param_found = True + break + + if not param_found: + action_result["action"]["parameters"].append({ + "name": "shuffle_action_logs", + "value": log_contents, + }) + + except Exception as e: + print(f"Failed adding parameter: {e}") + + # FIXME: Adding retries here. + try: + finished = False + for i in range (0, 5): + try: + ret = requests.post(url, headers=headers, json=action_result, timeout=10) + + self.logger.info(f"[DEBUG] Result: {ret.status_code} (break on 200)") + if ret.status_code == 200 or ret.status_code == 201: + finished = True + break + else: + self.logger.info(f"[DEBUG] RESP: {ret.text}") + + except (requests.exceptions.RequestException, TimeoutError) as e: + time.sleep(5) + continue + except requests.exceptions.ConnectionError as e: + time.sleep(5) + continue + except http.client.RemoteDisconnected as e: + time.sleep(5) + continue + except urllib3.exceptions.ProtocolError as e: + time.sleep(5) + continue + + time.sleep(5) + + if not finished: + # Not sure why this would work tho :) + action_result["status"] = "FAILURE" + action_result["result"] = f"POST error: {e}" + self.logger.info(f"[DEBUG] Before typeerror stream result: {e}") + ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) self.logger.info(f"""[DEBUG] Successful request result request: Status= {ret.status_code} & Response= {ret.text}. Action status: {action_result["status"]}""") except requests.exceptions.ConnectionError as e: @@ -821,7 +981,7 @@ class AppBase: returns = [] for item in value: self.logger.info("VALUE: %s" % item) - if len(item) != 36: + if len(item) != 36 and not item.startswith("file_"): self.logger.info("Bad length for file value %s" % item) continue #return { @@ -924,6 +1084,7 @@ class AppBase: #return value.json() return {"success": False} + # Wrapper for set_files def set_file(self, infiles): return self.set_files(infiles) @@ -1010,19 +1171,23 @@ class AppBase: } # Simple validation of parameters in general + replace_params = False try: tmp_parameters = action["parameters"] + for param in tmp_parameters: + if param["value"] == "SHUFFLE_AUTO_REMOVED": + replace_params = True except KeyError: action["parameters"] = [] except TypeError: pass self.action = copy.deepcopy(action) - self.logger.info("[DEBUG] Sending starting action result (EXECUTING)") + self.logger.info(f"[DEBUG] Sending starting action result (EXECUTING). Param replace: {replace_params}") headers = { "Content-Type": "application/json", - "Authorization": "Bearer %s" % self.authorization + "Authorization": f"Bearer {self.authorization}" } if len(self.action) == 0: @@ -1118,6 +1283,32 @@ class AppBase: self.full_execution = fullexecution + + try: + if replace_params == True: + for inner_action in self.full_execution["workflow"]["actions"]: + self.logger.info("[DEBUG] ID: %s vs %s" % (inner_action["id"], self.action["id"])) + + # In case of some kind of magic, we're just doing params + if inner_action["id"] == self.action["id"]: + self.logger.info("FOUND!") + + if isinstance(self.action, str): + self.logger.info("Params is in string object for self.action?") + else: + self.action["parameters"] = inner_action["parameters"] + self.action_result["action"]["parameters"] = inner_action["parameters"] + + if isinstance(self.original_action, str): + self.logger.info("Params for original actions is in string object?") + else: + self.original_action["parameters"] = inner_action["parameters"] + + break + + except Exception as e: + self.logger.info(f"[WARNING] Failed in replace params action parsing: {e}") + self.logger.info("[DEBUG] AFTER FULLEXEC stream result (init)") # Gets the value at the parenthesis level you want @@ -1489,6 +1680,11 @@ class AppBase: newvalue = [] firstitem = actualitem[0][0] seconditem = actualitem[0][1] + if isinstance(firstitem, int): + firstitem = str(firstitem) + if isinstance(seconditem, int): + seconditem = str(seconditem) + print("[DEBUG] ACTUAL PARSED: %s" % actualitem) # Means it's a single item -> continue @@ -1509,17 +1705,23 @@ class AppBase: newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:]) else: print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem)) - if firstitem.lower() == "max" or firstitem.lower() == "last" or firstitem.lower() == "end": - firstitem = len(basejson)-1 - elif firstitem.lower() == "min" or firstitem.lower() == "first": - firstitem = 0 + if isinstance(firstitem, str): + if firstitem.lower() == "max" or firstitem.lower() == "last" or firstitem.lower() == "end": + firstitem = len(basejson)-1 + elif firstitem.lower() == "min" or firstitem.lower() == "first": + firstitem = 0 + else: + firstitem = int(firstitem) else: firstitem = int(firstitem) - if seconditem.lower() == "max" or seconditem.lower() == "last" or firstitem.lower() == "end": - seconditem = len(basejson)-1 - elif seconditem.lower() == "min" or seconditem.lower() == "first": - seconditem = 0 + if isinstance(seconditem, str): + if seconditem.lower() == "max" or seconditem.lower() == "last" or firstitem.lower() == "end": + seconditem = len(basejson)-1 + elif seconditem.lower() == "min" or seconditem.lower() == "first": + seconditem = 0 + else: + seconditem = int(seconditem) else: seconditem = int(seconditem) @@ -1704,7 +1906,7 @@ class AppBase: basejson = json.loads(baseresult) except json.decoder.JSONDecodeError as e: try: - baseresult = baseresult.replace("\'", "\"") + #baseresult = baseresult.replace("\'", "\"") basejson = json.loads(baseresult) except json.decoder.JSONDecodeError as e: print("Parser issue with JSON: %s" % e) @@ -1768,9 +1970,10 @@ class AppBase: # return template #self.logger.info(globals()) - self.logger.info("[DEBUG] Running liquid with data of length %d" % len(template)) + if len(template) > 100: + self.logger.info("[DEBUG] Running liquid with data of length %d" % len(template)) #self.logger.info(f"[DEBUG] Data: {template}") - run = Liquid(template, mode="wild", from_file=False) + run = Liquid(template, mode="wild", from_file=False, filters=shuffle_filters.filters) # Can't handle self yet (?) ret = run.render(**globals()) @@ -1822,8 +2025,8 @@ class AppBase: self.action_result["status"] = "FAILURE" data = { "success": False, - "input": template, "reason": f"Failed to parse LiquidPy: {error_msg}", + "input": template, } try: self.action_result["result"] = json.dumps(data) @@ -1945,7 +2148,11 @@ class AppBase: except: self.logger.info("Error in initial replacement of escaped dollar!") - #self.logger.info("POST input value: %s" % parameter["value"]) + # Basic fix in case variant isn't set + try: + self.logger.info("[DEBUG] Parameter variant: %s" % parameter["variant"]) + except: + parameter["variant"] = "STATIC_VALUE" # Regex to find all the things if parameter["variant"] == "STATIC_VALUE": @@ -2176,7 +2383,7 @@ class AppBase: return True else: print("[DEBUG] Condition: can't handle %s yet. Setting to true" % check) - + return False def check_branch_conditions(action, fullexecution, self): @@ -2187,21 +2394,48 @@ class AppBase: except KeyError: return True, "" + + available_checks = [ + "=", + "equals", + "!=", + "does not equal", + ">", + "larger than", + "<", + "less than", + ">=", + "<=", + "startswith", + "endswith", + "contains", + "contains_any_of", + "re", + "matches regex", + ] + relevantbranches = [] + correct_branches = 0 + matching_branches = 0 for branch in fullexecution["workflow"]["branches"]: if branch["destination_id"] != action["id"]: continue + matching_branches += 1 # Remove anything without a condition try: if (branch["conditions"]) == 0 or branch["conditions"] == None: + correct_branches += 1 continue except KeyError: + correct_branches += 1 continue self.logger.info("[DEBUG] Relevant conditions: %s" % branch["conditions"]) successful_conditions = [] failed_conditions = [] + successful_conditions = 0 + total_conditions = len(branch["conditions"]) for condition in branch["conditions"]: self.logger.info("[DEBUG] Getting condition value of %s" % condition) @@ -2209,6 +2443,7 @@ class AppBase: sourcevalue = condition["source"]["value"] check, sourcevalue, is_loop = parse_params(action, fullexecution, condition["source"], self) if check: + continue return False, {"success": False, "reason": "Failed condition (1): %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)} #sourcevalue = sourcevalue.encode("utf-8") @@ -2217,28 +2452,11 @@ class AppBase: check, destinationvalue, is_loop = parse_params(action, fullexecution, condition["destination"], self) if check: + continue return False, {"success": False, "reason": "Failed condition (2): %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)} #destinationvalue = destinationvalue.encode("utf-8") destinationvalue = parse_wrapper_start(destinationvalue, self) - available_checks = [ - "=", - "equals", - "!=", - "does not equal", - ">", - "larger than", - "<", - "less than", - ">=", - "<=", - "startswith", - "endswith", - "contains", - "contains_any_of", - "re", - "matches regex", - ] if not condition["condition"]["value"] in available_checks: self.logger.warning("Skipping %s %s %s because %s is invalid." % (sourcevalue, condition["condition"]["value"], destinationvalue, condition["condition"]["value"])) @@ -2255,22 +2473,58 @@ class AppBase: except KeyError: pass - if not validation: - self.logger.info("Failed condition check for %s %s %s." % (sourcevalue, condition["condition"]["value"], destinationvalue)) - return False, {"success": False, "reason": "Failed condition (3): %s %s %s" % (sourcevalue, condition["condition"]["value"], destinationvalue)} + if validation == True: + successful_conditions += 1 + #if not validation: + # self.logger.info("Failed condition check for %s %s %s." % (sourcevalue, condition["condition"]["value"], destinationvalue)) + # return False, {"success": False, "reason": "Failed condition (3): %s %s %s" % (sourcevalue, condition["condition"]["value"], destinationvalue)} - # Make a general parser here, at least to get param["name"] = param["value"] in maparameter[string]string - #for condition in branch.conditons: + self.logger.info("CONDITIONS VS SUCCESS: %d vs %d" % (total_conditions, successful_conditions)) + + if total_conditions == successful_conditions: + correct_branches += 1 + if matching_branches == 0: + return True, "" + + if matching_branches > 0 and correct_branches > 0: + return True, "" + + self.logger.info("[DEBUG] Correct branches vs matching branches: %d vs %d" % (correct_branches, matching_branches)) + return False, {"success": False, "reason": "Minimum of one branch's conditions must be correct to continue. Total: %d of %d" % (correct_branches, matching_branches)} + + #Correct branches vs matching branches: 1 vs 1 + #if return True, "" + # + # + # + # + # CONT + # CONT + # CONT + # CONT + # CONT + # CONT + # CONT + # CONT + # CONT + # CONT + # CONT + # CONT + # CONT + # + # + # + # # THE START IS ACTUALLY RIGHT HERE :O # Checks whether conditions are met, otherwise set branchcheck, tmpresult = check_branch_conditions(action, fullexecution, self) - if isinstance(tmpresult, object) or isinstance(tmpresult, list): + if isinstance(tmpresult, object) or isinstance(tmpresult, list) or isinstance(tmpresult, dict): self.logger.info("[DEBUG] Fixing branch return as object -> string") try: #tmpresult = tmpresult.replace("'", "\"") @@ -2278,19 +2532,14 @@ class AppBase: except json.decoder.JSONDecodeError as e: self.logger.info(f"[WARNING] Failed condition parsing {tmpresult} to string") + # IF branches fail: Exit! if not branchcheck: self.logger.info("Failed one or more branch conditions.") self.action_result["result"] = tmpresult self.action_result["status"] = "SKIPPED" - try: - ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=self.action_result) - self.logger.info("Result: %d" % ret.status_code) - if ret.status_code != 200: - self.logger.info(ret.text) - except requests.exceptions.ConnectionError as e: - self.logger.exception(e) + self.action_result["completed_at"] = int(time.time()) - self.logger.info("\n\n[DEBUG] RETURNING BECAUSE A BRANCH FAILED: %s\n\n" % tmpresult) + self.send_result(self.action_result, headers, stream_path) return # Replace name cus there might be issues @@ -2612,7 +2861,13 @@ class AppBase: for i in range(0, curminlength): tmpitem = json.loads(json.dumps(parameter["value"])) for key, value in replacements.items(): - replacement = json.dumps(json.loads(value)[i]) + replacement = value + try: + replacement = json.dumps(json.loads(value)[i]) + except IndexError as e: + self.logger.info(f"[ERROR] Failed handling value parsing with index: {e}") + pass + if replacement.startswith("\"") and replacement.endswith("\""): replacement = replacement[1:len(replacement)-1] #except json.decoder.JSONDecodeError as e: @@ -2667,13 +2922,20 @@ class AppBase: multi_parameters[parameter["name"]] = resultarray else: # Parses things like int(value) - self.logger.info("[DEBUG] Normal parsing (not looping)")#with data %s" % value) + #self.logger.info("[DEBUG] Normal parsing (not looping)")#with data %s" % value) # This part has fucked over so many random JSON usages because of weird paranthesis parsing value = parse_wrapper_start(value, self) #self.logger.info("[DEBUG] Post return: %s" % value) #self.logger.info("POST data value: %s" % value) + + try: + if str(value).startswith("b'") and str(value).endswith("'"): + value = value[2:-1] + except Exception as e: + print(f"Value rawbytes Exception: {e}") + params[parameter["name"]] = value multi_parameters[parameter["name"]] = value @@ -2803,7 +3065,7 @@ class AppBase: except Exception as e: self.logger.warning("[ERROR] Failed to parse coroutine value for old app: {e}") - self.logger.info("\n[INFO] Returned from execution with types %s" % type(newres)) + self.logger.info("\n[INFO] Returned from execution with type(s) %s" % type(newres)) #self.logger.info("\n[INFO] Returned from execution with %s of types %s" % (newres, type(newres)))#, newres) if isinstance(newres, tuple): self.logger.info(f"[INFO] Handling return as tuple: {newres}") @@ -2946,6 +3208,11 @@ class AppBase: # Send the result :) self.send_result(self.action_result, headers, stream_path) + try: + self.log_capture_string.close() + except: + pass + return @classmethod diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 198177c4..4cfe3e2c 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -3,7 +3,7 @@ ### DEFAULT NAME=shuffle-app_sdk -VERSION=0.9.50 +VERSION=0.9.61 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly @@ -19,17 +19,17 @@ docker push ghcr.io/frikky/$NAME:nightly docker push ghcr.io/frikky/$NAME:latest #### KALI ### -NAME=shuffle-app_sdk_kali -docker build . -f Dockerfile_kali -t frikky/shuffle:app_sdk_kali -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION - -docker push frikky/shuffle:app_sdk_kali -docker push ghcr.io/frikky/$NAME:$VERSION -docker push ghcr.io/frikky/$NAME:nightly +#NAME=shuffle-app_sdk_kali +#docker build . -f Dockerfile_kali -t frikky/shuffle:app_sdk_kali -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION +# +#docker push frikky/shuffle:app_sdk_kali +#docker push ghcr.io/frikky/$NAME:$VERSION +#docker push ghcr.io/frikky/$NAME:nightly ### BLACKARCH ### -NAME=shuffle-app_sdk_blackarch -docker build . -f Dockerfile_blackarch -t frikky/shuffle:app_sdk_blackarch -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION - -docker push frikky/shuffle:app_sdk_blackarch -docker push ghcr.io/frikky/$NAME:$VERSION -docker push ghcr.io/frikky/$NAME:nightly +#NAME=shuffle-app_sdk_blackarch +#docker build . -f Dockerfile_blackarch -t frikky/shuffle:app_sdk_blackarch -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION +# +#docker push frikky/shuffle:app_sdk_blackarch +#docker push ghcr.io/frikky/$NAME:$VERSION +#docker push ghcr.io/frikky/$NAME:nightly diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 9b535c72..25acc65a 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -14,6 +14,7 @@ import ( "encoding/json" "errors" "fmt" + //"github.com/docker/docker" "github.com/docker/docker/api/types" //"github.com/docker/docker/api/types/container" @@ -417,7 +418,7 @@ func stopWebhook(image string, identifier string) error { // Starts a new webhook func handleStopHookDocker(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -502,7 +503,7 @@ var webhook = `{ // Starts a new webhook func handleDeleteHookDocker(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -619,7 +620,7 @@ func hookTest() { //https://stackoverflow.com/questions/23935141/how-to-copy-docker-images-from-one-host-to-another-without-using-a-repository func getDockerImage(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 0eb5e3b3..a0289ae9 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -1,34 +1,35 @@ module main -go 1.15 +go 1.16 -replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi //replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch require ( cloud.google.com/go/datastore v1.6.0 - cloud.google.com/go/pubsub v1.17.0 + cloud.google.com/go/iam v0.1.1 // indirect + cloud.google.com/go/pubsub v1.17.1 cloud.google.com/go/storage v1.18.2 github.com/basgys/goxml2json v1.1.0 github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 - github.com/docker/docker v20.10.9+incompatible + github.com/docker/docker v20.10.12+incompatible github.com/frikky/kin-openapi v0.41.0 - github.com/fsouza/go-dockerclient v1.7.4 + github.com/fsouza/go-dockerclient v1.7.7 github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.3.1 github.com/go-git/go-git/v5 v5.4.2 github.com/gorilla/mux v1.8.0 - github.com/h2non/filetype v1.1.1 + github.com/h2non/filetype v1.1.3 + github.com/nirasan/go-oauth-pkce-code-verifier v0.0.0-20170819232839-0fbfe93532da // indirect github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.1.79 - github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect + github.com/shuffle/shuffle-shared v0.2.7 go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 - google.golang.org/api v0.58.0 + golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce + google.golang.org/api v0.65.0 google.golang.org/appengine v1.6.7 - google.golang.org/grpc v1.41.0 + google.golang.org/grpc v1.43.0 gopkg.in/src-d/go-git.v4 v4.13.1 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 9e9473c5..a8b52797 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1,14 +1,17 @@ package main import ( + uuid "github.com/satori/go.uuid" "github.com/shuffle/shuffle-shared" "bufio" "bytes" "context" "crypto/md5" + //"crypto/tls" //"crypto/x509" + "encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -16,11 +19,13 @@ import ( "io" "io/ioutil" "log" + "math/rand" "net/http" "net/url" "os" "os/exec" "path/filepath" + //"regexp" "strings" "time" @@ -49,12 +54,14 @@ import ( "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" + + //cv "github.com/nirasan/go-oauth-pkce-code-verifier" + //githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http" // Random xj "github.com/basgys/goxml2json" newscheduler "github.com/carlescere/scheduler" - "github.com/satori/go.uuid" "golang.org/x/crypto/bcrypt" "gopkg.in/yaml.v3" @@ -706,7 +713,7 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) } func handleRegister(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -835,7 +842,7 @@ func handleCookie(request *http.Request) bool { } func handleInfo(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1118,7 +1125,7 @@ func increaseStatisticsField(ctx context.Context, fieldname, id string, amount i // FIXME - forward this to emails or whatever CRM system in use func handleContact(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1163,8 +1170,17 @@ func handleContact(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true, "message": "Thanks for reaching out. We will contact you soon!"}`))) } +func verifier() (*shuffle.CodeVerifier, error) { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + b := make([]byte, 32, 32) + for i := 0; i < 32; i++ { + b[i] = byte(r.Intn(255)) + } + return shuffle.CreateCodeVerifierFromBytes(b) +} + func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1186,14 +1202,74 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { return } - //ssoUrl = org.SSOConfig.SOSOEntrypoint - redirectUri := shuffle.SSOUrl + baseSSOUrl := "" + handled := []string{} + for _, user := range users { + if shuffle.ArrayContains(handled, user.ActiveOrg.Id) { + continue + } + + handled = append(handled, user.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) + if err != nil { + log.Printf("[WARNING] Error getting org in admin check: %s", err) + continue + } + + // No childorg setup, only parent org + if len(org.ManagerOrgs) > 0 || len(org.CreatorOrg) > 0 { + continue + } + + // Should run calculations + if len(org.SSOConfig.OpenIdAuthorization) > 0 { + log.Printf("[DEBUG] Found OpenID url (PKCE). Extra redirect check: %s", request.URL.String()) + baseSSOUrl = org.SSOConfig.OpenIdAuthorization + + codeChallenge := uuid.NewV4().String() + //h.Write([]byte(v.Value)) + verifier, verifiererr := verifier() + if verifiererr == nil { + codeChallenge = verifier.Value + } + + //log.Printf("[DEBUG] Got challenge value %s (pre state)", codeChallenge) + + // https://192.168.55.222:3443/api/v1/login_openid + //location := strings.Split(request.URL.String(), "/") + //redirectUrl := url.QueryEscape("http://localhost:5001/api/v1/login_openid") + redirectUrl := url.QueryEscape(fmt.Sprintf("http://%s/api/v1/login_openid", request.Host)) + if strings.Contains(request.Host, "shuffle-backend") && !strings.Contains(os.Getenv("BASE_URL"), "shuffle-backend") { + redirectUrl = url.QueryEscape(fmt.Sprintf("%s/api/v1/login_openid", os.Getenv("BASE_URL"))) + } + + state := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("org=%s&challenge=%s&redirect=%s", org.Id, codeChallenge, redirectUrl))) + + // has to happen after initial value is stored + if verifiererr == nil { + codeChallenge = verifier.CodeChallengeS256() + } + + //log.Printf("[DEBUG] Got challenge value %s (POST state)", codeChallenge) + + baseSSOUrl += fmt.Sprintf("?client_id=%s&response_type=code&scope=openid&redirect_uri=%s&state=%s&code_challenge_method=S256&code_challenge=%s", org.SSOConfig.OpenIdClientId, redirectUrl, state, codeChallenge) + break + } + + if len(org.SSOConfig.SSOEntrypoint) > 0 { + log.Printf("[DEBUG] Found SAML SSO url") + baseSSOUrl = org.SSOConfig.SSOEntrypoint + break + } + } + + //log.Printf("[DEBUG] OpenID URL: %s", baseSSOUrl) resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "redirect", "sso_url": "%s"}`, redirectUri))) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "redirect", "sso_url": "%s"}`, baseSSOUrl))) } func handleLogin(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1418,8 +1494,12 @@ func fixUserOrg(ctx context.Context, user *shuffle.User) *shuffle.User { } // Used for testing only. Shouldn't impact production. -func handleCors(resp http.ResponseWriter, request *http.Request) bool { - allowedOrigins := "http://localhost:3000" +/* +func shuffle.HandleCors(resp http.ResponseWriter, request *http.Request) bool { + // Used for Codespace dev + allowedOrigins := "https://frikky-shuffle-5gvr4xx62w64-3000.githubpreview.dev" + //origin := request.Header["Origin"] + //log.Printf("Origin: %s", origin) //allowedOrigins := "http://localhost:3002" resp.Header().Set("Vary", "Origin") @@ -1436,6 +1516,7 @@ func handleCors(resp http.ResponseWriter, request *http.Request) bool { return false } +*/ func parseWorkflowParameters(resp http.ResponseWriter, request *http.Request) (map[string]interface{}, error) { body, err := ioutil.ReadAll(request.Body) @@ -1580,7 +1661,7 @@ func SearchNested(obj interface{}, key string) (interface{}, bool) { } func handleSetHook(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1768,7 +1849,7 @@ func verifyHook(hook shuffle.Hook) (bool, string) { } func setSpecificSchedule(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1828,7 +1909,7 @@ func setSpecificSchedule(resp http.ResponseWriter, request *http.Request) { } func getSpecificWebhook(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1880,7 +1961,7 @@ func getSpecificWebhook(resp http.ResponseWriter, request *http.Request) { // Starts a new webhook func handleDeleteSchedule(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1935,7 +2016,7 @@ func handleDeleteSchedule(resp http.ResponseWriter, request *http.Request) { // Starts a new webhook func handleNewSchedule(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -2229,7 +2310,7 @@ func getSpecificSchedule(resp http.ResponseWriter, request *http.Request) { return } - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -2295,7 +2376,7 @@ func loadYaml(fileLocation string) (ApiYaml, error) { // This should ALWAYS come from an OUTPUT func executeSchedule(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -2788,7 +2869,7 @@ type Result struct { // r.HandleFunc("/api/v1/docs/{key}", getDocs).Methods("GET", "OPTIONS") func getOpenapi(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -3254,7 +3335,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) { // Creates an app from the app builder func verifySwagger(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -3316,11 +3397,6 @@ func createFs(basepath, pathname string) (billy.Filesystem, error) { return err } - //if strings.Contains(path, "yaml") { - // log.Printf("PATH: %s -> %s", path, fullpath) - // //log.Printf("DATA: %s", string(srcData)) - //} - dst, err := fs.Create(fullpath) if err != nil { log.Printf("Dst error: %s", err) @@ -3863,7 +3939,7 @@ func runInitEs(ctx context.Context) { } for _, schedule := range schedules { - if schedule.Environment == "cloud" { + if strings.ToLower(schedule.Environment) == "cloud" { log.Printf("Skipping cloud schedule") continue } @@ -4999,7 +5075,7 @@ func handleStopCloudSync(syncUrl string, org shuffle.Org) (*shuffle.Org, error) This is here to both enable and disable cloud sync features for an organization */ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -5101,8 +5177,19 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { _, err = handleStopCloudSync(syncPath, *org) if err != nil { + ret := shuffle.ResultChecker{ + Success: false, + Reason: fmt.Sprintf("%s", err), + } + resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + b, err := json.Marshal(ret) + if err != nil { + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + + resp.Write(b) } else { resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Successfully disabled cloud sync for org."}`))) @@ -5687,10 +5774,15 @@ func initHandlers() { dbclient, err = datastore.NewClient(ctx, gceProject, option.WithGRPCDialOption(grpc.WithNoProxy())) if err != nil { if elasticConfig == "" { - log.Fatalf("[ERROR] Database client error during init: %s. Env: SHUFFLE_ELASTIC=false", err) + log.Printf("[ERROR] Database client error during init: %s. Env: SHUFFLE_ELASTIC=false", err) } else { - log.Printf("[DEBUG] Database client error during init: %s. Here for backwards compatibility: not critical.", err) + if !strings.Contains(fmt.Sprintf("%s", err), "find default credentials") { + log.Printf("[DEBUG] Database client error info during init: %s. Here for backwards compatibility: not critical.", err) + } + dbclient = &datastore.Client{} } + } else { + //log.Printf("Database client initiated: %s", dbclient) } for { @@ -5783,10 +5875,13 @@ func initHandlers() { r.HandleFunc("/api/v1/apps/authentication/{appauthId}/config", shuffle.SetAuthenticationConfig).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/authentication/{appauthId}", shuffle.DeleteAppAuthentication).Methods("DELETE", "OPTIONS") - // Related to + // Related to NFT things r.HandleFunc("/api/v1/workflows/collections/load", shuffle.LoadCollections).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/collections/{key}", shuffle.HandleGetCollection).Methods("GET", "OPTIONS") + // Related to use-cases that are not directly workflows. + r.HandleFunc("/api/v1/workflows/usecases", shuffle.LoadUsecases).Methods("GET", "OPTIONS") + // Legacy app things r.HandleFunc("/api/v1/workflows/apps/validate", validateAppInput).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/apps", getWorkflowApps).Methods("GET", "OPTIONS") @@ -5868,7 +5963,8 @@ func initHandlers() { // Docker orborus specific - downloads an image r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/migrate_database", migrateDatabase).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/login_sso", shuffle.HandleSSO).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/login_sso", shuffle.HandleSSO).Methods("GET", "POST", "OPTIONS") + r.HandleFunc("/api/v1/login_openid", shuffle.HandleOpenId).Methods("GET", "OPTIONS") // Important for email, IDS etc. Create this by: // PS: For cloud, this has to use cloud storage. @@ -5887,6 +5983,9 @@ func initHandlers() { r.HandleFunc("/api/v1/notifications/clear", shuffle.HandleClearNotifications).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS") //r.HandleFunc("/api/v1/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/users/notifications", shuffle.HandleGetNotifications).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/users/notifications/clear", shuffle.HandleClearNotifications).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/users/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS") http.Handle("/", r) } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index c05c380d..27ca1615 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -21,6 +21,7 @@ import ( "github.com/docker/docker/api/types" dockerclient "github.com/docker/docker/client" + //gyaml "github.com/ghodss/yaml" "github.com/h2non/filetype" @@ -34,6 +35,7 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" + //"github.com/gorilla/websocket" //"google.golang.org/appengine" //"google.golang.org/appengine/memcache" @@ -142,7 +144,7 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode } func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -243,7 +245,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque // FIXME: Authenticate this one? Can org ID be auth enough? // (especially since we have a default: shuffle) func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -335,7 +337,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -393,7 +395,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { } func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -666,7 +668,7 @@ func handleExecutionStatistics(execution shuffle.WorkflowExecution) { } func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1034,7 +1036,7 @@ func cloudExecuteAction(execution shuffle.WorkflowExecution) error { } func executeWorkflow(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1131,7 +1133,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { } func stopSchedule(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1281,7 +1283,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { } func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1390,7 +1392,7 @@ func deleteSchedule(ctx context.Context, id string) error { } func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1641,7 +1643,7 @@ func setExampleresult(ctx context.Context, result shuffle.AppExecutionExample) e } func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1714,7 +1716,7 @@ func handleGetfile(resp http.ResponseWriter, request *http.Request) ([]byte, err // Basically a search for apps that aren't activated yet func getSpecificApps(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1789,7 +1791,7 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) { } func validateAppInput(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1910,7 +1912,7 @@ func loadGithubWorkflows(url, username, password, userId, branch, orgId string) } func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1972,7 +1974,7 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { } func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -2290,7 +2292,7 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra } func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } diff --git a/backend/tests/files.sh b/backend/tests/files.sh index 64417941..91171924 100755 --- a/backend/tests/files.sh +++ b/backend/tests/files.sh @@ -3,7 +3,7 @@ #curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -d '{"filename": "file.txt", "org_id": "b199646b-16d2-456d-9fd6-b9972e929466", "workflow_id": "global"}' # #echo -#curl http://localhost:5001/api/v1/files/e19cffe4-e2da-47e9-809e-904f5cb03687/upload -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -F 'shuffle_file=@files.sh' +curl http://localhost:5001/api/v1/apps/upload -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -F 'shuffle_file=@files.sh' # #curl http://localhost:5001/api/v1/files/1915981b-b897-4db1-8a2e-44bc34cead3b/content -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" #curl http://localhost:5001/api/v1/files/e19cffe4-e2da-47e9-809e-904f5cb03687 -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" diff --git a/docker-compose.yml b/docker-compose.yml index d5c27571..02e07e9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,7 +48,6 @@ services: - /var/run/docker.sock:/var/run/docker.sock environment: - SHUFFLE_WORKER_VERSION=nightly - - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} - DOCKER_API_VERSION=1.40 @@ -64,13 +63,12 @@ services: - SHUFFLE_SWARM_CONFIG=runn restart: unless-stopped opensearch: - image: opensearchproject/opensearch:1.2.3 + image: opensearchproject/opensearch:1.2.4 hostname: shuffle-opensearch container_name: shuffle-opensearch environment: - bootstrap.memory_lock=true - - "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - - plugins.security.disabled=true + - "OPENSEARCH_JAVA_OPTS=-Xms2048m -Xmx2048m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - cluster.routing.allocation.disk.threshold_enabled=false - cluster.name=shuffle-cluster - node.name=shuffle-opensearch @@ -96,4 +94,3 @@ networks: driver: bridge #driver: overlay -#driver: bridge diff --git a/frontend/package.json b/frontend/package.json index 5766f14f..e7ea8871 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,10 +1,11 @@ { "name": "shuffler", "homepage": "https://shuffler.io", - "version": "0.9.50", + "version": "0.9.61", "private": true, "dependencies": { "@babel/core": "^7.15.8", + "@emotion/is-prop-valid": "^1.1.1", "@emotion/react": "^11.7.0", "@emotion/styled": "^11.6.0", "@material-ui/core": "^4.5.2", @@ -66,6 +67,7 @@ "react-scripts": "^4.0.1", "react-shepherd": "^3.3.6", "reactstrap": "^7.1.0", + "reaviz": "^12.1.0", "shellwords": "^0.1.1", "simplebar": "^4.2.3", "styled-components": "^4.4.0", diff --git a/frontend/public/images/detectionframework.png b/frontend/public/images/detectionframework.png new file mode 100644 index 00000000..30da65a5 Binary files /dev/null and b/frontend/public/images/detectionframework.png differ diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index d27cdc39..424eb9d5 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -15,7 +15,7 @@ import theme from "./theme"; import Apps from "./views/Apps"; import AppCreator from "./views/AppCreator"; -import Dashboard from "./views/Dashboard"; +import Dashboard from "./views/Dashboard.jsx"; import AdminSetup from "./views/AdminSetup"; import Admin from "./views/Admin"; import Docs from "./views/Docs"; @@ -49,6 +49,12 @@ if (window.location.port === "3000") { //globalUrl = "http://localhost:5002" } +if (globalUrl.includes("githubpreview.dev")) { + //globalUrl = globalUrl.replace("3000", "5001") + globalUrl = "https://frikky-shuffle-5gvr4xx62w64-5001.githubpreview.dev" +} +console.log("global: ", globalUrl) + const App = (message, props) => { const [userdata, setUserData] = useState({}); @@ -70,7 +76,7 @@ const App = (message, props) => { checkLogin(); setDataset(true); } - }); + }, []); if ( isLoaded && @@ -78,17 +84,19 @@ const App = (message, props) => { !window.location.pathname.startsWith("/login") && !window.location.pathname.startsWith("/docs") && !window.location.pathname.startsWith("/detectionframework") && - !window.location.pathname.startsWith("/adminsetup") + !window.location.pathname.startsWith("/adminsetup") && + !window.location.pathname.startsWith("/usecases") ) { window.location = "/login"; } const getUserNotifications = () => { - fetch(`${globalUrl}/api/v1/notifications`, { + fetch(`${globalUrl}/api/v1/users/notifications`, { credentials: "include", headers: { "Content-Type": "application/json", }, + cors: "cors", }) .then((response) => response.json()) .then((responseJson) => { @@ -109,7 +117,7 @@ const App = (message, props) => { const checkLogin = () => { var baseurl = globalUrl; - fetch(baseurl + "/api/v1/users/getinfo", { + fetch(`${globalUrl}/api/v1/getinfo`, { credentials: "include", headers: { "Content-Type": "application/json", @@ -399,7 +407,7 @@ const App = (message, props) => { /> { {action.must_activate ? ( diff --git a/frontend/src/components/DetectionFramework.jsx b/frontend/src/components/DetectionFramework.jsx index e66d09b7..2d7fc528 100644 --- a/frontend/src/components/DetectionFramework.jsx +++ b/frontend/src/components/DetectionFramework.jsx @@ -1499,7 +1499,7 @@ const Framework = (props) => { /> :
- TBD: Coming in 1.0.0. + Coming in 1.0.0. Register for Shuffle cloud to try an early version now.
: null} diff --git a/frontend/src/components/Header.js b/frontend/src/components/Header.js index 773b5d2b..938e9b22 100644 --- a/frontend/src/components/Header.js +++ b/frontend/src/components/Header.js @@ -35,6 +35,7 @@ import { import { Analytics as AnalyticsIcon, + Lightbulb as LightbulbIcon, } from "@mui/icons-material"; //import LogoutIcon from '@mui/icons-material/Logout'; import { useAlert } from "react-alert"; @@ -466,6 +467,16 @@ const Header = (props) => { Get Started + { + event.preventDefault(); + handleClose(); + }} + > + + Use Cases + + { event.preventDefault(); diff --git a/frontend/src/components/NestedMenuItem.jsx b/frontend/src/components/NestedMenuItem.jsx new file mode 100644 index 00000000..e48c2988 --- /dev/null +++ b/frontend/src/components/NestedMenuItem.jsx @@ -0,0 +1,202 @@ +import React, {useState, useRef, useImperativeHandle} from 'react' +import {makeStyles} from '@material-ui/core/styles' +import Menu, {MenuProps} from '@material-ui/core/Menu' +import MenuItem, {MenuItemProps} from '@material-ui/core/MenuItem' +import ArrowRight from '@material-ui/icons/ArrowRight' +import clsx from 'clsx' + +export interface NestedMenuItemProps extends Omit { + /** + * Open state of parent ``, used to close decendent menus when the + * root menu is closed. + */ + parentMenuOpen: boolean + /** + * Component for the container element. + * @default 'div' + */ + component?: React.ElementType + /** + * Effectively becomes the `children` prop passed to the `` + * element. + */ + label?: React.ReactNode + /** + * @default + */ + rightIcon?: React.ReactNode + /** + * Props passed to container element. + */ + ContainerProps?: React.HTMLAttributes & + React.RefAttributes + /** + * Props passed to sub `` element + */ + MenuProps?: Omit + /** + * @see https://material-ui.com/api/list-item/ + */ + button?: true | undefined +} + +const TRANSPARENT = 'rgba(0,0,0,0)' +const useMenuItemStyles = makeStyles((theme) => ({ + root: (props: any) => ({ + backgroundColor: props.open ? theme.palette.action.hover : TRANSPARENT + }) +})) + +/** + * Use as a drop-in replacement for `` when you need to add cascading + * menu elements as children to this component. + */ +const NestedMenuItem = React.forwardRef< + HTMLLIElement | null, + NestedMenuItemProps +>(function NestedMenuItem(props, ref) { + const { + parentMenuOpen, + component = 'div', + label, + rightIcon = , + children, + className, + tabIndex: tabIndexProp, + MenuProps = {}, + ContainerProps: ContainerPropsProp = {}, + ...MenuItemProps + } = props + + const {ref: containerRefProp, ...ContainerProps} = ContainerPropsProp + + const menuItemRef = useRef(null) + useImperativeHandle(ref, () => menuItemRef.current) + + const containerRef = useRef(null) + useImperativeHandle(containerRefProp, () => containerRef.current) + + const menuContainerRef = useRef(null) + + const [isSubMenuOpen, setIsSubMenuOpen] = useState(false) + + const handleMouseEnter = (event: React.MouseEvent) => { + setIsSubMenuOpen(true) + + if (ContainerProps?.onMouseEnter) { + ContainerProps.onMouseEnter(event) + } + } + const handleMouseLeave = (event: React.MouseEvent) => { + setIsSubMenuOpen(false) + + if (ContainerProps?.onMouseLeave) { + ContainerProps.onMouseLeave(event) + } + } + + // Check if any immediate children are active + const isSubmenuFocused = () => { + const active = containerRef.current?.ownerDocument?.activeElement + for (const child of menuContainerRef.current?.children ?? []) { + if (child === active) { + return true + } + } + return false + } + + const handleFocus = (event: React.FocusEvent) => { + if (event.target === containerRef.current) { + setIsSubMenuOpen(true) + } + + if (ContainerProps?.onFocus) { + ContainerProps.onFocus(event) + } + } + + const handleKeyDown = (event: React.KeyboardEvent) => { + if (event.key === 'Escape') { + return + } + + if (isSubmenuFocused()) { + event.stopPropagation() + } + + const active = containerRef.current?.ownerDocument?.activeElement + + if (event.key === 'ArrowLeft' && isSubmenuFocused()) { + containerRef.current?.focus() + } + + if ( + event.key === 'ArrowRight' && + event.target === containerRef.current && + event.target === active + ) { + const firstChild = menuContainerRef.current?.children[0] as + | HTMLElement + | undefined + firstChild?.focus() + } + } + + const open = isSubMenuOpen && parentMenuOpen + const menuItemClasses = useMenuItemStyles({open}) + + // Root element must have a `tabIndex` attribute for keyboard navigation + let tabIndex + if (!props.disabled) { + tabIndex = tabIndexProp !== undefined ? tabIndexProp : -1 + } + + return ( +
+ + {label} + {rightIcon} + + { + setIsSubMenuOpen(false) + }} + > +
+ {children} +
+ +
+ ) +}) + +export default NestedMenuItem diff --git a/frontend/src/components/Oauth2Auth.jsx b/frontend/src/components/Oauth2Auth.jsx index 627696e7..77fc31f5 100644 --- a/frontend/src/components/Oauth2Auth.jsx +++ b/frontend/src/components/Oauth2Auth.jsx @@ -49,6 +49,8 @@ const MenuProps = { scrollX: "auto", }, }, + variant: "menu", + getContentAnchorEl: null, }; const AuthenticationOauth2 = (props) => { @@ -84,6 +86,7 @@ const AuthenticationOauth2 = (props) => { const [oauthUrl, setOauthUrl] = React.useState(""); const [buttonClicked, setButtonClicked] = React.useState(false); const [selectedScopes, setSelectedScopes] = React.useState([]); + const [offlineAccess, setOfflineAccess] = React.useState(true); const allscopes = authenticationType.scope !== undefined ? authenticationType.scope : []; @@ -111,8 +114,19 @@ const AuthenticationOauth2 = (props) => { setButtonClicked(true); console.log("SCOPES: ", scopes); + client_id = client_id.trim() + client_secret = client_secret.trim() + oauth_url = oauth_url.trim() + var resources = ""; if (scopes !== undefined && (scopes !== null) & (scopes.length > 0)) { + if (offlineAccess === true && !scopes.includes("offline_access")) { + if (authenticationType.redirect_uri.includes("microsoft")) { + console.log("Appending offline access") + scopes.push("offline_access") + } + } + resources = scopes.join(" "); //resources = scopes.join(","); } @@ -496,34 +510,49 @@ const AuthenticationOauth2 = (props) => { }} /> {allscopes.length === 0 ? null : ( - - Scopes - } - renderValue={(selected) => selected.join(", ")} - MenuProps={MenuProps} - > - {allscopes.map((data, index) => { - return ( - - -1} /> - - - ); - })} - - +
+ + Scopes + } + renderValue={(selected) => selected.join(", ")} + MenuProps={MenuProps} + > + {allscopes.map((data, index) => { + return ( + + -1} /> + + + ); + })} + + + + + { + setOfflineAccess(!offlineAccess) + }}/> + + +
)} )} diff --git a/frontend/src/components/OrgHeader.jsx b/frontend/src/components/OrgHeader.jsx index 49897638..a370af93 100644 --- a/frontend/src/components/OrgHeader.jsx +++ b/frontend/src/components/OrgHeader.jsx @@ -97,6 +97,30 @@ const OrgHeader = (props) => { ? "" : selectedOrganization.defaults.notification_workflow ); + const [openidClientId, setOpenidClientId] = React.useState( + selectedOrganization.sso_config === undefined + ? "" + : selectedOrganization.sso_config.client_id === undefined || + selectedOrganization.sso_config.client_id.length === 0 + ? "" + : selectedOrganization.sso_config.client_id + ); + const [openidAuthorization, setOpenidAuthorization] = React.useState( + selectedOrganization.sso_config === undefined + ? "" + : selectedOrganization.sso_config.openid_authorization === undefined || + selectedOrganization.sso_config.openid_authorization.length === 0 + ? "" + : selectedOrganization.sso_config.openid_authorization + ); + const [openidToken, setOpenidToken] = React.useState( + selectedOrganization.sso_config === undefined + ? "" + : selectedOrganization.sso_config.openid_token === undefined || + selectedOrganization.sso_config.openid_token.length === 0 + ? "" + : selectedOrganization.sso_config.openid_token + ) const [file, setFile] = React.useState(""); const [fileBase64, setFileBase64] = React.useState( @@ -145,6 +169,7 @@ const OrgHeader = (props) => { defaults, sso_config ) => { + const data = { name: name, description: description, @@ -216,6 +241,9 @@ const OrgHeader = (props) => { { sso_entrypoint: ssoEntrypoint, sso_certificate: ssoCertificate, + client_id: openidClientId, + openid_authorization: openidAuthorization, + openid_token: openidToken, } ) } @@ -548,79 +576,200 @@ const OrgHeader = (props) => { )} - - - SSO Entrypoint (IdP) - 0 - } - id="outlined-with-placeholder" - margin="normal" - variant="outlined" - placeholder="The entrypoint URL from your provider" - value={ssoEntrypoint} - onChange={(e) => { - setSsoEntrypoint(e.target.value); - }} - InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, - style: { - color: "white", - }, - }} - /> - - - - - SSO Certificate (X509) - { - setSsoCertificate(e.target.value); - }} - InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, - style: { - color: "white", - }, - }} - /> - - + {isCloud ? null : + + OpenID connect + + + + Client ID + 0 + } + id="outlined-with-placeholder" + margin="normal" + variant="outlined" + placeholder="The OpenID client ID from the identity provider" + value={openidClientId} + onChange={(e) => { + setOpenidClientId(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + Authorization URL + { + setOpenidAuthorization(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + Token URL + { + setOpenidToken(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + } + {isCloud ? null : + + SAML SSO (v1.1) + + + + SSO Entrypoint (IdP) + 0 + } + id="outlined-with-placeholder" + margin="normal" + variant="outlined" + placeholder="The entrypoint URL from your provider" + value={ssoEntrypoint} + onChange={(e) => { + setSsoEntrypoint(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + SSO Certificate (X509) + { + setSsoCertificate(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + } {/* {expanded ? diff --git a/frontend/src/components/PaperComponent.jsx b/frontend/src/components/PaperComponent.jsx new file mode 100644 index 00000000..76fb56ef --- /dev/null +++ b/frontend/src/components/PaperComponent.jsx @@ -0,0 +1,19 @@ +import React, {useState, useEffect, useLayoutEffect} from 'react'; + +import Draggable from "react-draggable"; +import { + Paper +} from "@material-ui/core"; + +const PaperComponent = (props) => { + return ( + + + + ) +} + +export default PaperComponent; diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 93f582c3..a0d1002a 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -1,11 +1,12 @@ import React, { useState, useEffect, useLayoutEffect } from "react"; import { makeStyles, createStyles } from "@material-ui/core/styles"; +import { validateJson, GetIconInfo } from "../views/Workflows.jsx"; import { GetParsedPaths } from "../views/Apps.jsx"; -import { GetIconInfo } from "../views/Workflows.jsx"; import { sortByKey } from "../views/AngularWorkflow.jsx"; import { useTheme } from "@material-ui/core/styles"; import NestedMenuItem from "material-ui-nested-menu-item"; +import { useAlert } from "react-alert"; import theme from '../theme'; //import NestedMenuItem from "./NestedMenu.jsx"; @@ -168,6 +169,7 @@ const ParsedAction = (props) => { //const theme = useTheme(); const classes = useStyles(); + const alert = useAlert() const [expansionModalOpen, setExpansionModalOpen] = React.useState(false); const [hideBody, setHideBody] = React.useState(true); @@ -178,23 +180,21 @@ const ParsedAction = (props) => { const [hiddenDescription, setHiddenDescription] = React.useState(true); useEffect(() => { - - //if (data.startsWith("${") && data.endsWith("}")) { - //} - // PARAM FIX - Gonna use the ID field, even though it's a hack - const paramcheck = selectedAction.parameters.find(param => param.name === "body") - console.log("LOADED! Change hideBody based on input? Action: ", selectedAction, paramcheck) - if (paramcheck !== undefined && paramcheck !== null) { - if (paramcheck.id === "TOGGLED"){ - setHideBody(false) - setActivateHidingBodyButton(false) - console.log("TOGGLED BODY!") - } else { - setHideBody(true) - - if (paramcheck.id === "UNTOGGLED") { + if (selectedAction.parameters !== null && selectedAction.parameters !== undefined) { + const paramcheck = selectedAction.parameters.find(param => param.name === "body") + //console.log("LOADED! Change hideBody based on input? Action: ", selectedAction, paramcheck) + if (paramcheck !== undefined && paramcheck !== null) { + if (paramcheck.id === "TOGGLED"){ + setHideBody(false) setActivateHidingBodyButton(false) - console.log("UNTOGGLED!") + console.log("TOGGLED BODY!") + } else { + setHideBody(true) + + if (paramcheck.id === "UNTOGGLED") { + setActivateHidingBodyButton(false) + console.log("UNTOGGLED!") + } } } } @@ -304,6 +304,8 @@ const ParsedAction = (props) => { }); }; + + const defineStartnode = () => { if (cy === undefined) { return; @@ -391,14 +393,46 @@ const ParsedAction = (props) => { if (actionlist.length === 0) { // FIXME: Have previous execution values in here - actionlist.push({ - type: "Execution Argument", - name: "Execution Argument", - value: "$exec", - highlight: "exec", - autocomplete: "exec", - example: "", - }); + if (workflowExecutions.length > 0) { + for (var key in workflowExecutions) { + if ( + workflowExecutions[key].execution_argument === undefined || + workflowExecutions[key].execution_argument === null || + workflowExecutions[key].execution_argument.length === 0 + ) { + continue; + } + + console.log("EXEC: ", workflowExecutions[key].execution_argument) + + const valid = validateJson(workflowExecutions[key].execution_argument) + console.log("VALID: ", valid) + if (valid.valid) { + actionlist.push({ + type: "Execution Argument", + name: "Execution Argument", + value: "$exec", + highlight: "exec", + autocomplete: "exec", + example: valid.result, + }) + break + } + } + + } + + if (actionlist.length === 0) { + actionlist.push({ + type: "Execution Argument", + name: "Execution Argument", + value: "$exec", + highlight: "exec", + autocomplete: "exec", + example: "", + }) + } + actionlist.push({ type: "Shuffle DB", name: "Shuffle DB", @@ -454,7 +488,8 @@ const ParsedAction = (props) => { continue; } - var exampledata = item.example === undefined ? "" : item.example; + var exampledata = item.example === undefined || item.example === null ? "" : item.example; + console.log("EXAMPLE: ", exampledata) // Find previous execution and their variables //exampledata === "" && if (workflowExecutions.length > 0) { @@ -471,55 +506,22 @@ const ParsedAction = (props) => { var foundResult = workflowExecutions[key].results.find( (result) => result.action.id === item.id ); - if (foundResult === undefined) { + if (foundResult === undefined || foundResult === null) { continue; } - foundResult.result = foundResult.result.trim(); - foundResult.result = foundResult.result - .split(" None") - .join(' "None"'); - foundResult.result = foundResult.result - .split(" False") - .join(" false"); - foundResult.result = foundResult.result - .split(" True") - .join(" true"); + if (foundResult.result !== undefined && foundResult.result !== null) { + foundResult = foundResult.result + } + console.log("VALID RESULT: ", foundResult) - var jsonvalid = true; - try { - const tmp = String(JSON.parse(foundResult.result)); - if ( - !foundResult.result.includes("{") && - !foundResult.result.includes("[") - ) { - jsonvalid = false; - } - } catch (e) { - try { - foundResult.result = foundResult.result - .split("'") - .join('"'); - const tmp = String(JSON.parse(foundResult.result)); - if ( - !foundResult.result.includes("{") && - !foundResult.result.includes("[") - ) { - jsonvalid = false; - } - } catch (e) { - jsonvalid = false; - } - } - - // Finds the FIRST json only - if (jsonvalid) { - exampledata = JSON.parse(foundResult.result); + const valid = validateJson(foundResult) + if (valid.valid) { + exampledata = valid.result; break; - } - //else { - // console.log("Invalid JSON: ", foundResult.result) - //} + } else { + exampledata = foundResult; + } } } @@ -528,6 +530,7 @@ const ParsedAction = (props) => { item.label === null || item.label === undefined ? "" : item.label.split(" ").join("_"); + const actionvalue = { type: "action", id: item.id, @@ -539,11 +542,65 @@ const ParsedAction = (props) => { } } + //console.log("ACTIONLIST: ", actionlist) setActionlist(actionlist); } } }); + + const calculateHelpertext = (input_data) => { + var helperText = "" + var looperText = "" + const found = input_data.match(/[$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,}/g) + + if (found !== null) { + try { + // When the found array is empty. + for (var i = 0; i < found.length; i++) { + const variableSplit = found[i].split(".#") + if ((variableSplit.length-1) > 1) { + //console.log("Larger than 1: ", variableSplit) + if (looperText.length === 0) { + looperText += "PS: Double looping (.#) may cause problems." + } + } + + var foundSlice = false + for (var j = 0; j < actionlist.length; j++) { + //console.log("ACTION: ", found[i], actionlist[j]) + //console.log("ACTION :", found[i].split(".")[0].slice(1,).toLowerCase(), actionlist[j].autocomplete.toLowerCase()) + if(found[i].split(".")[0].slice(1,).toLowerCase() == actionlist[j].autocomplete.toLowerCase()){ + //console.log("Found: ", found[i]) + // Validate path? + + foundSlice = true + } + } + + if (!foundSlice) { + if (!helperText.includes("Invalid variables")) { + helperText+= "Invalid variables: " + } + helperText+= found[i] + ", " + } + } + } catch (e) { + console.log("Parsing error: ", e) + } + } + + if (looperText.length > 0) { + if (helperText.length > 0) { + helperText += ". " + } + + helperText += looperText + } + + return helperText + } + const changeActionParameter = (event, count, data) => { //console.log("Action change: ", selectedAction, data) if (data.name.startsWith("${") && data.name.endsWith("}")) { @@ -1282,21 +1339,27 @@ const ParsedAction = (props) => { const clickedFieldId = "rightside_field_" + count; - const shufflecode = + const shufflecode = fieldCount !== count ? null : + ( + + ) // 0) { + baseHelperText = calculateHelpertext(data.value) + } var datafield = ( { //changeActionParameterCodemirror(event, count, data) changeActionParameter(event, count, data); }} - helperText={ + helperText={baseHelperText.length > 0 ? baseHelperText : selectedApp.generated && selectedApp.activated && data.name === "body" ? ( @@ -1423,15 +1486,7 @@ const ParsedAction = (props) => { ) : null } onBlur={(event) => { - // Super basic check - //if (event.target.value.startsWith("{")) { - // console.log("VALIDATING JSON") - // try { - // JSON.parse(event.target.value) - // } catch (e) { - // alert.error("Failed to parse json: ", e) - // } - //} + baseHelperText = calculateHelpertext(event.target.value) }} /> ); @@ -1536,6 +1591,9 @@ const ParsedAction = (props) => { datafield = ( { selectedApp.versions !== undefined && selectedApp.versions.length > 1 ? ( { SelectDisplayProps={{ style: { marginLeft: 10, + maxWidth: 250, }, }} fullWidth @@ -2581,6 +2653,7 @@ const ParsedAction = (props) => { > { setAuthenticationModalOpen(true); @@ -2597,6 +2670,9 @@ const ParsedAction = (props) => {
Environment { {/*setNewSelectedAction !== undefined ? { console.log("CHANGE NAMESPACE: ", event.target); @@ -3185,7 +3185,7 @@ const Admin = (props) => { } return ( - + { style={{ minWidth: 125, maxWidth: 125, overflow: "hidden" }} /> @@ -3650,7 +3650,26 @@ const Admin = (props) => { bgColor = "#1f2023"; } - console.log("Auth data: ", data) + //console.log("Auth data: ", data) + if (data.type === "oauth2") { + data.fields = [ + { + "key": "url", + "value": "Secret. Replaced during app execution!", + }, + { + "key": "client_id", + "value": "Secret. Replaced during app execution!", + }, + { + "key": "client_secret", + "value": "Secret. Replaced during app execution!", + }, + { + "key": "scope", + "value": "Secret. Replaced during app execution!", + }] + } return ( @@ -3720,11 +3739,11 @@ const Admin = (props) => { /> { const [userSettings, setUserSettings] = React.useState({}); const [subworkflow, setSubworkflow] = React.useState({}); const [subworkflowStartnode, setSubworkflowStartnode] = React.useState(""); - const [leftViewOpen, setLeftViewOpen] = React.useState(true); - const [leftBarSize, setLeftBarSize] = React.useState(350); + const [leftViewOpen, setLeftViewOpen] = React.useState(isMobile ? false : true); + const [leftBarSize, setLeftBarSize] = React.useState(isMobile ? 0 : 350); + const [creatorProfile, setCreatorProfile] = React.useState({}); + const [appGroup, setAppGroup] = React.useState([]); + const [triggerGroup, setTriggerGroup] = React.useState([]); const [executionText, setExecutionText] = React.useState(""); const [executionRequestStarted, setExecutionRequestStarted] = React.useState(false); @@ -333,6 +346,7 @@ const AngularWorkflow = (defaultprops) => { const [executionModalOpen, setExecutionModalOpen] = React.useState(false); const [executionModalView, setExecutionModalView] = React.useState(0); const [executionData, setExecutionData] = React.useState({}); + const [appsLoaded, setAppsLoaded] = React.useState(false); const [lastSaved, setLastSaved] = React.useState(true); @@ -340,8 +354,7 @@ const AngularWorkflow = (defaultprops) => { const [_, setUpdate] = useState(""); // Used for rendring, don't remove const [workflowExecutions, setWorkflowExecutions] = React.useState([]); - const [defaultEnvironmentIndex, setDefaultEnvironmentIndex] = - React.useState(0); + const [defaultEnvironmentIndex, setDefaultEnvironmentIndex] = React.useState(0); // This should all be set once, not on every iteration // Use states and don't update lol @@ -358,6 +371,7 @@ const AngularWorkflow = (defaultprops) => { const triggerEnvironments = isCloud ? ["cloud"] : ["onprem", "cloud"]; const unloadText = "Are you sure you want to leave without saving (CTRL+S)?"; const classes = useStyles(); + const cytoscapeWidth = isMobile ? bodyWidth - leftBarSize : bodyWidth - leftBarSize - 25 const [elements, setElements] = useState([]); @@ -544,13 +558,17 @@ const AngularWorkflow = (defaultprops) => { }) .then((responseJson) => { if ( - responseJson.apikey === undefined || + responseJson.success === true && + (responseJson.apikey === undefined || responseJson.apikey.length === 0 || - responseJson.apikey === null + responseJson.apikey === null) ) { generateApikey(); } - setUserSettings(responseJson); + + if (responseJson.success === true) { + setUserSettings(responseJson) + } }) .catch((error) => { console.log(error); @@ -644,7 +662,8 @@ const AngularWorkflow = (defaultprops) => { }); const newitem = removeParam("execution_id", cursearch); - props.history.push(curpath + newitem); + navigate(curpath + newitem) + //props.history.push(curpath + newitem); } } } @@ -673,7 +692,7 @@ const AngularWorkflow = (defaultprops) => { return response.json(); }) .then((responseJson) => { - console.log("RESPONSE: ", responseJson) + //console.log("RESPONSE: ", responseJson) handleUpdateResults(responseJson, executionRequest); }) .catch((error) => { @@ -720,200 +739,202 @@ const AngularWorkflow = (defaultprops) => { // Loop nodes and find results // Update on every interval? idk - if (JSON.stringify(responseJson) !== JSON.stringify(executionData)) { - // FIXME: If another is selected, don't edit.. - // Doesn't work because this is some async garbage - if ( - executionData.execution_id === undefined || - (responseJson.execution_id === executionData.execution_id && - responseJson.results !== undefined && - responseJson.results !== null) - ) { - if ( - executionData.status !== responseJson.status || - executionData.result !== responseJson.result || - executionData.results.length !== responseJson.results.length - ) { - setExecutionData(responseJson); - } else { - console.log("NOT updating state."); - } - } - } + ReactDOM.unstable_batchedUpdates(() => { + if (JSON.stringify(responseJson) !== JSON.stringify(executionData)) { + // FIXME: If another is selected, don't edit.. + // Doesn't work because this is some async garbage + if ( + executionData.execution_id === undefined || + (responseJson.execution_id === executionData.execution_id && + responseJson.results !== undefined && + responseJson.results !== null) + ) { + if ( + executionData.status !== responseJson.status || + executionData.result !== responseJson.result || + executionData.results.length !== responseJson.results.length + ) { + setExecutionData(responseJson); + } else { + console.log("NOT updating state."); + } + } + } - if (responseJson.execution_id !== executionRequest.execution_id) { - cy.elements().removeClass( - "success-highlight failure-highlight executing-highlight" - ); - return; - } + if (responseJson.execution_id !== executionRequest.execution_id) { + cy.elements().removeClass( + "success-highlight failure-highlight executing-highlight" + ); + return; + } - if (responseJson.results !== null && responseJson.results.length > 0) { - for (var key in responseJson.results) { - var item = responseJson.results[key]; - var currentnode = cy.getElementById(item.action.id); - if (currentnode.length === 0) { - continue; - } + if (responseJson.results !== null && responseJson.results.length > 0) { + for (var key in responseJson.results) { + var item = responseJson.results[key]; + var currentnode = cy.getElementById(item.action.id); + if (currentnode.length === 0) { + continue; + } - currentnode = currentnode[0]; - const outgoingEdges = currentnode.outgoers("edge"); - const incomingEdges = currentnode.incomers("edge"); + currentnode = currentnode[0]; + const outgoingEdges = currentnode.outgoers("edge"); + const incomingEdges = currentnode.incomers("edge"); - switch (item.status) { - case "EXECUTING": - currentnode.removeClass("not-executing-highlight"); - currentnode.removeClass("success-highlight"); - currentnode.removeClass("failure-highlight"); - currentnode.removeClass("shuffle-hover-highlight"); - currentnode.removeClass("awaiting-data-highlight"); - incomingEdges.addClass("success-highlight"); - currentnode.addClass("executing-highlight"); - break; - case "SKIPPED": - currentnode.removeClass("not-executing-highlight"); - currentnode.removeClass("success-highlight"); - currentnode.removeClass("failure-highlight"); - currentnode.removeClass("shuffle-hover-highlight"); - currentnode.removeClass("awaiting-data-highlight"); - currentnode.removeClass("executing-highlight"); - currentnode.addClass("skipped-highlight"); - break; - case "WAITING": - currentnode.removeClass("not-executing-highlight"); - currentnode.removeClass("success-highlight"); - currentnode.removeClass("failure-highlight"); - currentnode.removeClass("shuffle-hover-highlight"); - currentnode.removeClass("awaiting-data-highlight"); - currentnode.addClass("executing-highlight"); + switch (item.status) { + case "EXECUTING": + currentnode.removeClass("not-executing-highlight"); + currentnode.removeClass("success-highlight"); + currentnode.removeClass("failure-highlight"); + currentnode.removeClass("shuffle-hover-highlight"); + currentnode.removeClass("awaiting-data-highlight"); + incomingEdges.addClass("success-highlight"); + currentnode.addClass("executing-highlight"); + break; + case "SKIPPED": + currentnode.removeClass("not-executing-highlight"); + currentnode.removeClass("success-highlight"); + currentnode.removeClass("failure-highlight"); + currentnode.removeClass("shuffle-hover-highlight"); + currentnode.removeClass("awaiting-data-highlight"); + currentnode.removeClass("executing-highlight"); + currentnode.addClass("skipped-highlight"); + break; + case "WAITING": + currentnode.removeClass("not-executing-highlight"); + currentnode.removeClass("success-highlight"); + currentnode.removeClass("failure-highlight"); + currentnode.removeClass("shuffle-hover-highlight"); + currentnode.removeClass("awaiting-data-highlight"); + currentnode.addClass("executing-highlight"); - if (!visited.includes(item.action.label)) { - if (executionRunning) { - visited.push(item.action.label); - setVisited(visited); - } - } + if (!visited.includes(item.action.label)) { + if (executionRunning) { + visited.push(item.action.label); + setVisited(visited); + } + } - // FIXME - add outgoing nodes to executing - //const outgoingNodes = outgoingEdges.find().data().target - if (outgoingEdges.length > 0) { - outgoingEdges.addClass("success-highlight"); - } - break; - case "SUCCESS": - currentnode.removeClass("not-executing-highlight"); - currentnode.removeClass("executing-highlight"); - currentnode.removeClass("failure-highlight"); - currentnode.removeClass("shuffle-hover-highlight"); - currentnode.removeClass("awaiting-data-highlight"); - currentnode.addClass("success-highlight"); - incomingEdges.addClass("success-highlight"); - outgoingEdges.addClass("success-highlight"); + // FIXME - add outgoing nodes to executing + //const outgoingNodes = outgoingEdges.find().data().target + if (outgoingEdges.length > 0) { + outgoingEdges.addClass("success-highlight"); + } + break; + case "SUCCESS": + currentnode.removeClass("not-executing-highlight"); + currentnode.removeClass("executing-highlight"); + currentnode.removeClass("failure-highlight"); + currentnode.removeClass("shuffle-hover-highlight"); + currentnode.removeClass("awaiting-data-highlight"); + currentnode.addClass("success-highlight"); + incomingEdges.addClass("success-highlight"); + outgoingEdges.addClass("success-highlight"); - if ( - visited !== undefined && - visited !== null && - !visited.includes(item.action.label) - ) { - if (executionRunning) { - visited.push(item.action.label); - setVisited(visited); - } - } + if ( + visited !== undefined && + visited !== null && + !visited.includes(item.action.label) + ) { + if (executionRunning) { + visited.push(item.action.label); + setVisited(visited); + } + } - // FIXME - add outgoing nodes to executing - //const outgoingNodes = outgoingEdges.find().data().target - if (outgoingEdges.length > 0) { - for (var i = 0; i < outgoingEdges.length; i++) { - const edge = outgoingEdges[i]; - const targetnode = cy.getElementById(edge.data().target); - if ( - targetnode !== undefined && - !targetnode.classes().includes("success-highlight") && - !targetnode.classes().includes("failure-highlight") - ) { - targetnode.removeClass("not-executing-highlight"); - targetnode.removeClass("success-highlight"); - targetnode.removeClass("shuffle-hover-highlight"); - targetnode.removeClass("failure-highlight"); - targetnode.removeClass("awaiting-data-highlight"); - targetnode.addClass("executing-highlight"); - } - } - } - break; - case "FAILURE": - //When status comes as failure, allow user to start workflow execution - if (executionRunning) { - setExecutionRunning(false); - } + // FIXME - add outgoing nodes to executing + //const outgoingNodes = outgoingEdges.find().data().target + if (outgoingEdges.length > 0) { + for (var i = 0; i < outgoingEdges.length; i++) { + const edge = outgoingEdges[i]; + const targetnode = cy.getElementById(edge.data().target); + if ( + targetnode !== undefined && + !targetnode.classes().includes("success-highlight") && + !targetnode.classes().includes("failure-highlight") + ) { + targetnode.removeClass("not-executing-highlight"); + targetnode.removeClass("success-highlight"); + targetnode.removeClass("shuffle-hover-highlight"); + targetnode.removeClass("failure-highlight"); + targetnode.removeClass("awaiting-data-highlight"); + targetnode.addClass("executing-highlight"); + } + } + } + break; + case "FAILURE": + //When status comes as failure, allow user to start workflow execution + if (executionRunning) { + setExecutionRunning(false); + } - currentnode.removeClass("not-executing-highlight"); - currentnode.removeClass("executing-highlight"); - currentnode.removeClass("success-highlight"); - currentnode.removeClass("awaiting-data-highlight"); - currentnode.removeClass("shuffle-hover-highlight"); - currentnode.addClass("failure-highlight"); + currentnode.removeClass("not-executing-highlight"); + currentnode.removeClass("executing-highlight"); + currentnode.removeClass("success-highlight"); + currentnode.removeClass("awaiting-data-highlight"); + currentnode.removeClass("shuffle-hover-highlight"); + currentnode.addClass("failure-highlight"); - if (!visited.includes(item.action.label)) { - if ( - item.action.result !== undefined && - item.action.result !== null && - !item.action.result.includes("failed condition") - ) { - alert.error( - "Error for " + - item.action.label + - " with result " + - item.result - ); - } - visited.push(item.action.label); - setVisited(visited); - } - break; - case "AWAITING_DATA": - currentnode.removeClass("not-executing-highlight"); - currentnode.removeClass("executing-highlight"); - currentnode.removeClass("success-highlight"); - currentnode.removeClass("failure-highlight"); - currentnode.removeClass("shuffle-hover-highlight"); - currentnode.addClass("awaiting-data-highlight"); - break; - default: - console.log("DEFAULT?"); - break; - } - } - } + if (!visited.includes(item.action.label)) { + if ( + item.action.result !== undefined && + item.action.result !== null && + !item.action.result.includes("failed condition") + ) { + alert.error( + "Error for " + + item.action.label + + " with result " + + item.result + ); + } + visited.push(item.action.label); + setVisited(visited); + } + break; + case "AWAITING_DATA": + currentnode.removeClass("not-executing-highlight"); + currentnode.removeClass("executing-highlight"); + currentnode.removeClass("success-highlight"); + currentnode.removeClass("failure-highlight"); + currentnode.removeClass("shuffle-hover-highlight"); + currentnode.addClass("awaiting-data-highlight"); + break; + default: + console.log("DEFAULT?"); + break; + } + } + } - if ( - responseJson.status === "ABORTED" || - responseJson.status === "STOPPED" || - responseJson.status === "FAILURE" || - responseJson.status === "WAITING" - ) { - stop(); + if ( + responseJson.status === "ABORTED" || + responseJson.status === "STOPPED" || + responseJson.status === "FAILURE" || + responseJson.status === "WAITING" + ) { + stop(); - if (executionRunning) { - setExecutionRunning(false); - } + if (executionRunning) { + setExecutionRunning(false); + } - var curelements = cy.elements(); - for (var i = 0; i < curelements.length; i++) { - if (curelements[i].classes().includes("executing-highlight")) { - curelements[i].removeClass("executing-highlight"); - curelements[i].addClass("failure-highlight"); - } - } + var curelements = cy.elements(); + for (var i = 0; i < curelements.length; i++) { + if (curelements[i].classes().includes("executing-highlight")) { + curelements[i].removeClass("executing-highlight"); + curelements[i].addClass("failure-highlight"); + } + } - getWorkflowExecution(props.match.params.key, ""); - } else if (responseJson.status === "FINISHED") { - setExecutionRunning(false); - stop(); - getWorkflowExecution(props.match.params.key, ""); - setUpdate(Math.random()); - } + getWorkflowExecution(props.match.params.key, ""); + } else if (responseJson.status === "FINISHED") { + setExecutionRunning(false); + stop(); + getWorkflowExecution(props.match.params.key, ""); + setUpdate(Math.random()); + } + }) }; const sendStreamRequest = (body) => { @@ -958,234 +979,252 @@ const AngularWorkflow = (defaultprops) => { return; } - setSavingState(2); + setSavingState(2); - // This might not be the right course of action, but seems logical, as items could be running already - // Makes it possible to update with a version in current render - stop(); - var useworkflow = workflow; - if (curworkflow !== undefined) { - useworkflow = curworkflow; - } + // This might not be the right course of action, but seems logical, as items could be running already + // Makes it possible to update with a version in current render + stop(); + var useworkflow = workflow; + if (curworkflow !== undefined) { + useworkflow = curworkflow; + } - var cyelements = cy.elements(); - var newActions = []; - var newTriggers = []; - var newBranches = []; - var newVBranches = []; - var newComments = []; - for (var key in cyelements) { - if (cyelements[key].data === undefined) { - continue; - } + var cyelements = cy.elements(); + var newActions = []; + var newTriggers = []; + var newBranches = []; + var newVBranches = []; + var newComments = []; + for (var key in cyelements) { + if (cyelements[key].data === undefined) { + continue; + } - var type = cyelements[key].data()["type"]; - if (type === undefined) { - if ( - cyelements[key].data().source === undefined || - cyelements[key].data().target === undefined - ) { - continue; - } + var type = cyelements[key].data()["type"]; + if (type === undefined) { + if ( + cyelements[key].data().source === undefined || + cyelements[key].data().target === undefined + ) { + continue; + } - var parsedElement = { - id: cyelements[key].data().id, - source_id: cyelements[key].data().source, - destination_id: cyelements[key].data().target, - conditions: cyelements[key].data().conditions, - decorator: cyelements[key].data().decorator, - }; + var parsedElement = { + id: cyelements[key].data().id, + source_id: cyelements[key].data().source, + destination_id: cyelements[key].data().target, + conditions: cyelements[key].data().conditions, + decorator: cyelements[key].data().decorator, + }; - if (parsedElement.decorator) { - newVBranches.push(parsedElement); - } else { - newBranches.push(parsedElement); - } - } else { - if (type === "ACTION") { - const cyelement = cyelements[key].data(); - const elementid = - cyelement.id === undefined || cyelement.id === null - ? cyelement["_id"] - : cyelement.id; + if (parsedElement.decorator) { + newVBranches.push(parsedElement); + } else { + newBranches.push(parsedElement); + } + } else { + if (type === "ACTION") { + const cyelement = cyelements[key].data(); + const elementid = + cyelement.id === undefined || cyelement.id === null + ? cyelement["_id"] + : cyelement.id; - var curworkflowAction = useworkflow.actions.find( - (a) => - a !== undefined && - (a["id"] === elementid || a["_id"] === elementid) - ); - if (curworkflowAction === undefined) { - curworkflowAction = cyelements[key].data(); - } + var curworkflowAction = useworkflow.actions.find( + (a) => + a !== undefined && + (a["id"] === elementid || a["_id"] === elementid) + ); + if (curworkflowAction === undefined) { + curworkflowAction = cyelements[key].data(); + } - curworkflowAction.position = cyelements[key].position(); + curworkflowAction.position = cyelements[key].position(); - // workaround to fix some edgecases - if ( - curworkflowAction.parameters === "" || - curworkflowAction.parameters === null - ) { - curworkflowAction.parameters = []; - } + // workaround to fix some edgecases + if ( + curworkflowAction.parameters === "" || + curworkflowAction.parameters === null + ) { + curworkflowAction.parameters = []; + } - if ( - curworkflowAction.example === undefined || - curworkflowAction.example === "" || - curworkflowAction.example === null - ) { - if (cyelements[key].data().example !== undefined) { - curworkflowAction.example = cyelements[key].data().example; - } - } + if ( + curworkflowAction.example === undefined || + curworkflowAction.example === "" || + curworkflowAction.example === null + ) { + if (cyelements[key].data().example !== undefined) { + curworkflowAction.example = cyelements[key].data().example; + } + } - // Override just in this place - curworkflowAction.errors = []; - curworkflowAction.isValid = true; + // Override just in this place + curworkflowAction.errors = []; + curworkflowAction.isValid = true; - // Cleans up OpenAPI items - var newparams = []; - for (var key in curworkflowAction.parameters) { - const thisitem = curworkflowAction.parameters[key]; - if (thisitem.name.startsWith("${") && thisitem.name.endsWith("}")) { - continue; - } + // Cleans up OpenAPI items + var newparams = []; + for (var key in curworkflowAction.parameters) { + const thisitem = curworkflowAction.parameters[key]; + if (thisitem.name.startsWith("${") && thisitem.name.endsWith("}")) { + continue; + } - newparams.push(thisitem); - } + newparams.push(thisitem); + } - curworkflowAction.parameters = newparams; - newActions.push(curworkflowAction); - } else if (type === "TRIGGER") { - var curworkflowTrigger = useworkflow.triggers.find( - (a) => a.id === cyelements[key].data()["id"] - ); - if (curworkflowTrigger === undefined) { - curworkflowTrigger = cyelements[key].data(); - } + curworkflowAction.parameters = newparams; + newActions.push(curworkflowAction); + } else if (type === "TRIGGER") { + var curworkflowTrigger = useworkflow.triggers.find( + (a) => a.id === cyelements[key].data()["id"] + ); + if (curworkflowTrigger === undefined) { + curworkflowTrigger = cyelements[key].data(); + } - curworkflowTrigger.position = cyelements[key].position(); + curworkflowTrigger.position = cyelements[key].position(); - newTriggers.push(curworkflowTrigger); - } else if (type === "COMMENT") { - if (useworkflow.comments === undefined) { - useworkflow.comments = []; - } + newTriggers.push(curworkflowTrigger); + } else if (type === "COMMENT") { + if (useworkflow.comments === undefined) { + useworkflow.comments = []; + } - var curworkflowComment = useworkflow.comments.find( - (a) => a.id === cyelements[key].data()["id"] - ) + var curworkflowComment = useworkflow.comments.find( + (a) => a.id === cyelements[key].data()["id"] + ) - if (curworkflowComment === undefined) { - curworkflowComment = cyelements[key].data(); - } + if (curworkflowComment === undefined) { + curworkflowComment = cyelements[key].data(); + } - const parsedHeight = parseInt(curworkflowComment["height"]) - if (!isNaN(parsedHeight)) { - curworkflowComment.height = parsedHeight - } else { - curworkflowComment.width = 150 - } + const parsedHeight = parseInt(curworkflowComment["height"]) + if (!isNaN(parsedHeight)) { + curworkflowComment.height = parsedHeight + } else { + curworkflowComment.width = 150 + } - const parsedWidth = parseInt(curworkflowComment["width"]) - if (!isNaN(parsedWidth)) { - curworkflowComment.width = parsedWidth - } else { - curworkflowComment.width = 200 - } + const parsedWidth = parseInt(curworkflowComment["width"]) + if (!isNaN(parsedWidth)) { + curworkflowComment.width = parsedWidth + } else { + curworkflowComment.width = 200 + } - curworkflowComment.position = cyelements[key].position(); - //console.log(curworkflowComment) + curworkflowComment.position = cyelements[key].position(); + //console.log(curworkflowComment) - newComments.push(curworkflowComment); - } else { - alert.info("No handler for type: " + type); - } - } - } + newComments.push(curworkflowComment); + } else { + alert.info("No handler for type: " + type); + } + } + } - useworkflow.actions = newActions; - useworkflow.triggers = newTriggers; - useworkflow.branches = newBranches; - useworkflow.comments = newComments; - useworkflow.visual_branches = newVBranches; + useworkflow.actions = newActions; + useworkflow.triggers = newTriggers; + useworkflow.branches = newBranches; + useworkflow.comments = newComments; + useworkflow.visual_branches = newVBranches; - // Errors are backend defined - useworkflow.errors = []; - useworkflow.previously_saved = true; + // Errors are backend defined + useworkflow.errors = []; + useworkflow.previously_saved = true; - setLastSaved(true); - fetch(globalUrl + "/api/v1/workflows/" + props.match.params.key, { - method: "PUT", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - body: JSON.stringify(useworkflow), - credentials: "include", - }) - .then((response) => { - setSavingState(0); - if (response.status !== 200) { - console.log("Status not 200 for setting workflows :O!"); - } + if (cy !== undefined) { + // scale: 0.3, + // bg: "#27292d", + const cyImageData = cy.png({ + output: "base64uri", + maxWidth: 480, + maxHeight: 270, + }) - return response.json(); - }) - .then((responseJson) => { - if (executionArgument !== undefined && startNode !== undefined) { - console.log("Running execution AFTER saving"); - executeWorkflow(executionArgument, startNode, true); - return; - } + if (cyImageData !== undefined && cyImageData !== null && cyImageData.length > 0) { + useworkflow.image = cyImageData + } + } - if (!responseJson.success) { - console.log(responseJson); - alert.error("Failed to save: " + responseJson.reason); - } else { - if ( - responseJson.new_id !== undefined && - responseJson.new_id !== null - ) { - window.location.pathname = "/workflows/" + responseJson.new_id; - } + setLastSaved(true); + fetch(globalUrl + "/api/v1/workflows/" + props.match.params.key, { + method: "PUT", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(useworkflow), + credentials: "include", + }) + .then((response) => { + setSavingState(0); + if (response.status !== 200) { + console.log("Status not 200 for setting workflows :O!"); + } - success = true; - if (responseJson.errors !== undefined) { - workflow.errors = responseJson.errors; - if (responseJson.errors.length === 0) { - workflow.isValid = true; - workflow.is_valid = true; + return response.json(); + }) + .then((responseJson) => { + if (executionArgument !== undefined && startNode !== undefined) { + //console.log("Running execution AFTER saving"); + executeWorkflow(executionArgument, startNode, true); + return; + } - const cyelements = cy.elements(); - for (var i = 0; i < cyelements.length; i++) { - cyelements[i].removeStyle(); - cyelements[i].data().is_valid = true; - cyelements[i].data().errors = []; - } + if (!responseJson.success) { + console.log(responseJson); + if (responseJson.reason !== undefined && responseJson.reason !== null) { + alert.error("Failed to save: " + responseJson.reason); + } else { + alert.error("Failed to save. Please contact your admin if this is unexpected.") + } + } else { + if ( + responseJson.new_id !== undefined && + responseJson.new_id !== null + ) { + window.location.pathname = "/workflows/" + responseJson.new_id; + } - for (var key in workflow.actions) { - workflow.actions[key].is_valid = true; - workflow.actions[key].errors = []; - } - } + success = true; + if (responseJson.errors !== undefined) { + workflow.errors = responseJson.errors; + if (responseJson.errors.length === 0) { + workflow.isValid = true; + workflow.is_valid = true; - for (var key in workflow.errors) { - alert.info(workflow.errors[key]); - } + const cyelements = cy.elements(); + for (var i = 0; i < cyelements.length; i++) { + cyelements[i].removeStyle(); + cyelements[i].data().is_valid = true; + cyelements[i].data().errors = []; + } - setWorkflow(workflow); - } + for (var key in workflow.actions) { + workflow.actions[key].is_valid = true; + workflow.actions[key].errors = []; + } + } - setSavingState(1); - setTimeout(() => { - setSavingState(0); - }, 1500); - } - }) - .catch((error) => { - setSavingState(0); - alert.error(error.toString()); - }); + for (var key in workflow.errors) { + alert.info(workflow.errors[key]); + } + + setWorkflow(workflow); + } + + setSavingState(1); + setTimeout(() => { + setSavingState(0); + }, 1500); + } + }) + .catch((error) => { + setSavingState(0); + alert.error(error.toString()); + }); return success; }; @@ -1315,7 +1354,7 @@ const AngularWorkflow = (defaultprops) => { // This can be used to only show prioritzed ones later // Right now, it can prioritize authenticated ones //"Testing", - const internalIds = ["Shuffle Tools", "http"]; + const internalIds = ["Shuffle Tools", "http", "email"]; const getAppAuthentication = (reset, updateAction) => { fetch(globalUrl + "/api/v1/apps/authentication", { @@ -1328,7 +1367,7 @@ const AngularWorkflow = (defaultprops) => { }) .then((response) => { if (response.status !== 200) { - console.log("Status not 200 for apps :O!"); + console.log("Status not 200 for app auth :O!"); } return response.json(); @@ -1435,6 +1474,7 @@ const AngularWorkflow = (defaultprops) => { credentials: "include", }) .then((response) => { + setAppsLoaded(true) if (response.status !== 200) { console.log("Status not 200 for apps :O!"); } @@ -1442,6 +1482,26 @@ const AngularWorkflow = (defaultprops) => { return response.json(); }) .then((responseJson) => { + if (responseJson === null) { + console.log("No response") + const pretend_apps = [{ + "name": "TBD", + "app_name": "TBD", + "app_version": "TBD", + "description": "TBD", + "version": "TBD", + "large_image": "", + }] + setApps(pretend_apps) + setFilteredApps(pretend_apps) + setPrioritizedApps(pretend_apps); + return + } + + if (responseJson.success === false) { + return + } + // FIXME - handle versions on left bar //handleAppVersioning(responseJson) //var tmpapps = [] @@ -1477,7 +1537,8 @@ const AngularWorkflow = (defaultprops) => { } }) .catch((error) => { - alert.error(error.toString()); + setAppsLoaded(true) + alert.error("App loading error: "+error.toString()); }); }; @@ -1512,12 +1573,52 @@ const AngularWorkflow = (defaultprops) => { responseJson.errors = []; } + if (responseJson.actions === undefined || responseJson.actions === null) { + responseJson.actions = []; + } + + if (responseJson.triggers === undefined || responseJson.triggers === null) { + responseJson.triggers = []; + } + if (responseJson.public) { alert.info( - "This workflow is public. You will have to save it to make it your own!" + "This workflow is public. Save the workflow to " ); + + console.log("RESP: ", responseJson) + if (Object.getOwnPropertyNames(creatorProfile).length === 0) { + //getUserProfile("frikky") + getUserProfile(responseJson.id) + } + + //{appGroup.map((data, index) => { + //const [appGroup, setAppGroup] = React.useState([]); + var appsFound = [] + for (var key in responseJson.actions) { + const parsedAction = responseJson.actions[key] + if (parsedAction.large_image === undefined || parsedAction.large_image === null || parsedAction.large_image === "") { + continue + } + if (appsFound.findIndex(data => data.app_name === parsedAction.app_name) < 0){ + appsFound.push(parsedAction) + } + } + + setAppGroup(appsFound) + + appsFound = [] + for (var key in responseJson.triggers) { + const parsedAction = responseJson.triggers[key] + if (appsFound.findIndex(data => data.app_name === parsedAction.app_name) < 0){ + appsFound.push(parsedAction) + } + } + + setTriggerGroup(appsFound) } + // Appends SUBFLOWS. Does NOT run during normal grabbing of workflows. if (sourcenode.id !== undefined) { console.log("WORKFLOW: ", responseJson); @@ -1698,78 +1799,83 @@ const AngularWorkflow = (defaultprops) => { } */ - cy.removeListener("select"); - cy.on("select", "node", (e) => onNodeSelect(e, appAuthentication)); - cy.on("select", "edge", (e) => onEdgeSelect(e)); + //cy.removeListener("select"); + //cy.on("select", "node", (e) => onNodeSelect(e, appAuthentication)); + //cy.on("select", "edge", (e) => onEdgeSelect(e)); + // FIXME - check if they have value before overriding like this for no reason. // Would save a lot of time (400~ ms -> 30ms) //console.log("ACTION: ", selectedAction) //console.log("APP: ", selectedApp) - setSelectedAction({}); - setSelectedApp({}); - setSelectedTrigger({}); - setSelectedComment({}) - //setSelectedEdge({}) - // setSelectedTriggerIndex(-1) - //setSelectedActionEnvironment({}) - setSelectedEdge({}); - //setTriggerAuthentication({}) - //setSelectedTriggerIndex(-1) - //setTriggerFolders([]) + ReactDOM.unstable_batchedUpdates(() => { + setSelectedAction({}); + setSelectedApp({}); + setSelectedTrigger({}); + setSelectedComment({}) + setSelectedEdge({}); - // Can be used for right side view - setRightSideBarOpen(false); - setScrollConfig({ - top: 0, - left: 0, - selected: "", - }); - console.timeEnd("UNSELECT"); + setSelectedEdge({}) + setSelectedActionEnvironment({}) + setTriggerAuthentication({}) + setSelectedTriggerIndex(-1) + setTriggerFolders([]) + + // Can be used for right side view + setRightSideBarOpen(false); + setScrollConfig({ + top: 0, + left: 0, + selected: "", + }); + console.timeEnd("UNSELECT"); + }) }; const onEdgeSelect = (event) => { - setRightSideBarOpen(true); - setLastSaved(false); + ReactDOM.unstable_batchedUpdates(() => { + setRightSideBarOpen(true); + setLastSaved(false); - /* - // Used to not be able to edit trigger-based branches. - const triggercheck = workflow.triggers.find(trigger => trigger.id === event.target.data()["source"]) - if (triggercheck === undefined) { - */ - if ( - event.target.data("type") !== "COMMENT" && - event.target.data().decorator - ) { - alert.info("This edge can't be edited."); - } else { - //console.log("DATA: ", event.target.data()) - const destinationId = event.target.data("target"); - //console.log("DATA: ", event.target.data()) - const curaction = workflow.actions.find((a) => a.id === destinationId); - //console.log("ACTION: ", curaction) - if (curaction !== undefined && curaction !== null) { - if ( - curaction.app_name === "Shuffle Tools" && - curaction.name === "router" - ) { - alert.info("Router action can't have incoming conditions"); - event.target.unselect(); - return; - } - } + /* + // Used to not be able to edit trigger-based branches. + const triggercheck = workflow.triggers.find(trigger => trigger.id === event.target.data()["source"]) + if (triggercheck === undefined) { + */ + if ( + event.target.data("type") !== "COMMENT" && + event.target.data().decorator + ) { + alert.info("This edge can't be edited."); + } else { + //console.log("DATA: ", event.target.data()) + const destinationId = event.target.data("target"); + //console.log("DATA: ", event.target.data()) + const curaction = workflow.actions.find((a) => a.id === destinationId); + //console.log("ACTION: ", curaction) + if (curaction !== undefined && curaction !== null) { + if ( + curaction.app_name === "Shuffle Tools" && + curaction.name === "router" + ) { + alert.info("Router action can't have incoming conditions"); + event.target.unselect(); + return; + } + } - setSelectedEdgeIndex( - workflow.branches.findIndex( - (data) => data.id === event.target.data()["id"] - ) - ); - setSelectedEdge(event.target.data()); - } + setSelectedEdgeIndex( + workflow.branches.findIndex( + (data) => data.id === event.target.data()["id"] + ) + ); + setSelectedEdge(event.target.data()); + } - setSelectedAction({}); - setSelectedTrigger({}); + setSelectedAction({}); + setSelectedTrigger({}); + }) }; // Comparing locations between nodes and setting views @@ -2089,9 +2195,11 @@ const AngularWorkflow = (defaultprops) => { if (!lastSaved) { return unloadText; } else { - //document.removeEventListener("mousemove", onMouseUpdate, true); - document.removeEventListener("keydown", handleKeyDown, true); - document.removeEventListener("paste", handlePaste, true); + if (workflow.public === false) { + //document.removeEventListener("mousemove", onMouseUpdate, true); + document.removeEventListener("keydown", handleKeyDown, true); + document.removeEventListener("paste", handlePaste, true); + } } }); @@ -2099,360 +2207,367 @@ const AngularWorkflow = (defaultprops) => { // https://stackoverflow.com/questions/16677856/cy-onselect-callback-only-once // onNodeClick const onNodeSelect = (event, newAppAuth) => { - const data = event.target.data(); - if (data.isButton) { - if (data.buttonType === "delete") { - const parentNode = cy.getElementById(data.attachedTo); - if (parentNode !== null && parentNode !== undefined) { - removeNode(data.attachedTo) - //parentNode.remove() - } + // Otherwise everything is SUPER slow + ReactDOM.unstable_batchedUpdates(() => { + const data = event.target.data(); + if (data.isButton) { + if (data.buttonType === "delete") { + const parentNode = cy.getElementById(data.attachedTo); + if (parentNode !== null && parentNode !== undefined) { + removeNode(data.attachedTo) + //parentNode.remove() + } - return - } else if ( - data.buttonType === "set_startnode" && - data.type !== "TRIGGER" - ) { - const parentNode = cy.getElementById(data.attachedTo); - if (parentNode !== null && parentNode !== undefined) { - var oldstartnode = cy.getElementById(workflow.start); - if ( - oldstartnode !== null && - oldstartnode !== undefined && - oldstartnode.length > 0 - ) { - try { - oldstartnode[0].data("isStartNode", false); - } catch (e) { - console.log("Startnode error: ", e); - } - } - - workflow.start = parentNode.data("id"); - setLastSaved(false); - parentNode.data("isStartNode", true); - } - - //event.target.unselect(); - return - } else if (data.buttonType === "copy") { - console.log("COPY!"); - // 1. Find parent - // 2. Find branches for parent - // 3. Make a new node that's moved a little bit - const parentNode = cy.getElementById(data.attachedTo); - if (parentNode !== null && parentNode !== undefined) { - var newNodeData = JSON.parse(JSON.stringify(parentNode.data())); - newNodeData.id = uuidv4(); - if (newNodeData.position !== undefined) { - newNodeData.position = { - x: newNodeData.position.x + 100, - y: newNodeData.position.y + 100, - }; - } - - newNodeData.isStartNode = false; - newNodeData.errors = []; - newNodeData.is_valid = true; - newNodeData.isValid = true; - newNodeData.label = parentNode.data("label") + "_copy"; - - cy.add({ - group: "nodes", - data: newNodeData, - position: newNodeData.position, - }); - - // Readding the icon after moving the node - if ( - newNodeData.app_name !== "Testing" || - newNodeData.app_name !== "Shuffle Workflow" - ) { - } else { - const iconInfo = GetIconInfo(newNodeData); - const svg_pin = ``; - const svgpin_Url = encodeURI("data:image/svg+xml;utf-8," + svg_pin); - - const offset = newNodeData.isStartNode ? 36 : 44; - const decoratorNode = { - position: { - x: newNodeData.position.x + offset, - y: newNodeData.position.y + offset, - }, - locked: true, - data: { - isDescriptor: true, - isValid: true, - is_valid: true, - label: "", - image: svgpin_Url, - imageColor: iconInfo.iconBackgroundColor, - attachedTo: newNodeData.id, - }, - }; - - cy.add(decoratorNode).unselectify(); - } - - workflow.actions.push(newNodeData); - - const sourcebranches = workflow.branches.filter( - (foundbranch) => foundbranch.source_id === parentNode.data("id") - ); - const destinationbranches = workflow.branches.filter( - (foundbranch) => - foundbranch.destination_id === parentNode.data("id") - ); - - for (var key in sourcebranches) { - var newbranch = JSON.parse(JSON.stringify(sourcebranches[key])); - newbranch.id = uuidv4(); - newbranch.source_id = newNodeData.id; - - newbranch._id = newbranch.id; - newbranch.source = newbranch.source_id; - newbranch.target = newbranch.destination_id; - cy.add({ - group: "edges", - data: newbranch, - }); - } - - for (var key in destinationbranches) { - var newbranch = JSON.parse( - JSON.stringify(destinationbranches[key]) - ); - newbranch.id = uuidv4(); - newbranch.destination_id = newNodeData.id; - - newbranch._id = newbranch.id; - newbranch.source = newbranch.source_id; - newbranch.target = newbranch.destination_id; - cy.add({ - group: "edges", - data: newbranch, - }); - } - - //event.target.unselect(); return - } - } + } else if ( + data.buttonType === "set_startnode" && + data.type !== "TRIGGER" + ) { + const parentNode = cy.getElementById(data.attachedTo); + if (parentNode !== null && parentNode !== undefined) { + var oldstartnode = cy.getElementById(workflow.start); + if ( + oldstartnode !== null && + oldstartnode !== undefined && + oldstartnode.length > 0 + ) { + try { + oldstartnode[0].data("isStartNode", false); + } catch (e) { + console.log("Startnode error: ", e); + } + } - return; - } else if (data.isDescriptor) { - console.log("Can't select descriptor"); - event.target.unselect(); - return; - } + workflow.start = parentNode.data("id"); + setLastSaved(false); + parentNode.data("isStartNode", true); + } + + //event.target.unselect(); + setRightSideBarOpen(true); + return + } else if (data.buttonType === "copy") { + console.log("COPY!"); + // 1. Find parent + // 2. Find branches for parent + // 3. Make a new node that's moved a little bit + const parentNode = cy.getElementById(data.attachedTo); + if (parentNode !== null && parentNode !== undefined) { + var newNodeData = JSON.parse(JSON.stringify(parentNode.data())); + newNodeData.id = uuidv4(); + if (newNodeData.position !== undefined) { + newNodeData.position = { + x: newNodeData.position.x + 100, + y: newNodeData.position.y + 100, + }; + } - if (data.type === "ACTION") { - //var curaction = JSON.parse(JSON.stringify(data)) - // FIXME: Trust it to just work? - //event.target.data() - var curaction = workflow.actions.find((a) => a.id === data.id); - if (!curaction || curaction === undefined) { - console.log("NOT FOUND DATA: ", event.target.data()) - if (data.id !== undefined && data.app_name !== undefined) { - workflow.actions.push(data) + newNodeData.isStartNode = false; + newNodeData.errors = []; + newNodeData.is_valid = true; + newNodeData.isValid = true; + newNodeData.label = parentNode.data("label") + "_copy"; + + cy.add({ + group: "nodes", + data: newNodeData, + position: newNodeData.position, + }); + + // Readding the icon after moving the node + if ( + newNodeData.app_name !== "Testing" || + newNodeData.app_name !== "Shuffle Workflow" + ) { + } else { + const iconInfo = GetIconInfo(newNodeData); + const svg_pin = ``; + const svgpin_Url = encodeURI("data:image/svg+xml;utf-8," + svg_pin); + + const offset = newNodeData.isStartNode ? 36 : 44; + const decoratorNode = { + position: { + x: newNodeData.position.x + offset, + y: newNodeData.position.y + offset, + }, + locked: true, + data: { + isDescriptor: true, + isValid: true, + is_valid: true, + label: "", + image: svgpin_Url, + imageColor: iconInfo.iconBackgroundColor, + attachedTo: newNodeData.id, + }, + }; + + cy.add(decoratorNode).unselectify(); + } + + workflow.actions.push(newNodeData); + + const sourcebranches = workflow.branches.filter( + (foundbranch) => foundbranch.source_id === parentNode.data("id") + ); + const destinationbranches = workflow.branches.filter( + (foundbranch) => + foundbranch.destination_id === parentNode.data("id") + ); + + for (var key in sourcebranches) { + var newbranch = JSON.parse(JSON.stringify(sourcebranches[key])); + newbranch.id = uuidv4(); + newbranch.source_id = newNodeData.id; + + newbranch._id = newbranch.id; + newbranch.source = newbranch.source_id; + newbranch.target = newbranch.destination_id; + cy.add({ + group: "edges", + data: newbranch, + }); + } + + for (var key in destinationbranches) { + var newbranch = JSON.parse( + JSON.stringify(destinationbranches[key]) + ); + newbranch.id = uuidv4(); + newbranch.destination_id = newNodeData.id; + + newbranch._id = newbranch.id; + newbranch.source = newbranch.source_id; + newbranch.target = newbranch.destination_id; + cy.add({ + group: "edges", + data: newbranch, + }); + } + + //event.target.unselect(); + return + } + } + + return; + } else if (data.isDescriptor) { + console.log("Can't select descriptor"); + event.target.unselect(); + return; + } + + if (data.type === "ACTION") { + setSelectedComment({}) + //var curaction = JSON.parse(JSON.stringify(data)) + // FIXME: Trust it to just work? + //event.target.data() + var curaction = workflow.actions.find((a) => a.id === data.id); + if (!curaction || curaction === undefined) { + console.log("NOT FOUND DATA: ", event.target.data()) + if (data.id !== undefined && data.app_name !== undefined) { + workflow.actions.push(data) + setWorkflow(workflow) + curaction = data + } else { + alert.error("Action not found. Please remake it."); + event.target.remove(); + return; + } + } + + //var newapps = JSON.parse(JSON.stringify(apps)) + var newapps = apps + const curapp = newapps.find( + (a) => + a.name === curaction.app_name && + (a.app_version === curaction.app_version || + (a.loop_versions !== null && + a.loop_versions.includes(curaction.app_version))) + ); + if (!curapp || curapp === undefined) { + alert.error( + `App ${curaction.app_name}:${curaction.app_version} not found. Is it activated?` + ); + + const tmpapp = { + name: curaction.app_name, + app_name: curaction.app_name, + app_version: curaction.app_version, + id: curaction.app_id, + actions: [curaction], + }; + + setSelectedApp(tmpapp); + setSelectedAction(curaction); + } else { + setAuthenticationType( + curapp.authentication.type === "oauth2" && + curapp.authentication.redirect_uri !== undefined && + curapp.authentication.redirect_uri !== null + ? { + type: "oauth2", + redirect_uri: curapp.authentication.redirect_uri, + refresh_uri: curapp.authentication.refresh_uri, + token_uri: curapp.authentication.token_uri, + scope: curapp.authentication.scope, + client_id: curapp.authentication.client_id, + client_secret: curapp.authentication.client_secret, + } + : { + type: "", + } + ); + + const requiresAuth = curapp.authentication.required; //&& ((curapp.authentication.parameters !== undefined && curapp.authentication.parameters !== null) || (curapp.authentication.type === "oauth2" && curapp.authentication.redirect_uri !== undefined && curapp.authentication.redirect_uri !== null)) + setRequiresAuthentication(requiresAuth); + if (curapp.authentication.required) { + //console.log("App requires auth.") + // Setup auth here :) + const authenticationOptions = []; + var findAuthId = ""; + if ( + curaction.authentication_id !== null && + curaction.authentication_id !== undefined && + curaction.authentication_id.length > 0 + ) { + findAuthId = curaction.authentication_id; + } + + var tmpAuth = JSON.parse(JSON.stringify(newAppAuth)); + + for (var key in tmpAuth) { + var item = tmpAuth[key]; + + const newfields = {}; + for (var filterkey in item.fields) { + newfields[item.fields[filterkey].key] = + item.fields[filterkey].value; + } + + item.fields = newfields; + if (item.app.name === curapp.name) { + authenticationOptions.push(item); + if (item.id === findAuthId) { + curaction.selectedAuthentication = item; + } + } + } + + curaction.authentication = authenticationOptions; + if ( + curaction.selectedAuthentication === null || + curaction.selectedAuthentication === undefined || + curaction.selectedAuthentication.length === "" + ) { + curaction.selectedAuthentication = {}; + } + } else { + curaction.authentication = []; + curaction.authentication_id = ""; + curaction.selectedAuthentication = {}; + } + + if ( + curaction.parameters !== undefined && + curaction.parameters !== null && + curaction.parameters.length > 0 + ) { + for (var key in curaction.parameters) { + if ( + curaction.parameters[key].options !== undefined && + curaction.parameters[key].options !== null && + curaction.parameters[key].options.length > 0 && + curaction.parameters[key].value === "" + ) { + curaction.parameters[key].value = + curaction.parameters[key].options[0]; + } + } + } + + setSelectedApp(curapp); + setSelectedAction(curaction); + + cy.removeListener("drag"); + cy.removeListener("free"); + cy.on("drag", "node", (e) => onNodeDrag(e, curaction)); + cy.on("free", "node", (e) => onNodeDragStop(e, curaction)); + } + + console.log("Object: ", environments) + if (environments !== undefined && environments !== null && (typeof environments === "array" || typeof environments === "object")) { + var parsedenv = environments + if (typeof environments === "object") { + parsedenv = [environments] + } + + var env = parsedenv.find((a) => a.Name === curaction.environment); + if (!env || env === undefined) { + env = parsedenv[defaultEnvironmentIndex]; + } + + setSelectedActionEnvironment(env); + } + } else if (data.type === "TRIGGER") { + setSelectedComment({}) + if (workflow.triggers === null) { + workflow.triggers = [] + } + + var trigger_index = workflow.triggers.findIndex( + (a) => a.id === data.id + ); + + //console.log("Trigger: ", data, trigger_index) + if (trigger_index === -1) { + workflow.triggers.push(data) + trigger_index = workflow.triggers.length-1 setWorkflow(workflow) - curaction = data - } else { - alert.error("Action not found. Please remake it."); - event.target.remove(); - return; - } - } - - var newapps = JSON.parse(JSON.stringify(apps)) - const curapp = newapps.find( - (a) => - a.name === curaction.app_name && - (a.app_version === curaction.app_version || - (a.loop_versions !== null && - a.loop_versions.includes(curaction.app_version))) - ); - if (!curapp || curapp === undefined) { - alert.error( - `App ${curaction.app_name}:${curaction.app_version} not found. Is it activated?` - ); - - const tmpapp = { - name: curaction.app_name, - app_name: curaction.app_name, - app_version: curaction.app_version, - id: curaction.app_id, - actions: [curaction], - }; - - setSelectedApp(tmpapp); - setSelectedAction(curaction); - } else { - setAuthenticationType( - curapp.authentication.type === "oauth2" && - curapp.authentication.redirect_uri !== undefined && - curapp.authentication.redirect_uri !== null - ? { - type: "oauth2", - redirect_uri: curapp.authentication.redirect_uri, - refresh_uri: curapp.authentication.refresh_uri, - token_uri: curapp.authentication.token_uri, - scope: curapp.authentication.scope, - client_id: curapp.authentication.client_id, - client_secret: curapp.authentication.client_secret, - } - : { - type: "", - } - ); - - const requiresAuth = curapp.authentication.required; //&& ((curapp.authentication.parameters !== undefined && curapp.authentication.parameters !== null) || (curapp.authentication.type === "oauth2" && curapp.authentication.redirect_uri !== undefined && curapp.authentication.redirect_uri !== null)) - setRequiresAuthentication(requiresAuth); - if (curapp.authentication.required) { - //console.log("App requires auth.") - // Setup auth here :) - const authenticationOptions = []; - var findAuthId = ""; - if ( - curaction.authentication_id !== null && - curaction.authentication_id !== undefined && - curaction.authentication_id.length > 0 - ) { - findAuthId = curaction.authentication_id; - } - - var tmpAuth = JSON.parse(JSON.stringify(newAppAuth)); - - for (var key in tmpAuth) { - var item = tmpAuth[key]; - - const newfields = {}; - for (var filterkey in item.fields) { - newfields[item.fields[filterkey].key] = - item.fields[filterkey].value; - } - - item.fields = newfields; - if (item.app.name === curapp.name) { - authenticationOptions.push(item); - if (item.id === findAuthId) { - curaction.selectedAuthentication = item; - } - } - } - - curaction.authentication = authenticationOptions; - if ( - curaction.selectedAuthentication === null || - curaction.selectedAuthentication === undefined || - curaction.selectedAuthentication.length === "" - ) { - curaction.selectedAuthentication = {}; - } - } else { - curaction.authentication = []; - curaction.authentication_id = ""; - curaction.selectedAuthentication = {}; - } - - if ( - curaction.parameters !== undefined && - curaction.parameters !== null && - curaction.parameters.length > 0 - ) { - for (var key in curaction.parameters) { - if ( - curaction.parameters[key].options !== undefined && - curaction.parameters[key].options !== null && - curaction.parameters[key].options.length > 0 && - curaction.parameters[key].value === "" - ) { - curaction.parameters[key].value = - curaction.parameters[key].options[0]; - } - } - } - - setSelectedApp(curapp); - setSelectedAction(curaction); - - cy.removeListener("drag"); - cy.removeListener("free"); - cy.on("drag", "node", (e) => onNodeDrag(e, curaction)); - cy.on("free", "node", (e) => onNodeDragStop(e, curaction)); - } - - console.log("Object: ", environments) - if (environments !== undefined && environments !== null && (typeof environments === "array" || typeof environments === "object")) { - var parsedenv = environments - if (typeof environments === "object") { - parsedenv = [environments] } - var env = parsedenv.find((a) => a.Name === curaction.environment); - if (!env || env === undefined) { - env = parsedenv[defaultEnvironmentIndex]; - } + //console.log("Trigger2: ", data, trigger_index) + //if (data.id !== undefined && data.app_name !== undefined) { + // //newapps.push(data) + // workflow.actions.push(data) + // curaction = data + //} else { + // alert.error("Action not found. Please remake it."); + // event.target.remove(); + // return; + //} - setSelectedActionEnvironment(env); - } - } else if (data.type === "TRIGGER") { - if (workflow.triggers === null) { - workflow.triggers = [] - } + if (data.app_name === "Shuffle Workflow") { + getAvailableWorkflows(trigger_index); + getSettings(); + } else if (data.app_name === "Webhook") { + if (workflow.triggers[trigger_index].parameters !== undefined) { + workflow.triggers[trigger_index].parameters[0] = { + name: "url", + value: referenceUrl + "webhook_" + selectedTrigger.id, + }; + } + } - var trigger_index = workflow.triggers.findIndex( - (a) => a.id === data.id - ); + console.log("DATA: ", data) + setSelectedTriggerIndex(trigger_index); + setSelectedTrigger(data); + setSelectedActionEnvironment(data.env); + } else if (data.type === "COMMENT") { + setSelectedComment(data); + } else { + alert.error("Can't handle " + data.type); + return; + } - //console.log("Trigger: ", data, trigger_index) - if (trigger_index === -1) { - workflow.triggers.push(data) - trigger_index = workflow.triggers.length-1 - setWorkflow(workflow) - } - - //console.log("Trigger2: ", data, trigger_index) - //if (data.id !== undefined && data.app_name !== undefined) { - // //newapps.push(data) - // workflow.actions.push(data) - // curaction = data - //} else { - // alert.error("Action not found. Please remake it."); - // event.target.remove(); - // return; - //} - - if (data.app_name === "Shuffle Workflow") { - getAvailableWorkflows(trigger_index); - getSettings(); - } else if (data.app_name === "Webhook") { - if (workflow.triggers[trigger_index].parameters !== undefined) { - workflow.triggers[trigger_index].parameters[0] = { - name: "url", - value: referenceUrl + "webhook_" + selectedTrigger.id, - }; - } - } - - console.log("DATA: ", data) - setSelectedTriggerIndex(trigger_index); - setSelectedTrigger(data); - setSelectedActionEnvironment(data.env); - } else if (data.type === "COMMENT") { - setSelectedComment(data); - } else { - alert.error("Can't handle " + data.type); - return; - } - - setRightSideBarOpen(true); - setLastSaved(false); - setScrollConfig({ - top: 0, - left: 0, - selected: "", - }); + setRightSideBarOpen(true); + setLastSaved(false); + setScrollConfig({ + top: 0, + left: 0, + selected: "", + }); + }) }; const activateApp = (appid) => { @@ -3006,6 +3121,7 @@ const AngularWorkflow = (defaultprops) => { } } + if ( nodedata.parameters !== undefined && nodedata.parameters !== null && @@ -3375,7 +3491,7 @@ const AngularWorkflow = (defaultprops) => { }) .then((response) => { if (response.status !== 200) { - console.log("Status not 200 for apps :O!"); + console.log("Status not 200 for envs :O!"); if (isCloud) { setEnvironments([{ Name: "Cloud", Type: "cloud" }]); } else { @@ -3438,145 +3554,6 @@ const AngularWorkflow = (defaultprops) => { window.location.pathname = "/workflows/" + props.match.params.key; } - // eslint-disable-next-line react-hooks/exhaustive-deps - useEffect(() => { - if (firstrequest) { - setFirstrequest(false); - getWorkflow(props.match.params.key, {}); - getApps(); - getAppAuthentication(); - getEnvironments(); - getWorkflowExecution(props.match.params.key, ""); - getAvailableWorkflows(-1); - getSettings(); - - const cursearch = - typeof window === "undefined" || window.location === undefined - ? "" - : window.location.search; - - // FIXME: Don't check specific one here - const tmpExec = new URLSearchParams(cursearch).get("execution_highlight"); - if ( - tmpExec !== undefined && - tmpExec !== null && - tmpExec === "executions" - ) { - setExecutionModalOpen(true) - const newitem = removeParam("execution_highlight", cursearch); - props.history.push(curpath + newitem); - } - - const tmpView = new URLSearchParams(cursearch).get("view"); - if ( - tmpView !== undefined && - tmpView !== null && - tmpView === "executions" - ) { - setExecutionModalOpen(true); - - const newitem = removeParam("view", cursearch); - props.history.push(curpath + newitem); - } - return; - } - - // App length necessary cus of cy initialization - if ( - // First load - gets the workflow - elements.length === 0 && - workflow.actions !== undefined && - !graphSetup && - Object.getOwnPropertyNames(workflow).length > 0 - ) { - setGraphSetup(true); - setupGraph(); - } else if ( - // 2nd load - configures cytoscape - // - !established && - cy !== undefined && - apps !== null && - apps !== undefined && - apps.length > 0 && - Object.getOwnPropertyNames(workflow).length > 0 && - authLoaded - ) { - //This part has to load LAST, as it's kind of not async. - //This means we need everything else to happen first. - - setEstablished(true); - // Validate if the node is just a node lol - cy.edgehandles({ - handleNodes: (el) => { - if (el.isNode() && - !el.data("isButton") && - !el.data("isDescriptor") && - !el.data("isSuggestion") && - el.data("type") !== "COMMENT") { - return true - } - - return false - }, - preview: true, - toggleOffOnLeave: true, - loopAllowed: function (node) { - return false; - }, - }); - - cy.fit(null, 200); - - cy.on("boxselect", "node", (e) => { - if (e.target.data("isButton") || e.target.data("isDescriptor") || e.target.data("isSuggestion")) { - e.target.unselect(); - } - - e.target.addClass("selected"); - }); - - cy.on("boxstart", (e) => { - console.log("START"); - cy.removeListener("select"); - }); - - cy.on("boxend", (e) => { - console.log("END: ", cy) - var cydata = cy.$(":selected").jsons(); - if (cydata !== undefined && cydata !== null && cydata.length > 0) { - alert.success(`Selected ${cydata.length} element(s). CTRL+C to copy them.`); - } - }); - - cy.on("select", "node", (e) => { - onNodeSelect(e, appAuthentication); - }); - cy.on("select", "edge", (e) => onEdgeSelect(e)); - - cy.on("unselect", (e) => onUnselect(e)); - - cy.on("add", "node", (e) => onNodeAdded(e)); - cy.on("add", "edge", (e) => onEdgeAdded(e)); - cy.on("remove", "node", (e) => onNodeRemoved(e)); - cy.on("remove", "edge", (e) => onEdgeRemoved(e)); - - cy.on("mouseover", "edge", (e) => onEdgeHover(e)); - cy.on("mouseout", "edge", (e) => onEdgeHoverOut(e)); - cy.on("mouseover", "node", (e) => onNodeHover(e)); - cy.on("mouseout", "node", (e) => onNodeHoverOut(e)); - - // Handles dragging - cy.on("drag", "node", (e) => onNodeDrag(e, selectedAction)); - cy.on("free", "node", (e) => onNodeDragStop(e, selectedAction)); - - cy.on("cxttap", "node", (e) => onCtxTap(e)); - - document.title = "Workflow - " + workflow.name; - registerKeys(); - } - }) - const animationDuration = 150; const onNodeHoverOut = (event) => { const nodedata = event.target.data(); @@ -3802,7 +3779,7 @@ const AngularWorkflow = (defaultprops) => { //var parentNode = cy.$("#" + event.target.data("id")); //if (parentNode.data("isButton") || parentNode.data("buttonId")) return; - if (nodedata.app_name !== undefined) { + if (nodedata.app_name !== undefined && !workflow.public === true) { const allNodes = cy.nodes().jsons(); var found = false; @@ -4131,6 +4108,8 @@ const AngularWorkflow = (defaultprops) => { 'control-point-distance': edgeCurve.distance, 'control-point-weight': edgeCurve.weight, } + } else { + console.log("FAILED node curve handling") } return edge; @@ -4255,6 +4234,152 @@ const AngularWorkflow = (defaultprops) => { */ }; + // eslint-disable-next-line react-hooks/exhaustive-deps + //useEffect(() => { + if (firstrequest) { + setFirstrequest(false); + getWorkflow(props.match.params.key, {}); + getApps(); + getAppAuthentication(); + getEnvironments(); + getWorkflowExecution(props.match.params.key, ""); + getAvailableWorkflows(-1); + getSettings(); + + const cursearch = + typeof window === "undefined" || window.location === undefined + ? "" + : window.location.search; + + // FIXME: Don't check specific one here + const tmpExec = new URLSearchParams(cursearch).get("execution_highlight"); + if ( + tmpExec !== undefined && + tmpExec !== null && + tmpExec === "executions" + ) { + setExecutionModalOpen(true) + const newitem = removeParam("execution_highlight", cursearch); + navigate(curpath + newitem) + //props.history.push(curpath + newitem); + } + + const tmpView = new URLSearchParams(cursearch).get("view"); + if ( + tmpView !== undefined && + tmpView !== null && + tmpView === "executions" + ) { + setExecutionModalOpen(true); + + const newitem = removeParam("view", cursearch); + navigate(curpath + newitem) + //navigate(`?execution_highlight=${parsed_url}`) + //props.history.push(curpath + newitem); + } + return; + } + + // App length necessary cus of cy initialization + if ( + // First load - gets the workflow + elements.length === 0 && + workflow.actions !== undefined && + !graphSetup && + Object.getOwnPropertyNames(workflow).length > 0 + ) { + + setGraphSetup(true); + setupGraph(); + console.log("In graph setup") + } else if ( + // 2nd load - configures cytoscape + // + !established && + cy !== undefined && + ((apps !== null && + apps !== undefined && + apps.length > 0) || workflow.public === true) && + Object.getOwnPropertyNames(workflow).length > 0 && + authLoaded + ) { + + console.log("In POST graph setup!") + //This part has to load LAST, as it's kind of not async. + //This means we need everything else to happen first. + + setEstablished(true); + // Validate if the node is just a node lol + cy.edgehandles({ + handleNodes: (el) => { + if (el.isNode() && + !el.data("isButton") && + !el.data("isDescriptor") && + !el.data("isSuggestion") && + el.data("type") !== "COMMENT") { + return true + } + + return false + }, + preview: true, + toggleOffOnLeave: true, + loopAllowed: function (node) { + return false; + }, + }); + + cy.fit(null, 200); + + cy.on("boxselect", "node", (e) => { + if (e.target.data("isButton") || e.target.data("isDescriptor") || e.target.data("isSuggestion")) { + e.target.unselect(); + } + + e.target.addClass("selected"); + }); + + cy.on("boxstart", (e) => { + console.log("START"); + cy.removeListener("select"); + }); + + cy.on("boxend", (e) => { + console.log("END: ", cy) + var cydata = cy.$(":selected").jsons(); + if (cydata !== undefined && cydata !== null && cydata.length > 0) { + alert.success(`Selected ${cydata.length} element(s). CTRL+C to copy them.`); + } + }); + + cy.on("select", "node", (e) => { + onNodeSelect(e, appAuthentication); + }); + cy.on("select", "edge", (e) => onEdgeSelect(e)); + + cy.on("unselect", (e) => onUnselect(e)); + + cy.on("add", "node", (e) => onNodeAdded(e)); + cy.on("add", "edge", (e) => onEdgeAdded(e)); + cy.on("remove", "node", (e) => onNodeRemoved(e)); + cy.on("remove", "edge", (e) => onEdgeRemoved(e)); + + cy.on("mouseover", "edge", (e) => onEdgeHover(e)); + cy.on("mouseout", "edge", (e) => onEdgeHoverOut(e)); + cy.on("mouseover", "node", (e) => onNodeHover(e)); + cy.on("mouseout", "node", (e) => onNodeHoverOut(e)); + + // Handles dragging + cy.on("drag", "node", (e) => onNodeDrag(e, selectedAction)); + cy.on("free", "node", (e) => onNodeDragStop(e, selectedAction)); + + cy.on("cxttap", "node", (e) => onCtxTap(e)); + + document.title = "Workflow - " + workflow.name; + registerKeys(); + } + //}) + const stopSchedule = (trigger, triggerindex) => { fetch( globalUrl + @@ -4356,15 +4481,16 @@ const AngularWorkflow = (defaultprops) => { marginRight: 5, display: "flex", flexDirection: "column", - height: "100%", + minHeight: isMobile ? bodyHeight-appBarSize*4 : "100%", + maxHeight: isMobile ? bodyHeight-appBarSize*4 : "100%", }; const paperAppStyle = { borderRadius: theme.palette.borderRadius, - minHeight: 100, - maxHeight: 100, - minWidth: "100%", - maxWidth: "100%", + minHeight: isMobile ? 50 : 100, + maxHeight: isMobile ? 50 : 100, + minWidth: isMobile ? 50 : "100%", + maxWidth: isMobile ? 50 : "100%", marginTop: "5px", color: "white", backgroundColor: surfaceColor, @@ -4420,7 +4546,7 @@ const AngularWorkflow = (defaultprops) => { const variableScrollStyle = { margin: 15, overflow: "scroll", - height: "66vh", + height: isMobile ? "100%" : "66vh", overflowX: "auto", overflowY: "auto", flex: "10", @@ -4712,8 +4838,8 @@ const AngularWorkflow = (defaultprops) => { } const tabStyle = { - maxWidth: leftBarSize / 3, - minWidth: leftBarSize / 3, + maxWidth: isMobile ? leftBarSize : leftBarSize / 3, + minWidth: isMobile ? leftBarSize : leftBarSize / 3, flex: 1, textTransform: "none", }; @@ -4723,12 +4849,14 @@ const AngularWorkflow = (defaultprops) => { marginRight: 5, }; + const parsedHeight = isMobile ? bodyHeight - appBarSize*4 : bodyHeight - appBarSize - 50 return (
{thisview} @@ -4739,6 +4867,8 @@ const AngularWorkflow = (defaultprops) => { indicatorColor="primary" onChange={handleSetTab} aria-label="Left sidebar tab" + orientation={isMobile ? "vertical" : "horizontal"} + style={{}} > { - Apps + {isMobile ? null : Apps} } style={tabStyle} @@ -4757,7 +4887,7 @@ const AngularWorkflow = (defaultprops) => { - Triggers + {isMobile ? null : Triggers} } style={tabStyle} @@ -4768,7 +4898,7 @@ const AngularWorkflow = (defaultprops) => { - Variables + {isMobile ? null : Variables} } style={tabStyle} @@ -4881,12 +5011,12 @@ const AngularWorkflow = (defaultprops) => { {triggers.map((trigger, index) => { var imageline = trigger.large_image.length === 0 ? ( - + ) : ( ); @@ -4909,8 +5039,8 @@ const AngularWorkflow = (defaultprops) => { {}}>
{ >
{imageline} - - -

- {trigger.name} -

- - - {trigger.description} - - - + {isMobile ? null : + + +

+ {trigger.name} +

+ + + {trigger.description} + + + } + ); @@ -5049,6 +5181,8 @@ const AngularWorkflow = (defaultprops) => { // AUTHENTICATION if (app.authentication.required) { + console.log("App auth is required!") + // Setup auth here :) const authenticationOptions = []; var findAuthId = ""; @@ -5060,6 +5194,7 @@ const AngularWorkflow = (defaultprops) => { findAuthId = newAppData.authentication_id; } + console.log("Found auth: ", findAuthId) var tmpAuth = JSON.parse(JSON.stringify(appAuthentication)); for (var key in tmpAuth) { var item = tmpAuth[key]; @@ -5111,8 +5246,8 @@ const AngularWorkflow = (defaultprops) => { const appScrollStyle = { overflow: "scroll", - maxHeight: bodyHeight - appBarSize - 55 - 50, - minHeight: bodyHeight - appBarSize - 55 - 50, + maxHeight: isMobile ? bodyHeight-appBarSize*4 : bodyHeight - appBarSize - 55 - 50, + minHeight: isMobile ? bodyHeight-appBarSize*4 : bodyHeight - appBarSize - 55 - 50, marginTop: 1, overflowY: "auto", overflowX: "hidden", @@ -5121,6 +5256,9 @@ const AngularWorkflow = (defaultprops) => { const handleAppDrag = (e, app) => { const cycontainer = cy.container(); + //console.log("e: ", e) + //console.log("Offset: ", cycontainer) + // Chrome lol if ( e.pageX > cycontainer.offsetLeft && @@ -5217,7 +5355,7 @@ const AngularWorkflow = (defaultprops) => { parameters: parameters, isStartNode: false, large_image: app.large_image, - run_magic_output: true, + run_magic_output: false, authentication: [], execution_variable: undefined, example: example, @@ -5312,6 +5450,94 @@ const AngularWorkflow = (defaultprops) => { onMouseOut={() => { setHover(false); }} + onClick={() => { + if (isMobile) { + newNodeId = uuidv4(); + const actionType = "ACTION"; + const actionLabel = getNextActionName(app.name); + var parameters = null; + var example = ""; + var description = "" + + if ( + app.actions[0].parameters !== null && + app.actions[0].parameters.length > 0 + ) { + parameters = app.actions[0].parameters; + } + + if ( + app.actions[0].returns.example !== undefined && + app.actions[0].returns.example !== null && + app.actions[0].returns.example.length > 0 + ) { + example = app.actions[0].returns.example; + } + + if ( + app.actions[0].description !== undefined && + app.actions[0].description !== null && + app.actions[0].description.length > 0 + ) { + description = app.actions[0].description + } + + const parsedEnvironments = + environments === null || environments === [] + ? "cloud" + : environments[defaultEnvironmentIndex] === undefined + ? "cloud" + : environments[defaultEnvironmentIndex].Name; + + // activated: app.generated === true ? app.activated === false ? false : true : true, + const newAppData = { + app_name: app.name, + app_version: app.app_version, + app_id: app.id, + sharing: app.sharing, + private_id: app.private_id, + description: description, + environment: parsedEnvironments, + errors: [], + finished: false, + id_: newNodeId, + _id_: newNodeId, + id: newNodeId, + is_valid: true, + label: actionLabel, + type: actionType, + name: app.actions[0].name, + parameters: parameters, + isStartNode: false, + large_image: app.large_image, + run_magic_output: false, + authentication: [], + execution_variable: undefined, + example: example, + category: + app.categories !== null && + app.categories !== undefined && + app.categories.length > 0 + ? app.categories[0] + : "", + authentication_id: "", + finished: false, + }; + + const nodeToBeAdded = { + group: "nodes", + data: newAppData, + renderedPosition: { + x: 100, + y: 100, + }, + }; + + parsedApp = nodeToBeAdded; + cy.add(nodeToBeAdded); + + } + }} > { userDrag: "none", userSelect: "none", borderRadius: theme.palette.borderRadius, - height: 80, - width: 80, + height: isMobile ? 40 : 80, + width: isMobile ? 40 : 80, }} /> - - - - {newAppname} - - - - - Version: {app.app_version} - - - - - {app.description} - - - + {isMobile ? null : + + + + {newAppname} + + + + + Version: {app.app_version} + + + + + {app.description} + + + + } @@ -5554,12 +5782,14 @@ const AngularWorkflow = (defaultprops) => { } // Does this one find the wrong one? - var newSelectedAction = JSON.parse(JSON.stringify(selectedAction)) + //var newSelectedAction = JSON.parse(JSON.stringify(selectedAction)) + var newSelectedAction = selectedAction newSelectedAction.name = newaction.name; newSelectedAction.parameters = JSON.parse(JSON.stringify(newaction.parameters)) newSelectedAction.errors = []; newSelectedAction.isValid = true; newSelectedAction.is_valid = true; + //console.log(newSelectedAction) // Simmple action swap autocompleter if (selectedAction.parameters !== undefined && newSelectedAction.parameters !== undefined && selectedAction.id === newSelectedAction.id) { @@ -5701,6 +5931,7 @@ const AngularWorkflow = (defaultprops) => { event.target.value = event.target.value.replaceAll(".", ""); event.target.value = event.target.value.replaceAll(",", ""); event.target.value = event.target.value.replaceAll(" ", "_"); + selectedAction.label = event.target.value; setSelectedAction(selectedAction); }; @@ -5821,7 +6052,7 @@ const AngularWorkflow = (defaultprops) => { top: appBarSize + 25, right: 25, height: "80vh", - width: 365, + width: isMobile ? "100%" : 365, minWidth: 200, maxWidth: 600, maxHeight: "100vh", @@ -6078,7 +6309,6 @@ const AngularWorkflow = (defaultprops) => { }} fullWidth multiline={multiline} - rows={5} color="primary" defaultValue={data.value} placeholder={placeholder} @@ -6138,6 +6368,9 @@ const AngularWorkflow = (defaultprops) => { Autocomplete { name: "auth_headers", value: "", }; + workflow.triggers[selectedTriggerIndex].parameters[3] = { + name: "custom_response_body", + value: "", + }; setWorkflow(workflow); } else { // Always update @@ -8719,6 +8963,9 @@ const AngularWorkflow = (defaultprops) => {
Environment { return null; }; - const cytoscapeViewWidths = 850; + const cytoscapeViewWidths = isMobile ? 50 : 850; const bottomBarStyle = { position: "fixed", - right: 20, - left: leftBarSize, - bottom: 0, + right: isMobile ? 20 : 20, + bottom: isMobile ? undefined : 0, + top: isMobile ? appBarSize + 55 : undefined, + left: isMobile ? undefined : leftBarSize, minWidth: cytoscapeViewWidths, maxWidth: cytoscapeViewWidths, marginLeft: 20, @@ -9869,11 +10186,15 @@ const AngularWorkflow = (defaultprops) => { const topBarStyle = { position: "fixed", right: 0, - left: leftBarSize + 20, - top: appBarSize + 20, + left: isMobile ? 20 : leftBarSize + 20, + top: isMobile ? 30 : appBarSize + 20, }; const TopCytoscapeBar = (props) => { + if (workflow.public === true) { + return null + } + return (
@@ -9897,10 +10218,6 @@ const AngularWorkflow = (defaultprops) => {

{workflow.name}

- - {workflow.public ? ( -

Public Workflow PREVIEW

- ) : null} @@ -10013,7 +10330,30 @@ const AngularWorkflow = (defaultprops) => { - + {isMobile ? + + + + + + : null} + ); }; @@ -10102,7 +10442,7 @@ const AngularWorkflow = (defaultprops) => { return null; } - const boxSize = 100; + const boxSize = isMobile ? 50 : 100; const executionButton = executionRunning ? ( @@ -10114,7 +10454,7 @@ const AngularWorkflow = (defaultprops) => { abortExecution(); }} > - + @@ -10132,7 +10472,7 @@ const AngularWorkflow = (defaultprops) => { executeWorkflow(executionText, workflow.start, lastSaved); }} > - + @@ -10143,13 +10483,17 @@ const AngularWorkflow = (defaultprops) => { {executionButton}
- {workflow.public ? null : ( + {isMobile || workflow.public ? null : ( { return null; } + var defaultReturn = null if (Object.getOwnPropertyNames(selectedAction).length > 0) { if (Object.getOwnPropertyNames(selectedAction).length === 0) { return null; } - return ( - -
- -
- - ); + defaultReturn = + } else if (Object.getOwnPropertyNames(selectedComment).length > 0) { - return ( -
- -
- ); + defaultReturn = } else if (Object.getOwnPropertyNames(selectedTrigger).length > 0) { if (selectedTrigger.trigger_type === "SCHEDULE") { - return ( -
- -
- ); + defaultReturn = } else if (selectedTrigger.trigger_type === "WEBHOOK") { - return ( -
- -
- ); + defaultReturn = } else if (selectedTrigger.trigger_type === "SUBFLOW") { - return ( -
- -
- ); + defaultReturn = } else if (selectedTrigger.trigger_type === "EMAIL") { - return ( -
- -
- ); + defaultReturn = } else if (selectedTrigger.trigger_type === "USERINPUT") { - return ( -
- -
- ); + defaultReturn = } else if (selectedTrigger.trigger_type === undefined) { + //defaultReturn = return null; } else { console.log( @@ -10483,14 +10800,46 @@ const AngularWorkflow = (defaultprops) => { return null; } } else if (Object.getOwnPropertyNames(selectedEdge).length > 0) { - return ( -
- -
- ); + defaultReturn = } - return null; + const iOS = typeof navigator !== 'undefined' && /iPad|iPhone|iPod/.test(navigator.userAgent); + + const drawerBleeding = 56; + return ( + isMobile ? + { + console.log("Close!") + //setRightSideBarOpen(false) + cy.elements().unselect() + }} + disableSwipeToOpen={false} + ModalProps={{ + keepMounted: true, + }} + PaperProps={{ + style: { + maxHeight: "70%", + overflow: "auto", + } + }} + > + {defaultReturn} + + : + +
+ {defaultReturn} +
+ + ); + + //return null; }; // This can execute a workflow with firestore. Used for test, as datastore is old and stuff @@ -10499,66 +10848,211 @@ const AngularWorkflow = (defaultprops) => { // executeWorkflowWebsocket() //}}>Execute websocket // + + // Searhc by username, userId, workflow, appId should all work + const getUserProfile = (username) => { + fetch(`${globalUrl}/api/v1/users/creators/${username}`, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for WORKFLOW EXECUTION :O!"); + } - const leftView = leftViewOpen ? ( -
- -
- ) : ( -
-
{ - setLeftViewOpen(true); - setLeftBarSize(350); - }} - > - - - -
-
- ); - const executionPaperStyle = { - minWidth: "95%", - maxWidth: "95%", - marginTop: "5px", - color: "white", - marginBottom: 10, - padding: 5, - backgroundColor: surfaceColor, - cursor: "pointer", - display: "flex", - minHeight: 40, - maxHeight: 40, - }; + return response.json(); + }) + .then((responseJson) => { + console.log("Found creator: ", responseJson) + if (responseJson.success !== false) { + setCreatorProfile(responseJson) + } + }) + .catch((error) => { + console.log(error); + }) + } + + const leftView = workflow.public === true ? +
+ + {workflow.name} + + + This workflow is public and { + saveWorkflow() + }}>must be saved to be used in your organization. + + {Object.getOwnPropertyNames(creatorProfile).length !== 0 && creatorProfile.github_avatar !== undefined && creatorProfile.github_avatar !== null ? +
+ { + }}> + + + + + + Shared by {creatorProfile.github_username} + +
+ : null} +
+ {workflow.tags !== undefined && workflow.tags !== null && workflow.tags.length > 0 ? +
+ + Tags + +
+ {workflow.tags.map((tag, index) => { + if (index >= 3) { + return null; + } - const parsedExecutionArgument = () => { - var showResult = executionData.execution_argument.trim(); - const validate = validateJson(showResult); + return ( + + ); + })} +
+
+ : null } +
+ + Mitre Att&ck: + + + TBD + +
+ {appGroup.length > 0 ? +
+ + Apps + + + {appGroup.map((data, index) => { + return ( + + + + ) + })} + +
+ : null} + {triggerGroup.length > 0 ? +
+ + Triggers + + + {triggerGroup.map((data, index) => { + return ( + + ) + })} + +
+ : null} +
+ + Related Workflows: + + + TBD + +
+ {workflow.description !== undefined && workflow.description !== null && workflow.description.length > 0 ? +
+ + Description + + + {workflow.description} + +
+ : null} +
+ : isMobile && leftViewOpen ? +
+ +
+ : leftViewOpen ? ( +
+ +
+ ) : ( +
+
{ + setLeftViewOpen(true); + setLeftBarSize(350); + }} + > + + + +
+
+); - if (validate.valid) { - if (typeof validate.result === "string") { - try { - validate.result = JSON.parse(validate.result); - } catch (e) { - console.log("Error: ", e); - validate.valid = false; - } - } +const executionPaperStyle = { + minWidth: "95%", + maxWidth: "95%", + marginTop: "5px", + color: "white", + marginBottom: 10, + padding: 5, + backgroundColor: surfaceColor, + cursor: "pointer", + display: "flex", + minHeight: 40, + maxHeight: 40, +}; + +const parsedExecutionArgument = () => { + var showResult = executionData.execution_argument.trim(); + const validate = validateJson(showResult); + + if (validate.valid) { + if (typeof validate.result === "string") { + try { + validate.result = JSON.parse(validate.result); + } catch (e) { + console.log("Error: ", e); + validate.valid = false; + } + } return (
@@ -10934,26 +11428,53 @@ const AngularWorkflow = (defaultprops) => { var executionDelay = -75 const executionModal = ( - setExecutionModalOpen(false)} + onClose={() => { + setExecutionModalOpen(false) + }} style={{ resize: "both", overflow: "auto", zIndex: 10005 }} + hideBackdrop={false} + variant="temporary" + BackdropProps={{ + style: { + backgroundColor: "transparent", + } + }} PaperProps={{ style: { resize: "both", overflow: "auto", - minWidth: 420, - maxWidth: 420, + minWidth: isMobile ? "100%" : 420, + maxWidth: isMobile ? "100%" : 420, backgroundColor: "#1F2023", color: "white", fontSize: 18, zIndex: 10005, + borderLeft: theme.palette.defaultBorder, }, }} > + {isMobile ? + + { + e.preventDefault(); + setExecutionModalOpen(false) + }} + > + + + + : null} {executionModalView === 0 ? ( -
+
{ // Especially important IF the result is > 1 Mb in cloud var checkStarted = false if (isCloud && data.results !== undefined && data.results !== null && data.results.length > 0) { + + if (data.execution_argument !== undefined && data.execution_argument !== null && data.execution_argument.includes("too large")) { + setExecutionData({}); + checkStarted = true + start(); + setExecutionRunning(true); + setExecutionRequestStarted(false); + } else { for (var key in data.results) { if (data.results[key].status !== "SUCCESS") { continue @@ -11069,6 +11598,7 @@ const AngularWorkflow = (defaultprops) => { break } } + } } const cur_execution = { @@ -11158,7 +11688,7 @@ const AngularWorkflow = (defaultprops) => { )}
) : ( -
+
{ color="primary" style={{ float: "right", marginTop: 20, marginLeft: 10 }} onClick={() => { - console.log("DATA: ", executionData); + //console.log("DATA: ", executionData); executeWorkflow( executionData.execution_argument, executionData.start, @@ -11740,7 +12270,7 @@ const AngularWorkflow = (defaultprops) => { )}
)} - + ); // This sucks :) @@ -11774,48 +12304,50 @@ const AngularWorkflow = (defaultprops) => { validate.result = JSON.parse(validate.result); } + const AppResultVariable = ({data}) => { + const [open, setOpen] = React.useState(false) + const showVariable = data.value.length < 60 + console.log("Value: ", data.value) + + return ( +
+ { + if (!showVariable) { + setOpen(!open) + } + }} + > + + {data.name}: {showVariable ? data.value : null} + + + {open ? + + {data.value} + + : null} +
+ ) + } + var draggingDisabled = false; const codePopoutModal = !codeModalOpen ? null : ( - { - console.log(event); - console.log(event.srcElement); - if (!dragging) { - console.log("START"); - //setDragging(true); - } - }} - disabled={true} - onStop={(e) => { - console.log("STOP"); - if (!dragging) { - return; - } - - setDragging(false); - const newoffsetX = - parseInt(dragPosition.x) - parseInt(e.layerX - e.offsetX); - const newoffsetY = - parseInt(dragPosition.y) - parseInt(e.layerY - e.offsetY); - if ( - newoffsetX <= 40 && - newoffsetX >= -40 && - newoffsetY <= 40 && - newoffsetY >= -40 - ) { - console.log("SKIP X & Y"); - return; - } - - const newPosition = { - x: e.layerX - e.offsetX, - y: e.layerY - e.offsetY, - }; - setDragPosition(newPosition); - }} - position={dragPosition} - > { pointerEvents: "auto", backgroundColor: inputColor, color: "white", - minWidth: 650, + minWidth: isMobile ? "90%" : 650, padding: 30, maxHeight: 550, overflowY: "auto", overflowX: "hidden", zIndex: 10012, + border: theme.palette.defaultBorder, }, }} > @@ -11956,10 +12489,12 @@ const AngularWorkflow = (defaultprops) => {
{selectedResult.action.label} @@ -12041,7 +12576,7 @@ const AngularWorkflow = (defaultprops) => { variant="h6" style={{ marginBottom: 0, marginTop: 0 }} > - Variables + Variables (click to expand) {selectedResult.action.parameters.map((data, index) => { if (data.value.length === 0) { @@ -12057,14 +12592,7 @@ const AngularWorkflow = (defaultprops) => { } return ( -
- - {data.name}: {data.value} - -
+ ); })}
@@ -12072,7 +12600,6 @@ const AngularWorkflow = (defaultprops) => {
- ); const newView = ( @@ -12080,11 +12607,11 @@ const AngularWorkflow = (defaultprops) => {
+ {/*isMobile ? null : leftView*/} {leftView} {workflow.id === undefined || workflow.id === null || - apps.length === 0 ? ( - + appsLoaded === false ? (
{ maxZoom={2.0} wheelSensitivity={0.25} style={{ - width: bodyWidth - leftBarSize - 15, + width: cytoscapeWidth, height: bodyHeight - appBarSize - 5, backgroundColor: surfaceColor, }} @@ -12166,7 +12693,7 @@ const AngularWorkflow = (defaultprops) => { requiresAuthentication={requiresAuthentication} /> - +
); @@ -12184,19 +12711,28 @@ const AngularWorkflow = (defaultprops) => { return ( { setNewVariableName(""); setExecutionVariablesModalOpen(false); }} PaperProps={{ style: { + pointerEvents: "auto", backgroundColor: surfaceColor, color: "white", + border: theme.palette.defaultBorder, + maxWidth: "100%", }, }} > - + Execution Variable @@ -12328,6 +12864,12 @@ const AngularWorkflow = (defaultprops) => { return ( { setNewVariableName(""); @@ -12337,13 +12879,16 @@ const AngularWorkflow = (defaultprops) => { }} PaperProps={{ style: { + pointerEvents: "auto", backgroundColor: surfaceColor, color: "white", + border: theme.palette.defaultBorder, + maxWidth: isMobile ? bodyWidth-100 : "100%", }, }} > - + Workflow Variable @@ -12499,7 +13044,7 @@ const AngularWorkflow = (defaultprops) => { ) { return ( - + {selectedApp.name} does not require authentication @@ -12605,7 +13150,7 @@ const AngularWorkflow = (defaultprops) => { return (
- +
Authentication for {selectedApp.name}
@@ -12663,6 +13208,9 @@ const AngularWorkflow = (defaultprops) => { data.schema !== null && data.schema.type === "bool" ? ( - - Accounts - - - - - - - - - - - - -
- -
- - - - - - - - -
Total Shipments
- - 763,215 - - - -
- -
- - - - - - -
Daily Sales
- - {" "} - 3,500€ - - - -
- -
- - - - - - -
Completed Tasks
- - 12,100K - - - -
- -
- - - -
); diff --git a/frontend/src/views/Docs.jsx b/frontend/src/views/Docs.jsx index 91ac42da..d6c98b9e 100644 --- a/frontend/src/views/Docs.jsx +++ b/frontend/src/views/Docs.jsx @@ -102,7 +102,7 @@ const Docs = (defaultprops) => { maxHeight: "83vh", overflowX: "hidden", overflowY: "auto", - zIndex: 10003, + zIndex: 1000, }; const fetchDocList = () => { @@ -141,6 +141,11 @@ const Docs = (defaultprops) => { setData(responseJson.reason); document.title = "Shuffle " + docId + " documentation"; + if (responseJson.reason !== undefined && responseJson.reason !== null && responseJson.reason.includes("404: Not Found")) { + navigate("/docs") + return + } + if (responseJson.meta !== undefined) { setSelectedMeta(responseJson.meta); } @@ -352,7 +357,7 @@ const Docs = (defaultprops) => { } function Img(props) { - return {props.alt}; + return {props.alt}; } function CodeHandler(props) { @@ -517,7 +522,8 @@ const Docs = (defaultprops) => { const newname = item.charAt(0).toUpperCase() + item.substring(1).split("_").join(" ").split("-").join(" "); - const itemMatching = + + const itemMatching = props.match.params.key === undefined ? false : props.match.params.key.toLowerCase() === item.toLowerCase(); //const [tocLines, setTocLines] = React.useState([]); return ( diff --git a/frontend/src/views/GettingStarted.jsx b/frontend/src/views/GettingStarted.jsx index c9598828..c66e8daf 100644 --- a/frontend/src/views/GettingStarted.jsx +++ b/frontend/src/views/GettingStarted.jsx @@ -2,6 +2,7 @@ import React, { useEffect, useContext } from "react"; import { makeStyles } from "@material-ui/core/styles"; import { useTheme } from "@material-ui/core/styles"; +import ReactGA from 'react-ga'; import SecurityFramework from '../components/SecurityFramework.jsx'; import { @@ -67,7 +68,7 @@ import { DataGrid, GridToolbar } from "@material-ui/data-grid"; //import JSONPrettyMon from 'react-json-pretty/dist/monikai' import Dropzone from "../components/Dropzone"; -import { Link } from "react-router-dom"; +import { useNavigate, Link, useParams } from "react-router-dom"; import { useAlert } from "react-alert"; import ChipInput from "material-ui-chip-input"; import { v4 as uuidv4 } from "uuid"; @@ -109,263 +110,7 @@ const useStyles = makeStyles((theme) => ({ }, })); -// Takes an action in Shuffle and -// Returns information about the icon, the color etc to be used -// This can be used for actions of all types -export const GetIconInfo = (action) => { - // Finds the icon based on the action. Should be verbs. - const iconList = [ - { key: "cache_add", values: ["set_cache"] }, - { key: "cache_get", values: ["get_cache"] }, - { key: "filter", values: ["filter", "route", "router"] }, - { key: "merge", values: ["join", "merge"] }, - { - key: "search", - values: ["search", "find", "locate", "index", "analyze", "anal", "match", "check cache", "check", "verify", "validate"], - }, - { key: "list", values: ["list", "head", "options"] }, - { - key: "download", - values: [ - "capture", - "get", - "download", - "return", - "hello_world", - "curl", - "request", - "export", - "preview", - ], - }, - { key: "add", values: ["add", "accept", ] }, - { key: "delete", values: ["delete", "remove", "clear", "clean", "dismiss",] }, - { - key: "send", - values: [ - "send", - "dispatch", - "mail", - "forward", - "post", - "submit", - "mark", - "set", - "release", - ], - }, - { - key: "repeat", - values: ["repeat", "retry", "pause", "skip", "copy", "replicat", "demo", ], - }, - { key: "execute", values: ["execute", "run", "play", "raise"] }, - { key: "extract", values: ["extract", "unpack", "decompress", "open"] }, - { key: "inflate", values: ["inflate", "pack", "compress"] }, - { - key: "edit", - values: [ - "modify", - "update", - "create", - "edit", - "put", - "patch", - "change", - "replace", - "conver", - "map", - "format", - "escape", - "describe", - ], - }, - { - key: "compare", - values: ["compare", "convert", "to", "filter", "translate", "parse"], - }, - { key: "close", values: ["close", "stop", "cancel", "block"] }, - ]; - - var selectedKey = ""; - if (action.name === undefined || action.name === null) { - } else { - const actionname = action.name.toLowerCase(); - for (var key in iconList) { - //console.log(iconList[key], actionname) - const found = iconList[key].values.find((value) => - actionname.includes(value) - ); - if (found !== null && found !== undefined) { - selectedKey = iconList[key].key; - break; - } - } - } - - // Some of these are manually parsed or created instead of material ui - //M8 0C3.58 0 0 1.79 0 4C0 6.21 3.58 8 8 8C12.42 8 16 6.21 16 4C16 1.79 12.42 0 8 0ZM0 6V9C0 11.21 3.58 13 8 13C12.42 13 16 11.21 16 9V6C16 8.21 12.42 10 8 10C3.58 10 0 8.21 0 6ZM0 11V14C0 16.21 3.58 18 8 18C9.41 18 10.79 17.81 12 17.46V14.46C10.79 14.81 9.41 15 8 15C3.58 15 0 13.21 0 11ZM17 11V14H14V16H17V19H19V16H22V14H19V11 - //https://www.figma.com/file/uCfnMs5w6wnLx6ehPHEV74/Figma-Material-Design-System-v3_0?node-id=834%3A21 - //COLORS: https://www.pinterest.co.uk/pin/326299935499972946/ - const defaultColor = "#f76b1c"; - const defaultGradient = ["#fad961", "#f76b1c"]; - const parsedIcons = { - cache_add: { - icon: "M11 3C6.58 3 3 4.79 3 7C3 9.21 6.58 11 11 11C15.42 11 19 9.21 19 7C19 4.79 15.42 3 11 3ZM3 9V12C3 14.21 6.58 16 11 16C15.42 16 19 14.21 19 12V9C19 11.21 15.42 13 11 13C6.58 13 3 11.21 3 9ZM3 14V17C3 19.21 6.58 21 11 21C12.41 21 13.79 20.81 15 20.46V17.46C13.79 17.81 12.41 18 11 18C6.58 18 3 16.21 3 14ZM20 14V17H17V19H20V22H22V19H25V17H22V14", - iconColor: "white", - iconBackgroundColor: "#8acc3f", - originalIcon: "", - fillGradient: ["#8acc3f", "#459622"], - }, - cache_get: { - icon: "M12 2C7.58 2 4 3.79 4 6C4 8.06 7.13 9.74 11.15 9.96C12.45 8.7 14.19 8 16 8C16.8 8 17.59 8.14 18.34 8.41C19.37 7.74 20 6.91 20 6C20 3.79 16.42 2 12 2ZM4 8V11C4 12.68 6.08 14.11 9 14.71C9.06 13.7 9.32 12.72 9.77 11.82C6.44 11.34 4 9.82 4 8ZM15.93 9.94C14.75 9.95 13.53 10.4 12.46 11.46C8.21 15.71 13.71 22.5 18.75 19.17L23.29 23.71L24.71 22.29L20.17 17.75C22.66 13.97 19.47 9.93 15.93 9.94ZM15.9 12C17.47 11.95 19 13.16 19 15C19 15.7956 18.6839 16.5587 18.1213 17.1213C17.5587 17.6839 16.7956 18 16 18C13.33 18 12 14.77 13.88 12.88C14.47 12.29 15.19 12 15.9 12ZM4 13V16C4 18.05 7.09 19.72 11.06 19.95C10.17 19.07 9.54 17.95 9.22 16.74C6.18 16.17 4 14.72 4 13Z", - iconColor: "white", - iconBackgroundColor: "#8acc3f", - originalIcon: "", - fillGradient: ["#8acc3f", "#459622"], - }, - repeat: { - icon: "M19 8l-4 4h3c0 3.31-2.69 6-6 6-1.01 0-1.97-.25-2.8-.7l-1.46 1.46C8.97 19.54 10.43 20 12 20c4.42 0 8-3.58 8-8h3l-4-4zM6 12c0-3.31 2.69-6 6-6 1.01 0 1.97.25 2.8.7l1.46-1.46C15.03 4.46 13.57 4 12 4c-4.42 0-8 3.58-8 8H1l4 4 4-4H6z", - iconColor: "white", - iconBackgroundColor: defaultColor, - originalIcon: , - }, - add: { - icon: "M19 13h-6v6h-2v-6H5v-2h6V5h2v6h6v2z", - iconColor: "white", - iconBackgroundColor: defaultColor, - originalIcon: , - }, - edit: { - icon: "M3 17.25V21h3.75L17.81 9.94l-3.75-3.75L3 17.25zM20.71 7.04c.39-.39.39-1.02 0-1.41l-2.34-2.34a.9959.9959 0 00-1.41 0l-1.83 1.83 3.75 3.75 1.83-1.83z", - iconColor: "white", - iconBackgroundColor: defaultColor, - originalIcon: , - }, - filter: { - icon: "M4.25 5.61C6.27 8.2 10 13 10 13v6c0 .55.45 1 1 1h2c.55 0 1-.45 1-1v-6s3.72-4.8 5.74-7.39c.51-.66.04-1.61-.79-1.61H5.04c-.83 0-1.3.95-.79 1.61z", - iconColor: "white", - iconBackgroundColor: "#f5515f", - originalIcon: "", - fillGradient: ["#f5515f", "#a1051d"], - }, - merge: { - icon: "M17 20.41 18.41 19 15 15.59 13.59 17 17 20.41zM7.5 8H11v5.59L5.59 19 7 20.41l6-6V8h3.5L12 3.5 7.5 8z", - iconColor: "white", - iconBackgroundColor: "#f5515f", - originalIcon: "", - fillGradient: ["#f5515f", "#a1051d"], - }, - compare: { - icon: "M10 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h5v2h2V1h-2v2zm0 15H5l5-6v6zm9-15h-5v2h5v13l-5-6v9h5c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2z", - iconColor: "white", - iconBackgroundColor: defaultColor, - originalIcon: , - }, - extract: { - icon: "M3 3h18v2H3z", - iconColor: "white", - iconBackgroundColor: defaultColor, - originalIcon: , - }, - inflate: { - icon: "M6 19h12v2H6z", - iconColor: "white", - iconBackgroundColor: defaultColor, - originalIcon: , - }, - list: { - icon: "M3 9h14V7H3v2zm0 4h14v-2H3v2zm0 4h14v-2H3v2zm16 0h2v-2h-2v2zm0-10v2h2V7h-2zm0 6h2v-2h-2v2z", - iconColor: "white", - iconBackgroundColor: defaultColor, - originalIcon: , - }, - execute: { - icon: "M8 5v14l11-7z", - iconColor: "white", - iconBackgroundColor: defaultColor, - originalIcon: , - }, - delete: { - icon: "M6 19c0 1.1.9 2 2 2h8c1.1 0 2-.9 2-2V7H6v12zM19 4h-3.5l-1-1h-5l-1 1H5v2h14V4z", - iconColor: "white", - iconBackgroundColor: "#03030e", - originalIcon: , - fillGradient: ["#03030e", "#205d66"], - }, - close: { - icon: "M19 6.41L17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z", - iconColor: "white", - iconBackgroundColor: "#03030e", - originalIcon: , - fillGradient: ["#03030e", "#205d66"], - }, - send: { - icon: "M2.01 21L23 12 2.01 3 2 10l15 2-15 2z", - iconColor: "white", - iconBackgroundColor: "#0373da", - originalIcon: , - fillGradient: ["#0bc8bf", "#0373da"], - }, - download: { - icon: "M19 9h-4V3H9v6H5l7 7 7-7zM5 18v2h14v-2H5z", - iconColor: "white", - iconBackgroundColor: "#0373da", - originalIcon: , - fillGradient: ["#0bc8bf", "#0373da"], - }, - search: { - icon: "M15.5 14h-.79l-.28-.27C15.41 12.59 16 11.11 16 9.5 16 5.91 13.09 3 9.5 3S3 5.91 3 9.5 5.91 16 9.5 16c1.61 0 3.09-.59 4.23-1.57l.27.28v.79l5 4.99L20.49 19l-4.99-5zm-6 0C7.01 14 5 11.99 5 9.5S7.01 5 9.5 5 14 7.01 14 9.5 11.99 14 9.5 14z", - iconColor: "white", - iconBackgroundColor: "green", - originalIcon: , - }, - }; - - var selectedItem = parsedIcons[selectedKey]; - if (selectedItem === undefined || selectedItem === null) { - return { - icon: "", - iconColor: "", - iconBackground: "black", - originalIcon: "", - }; - } - - if (selectedItem.fillGradient === undefined) { - selectedItem.fillGradient = defaultGradient; - selectedItem.iconBackgroundColor = defaultColor; - } - - if (selectedItem.icon === "" || selectedItem.icon === undefined) { - console.log( - `MISSING PATH FOR ${selectedKey} (find in scope): `, - selectedItem.originalIcon.type.type - ); - } - - if ( - (selectedItem.originalIcon === undefined || - selectedItem.originalIcon === "") && - selectedItem.icon !== "" && - selectedItem.icon !== undefined - ) { - const svg_pin = ( - - - - ); - selectedItem.originalIcon = svg_pin; - } - - return selectedItem; -}; - + const chipStyle = { backgroundColor: "#3d3f43", marginRight: 5, @@ -377,62 +122,6 @@ const chipStyle = { color: "white", }; -export const validateJson = (showResult) => { - //showResult = showResult.split(" None").join(" \"None\"") - showResult = showResult.split(" False").join(" false"); - showResult = showResult.split(" True").join(" true"); - - var jsonvalid = true; - try { - if (!showResult.includes("{") && !showResult.includes("[")) { - jsonvalid = false; - } - } catch (e) { - showResult = showResult.split("'").join('"'); - - try { - if (!showResult.includes("{") && !showResult.includes("[")) { - jsonvalid = false; - } - } catch (e) { - jsonvalid = false; - } - } - - var result = showResult; - try { - result = jsonvalid ? JSON.parse(showResult) : showResult; - } catch (e) { - ////console.log("Failed parsing JSON even though its valid: ", e) - jsonvalid = false; - } - - if (jsonvalid === false) { - - if (typeof showResult === 'string') { - showResult = showResult.trim() - } - - try { - var newstr = showResult.replaceAll("'", '"') - - //console.log("Try replacements and trimming with new value: ", newstr) - result = JSON.parse(newstr) - jsonvalid = true - } catch (e) { - - //console.log("Failed parsing JSON even though its valid (2): ", e) - jsonvalid = false - } - } - - //console.log("VALID: ", jsonvalid, result) - return { - valid: jsonvalid, - result: result, - }; -}; - const GettingStarted = (props) => { const { globalUrl, isLoggedIn, isLoaded, userdata } = props; @@ -440,6 +129,7 @@ const GettingStarted = (props) => { const theme = useTheme(); const alert = useAlert(); const classes = useStyles(theme); + let navigate = useNavigate(); const imgSize = 60; const referenceUrl = globalUrl + "/api/v1/hooks/"; @@ -459,8 +149,8 @@ const GettingStarted = (props) => { "https://github.com/frikky/shuffle-workflows" ); const [downloadBranch, setDownloadBranch] = React.useState("master"); - const [loadWorkflowsModalOpen, setLoadWorkflowsModalOpen] = - React.useState(false); + const [loadWorkflowsModalOpen, setLoadWorkflowsModalOpen] = React.useState(false); + const [videoViewOpen, setVideoViewOpen] = React.useState(false); const [exportModalOpen, setExportModalOpen] = React.useState(false); const [exportData, setExportData] = React.useState(""); @@ -825,6 +515,8 @@ const GettingStarted = (props) => { credentials: "include", }) .then((response) => { + setVideoViewOpen(true) + if (response.status !== 200) { console.log("Status not 200 for workflows :O!: ", response.status); @@ -879,6 +571,8 @@ const GettingStarted = (props) => { } }) .catch((error) => { + setVideoViewOpen(true) + alert.error(error.toString()); }); }; @@ -2446,70 +2140,27 @@ const GettingStarted = (props) => { maxWidth: 600, }; - const WorkflowView = () => { - /* - if (workflows.length === 0) { - return ( -
- -
-

Welcome to Shuffle

-
-
-

- Shuffle is a flexible, easy to use, automation platform - allowing users to integrate their services and devices freely. - It's made to significantly reduce the amount of manual labor, - and is focused on security applications.{" "} - - Click here to learn more. - -

-
-
- If you want to jump straight into it, click here to create your - first workflow: -
-
- - - - ..OR - - {workflowButtons} - -
- -
- ); - } - */ + const WorkflowView = () => { var workflowDelay = -150 var appDelay = -75 const textSpacingDiff = 8 const textType = "body2" - // Discover use-cases made by other creators! + // Discover use-cases made by us and other creators! const steps = [ { html: ( - - Find your integrations by following our simple detection framework! + { + if (isCloud) { + ReactGA.event({ + category: "getting-started", + action: `integerations_find_click`, + }) + } + }}> + Find relevant apps and start your automation journey ), tutorial: "find_integrations", @@ -2517,31 +2168,50 @@ const GettingStarted = (props) => { { html: - Discover { + Discover Use Case ideas and  + { + + if (isCloud) { + navigate(`/search?tab=workflows`) + + ReactGA.event({ + category: "getting-started", + action: `workflow_find_click`, + }) + return + } else { alert.success("TBD: Coming in version 1.0.0"); + } - const ele = document.getElementById("shuffle_search_field") - if (ele !== undefined && ele !== null) { - console.log("Found ele: ", ele) - ele.focus() - ele.style.borderColor = "#f86a3e" - ele.style.borderWidth = "2px" + const ele = document.getElementById("shuffle_search_field") + if (ele !== undefined && ele !== null) { + console.log("Found ele: ", ele) + ele.focus() + ele.style.borderColor = "#f86a3e" + ele.style.borderWidth = "2px" - } else { - alert.success("TBD: Coming in version 1.0.0"); - } - }}> - use-cases made by other creators! + } else { + //alert.success("TBD: Coming in version 1.0.0"); + } + }}> + workflows made by other creators! , tutorial: "discover_workflows", }, { html: ( - + { + if (isCloud) { + ReactGA.event({ + category: "getting-started", + action: `create_workflow_click`, + }) + } + }}> Learn to use Shuffle by  {setModalOpen(true)}}> creating your first workflow - and reading the docs. + and reading the docs. ), tutorial: "learn_shuffle", @@ -2557,6 +2227,55 @@ const GettingStarted = (props) => { return (
+ {isCloud ? + { + setVideoViewOpen(false) + }} + PaperProps={{ + style: { + backgroundColor: surfaceColor, + color: "white", + minWidth: 560, + minHeight: 415, + textAlign: "center", + }, + }} + > + + Welcome to Shuffle! + + + + { + e.preventDefault(); + setVideoViewOpen(false) + }} + > + + + + + + + : null}
Getting Started with Shuffle diff --git a/frontend/src/views/LoginPage.jsx b/frontend/src/views/LoginPage.jsx index 8928f166..0e6c6b51 100644 --- a/frontend/src/views/LoginPage.jsx +++ b/frontend/src/views/LoginPage.jsx @@ -1,5 +1,5 @@ /* eslint-disable react/no-multi-comp */ -import React, { useState } from "react"; +import React, { useState, useEffect } from "react"; import { makeStyles } from "@material-ui/styles"; import { useInterval } from "react-powerhooks"; @@ -32,9 +32,6 @@ const useStyles = makeStyles({ }); const LoginDialog = (props) => { - const theme = useTheme(); - let navigate = useNavigate(); - const { globalUrl, isLoaded, @@ -44,6 +41,11 @@ const LoginDialog = (props) => { register, checkLogin, } = props; + + const theme = useTheme(); + let navigate = useNavigate(); + const classes = useStyles(); + const [username, setUsername] = useState(""); const [password, setPassword] = useState(""); const [firstRequest, setFirstRequest] = useState(true); @@ -54,9 +56,13 @@ const LoginDialog = (props) => { const [MFAField, setMFAField] = useState(false); const [MFAValue, setMFAValue] = useState(""); + // Used to swap from login to register. True = login, false = register - const classes = useStyles(); + useEffect(() => { + checkAdmin() + }, [loginViewLoading]) + // Error messages etc const [loginInfo, setLoginInfo] = useState(""); @@ -143,7 +149,7 @@ const LoginDialog = (props) => { var baseurl = globalUrl; if (register) { - var url = baseurl + "/api/v1/users/login"; + var url = baseurl + "/api/v1/login"; fetch(url, { mode: "cors", method: "POST", @@ -180,7 +186,8 @@ const LoginDialog = (props) => { setIsLoggedIn(true); - navigate("/workflows") + //navigate("/workflows") + window.location.href = "/workflows" } }) ) @@ -264,6 +271,7 @@ const LoginDialog = (props) => { }} />
+ {loginViewLoading ? (
{
+ + + By connecting your Github account, you agree to our Terms of Service, and acknowledge that your non-sensitive data will be turned into a creator account. This enables you to earn a passive income from Shuffle. This IS reversible. + + + : null}
@@ -859,7 +866,7 @@ const Settings = (props) => { handleEthereumConnection(); }} > - Authenticate + Authenticate Metamask Wallet )}
@@ -917,11 +924,11 @@ const Settings = (props) => { }; const handleGithubConnection = () => { - console.log("GITHUB CONNECT WOO") + console.log("GITHUB CONNECT WOO: ", isCloud) //result = RestClient.post('https://github.com/login/oauth/access_token', console.log("HOST: ", window.location.host); - console.log("HOST: ", window.location); + console.log("Location: ", window.location); const redirectUri = isCloud ? window.location.host === "localhost:3002" ? "http%3A%2F%2Flocalhost:3002%2Fset_authentication" @@ -931,10 +938,11 @@ const Settings = (props) => { : `https%3A%2F%2F${window.location.host}%2Fset_authentication` + console.log("redirect: ", redirectUri) const client_id = "3d272b1b782b100b1e61" const username = userdata.id; - const scopes = "user:email"; + const scopes = "read:user"; const url = `https://github.com/login/oauth/authorize?access_type=offline&prompt=consent&client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&scope=${scopes}&state=username%3D${username}%26type%3Dgithub` diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index b1e7f13c..9ccf18b5 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -6,12 +6,16 @@ import { Navigate } from "react-router-dom"; import SecurityFramework from '../components/SecurityFramework.jsx'; import { ShepherdTour, ShepherdTourContext } from 'react-shepherd' +import { isMobile } from "react-device-detect" import { Badge, Avatar, Grid, + InputLabel, + Select, + ListSubheader, Paper, Tooltip, Divider, @@ -31,8 +35,15 @@ import { DialogTitle, DialogActions, DialogContent, + OutlinedInput, + Checkbox, + ListItemText, } from "@material-ui/core"; +import { + AvatarGroup, +} from "@mui/material" + import { GridOn as GridOnIcon, List as ListIcon, @@ -58,6 +69,8 @@ import { Publish as PublishIcon, CloudUpload as CloudUploadIcon, CloudDownload as CloudDownloadIcon, + ExpandLess as ExpandLessIcon, + ExpandMore as ExpandMoreIcon, } from "@material-ui/icons"; import NestedMenuItem from "material-ui-nested-menu-item"; @@ -381,9 +394,27 @@ const chipStyle = { }; export const validateJson = (showResult) => { - //showResult = showResult.split(" None").join(" \"None\"") - showResult = showResult.split(" False").join(" false"); - showResult = showResult.split(" True").join(" true"); + //console.log("INPUT: ", showResult, typeof showResult) + if (typeof showResult === 'string') { + //showResult = showResult.split(" None").join(" \"None\"") + showResult = showResult.split(" False").join(" false"); + showResult = showResult.split(" True").join(" true"); + //return { + // valid: false, + // result: showResult, + //}; + } + + //if (typeof showResult === undefined) { + + //} + + if (typeof showResult === "object" || typeof showResult === "array") { + return { + valid: true, + result: showResult, + }; + } var jsonvalid = true; try { @@ -453,6 +484,8 @@ const Workflows = (props) => { var upload = ""; const [workflows, setWorkflows] = React.useState([]); + const [_, setUpdate] = React.useState(""); // Used for rendering, don't remove + const [selectedUsecases, setSelectedUsecases] = React.useState([]); const [filteredWorkflows, setFilteredWorkflows] = React.useState([]); const [selectedWorkflow, setSelectedWorkflow] = React.useState({}); const [workflowDone, setWorkflowDone] = React.useState(false); @@ -488,6 +521,8 @@ const Workflows = (props) => { const [actionImageList, setActionImageList] = React.useState([]); const [firstLoad, setFirstLoad] = React.useState(true); + const [showMoreClicked, setShowMoreClicked] = React.useState(false); + const [usecases, setUsecases] = React.useState([]); const isCloud = window.location.host === "localhost:3002" || @@ -834,7 +869,7 @@ const Workflows = (props) => { console.log("Status not 200 for workflows :O!: ", response.status); if (isCloud) { - window.location.pathname = "/login"; + window.location.pathname = "/search?tab=workflows"; } alert.info("Failed getting workflows."); @@ -847,8 +882,10 @@ const Workflows = (props) => { .then((responseJson) => { if (responseJson !== undefined) { setWorkflows(responseJson); + fetchUsecases(responseJson) if (responseJson !== undefined) { + var actionnamelist = []; var parsedactionlist = []; for (var key in responseJson) { @@ -888,6 +925,83 @@ const Workflows = (props) => { }); }; + const handleKeysetting = (categorydata, workflows) => { + console.log("Workflows: ", workflows) + //workflows[0].category = ["detect"] + //workflows[0].usecase_ids = ["Correlate tickets"] + + if (workflows !== undefined && workflows !== null) { + var newcategories = [] + for (var key in categorydata) { + var category = categorydata[key] + category.matches = [] + + for (var subcategorykey in category.list) { + var subcategory = category.list[subcategorykey] + subcategory.matches = [] + + for (var workflowkey in workflows) { + const workflow = workflows[workflowkey] + + if (workflow.usecase_ids !== undefined && workflow.usecase_ids !== null) { + for (var usecasekey in workflow.usecase_ids) { + if (workflow.usecase_ids[usecasekey].toLowerCase() === subcategory.name.toLowerCase()) { + console.log("Got match: ", workflow.usecase_ids[usecasekey]) + + category.matches.push({ + "workflow": workflow.id, + "category": subcategory.name, + }) + subcategory.matches.push(workflow.id) + break + } + } + } + + if (subcategory.matches.length > 0) { + break + } + } + } + + newcategories.push(category) + } + + console.log("Categories: ", newcategories) + setUsecases(newcategories) + } else { + setUsecases(categorydata) + } + } + + const fetchUsecases = (workflows) => { + fetch(globalUrl + "/api/v1/workflows/usecases", { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for usecases"); + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success !== false) { + console.log("Usecases: ", responseJson) + handleKeysetting(responseJson, workflows) + } + }) + .catch((error) => { + //alert.error("ERROR: " + error.toString()); + console.log("ERROR: " + error.toString()); + }); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps useEffect(() => { if (workflows.length <= 0) { @@ -896,7 +1010,6 @@ const Workflows = (props) => { setView(tmpView); } - //setFirstrequest(false); getAvailableWorkflows(); } }, []) @@ -905,8 +1018,8 @@ const Workflows = (props) => { color: "#ffffff", width: "100%", display: "flex", - minWidth: 1024, - maxWidth: 1024, + minWidth: isMobile ? "100%" : 1024, + maxWidth: isMobile ? "100%" : 1024, margin: "auto", }; @@ -926,6 +1039,7 @@ const Workflows = (props) => { flexDirection: "column", }; + //flexDirection: !isMobile ? "column" : "row", const paperAppContainer = { display: "flex", flexWrap: "wrap", @@ -1270,7 +1384,7 @@ const Workflows = (props) => { }; return ( - + { ); }; + const getWorkflowAppgroup = (data) => { + if (data.actions === undefined || data.actions === null) { + return [] + } + + var appsFound = [] + for (var key in data.actions) { + const parsedAction = data.actions[key] + if (parsedAction.large_image === undefined || parsedAction.large_image === null || parsedAction.large_image === "") { + continue + } + + if (parsedAction.app_name === "Shuffle Tools" || parsedAction.app_id === "bc78f35c6c6351b07a09b7aed5d29652") { + continue + } + + if (appsFound.findIndex(data => data.app_name === parsedAction.app_name) < 0){ + appsFound.push(parsedAction) + } + } + + return appsFound + } + const WorkflowPaper = (props) => { const { data } = props; const [open, setOpen] = React.useState(false); @@ -1321,6 +1459,7 @@ const Workflows = (props) => { } const actions = data.actions !== null ? data.actions.length : 0; + const appGroup = getWorkflowAppgroup(data) const [triggers, subflows] = getWorkflowMeta(data); const workflowMenuButtons = ( @@ -1345,6 +1484,11 @@ const Workflows = (props) => { if (data.tags !== undefined && data.tags !== null) { setNewWorkflowTags(JSON.parse(JSON.stringify(data.tags))); } + + console.log("Editing: ", data) + if (data.usecase_ids !== undefined && data.usecase_ids !== null && data.usecase_ids.length > 0) { + setSelectedUsecases(data.usecase_ids) + } }} key={"change"} > @@ -1543,22 +1687,49 @@ const Workflows = (props) => { - - - - - {actions} - - - + {appGroup.length > 0 ? +
+ + {appGroup.map((data, index) => { + return ( +
{ + addFilter(data.app_name); + }} + > + + + +
+ ) + })} + +
+ : + + + + + {actions} + + + + } { justifyContent: "left", overflow: "hidden", marginTop: 5, + maxHeight: 28, + overflow: "hidden", }} > - {data.tags !== undefined + {data.tags !== undefined && data.tags !== null ? data.tags.map((tag, index) => { if (index >= 3) { return null; @@ -1709,7 +1882,8 @@ const Workflows = (props) => { tags, defaultReturnValue, editingWorkflow, - redirect + redirect, + currentUsecases, ) => { var method = "POST"; var extraData = ""; @@ -1736,6 +1910,12 @@ const Workflows = (props) => { workflowdata["default_return_value"] = defaultReturnValue; } + if (currentUsecases !== undefined && currentUsecases !== null) { + workflowdata["usecase_ids"] = currentUsecases + //workflows[0].category = ["detect"] + //workflows[0].usecase_ids = ["Correlate tickets"] + } + return fetch(globalUrl + "/api/v1/workflows" + extraData, { method: method, headers: { @@ -1982,30 +2162,54 @@ const Workflows = (props) => { const data = params.row.record; const actions = data.actions !== null ? data.actions.length : 0; let [triggers, subflows] = getWorkflowMeta(data); + const appGroup = getWorkflowAppgroup(data) return (
- - - - - {actions} - - - + {appGroup.length > 0 ? +
+ + {appGroup.map((data, index) => { + return ( +
{ + addFilter(data.app_name); + }} + > + + + +
+ ) + })} + +
+ : + + + + + {actions} + + + + } { return
{workflowData}
; }; + var total_count = 0 const modalView = modalOpen ? ( { style: { backgroundColor: surfaceColor, color: "white", - minWidth: "800px", + minWidth: isMobile ? "90%" : "800px", + maxWidth: isMobile ? "90%" : "800px", }, }} > @@ -2231,6 +2437,7 @@ const Workflows = (props) => { }} color="primary" placeholder="Name" + required margin="dense" defaultValue={newWorkflowName} autoFocus @@ -2246,47 +2453,117 @@ const Workflows = (props) => { color="primary" defaultValue={newWorkflowDescription} placeholder="Description" - rows="3" multiline margin="dense" fullWidth /> - { - newWorkflowTags.push(chip); - setNewWorkflowTags(newWorkflowTags); - }} - onDelete={(chip, index) => { - newWorkflowTags.splice(index, 1); - setNewWorkflowTags(newWorkflowTags); - }} - /> +
+ { + newWorkflowTags.push(chip); + setNewWorkflowTags(newWorkflowTags); + }} + onDelete={(chip, index) => { + newWorkflowTags.splice(index, 1); + setNewWorkflowTags(newWorkflowTags); + }} + /> + {usecases !== null && usecases !== undefined && usecases.length > 0 ? + + Usecases + + + : null} +
+ + {showMoreClicked ? + + setDefaultReturnValue(event.target.value)} + InputProps={{ + style: { + color: "white", + }, + }} + color="primary" + defaultValue={defaultReturnValue} + placeholder="Default return value (used for Subflows if the subflow fails)" + rows="3" + multiline + margin="dense" + fullWidth + /> + + : null} + + { + setShowMoreClicked(!showMoreClicked); + }} + > + {showMoreClicked ? : } + + - setDefaultReturnValue(event.target.value)} - InputProps={{ - style: { - color: "white", - }, - }} - color="primary" - defaultValue={defaultReturnValue} - placeholder="Default return value (used for Subflows if the subflow fails)" - rows="3" - multiline - margin="dense" - fullWidth - /> ) : ( @@ -3024,7 +3360,7 @@ const Workflows = (props) => { */} 1366 ? 1366 : 1200, + maxWidth: window.innerWidth > 1366 ? 1366 : isMobile ? "100%" : 1200, margin: "auto", padding: 20, }} diff --git a/functions/extensions/aws-s3-lambda/README.md b/functions/extensions/aws-s3-lambda/README.md new file mode 100644 index 00000000..f665851a --- /dev/null +++ b/functions/extensions/aws-s3-lambda/README.md @@ -0,0 +1,2 @@ +# AWS Lambda forwarder to Shuffle +This function is made to forward S3 notifications to Shuffle to run a workflow when an object is made or updated. diff --git a/functions/extensions/aws-s3-lambda/s3_function.py b/functions/extensions/aws-s3-lambda/s3_function.py new file mode 100644 index 00000000..82838a4e --- /dev/null +++ b/functions/extensions/aws-s3-lambda/s3_function.py @@ -0,0 +1,21 @@ +import json +import urllib.parse +import urllib3 +import os + +print('Loading function') + +def lambda_handler(event, context): + # Get the object from the event and show its content type + bucket = event['Records'][0]['s3']['bucket']['name'] + + webhook = os.environ.get("SHUFFLE_WEBHOOK") + if not webhook: + return "No webhook environment defined: SHUFFLE_WEBHOOK" + + http = urllib3.PoolManager() + ret = http.request('POST', webhook, body=json.dumps(event["Records"][0]).encode("utf-8")) + if ret.status != 200: + return "Bad status code for webhook: %d" % ret.status_code + + print("Status code: %d\nData: %s" % (ret.status, ret.data)) diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index f0186a6e..57beb89e 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.9.50 +VERSION=0.9.61 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 1c48f902..ee733a77 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -37,9 +37,11 @@ import ( "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/network" "github.com/docker/docker/api/types/swarm" + //"github.com/docker/docker/api/types/filters" dockerclient "github.com/docker/docker/client" - "github.com/satori/go.uuid" + uuid "github.com/satori/go.uuid" + //network "github.com/docker/docker/api/types/network" //natting "github.com/docker/go-connections/nat" "github.com/mackerelio/go-osstat/cpu" @@ -63,7 +65,7 @@ var baseimagename = os.Getenv("SHUFFLE_BASE_IMAGE_NAME") var baseimageregistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY") var baseimagetagsuffix = os.Getenv("SHUFFLE_BASE_IMAGE_TAG_SUFFIX") -var orgId = os.Getenv("ORG_ID") +//var orgId = os.Getenv("ORG_ID") var baseUrl = os.Getenv("BASE_URL") var environment = os.Getenv("ENVIRONMENT_NAME") var dockerApiVersion = os.Getenv("DOCKER_API_VERSION") @@ -199,7 +201,7 @@ func deployServiceWorkers(image string) { // Looks for and cleans up all existing items in swarm we can't re-use (Shuffle only) cleanupExistingNodes(ctx) // frikky@debian:~/git/shuffle/functions/onprem/worker$ docker service create --replicas 5 --name shuffle-workers --env SHUFFLE_SWARM_CONFIG=run --publish published=33333,target=33333 ghcr.io/frikky/shuffle-worker:nightly - networkName := "shuffle-executions" + networkName := "shuffle_swarm_executions" if len(swarmNetworkName) > 0 { networkName = swarmNetworkName } @@ -538,8 +540,14 @@ func deployWorker(image string, identifier string, env []string, executionReques nil, identifier+"-2", ) + if err != nil { + log.Printf("[ERROR] Failed to CREATE container (2): %s", err) + } err = dockercli.ContainerStart(context.Background(), cont.ID, containerStartOptions) + if err != nil { + log.Printf("[ERROR] Failed to start container (2): %s", err) + } } else { log.Printf("[ERROR] Failed initial container start. Quitting as this is NOT a simple network issue. Err: %s", err) } @@ -759,8 +767,12 @@ func main() { //baseUrl = "http://localhost:5001" } - if orgId == "" { - log.Printf("[ERROR] Org not defined. Set variable ORG_ID based on your org") + //if orgId == "" { + // log.Printf("[ERROR] Org not defined. Set variable ORG_ID based on your org") + // os.Exit(3) + //} + if environment == "" { + log.Printf("[ERROR] Environment not defined. Set variable ENVIRONMENT_NAME to configure it.") os.Exit(3) } @@ -797,7 +809,7 @@ func main() { // Run by default from now zombiecheck(ctx, workerTimeout) - log.Printf("[INFO] Running towards %s with Org %s", baseUrl, orgId) + log.Printf("[INFO] Running towards %s (BASE_URL) with environment name %s", baseUrl, environment) httpProxy := os.Getenv("HTTP_PROXY") httpsProxy := os.Getenv("HTTPS_PROXY") @@ -845,6 +857,8 @@ func main() { } } + client.Timeout = 10 * time.Second + fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl) req, err := http.NewRequest( "GET", @@ -859,8 +873,8 @@ func main() { zombiecounter := 0 req.Header.Add("Content-Type", "application/json") - req.Header.Add("Org-Id", orgId) - log.Printf("[INFO] Waiting for executions at %s with Org ID %s", fullUrl, orgId) + req.Header.Add("Org-Id", environment) + log.Printf("[INFO] Waiting for executions at %s with Environment %s", fullUrl, environment) hasStarted := false for { //go getStats() @@ -1045,7 +1059,7 @@ func main() { } result.Header.Add("Content-Type", "application/json") - result.Header.Add("Org-Id", orgId) + result.Header.Add("Org-Id", environment) resultResp, err := client.Do(result) if err != nil { diff --git a/functions/onprem/orborus/proxy_server.py b/functions/onprem/orborus/proxy_server.py new file mode 100644 index 00000000..8020616c --- /dev/null +++ b/functions/onprem/orborus/proxy_server.py @@ -0,0 +1,54 @@ +# +# curl -H "Org-id: Shuffle" --proxy "http://192.168.86.45:8081" http://192.168.86.45:5001/api/v1/workflows/queue + +import SocketServer +import SimpleHTTPServer +import requests +import json + +PORT = 8082 + +class MyProxy(SimpleHTTPServer.SimpleHTTPRequestHandler): + def do_GET(self): + url=self.path[:] + allheaders = {} + for item in ("%s" % self.headers).split("\n"): + headersplit = item.split(":") + if len(headersplit) == 2: + allheaders[headersplit[0]] = (headersplit[1][:-1]).strip() + + ret = requests.get(url, headers=allheaders) + print("RESP (%s) - %d - %s" % (url, ret.status_code, ret.text)) + + self.send_response(ret.status_code) + self.end_headers() + self.wfile.write(ret.text) + + def do_POST(self): + url=self.path[:] + allheaders = {} + for item in ("%s" % self.headers).split("\n"): + headersplit = item.split(":") + if len(headersplit) == 2: + allheaders[headersplit[0]] = (headersplit[1][:-1]).strip() + + length = int(self.headers.getheader('content-length')) + try: + message = json.loads(self.rfile.read(length)) + print("Got message: %s" % message) + ret = requests.post(url, headers=allheaders, json=message) + except: + message = self.rfile.read(length) + print("Got message: %s" % message) + ret = requests.post(url, headers=allheaders, data=message) + + print("RESP (%s) - %d - %s" % (url, ret.status_code, ret.text)) + + self.send_response(ret.status_code) + self.end_headers() + self.wfile.write(ret.text) + +httpd = SocketServer.ForkingTCPServer(('', PORT), MyProxy) +print("Now serving at %d" % PORT) +httpd.serve_forever() + diff --git a/functions/onprem/orborus/run.sh b/functions/onprem/orborus/run.sh index 48e5b701..eac60eec 100644 --- a/functions/onprem/orborus/run.sh +++ b/functions/onprem/orborus/run.sh @@ -1,9 +1,10 @@ docker run \ - --env ORG_ID=$ORG_ID \ + --env DOCKER_API_VERSION=1.40 \ --env ENVIRONMENT_NAME="Shuffle" \ - --env BASE_URL=http://shuffle-backend:5001 \ - --env DOCKER_API_VERSION=1.42 \ - --env RUNNING_MODE="Docker" \ - --network "shuffle_shuffle" \ + --env BASE_URL="http://192.168.86.45:5001" \ + --env HTTP_PROXY="http://192.168.86.45:8082" \ + --env HTTPS_PROXY="https://192.168.86.45:8082" \ + --env SHUFFLE_PASS_WORKER_PROXY=true \ + --env SHUFFLE_PASS_APP_PROXY=true \ -v /var/run/docker.sock:/var/run/docker.sock \ - frikky/shuffle:orborus + ghcr.io/frikky/shuffle-orborus:nightly diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 87919a26..e828cc0e 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.9.50 +VERSION=0.9.59 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index 5a1d006a..4874096e 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -595,6 +595,8 @@ github.com/shuffle/shuffle-shared v0.1.78 h1://YsgQ85Ep40AA3pLUXb+85BUrNz5sqGqh0 github.com/shuffle/shuffle-shared v0.1.78/go.mod h1:cW8LBv8P24rCPyJqGV6czxqrpnrv/R1d97EOqNgIvSk= github.com/shuffle/shuffle-shared v0.1.81 h1:/lOt7NSuMTWlRzgOKg2e7j95eakg3MgW6i/4Fp30kd4= github.com/shuffle/shuffle-shared v0.1.81/go.mod h1:cW8LBv8P24rCPyJqGV6czxqrpnrv/R1d97EOqNgIvSk= +github.com/shuffle/shuffle-shared v0.1.83 h1:xfmcqceBGXJVUyZNBmI+c6RKndrtKJyBIqdHD86v/XA= +github.com/shuffle/shuffle-shared v0.1.83/go.mod h1:cW8LBv8P24rCPyJqGV6czxqrpnrv/R1d97EOqNgIvSk= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 7460161f..e3d988ef 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -358,6 +358,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] log.Printf("\n\n[DEBUG] Result for %s already found - returning\n\n", newExecId) return nil } + cacheData := []byte("1") err = shuffle.SetCache(ctx, newExecId, cacheData) if err != nil { @@ -371,17 +372,19 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] waitTime := time.Duration(action.ExecutionDelay) * time.Second time.AfterFunc(waitTime, func() { - DeployContainer(ctx, cli, config, hostConfig, identifier, workflowExecution) + DeployContainer(ctx, cli, config, hostConfig, identifier, workflowExecution, newExecId) }) } else { - log.Printf("[DEBUG] Running app %s in docker NORMALLY as there is no delay set", action.Name) - return DeployContainer(ctx, cli, config, hostConfig, identifier, workflowExecution) + log.Printf("[DEBUG] Running app %s in docker NORMALLY as there is no delay set with identifier %s", action.Name, identifier) + returnvalue := DeployContainer(ctx, cli, config, hostConfig, identifier, workflowExecution, newExecId) + log.Printf("[DEBUG] Normal deploy ret: %s", returnvalue) + return returnvalue } return nil } -func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *container.Config, hostConfig *container.HostConfig, identifier string, workflowExecution shuffle.WorkflowExecution) error { +func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *container.Config, hostConfig *container.HostConfig, identifier string, workflowExecution shuffle.WorkflowExecution, newExecId string) error { cont, err := cli.ContainerCreate( ctx, config, @@ -396,6 +399,11 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont if !strings.Contains(err.Error(), "Conflict. The container name") { log.Printf("[ERROR] Container CREATE error (1): %s", err) + cacheErr := shuffle.DeleteCache(ctx, newExecId) + if cacheErr != nil { + log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, cacheErr) + } + return err } else { parsedUuid := uuid.NewV4() @@ -414,6 +422,12 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont if err != nil { log.Printf("[ERROR] Container create error (2): %s", err) + + cacheErr := shuffle.DeleteCache(ctx, newExecId) + if cacheErr != nil { + log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, cacheErr) + } + return err } @@ -445,6 +459,12 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont if err != nil { log.Printf("[ERROR] Container create error (3): %s", err) + + cacheErr := shuffle.DeleteCache(ctx, newExecId) + if cacheErr != nil { + log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, cacheErr) + } + return err } @@ -454,6 +474,12 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont if err != nil { log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err) + + cacheErr := shuffle.DeleteCache(ctx, newExecId) + if cacheErr != nil { + log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, cacheErr) + } + //shutdown(workflowExecution, workflowExecution.Workflow.ID, true) return err } @@ -1163,7 +1189,6 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { // if everything is generated during execution //log.Printf("[DEBUG][%s] Deployed with CALLBACK_URL %s and BASE_URL %s", workflowExecution.ExecutionId, appCallbackUrl, baseUrl) env := []string{ - fmt.Sprintf("ACTION=%s", string(actionData)), fmt.Sprintf("EXECUTIONID=%s", workflowExecution.ExecutionId), fmt.Sprintf("AUTHORIZATION=%s", workflowExecution.Authorization), fmt.Sprintf("CALLBACK_URL=%s", baseUrl), @@ -1171,6 +1196,40 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { fmt.Sprintf("TZ=%s", timezone), } + if len(actionData) >= 100000 { + log.Printf("[WARNING] Omitting some data from action execution. Length: %d. Fix in SDK!", len(actionData)) + newParams := []shuffle.WorkflowAppActionParameter{} + for _, param := range action.Parameters { + paramData, err := json.Marshal(param) + if err != nil { + log.Printf("[WARNING] Failed to marshal param %s: %s", param.Name, err) + newParams = append(newParams, param) + continue + } + + if len(paramData) >= 50000 { + log.Printf("[WARNING] Removing a lot of data from param %s with length %d", param.Name, len(paramData)) + param.Value = "SHUFFLE_AUTO_REMOVED" + } + + newParams = append(newParams, param) + } + + action.Parameters = newParams + actionData, err = json.Marshal(action) + if err == nil { + log.Printf("[DEBUG] Ran data replace on action %s. new length: %d", action.Name, len(actionData)) + } else { + log.Printf("[WARNING] Failed to marshal new actionData: %s", err) + + } + } else { + log.Printf("[DEBUG] Actiondata is NOT 100000 in length. Adding as normal.") + } + + actionEnv := fmt.Sprintf("ACTION=%s", string(actionData)) + env = append(env, actionEnv) + if strings.ToLower(os.Getenv("SHUFFLE_PASS_APP_PROXY")) == "true" { //log.Printf("APPENDING PROXY TO THE APP!") env = append(env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) @@ -1296,6 +1355,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { } else { err = deployApp(dockercli, images[0], identifier, env, workflowExecution, action) + log.Printf("[DEBUG] Failed deploying app? %s", err) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { log.Printf("[DEBUG] Shutting down (9)") @@ -1517,7 +1577,7 @@ func executionInit(workflowExecution shuffle.WorkflowExecution) error { onpremApps := []string{} toExecuteOnprem := []string{} for _, action := range workflowExecution.Workflow.Actions { - if action.Environment != environment { + if strings.ToLower(action.Environment) != strings.ToLower(environment) { continue } @@ -1599,7 +1659,7 @@ func handleDefaultExecution(client *http.Client, req *http.Request, workflowExec err := executionInit(workflowExecution) if err != nil { - log.Printf("[INFO] Workflow setup failed: %s", workflowExecution.ExecutionId, err) + log.Printf("[INFO] Workflow setup failed for %s: %s", workflowExecution.ExecutionId, err) log.Printf("[DEBUG] Shutting down (18)") shutdown(workflowExecution, "", "", true) } diff --git a/functions/usecases/README.md b/functions/usecases/README.md new file mode 100644 index 00000000..553034c2 --- /dev/null +++ b/functions/usecases/README.md @@ -0,0 +1,16 @@ +# Mindmap exporter +Shuffle has a mindmap for Workflow use-cases. These can be changed and exported, with the most important piece being that they're explorable and editable. This has and will come in handy for us as we build it into the product. + +https://www.mindmeister.com/map/2172644474 + +## Editing the Mindmap +There are a few categories. To edit them, click the small plus next to the branch you want to change. + +## Exporting the Mindmap +Click "Export as RTF" in the top left corner of the URL. Download it there. + +## Generating the Shuffle-comaptible mindmap +1. Move the rtf file here +2. Rename it categories.rtf +3. Run the read_categories.py file (python3 read_categories.py) +4. You now have a file called categories.json locally with all the categories in JSON format, ready to be used in graphs. diff --git a/functions/usecases/categories.json b/functions/usecases/categories.json new file mode 100644 index 00000000..e438a380 --- /dev/null +++ b/functions/usecases/categories.json @@ -0,0 +1,260 @@ +[ + { + "name": "1. Collect & Distribute", + "color": "#c51152", + "list": [ + { + "name": "2-way Ticket synchronization", + "items": {} + }, + { + "name": "Email management", + "items": { + "name": "Release a quarantined message", + "items": {} + } + }, + { + "name": "EDR to ticket", + "items": { + "name": "Get host information", + "items": {} + } + }, + { + "name": "SIEM to ticket", + "items": {} + }, + { + "name": "ChatOps", + "items": {} + }, + { + "name": "Threat Intel received", + "items": {} + }, + { + "name": "Domain investigation with LetsEncrypt", + "items": {} + }, + { + "name": "Botnet tracker", + "items": {} + }, + { + "name": "Get running containers", + "items": {} + }, + { + "name": "Assign tickets", + "items": {} + }, + { + "name": "Firewall alerts", + "items": { + "name": "URL filtering", + "items": {} + } + }, + { + "name": "IDS/IPS alerts", + "items": { + "name": "Manage policies", + "items": {} + } + }, + { + "name": "Deduplicate information", + "items": {} + }, + { + "name": "Correlate information", + "items": {} + } + ] + }, + { + "name": "2. Enrich", + "color": "#f4c20d", + "list": [ + { + "name": "Internal Enrichment", + "items": { + "name": "...", + "items": {} + } + }, + { + "name": "External historical Enrichment", + "items": { + "name": "...", + "items": {} + } + }, + { + "name": "Realtime", + "items": { + "name": "Analyze screenshots", + "items": {} + } + }, + { + "name": "Ticketing webhook verification", + "items": {} + } + ] + }, + { + "name": "3. Detect", + "color": "#3cba54", + "list": [ + { + "name": "Search SIEM (Sigma)", + "items": { + "name": "Endpoint", + "items": {} + } + }, + { + "name": "Search EDR (OSQuery)", + "items": {} + }, + { + "name": "Search emails (Phish)", + "items": { + "name": "Check headers and IOCs", + "items": {} + } + }, + { + "name": "Search IOCs (ioc-finder)", + "items": {} + }, + { + "name": "Search files (Yara)", + "items": {} + }, + { + "name": "Correlate tickets", + "items": {} + }, + { + "name": "Honeypot access", + "items": { + "name": "...", + "items": {} + } + } + ] + }, + { + "name": "4. Respond", + "color": "#4a148c", + "list": [ + { + "name": "Eradicate malware", + "items": {} + }, + { + "name": "Quarantine host(s)", + "items": {} + }, + { + "name": "Trigger scans", + "items": {} + }, + { + "name": "Update indicators (FW, EDR, SIEM...)", + "items": {} + }, + { + "name": "Autoblock activity when threat intel is received", + "items": {} + }, + { + "name": "Lock/Delete/Reset account", + "items": {} + }, + { + "name": "Lock vault", + "items": {} + }, + { + "name": "Increase authentication", + "items": {} + }, + { + "name": "Get policies from assets", + "items": {} + } + ] + }, + { + "name": "5. Verify", + "color": "#4885ed", + "list": [ + { + "name": "Discover vulnerabilities", + "items": {} + }, + { + "name": "Discover assets", + "items": {} + }, + { + "name": "Ensure policies are followed", + "items": {} + }, + { + "name": "Find Inactive users", + "items": {} + }, + { + "name": "Ensure access rights match HR systems", + "items": {} + }, + { + "name": "Ensure onboarding is followed", + "items": {} + }, + { + "name": "Third party apps in SaaS", + "items": {} + }, + { + "name": "Devices used for your cloud account", + "items": {} + }, + { + "name": "Too much access in GCP/Azure/AWS/ other clouds", + "items": {} + }, + { + "name": "Certificate validation", + "items": {} + }, + { + "name": "Monitor new DNS entries for domain with passive DNS", + "items": {} + }, + { + "name": "Monitor and track password dumps", + "items": {} + }, + { + "name": "Monitor for mentions of domain on darknet sites", + "items": {} + }, + { + "name": "Reporting", + "items": { + "name": "Monthly reports", + "items": { + "name": "...", + "items": {} + } + } + } + ] + } +] \ No newline at end of file diff --git a/functions/usecases/categories.rtf b/functions/usecases/categories.rtf new file mode 100644 index 00000000..e19cf26a --- /dev/null +++ b/functions/usecases/categories.rtf @@ -0,0 +1,555 @@ +{\rtf1\ansi\deff0\deflang2057\plain\fs24\fet1 +{\fonttbl +{\f0\froman Arial;} +} +{\info +{\createim\yr2022\mo2\dy20\hr1\min15} +} + +\paperw11907\paperh16840\margl1800\margr1800\margt1440\margb1440 +\slmult0\ltrpar\li0 +{\b\fs28 +Shuffle categories +} +\par\pard\plain +\slmult0\ltrpar\li200 +{\fs24 +1. Collect & Distribute +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +2-way Ticket synchronization +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Email management +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Attachments +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Manage senders +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Manage URLs +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Encode & Decode URLs +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Release a quarantined message +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +EDR to ticket +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Fetch incidents & events +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Quarantine files +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Quarantine host (respond) +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Get host information +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +SIEM to ticket +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +ChatOps +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Threat Intel received +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Domain investigation with LetsEncrypt +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Botnet tracker +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Get running containers +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Assign tickets +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Firewall alerts +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Block/accept policies +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Add addresses and ports to groups +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Support custom URL categories +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Fetch logs for specific address +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +URL filtering +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +IDS/IPS alerts +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Get/Fetch alerts +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Receive alerts real-time +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Get PCAP files +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Get network logs +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Manage policies +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Deduplicate information +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Correlate information +} +\par\pard\plain +\slmult0\ltrpar\li200 +{\fs24 +3. Detect +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Search SIEM (Sigma) +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Network +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Endpoint +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Search EDR (OSQuery) +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Search emails (Phish) +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Check malware +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Check targeted +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Check headers and IOCs +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Search IOCs (ioc-finder) +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Search files (Yara) +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Correlate tickets +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Honeypot access +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +S3 Honeypot +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +SSH Honeypot +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +FTP honeypot +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Network honeypot +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +... +} +\par\pard\plain +\slmult0\ltrpar\li200 +{\fs24 +rich +} +\par\pard\plain +\slmult0\ltrpar\li200 +{\fs24 +5. Verify +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Discover vulnerabilities +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Discover assets +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Ensure policies are followed +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Find Inactive users +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Ensure access rights match HR systems +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Ensure onboarding is followed +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Third party apps in SaaS +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Devices used for your cloud account +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Too much access in GCP/Azure/AWS/ other clouds +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Certificate validation +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Monitor new DNS entries for domain with passive DNS +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Monitor and track password dumps +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Monitor for mentions of domain on darknet sites +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Reporting +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Automation time saved +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Automation money saved +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Incident response report +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Department cost +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Monthly reports +} +\par\pard\plain +\slmult0\ltrpar\li800 +{\fs24 +EDR alerts +} +\par\pard\plain +\slmult0\ltrpar\li800 +{\fs24 +SIEM alerts +} +\par\pard\plain +\slmult0\ltrpar\li800 +{\fs24 +Emails quarantined +} +\par\pard\plain +\slmult0\ltrpar\li800 +{\fs24 +... +} +\par\pard\plain +\slmult0\ltrpar\li200 +{\fs24 +4. Respond +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Eradicate malware +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Quarantine host(s) +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Trigger scans +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Update indicators (FW, EDR, SIEM...) +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Autoblock activity when threat intel is received +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Lock/Delete/Reset account +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Lock vault +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Increase authentication +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Get policies from assets +} +\par\pard\plain +\slmult0\ltrpar\li200 +{\fs24 +2. Enrich +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Internal Enrichment +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Users +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Hostnames +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +IPs +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Departments +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Role +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Software +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +... +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +External historical Enrichment +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +IPs +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +URLs +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Hashes +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Files +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +... +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Realtime +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +File detonation +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +URL detonation +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +PCAP analysis +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Analyze screenshots +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Ticketing webhook verification +} +\par\pard\plain +} \ No newline at end of file diff --git a/functions/usecases/read_categories.py b/functions/usecases/read_categories.py new file mode 100644 index 00000000..65df43c9 --- /dev/null +++ b/functions/usecases/read_categories.py @@ -0,0 +1,66 @@ +data = "" +with open("categories.rtf", "r") as tmp: + data = tmp.read() + +fixed_json = [] +linearity = 0 +heading = "" +subheading = "" +subsubheading = "" + +cnt = -1 +subcnt = -1 + +colors = ["#c51152", "#3cba54", "#4885ed", "#4a148c", "#f4c20d"] +for line in data.split("\n"): + if line == "rich": + continue + + if "li" in line: + lisplit = line.split("\\") + try: + linearity = int(lisplit[-1][2]) + except: + pass + + #print("Linearity: %s" % linearity) + + if line.startswith("{") or line.startswith("}"): + continue + + if line.startswith("\\"): + continue + + if linearity == 0: + continue + + if linearity == 2: + #if cnt >= 0: + # for key, value in fixed_json[cnt].items(): + # print(key, value) + + + cnt += 1 + subcnt = -1 + fixed_json.append({"name": line, "color": colors[cnt], "list": []}) + heading = line + elif linearity == 4: + subheading = line + fixed_json[cnt]["list"].append({"name": line, "items": {}}) + subcnt += 1 + elif linearity == 6: + fixed_json[cnt]["list"][subcnt]["items"] = {"name": line, "items": {}} + elif linearity == 8: + fixed_json[cnt]["list"][subcnt]["items"]["items"] = {"name": line, "items": {}} + else: + print("No handler for %s" % line) + +#print(line) +#print(data) +import json +filename = "categories.json" +fixed_json.sort(key=lambda x: x["name"]) +with open(filename, "w+") as tmp: + tmp.write(json.dumps(fixed_json, indent=4)) + +print("Wrote to file %s" % filename)