diff --git a/.env b/.env old mode 100644 new mode 100755 index 307c9d35..07deaf4a --- a/.env +++ b/.env @@ -2,6 +2,9 @@ ORG_ID=Shuffle ENVIRONMENT_NAME=Shuffle +# Sanitize liquid.py input +LIQUID_SANITIZE_INPUT=true + # Remote github config for first load SHUFFLE_DOWNLOAD_WORKFLOW_LOCATION= @@ -55,14 +58,28 @@ SHUFFLE_ORBORUS_STARTUP_DELAY= # Used for setting up a startup delay for Orbor SHUFFLE_BASE_IMAGE_NAME=shuffle SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io -SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-1.0.0" +SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-1.1.0" + +## shuffle_memcached (for distributed caching) +## shuffle_SWARM_CONFIG (run vs not run) +## shuffle_Scale_Replicas (workers/node) +## shuffle_App_Replicas (apps/node) + +SHUFFLE_SWARM_BRIDGE_DEFAULT_MTU=1500 # 1500 by default +# The eth0 interface inside a container corresponds +# to the virtual Ethernet interface that connects +# the container to the docker0 +SHUFFLE_SWARM_BRIDGE_DEFAULT_INTERFACE=eth0 # Used for auto-cleanup of containers. REALLY important at scale. SHUFFLE_CONTAINER_AUTO_CLEANUP=false SHUFFLE_ELASTIC=true SHUFFLE_LOGS_DISABLED=false -SHUFFLE_CHAT_DISABLED=false # Controls support chat +SHUFFLE_CHAT_DISABLED=false # Controls support chat SHUFFLE_RERUN_SCHEDULE=300 +SHUFFLE_DISABLE_RERUN_AND_ABORT=false +SHUFFLE_WORKER_SERVER_URL= # Definition in case Worker & Orborus is talking to the wrong server +SHUFFLE_ORBORUS_PULL_TIME= # Definition in case Orborus is pulling too often/not often enough # DATABASE CONFIGURATIONS DATASTORE_EMULATOR_HOST=shuffle-database:8000 diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md old mode 100644 new mode 100755 diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml old mode 100644 new mode 100755 diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md old mode 100644 new mode 100755 diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md old mode 100644 new mode 100755 diff --git a/.github/install-guide.html b/.github/install-guide.html old mode 100644 new mode 100755 diff --git a/.github/install-guide.md b/.github/install-guide.md old mode 100644 new mode 100755 index 2e97bbdc..48b3689e --- a/.github/install-guide.md +++ b/.github/install-guide.md @@ -91,7 +91,7 @@ npm start ## Backend - Golang http://localhost:5001 - REST API - requires [>=go1.13](https://golang.org/dl/) ```bash -export SHUFFLE_OPENSEARCH_URL="http://localhost:9200" +export SHUFFLE_OPENSEARCH_URL="https://localhost:9200" export SHUFFLE_ELASTIC=true export SHUFFLE_OPENSEARCH_USERNAME=admin export SHUFFLE_OPENSEARCH_PASSWORD=admin diff --git a/.github/push_nightly.sh b/.github/push_nightly.sh index 92d5320b..b3660193 100644 --- a/.github/push_nightly.sh +++ b/.github/push_nightly.sh @@ -2,40 +2,79 @@ # Done manually for now since GHCR isn't being pushed to easily with the current Github action CI. Nightly = Latest IF we run hotfixes on latest ### Pull latest from ghcr CI/CD -#docker pull ghcr.io/shuffle/shuffle-app_sdk:nightly -#docker pull ghcr.io/shuffle/shuffle-worker:nightly -#docker pull ghcr.io/shuffle/shuffle-orborus:nightly -#docker pull ghcr.io/shuffle/shuffle-frontend:nightly +docker pull ghcr.io/shuffle/shuffle-app_sdk:nightly +docker pull ghcr.io/shuffle/shuffle-worker:nightly +docker pull ghcr.io/shuffle/shuffle-orborus:nightly +docker pull ghcr.io/shuffle/shuffle-frontend:nightly #docker pull ghcr.io/shuffle/shuffle-backend:nightly +# +### NIGHTLY releases +docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly ghcr.io/frikky/shuffle-app_sdk:nightly +docker tag ghcr.io/shuffle/shuffle-worker:nightly ghcr.io/frikky/shuffle-worker:nightly +docker tag ghcr.io/shuffle/shuffle-orborus:nightly ghcr.io/frikky/shuffle-orborus:nightly +docker tag ghcr.io/shuffle/shuffle-frontend:nightly ghcr.io/frikky/shuffle-frontend:nightly +docker tag ghcr.io/shuffle/shuffle-backend:nightly ghcr.io/frikky/shuffle-backend:nightly + +docker push ghcr.io/frikky/shuffle-app_sdk:nightly +docker push ghcr.io/frikky/shuffle-worker:nightly +docker push ghcr.io/frikky/shuffle-orborus:nightly +docker push ghcr.io/frikky/shuffle-frontend:nightly +docker push ghcr.io/frikky/shuffle-backend:nightly ### LATEST releases: -#docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly ghcr.io/shuffle/shuffle-app_sdk:latest -#docker tag ghcr.io/shuffle/shuffle-worker:nightly ghcr.io/shuffle/shuffle-worker:latest -#docker tag ghcr.io/shuffle/shuffle-orborus:nightly ghcr.io/shuffle/shuffle-orborus:latest -#docker tag ghcr.io/shuffle/shuffle-frontend:nightly ghcr.io/shuffle/shuffle-frontend:latest -#docker tag ghcr.io/shuffle/shuffle-backend:nightly ghcr.io/shuffle/shuffle-backend:latest -# -#docker push ghcr.io/shuffle/shuffle-app_sdk:latest -#docker push ghcr.io/shuffle/shuffle-worker:latest -#docker push ghcr.io/shuffle/shuffle-orborus:latest -#docker push ghcr.io/shuffle/shuffle-frontend:latest -#docker push ghcr.io/shuffle/shuffle-backend:latest +## shuffle/shuffle +docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly ghcr.io/shuffle/shuffle-app_sdk:latest +docker tag ghcr.io/shuffle/shuffle-worker:nightly ghcr.io/shuffle/shuffle-worker:latest +docker tag ghcr.io/shuffle/shuffle-orborus:nightly ghcr.io/shuffle/shuffle-orborus:latest +docker tag ghcr.io/shuffle/shuffle-frontend:nightly ghcr.io/shuffle/shuffle-frontend:latest +docker tag ghcr.io/shuffle/shuffle-backend:nightly ghcr.io/shuffle/shuffle-backend:latest + +docker push ghcr.io/shuffle/shuffle-app_sdk:latest +docker push ghcr.io/shuffle/shuffle-worker:latest +docker push ghcr.io/shuffle/shuffle-orborus:latest +docker push ghcr.io/shuffle/shuffle-frontend:latest +docker push ghcr.io/shuffle/shuffle-backend:latest + +## frikky/shuffle +docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly ghcr.io/frikky/shuffle-app_sdk:latest +docker tag ghcr.io/shuffle/shuffle-worker:nightly ghcr.io/frikky/shuffle-worker:latest +docker tag ghcr.io/shuffle/shuffle-orborus:nightly ghcr.io/frikky/shuffle-orborus:latest +docker tag ghcr.io/shuffle/shuffle-frontend:nightly ghcr.io/frikky/shuffle-frontend:latest +docker tag ghcr.io/shuffle/shuffle-backend:nightly ghcr.io/frikky/shuffle-backend:latest + +docker push ghcr.io/frikky/shuffle-app_sdk:latest +docker push ghcr.io/frikky/shuffle-worker:latest +docker push ghcr.io/frikky/shuffle-orborus:latest +docker push ghcr.io/frikky/shuffle-frontend:latest +docker push ghcr.io/frikky/shuffle-backend:latest ### 1.1.0 releases: -#docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly ghcr.io/shuffle/shuffle-app_sdk:1.1.0 -#docker tag ghcr.io/shuffle/shuffle-worker:nightly ghcr.io/shuffle/shuffle-worker:1.1.0 -#docker tag ghcr.io/shuffle/shuffle-orborus:nightly ghcr.io/shuffle/shuffle-orborus:1.1.0 -#docker tag ghcr.io/shuffle/shuffle-frontend:nightly ghcr.io/shuffle/shuffle-frontend:1.1.0 -#docker tag ghcr.io/shuffle/shuffle-backend:nightly ghcr.io/shuffle/shuffle-backend:1.1.0 -# -#docker push ghcr.io/shuffle/shuffle-app_sdk:1.1.0 -#docker push ghcr.io/shuffle/shuffle-worker:1.1.0 -#docker push ghcr.io/shuffle/shuffle-orborus:1.1.0 -#docker push ghcr.io/shuffle/shuffle-frontend:1.1.0 -#docker push ghcr.io/shuffle/shuffle-backend:1.1.0 +## shuffle/shuffle +docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly ghcr.io/shuffle/shuffle-app_sdk:1.1.0 +docker tag ghcr.io/shuffle/shuffle-worker:nightly ghcr.io/shuffle/shuffle-worker:1.1.0 +docker tag ghcr.io/shuffle/shuffle-orborus:nightly ghcr.io/shuffle/shuffle-orborus:1.1.0 +docker tag ghcr.io/shuffle/shuffle-frontend:nightly ghcr.io/shuffle/shuffle-frontend:1.1.0 +docker tag ghcr.io/shuffle/shuffle-backend:nightly ghcr.io/shuffle/shuffle-backend:1.1.0 +docker push ghcr.io/shuffle/shuffle-app_sdk:1.1.0 +docker push ghcr.io/shuffle/shuffle-worker:1.1.0 +docker push ghcr.io/shuffle/shuffle-orborus:1.1.0 +docker push ghcr.io/shuffle/shuffle-frontend:1.1.0 +docker push ghcr.io/shuffle/shuffle-backend:1.1.0 +## frikky/shuffle +docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly ghcr.io/frikky/shuffle-app_sdk:1.1.0 +docker tag ghcr.io/shuffle/shuffle-worker:nightly ghcr.io/frikky/shuffle-worker:1.1.0 +docker tag ghcr.io/shuffle/shuffle-orborus:nightly ghcr.io/frikky/shuffle-orborus:1.1.0 +docker tag ghcr.io/shuffle/shuffle-frontend:nightly ghcr.io/frikky/shuffle-frontend:1.1.0 +docker tag ghcr.io/shuffle/shuffle-backend:nightly ghcr.io/frikky/shuffle-backend:1.1.0 + +docker push ghcr.io/frikky/shuffle-app_sdk:1.1.0 +docker push ghcr.io/frikky/shuffle-worker:1.1.0 +docker push ghcr.io/frikky/shuffle-orborus:1.1.0 +docker push ghcr.io/frikky/shuffle-frontend:1.1.0 +docker push ghcr.io/frikky/shuffle-backend:1.1.0 ### Manage worker-scale upload (Requires auth) # This is supposed to be unavailable, and only be downloadable by customers diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml old mode 100644 new mode 100755 diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml new file mode 100755 index 00000000..716b42db --- /dev/null +++ b/.github/workflows/docker-build.yaml @@ -0,0 +1,66 @@ +name: docker-build + +on: + push: + branches: launch +jobs: + main: + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + include: + - app: frontend + path: frontend + version: 1.0.0 + experimental: true + - app: backend + path: backend + version: 1.0.0 + experimental: false + - app: orborus + path: functions/onprem/orborus + version: 1.0.0 + experimental: false + - app: database + path: backend/database + version: 1.0.0 + experimental: false + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} +# Use below configuration for ghcr.io +# with: +# registry: ghcr.io +# username: ${{ github.repository_owner }} +# password: ${{ secrets.CR_PAT }} + - + name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + env: + BUILDX_NO_DEFAULT_LOAD: true + with: + context: ${{ matrix.path }}/ + file: ${{ matrix.path }}/Dockerfile + platforms: linux/amd64,linux/arm64 + #,linux/386 - no node image I guess? + push: true + tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.app }}:${{ matrix.version }} + - + name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/snyk-container-analysis.yml b/.github/workflows/snyk-container-analysis.yml old mode 100644 new mode 100755 diff --git a/.github/workflows/snyk-infrastructure-analysis.yml b/.github/workflows/snyk-infrastructure-analysis.yml old mode 100644 new mode 100755 diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/SECURITY.md b/SECURITY.md old mode 100644 new mode 100755 diff --git a/backend/Dockerfile b/backend/Dockerfile old mode 100644 new mode 100755 diff --git a/backend/README.md b/backend/README.md old mode 100644 new mode 100755 diff --git a/backend/app_gen/LICENSE b/backend/app_gen/LICENSE old mode 100644 new mode 100755 diff --git a/backend/app_gen/openapi-parsers/misp.py b/backend/app_gen/openapi-parsers/misp.py old mode 100644 new mode 100755 diff --git a/backend/app_gen/openapi-parsers/swimlane.py b/backend/app_gen/openapi-parsers/swimlane.py old mode 100644 new mode 100755 diff --git a/backend/app_gen/openapi/README.md b/backend/app_gen/openapi/README.md old mode 100644 new mode 100755 diff --git a/backend/app_gen/openapi/baseline/Dockerfile b/backend/app_gen/openapi/baseline/Dockerfile old mode 100644 new mode 100755 diff --git a/backend/app_gen/openapi/baseline/requirements.txt b/backend/app_gen/openapi/baseline/requirements.txt old mode 100644 new mode 100755 diff --git a/backend/app_gen/openapi/test.go b/backend/app_gen/openapi/test.go old mode 100644 new mode 100755 diff --git a/backend/app_gen/openapi/testGCP.go b/backend/app_gen/openapi/testGCP.go old mode 100644 new mode 100755 diff --git a/backend/app_gen/python-lib/README.md b/backend/app_gen/python-lib/README.md old mode 100644 new mode 100755 diff --git a/backend/app_gen/python-lib/baseline/Dockerfile b/backend/app_gen/python-lib/baseline/Dockerfile old mode 100644 new mode 100755 diff --git a/backend/app_gen/python-lib/baseline/docker-compose.yml b/backend/app_gen/python-lib/baseline/docker-compose.yml old mode 100644 new mode 100755 diff --git a/backend/app_gen/python-lib/baseline/env.txt b/backend/app_gen/python-lib/baseline/env.txt old mode 100644 new mode 100755 diff --git a/backend/app_gen/python-lib/baseline/requirements.txt b/backend/app_gen/python-lib/baseline/requirements.txt old mode 100644 new mode 100755 diff --git a/backend/app_gen/python-lib/generator.py b/backend/app_gen/python-lib/generator.py old mode 100644 new mode 100755 diff --git a/backend/app_gen/python-lib/requirements.txt b/backend/app_gen/python-lib/requirements.txt old mode 100644 new mode 100755 diff --git a/backend/app_sdk/Dockerfile b/backend/app_sdk/Dockerfile old mode 100644 new mode 100755 index 25c9aa40..d3709032 --- a/backend/app_sdk/Dockerfile +++ b/backend/app_sdk/Dockerfile @@ -1,5 +1,6 @@ #FROM python:3.9.1-alpine as base FROM python:3.10.0-alpine as base +#FROM python:3.11.3-alpine as base FROM base as builder RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev tzdata coreutils diff --git a/backend/app_sdk/Dockerfile_blackarch b/backend/app_sdk/Dockerfile_blackarch old mode 100644 new mode 100755 diff --git a/backend/app_sdk/Dockerfile_kali b/backend/app_sdk/Dockerfile_kali old mode 100644 new mode 100755 diff --git a/backend/app_sdk/LICENSE b/backend/app_sdk/LICENSE old mode 100644 new mode 100755 diff --git a/backend/app_sdk/README.md b/backend/app_sdk/README.md old mode 100644 new mode 100755 diff --git a/backend/app_sdk/__init__.py b/backend/app_sdk/__init__.py old mode 100644 new mode 100755 diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py old mode 100644 new mode 100755 index 2f23755f..b0ffee23 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -18,6 +18,10 @@ import urllib.parse import jinja2 import datetime import dateutil + +import threading +import concurrent.futures + from io import StringIO as StringBuffer from io import BytesIO from liquid import Liquid, defaults @@ -138,13 +142,47 @@ def escape(a): a = str(a) return json_escape(a) +@shuffle_filters.register +def neat_json(a): + try: + a = json.loads(a) + except: + pass + + return json.dumps(a, indent=4, sort_keys=True) + @shuffle_filters.register def flatten(a): a = list(a) - flat_list = [a for xs in xss for a in xs] + flat_list = [a for xs in a for a in xs] return flat_list +@shuffle_filters.register +def last(a): + try: + a = json.loads(a) + except: + pass + + if len(a) == 0: + return "" + + return a[-1] + +@shuffle_filters.register +def first(a): + try: + a = json.loads(a) + except: + pass + + if len(a) == 0: + return "" + + return a[0] + + @shuffle_filters.register def csv_parse(a): a = str(a) @@ -186,9 +224,37 @@ def csv_parse(a): allitems.append(fullitem) - return allitems + try: + return json.dumps(allitems) + except: + print("[ERROR] Failed dumping from JSON in csv parse") + return allitems + +@shuffle_filters.register +def parse_csv(a): + return csv_parse(a) + +@shuffle_filters.register +def format_csv(a): + return csv_parse(a) + +@shuffle_filters.register +def csv_format(a): + return csv_parse(a)@standard_filter_manager.register + +@shuffle_filters.register +def split(base, sep): + if not sep: + try: + return json.dumps(list(base)) + except: + return list(base) + + try: + return json.dumps(base.split(sep)) + except: + return base.split(sep) -#print(standard_filter_manager.filters) #print(shuffle_filters.filters) #print(Liquid("{{ '10' | plus: 1}}", filters=shuffle_filters.filters).render()) #print(Liquid("{{ '10' | minus: 1}}", filters=shuffle_filters.filters).render()) @@ -205,6 +271,7 @@ def csv_parse(a): ### ### + class AppBase: __version__ = None app_name = None @@ -327,13 +394,14 @@ class AppBase: new_input = fixed_return except Exception as e: - self.logger.info(f"[ERROR] Failed to run magic parser (2): {e}") + # Not used anymore + #self.logger.info(f"[ERROR] Failed to run magic parser (2): {e}") return input_data try: new_input = input_data.split() except Exception as e: - self.logger.info(f"[ERROR] Failed to run magic parser during split (1): {e}") + self.logger.info(f"[ERROR] Failed to run parser during split (1): {e}") return input_data # Won't ever touch this one? @@ -389,9 +457,11 @@ class AppBase: else: self.logger.warning(f"[WARNING] Magic output not defined.") except KeyError as e: - self.logger.warning(f"[DEBUG] Failed to run magic autoparser (send result) - keyerror: {e}") + #self.logger.warning(f"[DEBUG] Failed to run magic autoparser (send result) - keyerror: {e}") + pass except Exception as e: - self.logger.warning(f"[DEBUG] Failed to run magic autoparser (send result): {e}") + #self.logger.warning(f"[DEBUG] Failed to run magic autoparser (send result): {e}") + pass # Try it with some magic @@ -401,15 +471,6 @@ class AppBase: # FIXME: Add cleanup of parameters to not send to frontend here params = {} - #action = action_result["action"] - #try: - # for item in action["authentication"]: - # for action["parameters"] - # self.logger.info("AUTH: ", key, value) - # params[item["key"]] = item["value"] - #except KeyError: - # self.logger.info("No authentication specified!") - # pass # I wonder if this actually works self.logger.info(f"[DEBUG] Before last stream result") @@ -417,7 +478,7 @@ class AppBase: self.logger.info(f"[INFO] URL FOR RESULT (URL): {url}") try: - log_contents = "disabled: add env SHUFFLE_LOGS_DISABLED=true to Orborus to re-enable logs for apps" + log_contents = "disabled: add env SHUFFLE_LOGS_DISABLED=true to Orborus to re-enable logs for apps. Can not be enabled natively in Cloud except in Hybrid mode." if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true": log_contents = self.log_capture_string.getvalue() @@ -446,9 +507,9 @@ class AppBase: finished = False for i in range (0, 10): try: - ret = requests.post(url, headers=headers, json=action_result, timeout=10) + ret = requests.post(url, headers=headers, json=action_result, timeout=10, verify=False) - self.logger.info(f"[DEBUG] Result: {ret.status_code} (break on 200)") + self.logger.info(f"[DEBUG] Result: {ret.status_code} (break on 200 or 201)") if ret.status_code == 200 or ret.status_code == 201: finished = True break @@ -493,7 +554,7 @@ class AppBase: action_result["status"] = "FAILURE" action_result["result"] = json.dumps({"success": False, "reason": "POST error: Failed connecting to %s over 10 retries to the backend" % url}) self.logger.info(f"[DEBUG] Before typeerror stream result - NOT finished after 10 requests") - ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) + ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False) self.logger.info(f"""[DEBUG] Successful request result request: Status= {ret.status_code} & Response= {ret.text}. Action status: {action_result["status"]}""") except requests.exceptions.ConnectionError as e: @@ -503,7 +564,7 @@ class AppBase: action_result["result"] = json.dumps({"success": False, "reason": "Typeerror when sending to backend URL %s" % url}) self.logger.info(f"[DEBUG] Before typeerror stream result: {e}") - ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) + ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False) #self.logger.info(f"[DEBUG] Result: {ret.status_code}") #if ret.status_code != 200: # pr @@ -632,7 +693,7 @@ class AppBase: #self.logger.info(f"RET: {ret.text}") #self.logger.info(f"ID: {ret.status_code}") url = f"{self.url}/api/v1/orgs/{org_id}/validate_app_values" - ret = requests.post(url, json=data) + ret = requests.post(url, json=data, verify=False) if ret.status_code == 200: json_value = ret.json() if len(json_value["found"]) > 0: @@ -971,11 +1032,9 @@ class AppBase: for subparams in param_multiplier: #self.logger.info(f"SUBPARAMS IN MULTI: {subparams}") try: - #tmp = await func(**subparams) while True: try: - #tmp = await func(**subparams) tmp = func(**subparams) break except TypeError as e: @@ -988,7 +1047,7 @@ class AppBase: try: del subparams[field] - self.logger.info("Removed field invalid field %s" % field) + self.logger.info("Removed invalid field %s (1)" % field) except KeyError: break else: @@ -1015,6 +1074,7 @@ class AppBase: # An attempt at decomposing coroutine results + # Backwards compatibility try: if asyncio.iscoroutine(tmp): self.logger.info("[DEBUG] In coroutine (2)") @@ -1028,7 +1088,8 @@ class AppBase: tmp = asyncio.run(parse_value(tmp)) else: - self.logger.info("[DEBUG] Not in coroutine (2)") + #self.logger.info("[DEBUG] Not in coroutine (2)") + pass except Exception as e: self.logger.warning("[ERROR] Failed to parse coroutine value for old app: {e}") @@ -1102,10 +1163,11 @@ class AppBase: get_path = "/api/v1/files/namespaces/%s?execution_id=%s&ids=true" % (category, self.full_execution["execution_id"]) headers = { - "Authorization": "Bearer %s" % self.authorization + "Authorization": "Bearer %s" % self.authorization, + "User-Agent": "Shuffle 1.1.0", } - ret = requests.get("%s%s" % (self.url, get_path), headers=headers) + ret = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False) return ret.json() #if ret1.status_code != 200: # return { @@ -1127,10 +1189,11 @@ class AppBase: get_path = "/api/v1/files/namespaces/%s?execution_id=%s" % (namespace, self.full_execution["execution_id"]) headers = { - "Authorization": "Bearer %s" % self.authorization + "Authorization": "Bearer %s" % self.authorization, + "User-Agent": "Shuffle 1.1.0", } - ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers) + ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False) if ret1.status_code != 200: return None @@ -1189,10 +1252,11 @@ class AppBase: get_path = "/api/v1/files/%s?execution_id=%s" % (item, full_execution["execution_id"]) headers = { "Content-Type": "application/json", - "Authorization": "Bearer %s" % self.authorization + "Authorization": "Bearer %s" % self.authorization, + "User-Agent": "Shuffle 1.1.0", } - ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers) + ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False) self.logger.info("RET1 (file get): %s" % ret1.text) if ret1.status_code != 200: returns.append({ @@ -1203,7 +1267,7 @@ class AppBase: continue content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"]) - ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers) + ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers, verify=False) self.logger.info("RET2 (file get) done") if ret2.status_code == 200: tmpdata = ret1.json() @@ -1239,7 +1303,7 @@ class AppBase: "value": str(value), } - response = requests.post(url, json=data) + response = requests.post(url, json=data, verify=False) try: allvalues = response.json() allvalues["key"] = key @@ -1261,7 +1325,7 @@ class AppBase: "key": key, } - value = requests.post(url, json=data) + value = requests.post(url, json=data, verify=False) try: allvalues = value.json() self.logger.info("VAL1: ", allvalues) @@ -1291,7 +1355,8 @@ class AppBase: org_id = full_execution["workflow"]["execution_org"]["id"] headers = { "Content-Type": "application/json", - "Authorization": "Bearer %s" % self.authorization + "Authorization": "Bearer %s" % self.authorization, + "User-Agent": "Shuffle 1.1.0", } if not isinstance(infiles, list): @@ -1314,7 +1379,7 @@ class AppBase: self.logger.info(f"KeyError in file setup: {e}") pass - ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data) + ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data, verify=False) #self.logger.info(f"Ret CREATE: {ret.text}") cur_id = "" if ret.status_code == 200: @@ -1337,6 +1402,7 @@ class AppBase: new_headers = { "Authorization": f"Bearer {self.authorization}", + "User-Agent": "Shuffle 1.1.0", } upload_path = "/api/v1/files/%s/upload?execution_id=%s" % (cur_id, full_execution["execution_id"]) @@ -1345,7 +1411,7 @@ class AppBase: files={"shuffle_file": (filename, curfile["data"])} #open(filename,'rb')} - ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers) + ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers, verify=False) self.logger.info("Ret UPLOAD: %s" % ret.text) self.logger.info("Ret2 UPLOAD: %d" % ret.status_code) @@ -1383,7 +1449,8 @@ class AppBase: headers = { "Content-Type": "application/json", - "Authorization": f"Bearer {self.authorization}" + "Authorization": f"Bearer {self.authorization}", + "User-Agent": "Shuffle 1.1.0", } if len(self.action) == 0: @@ -1411,7 +1478,7 @@ class AppBase: # FIXME: Shouldn't skip this, but it's good for minimzing API calls #try: - # ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) + # ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False) # self.logger.info("Workflow: %d" % ret.status_code) # if ret.status_code != 200: # self.logger.info(ret.text) @@ -1426,7 +1493,7 @@ class AppBase: # If found, we get the full results list from backend fullexecution = {} if isinstance(self.full_execution, str) and len(self.full_execution) == 0: - self.logger.info("[DEBUG] NO EXECUTION - LOADING!") + #self.logger.info("[DEBUG] NO EXECUTION - LOADING!") try: failed = False rettext = "" @@ -1440,7 +1507,8 @@ class AppBase: ret = requests.post( "%s/api/v1/streams/results" % (self.base_url), headers=headers, - json=tmpdata + json=tmpdata, + verify=False ) if ret.status_code == 200: @@ -1448,12 +1516,13 @@ class AppBase: failed = False break - elif ret.status_code == 500: + #elif ret.status_code == 500 or ret.status_code == 400: + elif ret.status_code >= 400: self.logger.info("[ERROR] (fails: %d) Error in app with status code %d for results (1). RETRYING because results can't be handled" % (i+1, ret.status_code)) rettext = ret.text failed = True - time.sleep(10) + time.sleep(8) continue else: @@ -1461,6 +1530,7 @@ class AppBase: rettext = ret.text failed = True + time.sleep(8) break if failed: @@ -1497,6 +1567,13 @@ class AppBase: self.full_execution = fullexecution + #try: + # if "backend_url" in self.full_execution: + # self.url = self.full_execution["backend_url"] + # self.base_url = self.full_execution["backend_url"] + #except KeyError: + # pass + try: if replace_params == True: for inner_action in self.full_execution["workflow"]["actions"]: @@ -1522,7 +1599,7 @@ class AppBase: except Exception as e: self.logger.info(f"[WARNING] Failed in replace params action parsing: {e}") - self.logger.info("[DEBUG] AFTER FULLEXEC stream result (init)") + self.logger.info(f"[DEBUG] AFTER FULLEXEC stream result (init): {self.current_execution_id}") # Gets the value at the parenthesis level you want def parse_nested_param(string, level): @@ -1708,7 +1785,7 @@ class AppBase: # Because liquid can handle ALL of this now. # Implemented for >0.9.25 - self.logger.info("[DEBUG] Skipping parser because use of its been deprecated >0.9.25 due to Liquid implementation") + #self.logger.info("[DEBUG] Skipping parser because use of its been deprecated >0.9.25 due to Liquid implementation") return data, False wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length", "lenght", "join", "replace"] @@ -1761,7 +1838,7 @@ class AppBase: else: parse_string = inner_result - print("PARSE STRING: %s" % parse_string) + #print("PARSE STRING: %s" % parse_string) return parse_string, True # Looks for parantheses to grab special cases within a string, e.g: @@ -1900,7 +1977,7 @@ class AppBase: if isinstance(seconditem, int): seconditem = str(seconditem) - print("[DEBUG] ACTUAL PARSED: %s" % actualitem) + #print("[DEBUG] ACTUAL PARSED: %s" % actualitem) # Means it's a single item -> continue if seconditem == "": @@ -2022,13 +2099,13 @@ class AppBase: actionname_lower = parsersplit[0][1:].lower() #Actionname: Start_node - print(f"\n[INFO] Actionname: {actionname_lower}") + #print(f"\n[INFO] Actionname: {actionname_lower}") # 1. Find the action baseresult = "" appendresult = "" - print("[INFO] Parsersplit length: %d" % len(parsersplit)) + #print("[INFO] Parsersplit length: %d" % len(parsersplit)) if (actionname_lower.startswith("exec ") or actionname_lower.startswith("webhook ") or actionname_lower.startswith("schedule ") or actionname_lower.startswith("userinput ") or actionname_lower.startswith("email_trigger ") or actionname_lower.startswith("trigger ")) and len(parsersplit) == 1: record = False for char in actionname_lower: @@ -2071,7 +2148,6 @@ class AppBase: print("[DEBUG] No results to get values from.") baseresult = "$" + parsersplit[0][1:] - print("[DEBUG] BEFORE VARIABLES!") if len(baseresult) == 0: try: for variable in execution_data["workflow"]["workflow_variables"]: @@ -2082,13 +2158,12 @@ class AppBase: break except KeyError as e: - print("[INFO] KeyError wf variables: %s" % e) + #print("[INFO] KeyError wf variables: %s" % e) pass except TypeError as e: - print("[INFO] TypeError wf variables: %s" % e) + #print("[INFO] TypeError wf variables: %s" % e) pass - print("[DEBUG] BEFORE EXECUTION VAR") if len(baseresult) == 0: try: for variable in execution_data["execution_variables"]: @@ -2106,14 +2181,14 @@ class AppBase: except KeyError as error: print(f"[DEBUG] KeyError in JSON: {error}") - print(f"[INFO] After first trycatch. Baseresult")#, baseresult) + #print(f"[INFO] After first trycatch. Baseresult")#, baseresult) # 2. Find the JSON data # Returns if there isn't any JSON in the base ($nodename) if len(baseresult) == 0: return ""+appendresult, False - print("[INFO] After second return") + #print("[INFO] After second return") # Returns if the result is JUST something like $nodename, not $nodename.value if len(parsersplit) == 1: returndata = str(baseresult)+str(appendresult) @@ -2124,7 +2199,7 @@ class AppBase: baseresult = baseresult.replace(" False", " false,") # Tries to actually read it as JSON with some stupid formatting - print("[INFO] After third parser return - Formatted")#, baseresult) + #print("[INFO] After third parser return - Formatted")#, baseresult) basejson = {} try: basejson = json.loads(baseresult) @@ -2148,7 +2223,6 @@ class AppBase: print("[WARNING] Parseditem issue: %s" % e) pass - print("[DEBUG] DATA: (%s) %s" % (type(data), data)) if is_loop: print("[DEBUG] DATA IS A LOOP - SHOULD WRAP") if parsersplit[-1] == "#": @@ -2160,7 +2234,6 @@ class AppBase: parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data)) - print("[DEBUG] Before last return with %s" % appendresult) returndata = str(parseditem)+str(appendresult) # New in 0.8.97: Don't return items without lists @@ -2409,9 +2482,11 @@ class AppBase: newvalue[key] = recurse_cleanup_script(value) except json.decoder.JSONDecodeError as e: - print(f"[WARNING] Failed JSON replacement for OpenAPI keys (3) {e}") + # Since here the data isn't at all JSON compatible..? + # Seems to happen with newlines in variables being parsed in as strings? + print(f"[ERROR] Failed JSON replacement for OpenAPI keys (3) {e}. Value: {data}") except Exception as e: - print(f"[WARNING] Failed as an exception (1): {e}") + print(f"[ERROR] Failed as an exception (1): {e}") try: for deletekey in deletekeys: @@ -2476,9 +2551,17 @@ class AppBase: except: self.logger.info("Error in initial replacement of escaped dollar!") - # Basic fix in case variant isn't set + paramname = "" try: - self.logger.info(f"[DEBUG] Parameter variant: {parameter['variant']} of length {len(parameter['value'])}") + paramname = parameter["name"] + except: + pass + + # Basic fix in case variant isn't set + # Variant is ALWAYS STATIC_VALUE from mid 2021~ + try: + #self.logger.info(f"[DEBUG] Parameter '{paramname}' of length {len(parameter['value'])}") + parameter["variant"] = parameter["variant"] except: parameter["variant"] = "STATIC_VALUE" @@ -2651,19 +2734,6 @@ class AppBase: return True return False - #if tmp == "[]": - # tmp = [] - - #if type(tmp) == list and len(tmp) == 0 and not flip: - # new_list.append(item) - #elif type(tmp) == list and len(tmp) > 0 and flip: - # new_list.append(item) - #elif type(tmp) == str and not tmp and not flip: - # new_list.append(item) - #elif type(tmp) == str and tmp and flip: - # new_list.append(item) - #else: - # failed_list.append(item) elif check.lower() == "contains_any_of": newvalue = [destinationvalue.lower()] @@ -2729,6 +2799,7 @@ class AppBase: return True, "" except Exception as error: self.logger.info(f"[WARNING] Failed checking startnode: {error}") + return True, "" available_checks = [ "=", @@ -2783,9 +2854,6 @@ class AppBase: correct_branches += 1 continue - # FIXME: Check if the previous node has a result or not - - #self.logger.info("[DEBUG] Relevant conditions: %s" % branch["conditions"]) successful_conditions = [] failed_conditions = [] successful_conditions = 0 @@ -2798,27 +2866,20 @@ class AppBase: check, sourcevalue, is_loop = parse_params(action, fullexecution, condition["source"], self) if check: continue - return False, {"success": False, "reason": "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)} - #sourcevalue = sourcevalue.encode("utf-8") sourcevalue = parse_wrapper_start(sourcevalue, self) destinationvalue = condition["destination"]["value"] check, destinationvalue, is_loop = parse_params(action, fullexecution, condition["destination"], self) if check: continue - return False, {"success": False, "reason": "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)} - #destinationvalue = destinationvalue.encode("utf-8") destinationvalue = parse_wrapper_start(destinationvalue, self) if not condition["condition"]["value"] in available_checks: self.logger.warning("Skipping %s %s %s because %s is invalid." % (sourcevalue, condition["condition"]["value"], destinationvalue, condition["condition"]["value"])) continue - #self.logger.info(destinationvalue) - # NEGATE - # Configuration = negated because of WorkflowAppActionParam.. validation = run_validation(sourcevalue, condition["condition"]["value"], destinationvalue) try: @@ -2830,12 +2891,6 @@ class AppBase: if validation == True: successful_conditions += 1 - #if not validation: - # self.logger.info("Failed condition check for %s %s %s." % (sourcevalue, condition["condition"]["value"], destinationvalue)) - # return False, {"success": False, "reason": "Failed condition (3): %s %s %s" % (sourcevalue, condition["condition"]["value"], destinationvalue)} - - #self.logger.info("CONDITIONS VS SUCCESS: %d vs %d" % (total_conditions, successful_conditions)) - if total_conditions == successful_conditions: correct_branches += 1 @@ -2845,15 +2900,9 @@ class AppBase: if matching_branches > 0 and correct_branches > 0: return True, "" - # FIXME: Check if previous branches are at all finished - self.logger.info("[DEBUG] Correct branches vs matching branches: %d vs %d" % (correct_branches, matching_branches)) return False, {"success": False, "reason": "Minimum of one branch's conditions must be correct to continue. Total: %d of %d" % (correct_branches, matching_branches)} - #Correct branches vs matching branches: 1 vs 1 - #if - return True, "" - # # @@ -2881,7 +2930,7 @@ class AppBase: # Checks whether conditions are met, otherwise set branchcheck, tmpresult = check_branch_conditions(action, fullexecution, self) if isinstance(tmpresult, object) or isinstance(tmpresult, list) or isinstance(tmpresult, dict): - self.logger.info("[DEBUG] Fixing branch return as object -> string") + #self.logger.info("[DEBUG] Fixing branch return as object -> string") try: #tmpresult = tmpresult.replace("'", "\"") tmpresult = json.dumps(tmpresult) @@ -2904,7 +2953,8 @@ class AppBase: if " " in actionname: actionname.replace(" ", "_", -1) - #print(action) + #print("ACTION: ", action) + #print("exec: ", self.full_execution) #if action.generated: # actionname = actionname.lower() @@ -2916,7 +2966,8 @@ class AppBase: self.action_result["status"] = "FAILURE" self.action_result["result"] = json.dumps({ "success": False, - "reason": f"Function {actionname} doesn't exist.", + "reason": f"Function {actionname} doesn't exist, or the App is out of date.", + "details": "If this persists, please restart delete the Docker image locally, restart your Orborus instance and then try again to force-download the latest version. Contact support@shuffler.io with this data if the issue persists.", }) elif callable(func): try: @@ -2931,12 +2982,14 @@ class AppBase: # What variables are necessary here tho hmm params = {} - try: - for item in action["authentication"]: - #self.logger.info("AUTH: ", key, value) - params[item["key"]] = item["value"] - except KeyError: - self.logger.info("[DEBUG] No authentication specified!") + # This replacement should happen in backend as part of params + # error log is useless + #try: + # for item in action["authentication"]: + # self.logger.info("AUTH PARAM: ", key, value) + # #params[item["key"]] = item["value"] + #except KeyError as e: + # self.logger.info(f"[WARNING] No authentication specified! Is this correct? err: {e}") # Fixes OpenAPI body parameters for later. newparams = [] @@ -3007,13 +3060,12 @@ class AppBase: pass - self.logger.info(f"""HANDLING {action["parameters"][counter]["value"]}""") + self.logger.info(f"""HANDLING BODY: {action["parameters"][counter]["value"]}""") action["parameters"][counter]["value"] = recurse_cleanup_script(action["parameters"][counter]["value"]) #self.logger.info(action["parameters"]) # This seems redundant now - self.logger.info("[DEBUG] Pre parameters") for parameter in newparams: action["parameters"].append(parameter) @@ -3035,7 +3087,6 @@ class AppBase: # Multi_parameter has the data for each. variable minlength = 0 - self.logger.info("[DEBUG] Pre-loading parameters") multi_parameters = json.loads(json.dumps(params)) multiexecution = False multi_execution_lists = [] @@ -3293,9 +3344,6 @@ class AppBase: # This part has fucked over so many random JSON usages because of weird paranthesis parsing value = parse_wrapper_start(value, self) - #self.logger.info("[DEBUG] Post return: %s" % value) - - #self.logger.info("POST data value: %s" % value) try: if str(value).startswith("b'") and str(value).endswith("'"): @@ -3428,8 +3476,73 @@ class AppBase: break try: - newres = func(**params) + #try: + # Individual functions shouldn't take longer than this + # This is an attempt to make timeouts occur less, incentivizing users to make use efficient API's + # PS: Not implemented for lists - only single actions as of May 2023 + timeout = 30 + + # Check if current app is Shuffle Tools, then set to 55 due to certain actions being slow (ioc parser..) + #uu In general, this should be disabled for onprem + if self.action["app_name"].lower() == "shuffle tools": + timeout = 55 + + timeout = 30 + + try: + executor = concurrent.futures.ThreadPoolExecutor() + future = executor.submit(func, **params) + newres = future.result(timeout) + + if not future.done(): + # The future is still running, so we need to cancel it + future.cancel() + newres = json.dumps({ + "success": False, + "reason": "Timeout error within %d seconds. This happens if we can't reach or use the API you're trying to use within the time limit." % timeout, + "exception": str(e), + }) + + else: + # The future is done, so we can just get the result from newres :) + #newres = future.result() + #print("Future is done!") + pass + + except concurrent.futures.TimeoutError as e: + newres = json.dumps({ + "success": False, + "reason": "Timeout error within %d seconds (2). This happens if we can't reach or use the API you're trying to use within the time limit" % timeout + }) + break + + + + #thread = threading.Thread(target=func, args=(**params,)) + #thread.start() + + #thread.join(timeout) + + #if thread.is_alive(): + # # The thread is still running, so we need to stop it + # # You can handle this as needed, such as raising an exception + # timeout_handler() + + + #with Timeout(timeout): + # newres = func(**params) + # break + #except Timeout.Timeout as e: + # self.logger.info(f"[DEBUG] Timeout error: {e}") + # newres = json.dumps({ + # "success": False, + # "reason": "Timeout error within %d seconds. This typically happens if we can't reach the API you're trying to reach." % timeout, + # "exception": str(e), + # }) + + # break + except TypeError as e: newres = "" self.logger.info(f"[DEBUG] Got exec type error: {e}") @@ -3442,7 +3555,7 @@ class AppBase: errorstring = f"{e}" if "the JSON object must be" in errorstring: - self.logger.info("[ERROR] Something is wrong with the input for this function. Are lists and JSON data handled parsed properly (0)?") + self.logger.info("[ERROR] Something is wrong with the input for this function. Are lists and JSON data handled parsed properly (0)? the JSON object must be in...") try: e = json.loads(f"{e}") except: @@ -3461,7 +3574,7 @@ class AppBase: try: del params[field] - self.logger.info("[WARNING] Removed field invalid field %s" % field) + self.logger.info("[WARNING] Removed invalid field %s (2)" % field) except KeyError: break else: @@ -3472,7 +3585,7 @@ class AppBase: }) break except Exception as e: - self.logger.info("[ERROR] Something is wrong with the input for this function. Are lists and JSON data handled parsed properly (1)?") + self.logger.info(f"[ERROR] Something is wrong with the input for this function. Are lists and JSON data handled parsed properly (1)? err: {e}") try: e = json.loads(f"{e}") @@ -3499,7 +3612,8 @@ class AppBase: newres = asyncio.run(parse_value(newres)) else: - self.logger.info("[DEBUG] Not in coroutine (1)") + #self.logger.info("[DEBUG] Not in coroutine (1)") + pass except Exception as e: self.logger.warning("[ERROR] Failed to parse coroutine value for old app: {e}") @@ -3553,7 +3667,7 @@ class AppBase: result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) self.logger.info("Can't handle type %s value from function" % (type(newres))) - self.logger.info("[INFO] POST NEWRES RESULT!")#, result) + #self.logger.info("[INFO] POST NEWRES RESULT!")#, result) else: #self.logger.info("[INFO] APP_SDK DONE: Starting MULTI execution (length: %d) with values %s" % (minlength, multi_parameters)) # 1. Use number of executions based on the arrays being similar @@ -3712,7 +3826,6 @@ class AppBase: def execute(): if request.method == "POST": #print(request.get_json(force=True)) - #print("DATA: ", request.data) requestdata = {} try: requestdata = json.loads(request.data) @@ -3725,7 +3838,6 @@ class AppBase: #logger.info(f"[DEBUG] Datatype: {type(requestdata)}: {requestdata}") # Remaking class for each request - #print(f"APP: {app}") app = cls(redis=None, logger=logger, console_logger=logger) extra_info = "" @@ -3754,7 +3866,7 @@ class AppBase: # BASE URL (backend) try: app.url = requestdata["url"] - logger.info(f"BACKEND URL: {app.url}") + logger.info(f"BACKEND URL (url): {app.url}") except Exception as e: logger.info(f"[ERROR] Failed parsing url (backend): {e}") extra_info += f"\n{e}" @@ -3762,7 +3874,7 @@ class AppBase: # URL (worker) try: app.base_url = requestdata["base_url"] - logger.info(f"WORKER URL: {app.base_url}") + logger.info(f"WORKER URL (base url): {app.base_url}") except Exception as e: logger.info(f"[ERROR] Failed parsing base url (worker): {e}") extra_info += f"\n{e}" @@ -3840,11 +3952,7 @@ class AppBase: else: self.logger.info("ACTION TYPE (unhandled): %s" % type(action)) - #await app.execute_action(app.action) app.execute_action(app.action) - #app.run(host="0.0.0.0", port=33334) - if __name__ == "__main__": AppBase.run() - #asyncio.run(AppBase.run(), debug=True) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh old mode 100644 new mode 100755 index 1ff45c75..dd540b94 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -2,7 +2,7 @@ ### DEFAULT NAME=shuffle-app_sdk -VERSION=1.1.0 +VERSION=1.2.0 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly -t shuffle/shuffle:app_sdk -t shuffle/$NAME:$VERSION -t docker.pkg.github.com/shuffle/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:nightly diff --git a/backend/app_sdk/requirements.txt b/backend/app_sdk/requirements.txt old mode 100644 new mode 100755 diff --git a/backend/build.sh b/backend/build.sh old mode 100644 new mode 100755 index bb1f4a28..17c372c7 --- a/backend/build.sh +++ b/backend/build.sh @@ -1,10 +1,10 @@ #!/bin/sh docker stop shuffle-backend docker rm shuffle-backend -docker rmi frikky/shuffle:backend +docker rmi ghcr.io/shuffle/shuffle-backend:nightly -docker build . -t frikky/shuffle:backend -docker push frikky/shuffle:backend +docker build . -t ghcr.io/shuffle/shuffle-backend:nightly +docker push ghcr.io/shuffle/shuffle-backend:nightly echo "Starting server" #docker run -it \ diff --git a/backend/database/opensearch/docker-compose.yml b/backend/database/opensearch/docker-compose.yml old mode 100644 new mode 100755 diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go old mode 100644 new mode 100755 index f9c27eef..a6324faf --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -241,16 +241,15 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin BuildArgs: map[string]*string{}, Labels: labels, } - // NetworkMode: "host", httpProxy := os.Getenv("HTTP_PROXY") if len(httpProxy) > 0 { - buildOptions.BuildArgs["http_proxy"] = &httpProxy + buildOptions.BuildArgs["HTTP_PROXY"] = &httpProxy } httpsProxy := os.Getenv("HTTPS_PROXY") if len(httpProxy) > 0 { - buildOptions.BuildArgs["https_proxy"] = &httpsProxy + buildOptions.BuildArgs["HTTPS_PROXY"] = &httpsProxy } // Build the actual image @@ -352,7 +351,7 @@ func buildImage(tags []string, dockerfileFolder string) error { httpProxy := os.Getenv("HTTP_PROXY") if len(httpProxy) > 0 { - buildOptions.BuildArgs["http_proxy"] = &httpProxy + buildOptions.BuildArgs["HTTP_PROXY"] = &httpProxy } httpsProxy := os.Getenv("HTTPS_PROXY") if len(httpProxy) > 0 { @@ -386,170 +385,6 @@ func buildImage(tags []string, dockerfileFolder string) error { return nil } -// FIXME - very specific for webhooks. Make it easier? -func stopWebhook(image string, identifier string) error { - ctx := context.Background() - - containername := fmt.Sprintf("%s-%s", image, identifier) - - cli, err := client.NewEnvClient() - if err != nil { - log.Println("Unable to create docker client") - return err - } - - // containers, err := cli.ContainerList(ctx, types.ContainerListOptions{ - // All: true, - // }) - - if err := cli.ContainerStop(ctx, containername, nil); err != nil { - log.Printf("Unable to stop container %s - running removal anyway, just in case: %s", containername, err) - } - - removeOptions := types.ContainerRemoveOptions{ - RemoveVolumes: true, - Force: true, - } - - if err := cli.ContainerRemove(ctx, containername, removeOptions); err != nil { - log.Printf("Unable to remove container: %s", err) - } - - return nil -} - -// Starts a new webhook -func handleStopHookDocker(resp http.ResponseWriter, request *http.Request) { - cors := shuffle.HandleCors(resp, request) - if cors { - return - } - - location := strings.Split(request.URL.String(), "/") - - var fileId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if len(fileId) != 32 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) - return - } - - ctx := context.Background() - hook, err := shuffle.GetHook(ctx, fileId) - if err != nil { - log.Printf("Failed getting hook %s (stop docker): %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Printf("Status: %s", hook.Status) - log.Printf("Running: %t", hook.Running) - if !hook.Running { - message := fmt.Sprintf("Error: %s isn't running", hook.Id) - log.Println(message) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "%s"}`, message))) - return - } - - hook.Status = "stopped" - hook.Running = false - hook.Actions = []shuffle.HookAction{} - err = shuffle.SetHook(ctx, *hook) - if err != nil { - log.Printf("Failed setting hook: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - image := "webhook" - - // This is here to force stop and remove the old webhook - err = stopWebhook(image, fileId) - if err != nil { - log.Printf("Container stop issue for %s-%s: %s", image, fileId, err) - } - - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true, "message": "Stopped webhook"}`)) -} - -// THis is an example -// Can also be used as base data? -var webhook = `{ - "id": "d6ef8912e8bd37776e654cbc14c2629c", - "info": { - "url": "http://localhost:5001", - "name": "TheHive", - "description": "Webhook for TheHive" - }, - "transforms": {}, - "actions": {}, - "type": "webhook", - "running": false, - "status": "stopped" -}` - -// Starts a new webhook -func handleDeleteHookDocker(resp http.ResponseWriter, request *http.Request) { - ctx := context.Background() - cors := shuffle.HandleCors(resp, request) - if cors { - return - } - - location := strings.Split(request.URL.String(), "/") - - var fileId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if len(fileId) != 32 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) - return - } - - err := shuffle.DeleteKey(ctx, "hooks", fileId) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "message": "Can't delete"}`)) - return - } - - image := "webhook" - - // This is here to force stop and remove the old webhook - err = stopWebhook(image, fileId) - if err != nil { - log.Printf("Container stop issue for %s-%s: %s", image, fileId, err) - resp.Write([]byte(`{"success": false, "message": "Couldn't stop webhook"}`)) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true, "message": "Deleted webhook"}`)) -} - // Checks if an image exists func imageCheckBuilder(images []string) error { //log.Printf("[FIXME] ImageNames to check: %#v", images) @@ -595,32 +430,7 @@ func imageCheckBuilder(images []string) error { return nil } -func hookTest() { - var hook shuffle.Hook - err := json.Unmarshal([]byte(webhook), &hook) - log.Println(webhook) - if err != nil { - log.Printf("Failed hook unmarshaling: %s", err) - return - } - - ctx := context.Background() - err = shuffle.SetHook(ctx, hook) - if err != nil { - log.Printf("Failed setting hook: %s", err) - } - - returnHook, err := shuffle.GetHook(ctx, hook.Id) - if err != nil { - log.Printf("Failed getting hook %s (test): %s", hook.Id, err) - } - - if len(returnHook.Id) > 0 { - log.Printf("Success! - %s", returnHook.Id) - } -} - -//https://stackoverflow.com/questions/23935141/how-to-copy-docker-images-from-one-host-to-another-without-using-a-repository +// https://stackoverflow.com/questions/23935141/how-to-copy-docker-images-from-one-host-to-another-without-using-a-repository func getDockerImage(resp http.ResponseWriter, request *http.Request) { cors := shuffle.HandleCors(resp, request) if cors { @@ -643,14 +453,9 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { return } - type requestCheck struct { - Name string `datastore:"name" json:"name" yaml:"name"` - } - // This has to be done in a weird way because Datastore doesn't // support map[string]interface and similar (openapi3.Swagger) - var version requestCheck - + var version shuffle.DockerRequestCheck err = json.Unmarshal(body, &version) if err != nil { resp.WriteHeader(422) @@ -684,8 +489,11 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { alternativeName = strings.Join(alternativeNameSplit[1:3], "/") } + log.Printf("[INFO] Trying to download image: %s. Alt: %s", version.Name, alternativeName) + for _, image := range images { for _, tag := range image.RepoTags { + //log.Printf("[DEBUG] Tag: %s", tag) if strings.ToLower(tag) == strings.ToLower(version.Name) { img = image tagFound = tag @@ -699,11 +507,34 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { } } + pullOptions := types.ImagePullOptions{} + if len(img.ID) == 0 { + _, err := dockercli.ImagePull(context.Background(), version.Name, pullOptions) + if err == nil { + tagFound = version.Name + img.ID = version.Name + img2.ID = version.Name + + dockercli.ImageTag(ctx, version.Name, alternativeName) + } + } + + if len(img2.ID) == 0 { + _, err := dockercli.ImagePull(context.Background(), alternativeName, pullOptions) + if err == nil { + tagFound = alternativeName + img.ID = alternativeName + img2.ID = alternativeName + + dockercli.ImageTag(ctx, alternativeName, version.Name) + } + } + // REBUILDS THE APP if len(img.ID) == 0 { if len(img2.ID) == 0 { workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 0, 0) - //log.Printf("[INFO] Getting workflowapps for a rebuild. Got %d with err %#v", len(workflowapps), err) + log.Printf("[INFO] Getting workflowapps for a rebuild. Got %d with err %#v", len(workflowapps), err) if err == nil { imageName := "" imageVersion := "" @@ -728,7 +559,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { foundApp := shuffle.WorkflowApp{} imageName = strings.ToLower(imageName) imageVersion = strings.ToLower(imageVersion) - log.Printf("[DEBUG] Looking for appname %s with version %s", imageName, imageVersion) + log.Printf("[DEBUG] Docker Looking for appname %s with version %s", imageName, imageVersion) for _, app := range workflowapps { if strings.ToLower(strings.Replace(app.Name, " ", "_", -1)) == imageName && app.AppVersion == imageVersion { @@ -848,7 +679,7 @@ func handleRemoteDownloadApp(resp http.ResponseWriter, ctx context.Context, user app := tmpapp{} err := json.Unmarshal(respBody, &app) if err != nil || app.Success == false || len(app.OpenAPI) == 0 { - log.Printf("[ERROR] Failed app unmarshal during auto-download. Success%#v. Applength: %d: %s", app.Success, len(app.OpenAPI), err) + log.Printf("[ERROR] Failed app unmarshal during auto-download. Success: %#v. Applength: %d: %s", app.Success, len(app.OpenAPI), err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) return @@ -966,37 +797,29 @@ func activateWorkflowAppDocker(resp http.ResponseWriter, request *http.Request) } } - if app.Sharing || app.Public { - org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) - if err == nil { - added := false - if !shuffle.ArrayContains(org.ActiveApps, app.ID) { - org.ActiveApps = append(org.ActiveApps, app.ID) - added = true - } - - if added { - err = shuffle.SetOrg(ctx, *org, org.Id) - if err != nil { - log.Printf("[WARNING] Failed setting org when autoadding apps on save: %s", err) - } else { - log.Printf("[INFO] Added public app %s (%s) to org %s (%s)", app.Name, app.ID, user.ActiveOrg.Name, user.ActiveOrg.Id) - cacheKey := fmt.Sprintf("apps_%s", user.Id) - shuffle.DeleteCache(ctx, cacheKey) - } - } - } - } else { - log.Printf("[WARNING] User is trying to activate %s which is NOT public", app.Name) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) + // Just making sure it's being built properly + if app == nil { + log.Printf("[WARNING] App is nil. This shouldn't happen. Starting remote download(3)") + handleRemoteDownloadApp(resp, ctx, user, fileId) return } - log.Printf("[DEBUG] App %s (%s) activated for org %s by user %s", app.Name, app.ID, user.ActiveOrg.Id, user.Username) + // Check the app.. hmm + openApiApp, err := shuffle.GetOpenApiDatastore(ctx, app.ID) + if err != nil { + log.Printf("[WARNING] Error getting app %s (openapi config): %s", app.ID, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Couldn't find app OpenAPI"}`)) + return + } - // If onprem, it should autobuild the container(s) from here + log.Printf("[INFO] User %s (%s) is activating %s. Public: %t, Shared: %t", user.Username, user.Id, app.Name, app.Public, app.Sharing) + buildSwaggerApp(resp, []byte(openApiApp.Body), user, true) - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) + //app.Active = true + //app.Generated = true + //app, err := shuffle.SetApp(ctx, app) + + //resp.WriteHeader(200) + //resp.Write([]byte(`{"success": true}`)) } diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod old mode 100644 new mode 100755 index 822b91e0..b43db79b --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -1,100 +1,105 @@ -module main +module shuffle-shared + +replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared go 1.19 -//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared - require ( - cloud.google.com/go/datastore v1.10.0 - cloud.google.com/go/pubsub v1.28.0 - cloud.google.com/go/storage v1.28.1 + cloud.google.com/go/datastore v1.11.0 + cloud.google.com/go/pubsub v1.31.0 + cloud.google.com/go/storage v1.30.1 github.com/basgys/goxml2json v1.1.0 github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 - github.com/docker/docker v20.10.21+incompatible + github.com/docker/docker v24.0.2+incompatible github.com/frikky/kin-openapi v0.42.0 - github.com/fsouza/go-dockerclient v1.9.0 + github.com/fsouza/go-dockerclient v1.9.7 github.com/ghodss/yaml v1.0.0 - github.com/go-git/go-billy/v5 v5.3.1 - github.com/go-git/go-git/v5 v5.5.0 + github.com/go-git/go-billy/v5 v5.4.1 + github.com/go-git/go-git/v5 v5.7.0 github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.3.35 - golang.org/x/crypto v0.3.0 - google.golang.org/api v0.103.0 + github.com/shuffle/shuffle-shared v0.4.19 + golang.org/x/crypto v0.9.0 + google.golang.org/api v0.125.0 google.golang.org/appengine v1.6.7 - google.golang.org/grpc v1.51.0 + google.golang.org/grpc v1.55.0 gopkg.in/src-d/go-git.v4 v4.13.1 gopkg.in/yaml.v3 v3.0.1 ) require ( - cloud.google.com/go v0.105.0 // indirect - cloud.google.com/go/compute v1.13.0 // indirect - cloud.google.com/go/compute/metadata v0.2.1 // indirect - cloud.google.com/go/iam v0.7.0 // indirect + cloud.google.com/go v0.110.2 // indirect + cloud.google.com/go/compute v1.19.3 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v1.0.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/Microsoft/go-winio v0.6.0 // indirect - github.com/Microsoft/hcsshim v0.9.3 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect - github.com/acomagu/bufpipe v1.0.3 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect + github.com/acomagu/bufpipe v1.0.4 // indirect github.com/adrg/strutil v0.2.3 // indirect github.com/algolia/algoliasearch-client-go/v3 v3.18.1 // indirect + github.com/bitly/go-simplejson v0.5.0 // indirect github.com/bradfitz/gomemcache v0.0.0-20221031212613-62deef7fc822 // indirect github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect - github.com/cloudflare/circl v1.1.0 // indirect - github.com/containerd/cgroups v1.0.3 // indirect - github.com/containerd/containerd v1.6.6 // indirect - github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/cloudflare/circl v1.3.3 // indirect + github.com/containerd/containerd v1.6.18 // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect - github.com/frikky/go-elasticsearch/v8 v8.13.1 // indirect - github.com/go-git/gcfg v1.5.0 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/swag v0.19.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-github/v28 v28.1.1 // indirect github.com/google/go-querystring v1.0.0 // indirect + github.com/google/s2a-go v0.1.4 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect - github.com/googleapis/gax-go/v2 v2.7.0 // indirect - github.com/imdario/mergo v0.3.13 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/googleapis/gax-go/v2 v2.10.0 // indirect + github.com/imdario/mergo v0.3.15 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/mailru/easyjson v0.7.0 // indirect - github.com/moby/sys/mount v0.3.3 // indirect - github.com/moby/sys/mountinfo v0.6.2 // indirect + github.com/klauspost/compress v1.11.13 // indirect + github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e // indirect + github.com/moby/patternmatcher v0.5.0 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect - github.com/opencontainers/runc v1.1.2 // indirect + github.com/opencontainers/runc v1.1.5 // indirect + github.com/opensearch-project/opensearch-go v1.1.0 // indirect + github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect - github.com/pjbgf/sha1cd v0.2.0 // indirect + github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/sergi/go-diff v1.1.0 // indirect github.com/sirupsen/logrus v1.8.1 // indirect - github.com/skeema/knownhosts v1.1.0 // indirect + github.com/skeema/knownhosts v1.1.1 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/src-d/gcfg v1.4.0 // indirect - github.com/xanzy/ssh-agent v0.3.2 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect go.opencensus.io v0.24.0 // indirect go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/net v0.2.0 // indirect - golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.2.0 // indirect - golang.org/x/text v0.4.0 // indirect - golang.org/x/tools v0.1.12 // indirect + golang.org/x/mod v0.8.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/oauth2 v0.8.0 // indirect + golang.org/x/sync v0.2.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect + golang.org/x/tools v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/protobuf v1.30.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + ) diff --git a/backend/go-app/main.go b/backend/go-app/main.go old mode 100644 new mode 100755 index 4c9b21b5..b1304b4f --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -13,7 +13,7 @@ import ( //"crypto/tls" //"crypto/x509" - "encoding/base64" + //"encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -21,7 +21,6 @@ import ( "io" "io/ioutil" "log" - "math/rand" "net/http" "net/url" "os" @@ -38,9 +37,6 @@ import ( "cloud.google.com/go/storage" "google.golang.org/appengine/mail" - //"github.com/elastic/go-elasticsearch/v7" - //"github.com/elastic/go-elasticsearch/v8/esapi" - "github.com/frikky/kin-openapi/openapi2" "github.com/frikky/kin-openapi/openapi2conv" "github.com/frikky/kin-openapi/openapi3" @@ -82,18 +78,14 @@ import ( var gceProject = "shuffle" var bucketName = "shuffler.appspot.com" var baseAppPath = "/home/frikky/git/shaffuru/tmp/apps" + var baseDockerName = "frikky/shuffle" var registryName = "registry.hub.docker.com" var runningEnvironment = "onprem" var syncUrl = "https://shuffler.io" - -//var syncUrl = "http://localhost:5002" var syncSubUrl = "https://shuffler.io" -//var syncUrl = "http://localhost:5002" -//var syncSubUrl = "https://050196912a9d.ngrok.io" - var dbclient *datastore.Client type Userapi struct { @@ -701,9 +693,9 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) for tutorialIndex, tutorial := range neworg.Tutorials { if tutorial.Name == "Invite teammates" { neworg.Tutorials[tutorialIndex].Description = fmt.Sprintf("%d users are in your org. Org name and Image change next.", len(neworg.Users)) - if len(neworg.Users) > 0 { + if len(neworg.Users) > 1 { neworg.Tutorials[tutorialIndex].Done = true - neworg.Tutorials[tutorialIndex].Link = "/admin" + neworg.Tutorials[tutorialIndex].Link = "/admin?tab=users" } break @@ -796,7 +788,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { CloudSync: false, } - err = shuffle.SetOrg(ctx, newOrg, orgId) + err = shuffle.SetOrg(ctx, newOrg, newOrg.Id) if err != nil { log.Printf("[WARNING] Failed setting init organization: %s", err) } else { @@ -948,48 +940,104 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { }) // Updating user info if there's something wrong - if (len(userInfo.ActiveOrg.Name) == 0 || len(userInfo.ActiveOrg.Id) == 0) && len(userInfo.Orgs) > 0 { - _, err := shuffle.GetOrg(ctx, userInfo.Orgs[0]) - if err != nil { + if len(userInfo.ActiveOrg.Name) == 0 || len(userInfo.ActiveOrg.Id) == 0 { + if len(userInfo.Orgs) == 0 || (len(userInfo.Orgs) > 0 && userInfo.Orgs[0] == "") { orgs, err := shuffle.GetAllOrgs(ctx) - if err == nil { - newStringOrgs := []string{} - newOrgs := []shuffle.Org{} + log.Printf("[INFO] Fixing organization for user %s (%s). Found orgs: %d", userInfo.Username, userInfo.Id, len(orgs)) + if err == nil && len(orgs) > 0 { for _, org := range orgs { - if strings.ToLower(org.Name) == strings.ToLower(userInfo.Orgs[0]) { - newOrgs = append(newOrgs, org) - newStringOrgs = append(newStringOrgs, org.Id) + if len(org.Id) == 0 { + continue } - } - if len(newOrgs) > 0 { + // Prolly some way here to jump into another org + // when you have access to the DB userInfo.ActiveOrg = shuffle.OrgMini{ - Id: newOrgs[0].Id, - Name: newOrgs[0].Name, - } - - userInfo.Orgs = newStringOrgs - - err = shuffle.SetUser(ctx, &userInfo, true) - if err != nil { - log.Printf("Error patching User for activeOrg: %s", err) - } else { - log.Printf("Updated the users' org") + Name: org.Name, + Id: org.Id, + Role: "admin", } + userInfo.Orgs = []string{org.Id} + break } - } else { - log.Printf("Failed getting orgs for user. Major issue.: %s", err) } - } else { - // 1. Check if the org exists by ID - // 2. if it does, overwrite user - userInfo.ActiveOrg = shuffle.OrgMini{ - Id: userInfo.Orgs[0], + // Make a new one in case we couldn't find one + if len(userInfo.ActiveOrg.Id) == 0 { + orgSetupName := "default" + orgId := uuid.NewV4().String() + newOrg := shuffle.Org{ + Name: orgSetupName, + Id: orgId, + Org: orgSetupName, + Users: []shuffle.User{}, + Roles: []string{"admin", "user"}, + CloudSync: false, + } + + err = shuffle.SetOrg(ctx, newOrg, newOrg.Id) + if err == nil { + userInfo.ActiveOrg = shuffle.OrgMini{ + Name: newOrg.Name, + Id: newOrg.Id, + Role: "admin", + } + userInfo.Orgs = []string{newOrg.Id} + } else { + log.Printf("[WARNING] Failed to set new org: %s", err) + } } + + // Set user err = shuffle.SetUser(ctx, &userInfo, true) if err != nil { - log.Printf("[INFO] Error patching User for activeOrg: %s", err) + log.Printf("[WARNING] Failed fixing org info for user %s (%s)", userInfo.Username, userInfo.Id) + } else { + log.Printf("[INFO] Set organization for %s (%s) to be %s (%s)", userInfo.Username, userInfo.Id, userInfo.ActiveOrg.Name, userInfo.ActiveOrg.Id) + } + } else if len(userInfo.Orgs) > 0 && userInfo.Orgs[0] != "" { + _, err := shuffle.GetOrg(ctx, userInfo.Orgs[0]) + if err != nil { + orgs, err := shuffle.GetAllOrgs(ctx) + if err == nil { + newStringOrgs := []string{} + newOrgs := []shuffle.Org{} + for _, org := range orgs { + if strings.ToLower(org.Name) == strings.ToLower(userInfo.Orgs[0]) { + newOrgs = append(newOrgs, org) + newStringOrgs = append(newStringOrgs, org.Id) + } + } + + if len(newOrgs) > 0 { + userInfo.ActiveOrg = shuffle.OrgMini{ + Id: newOrgs[0].Id, + Name: newOrgs[0].Name, + } + + userInfo.Orgs = newStringOrgs + + err = shuffle.SetUser(ctx, &userInfo, true) + if err != nil { + log.Printf("Error patching User for activeOrg: %s", err) + } else { + log.Printf("Updated the users' org") + } + } + } else { + log.Printf("Failed getting orgs for user. Major issue.: %s", err) + } + + } else { + // 1. Check if the org exists by ID + // 2. if it does, overwrite user + userInfo.ActiveOrg = shuffle.OrgMini{ + Id: userInfo.Orgs[0], + } + err = shuffle.SetUser(ctx, &userInfo, true) + if err != nil { + log.Printf("[INFO] Error patching User for activeOrg: %s", err) + } } } } @@ -1052,15 +1100,18 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { chatDisabled = true } + userOrgs = shuffle.SortOrgList(userOrgs) orgPriorities := org.Priorities - if len(org.Priorities) < 5 { - log.Printf("[WARNING] Should find and add priorities as length is less than 5 for org %s", userInfo.ActiveOrg.Id) + if len(org.Priorities) < 10 { + log.Printf("[WARNING] Should find and add priorities as length is less than 10 for org %s", userInfo.ActiveOrg.Id) newPriorities, err := shuffle.GetPriorities(ctx, userInfo, org) if err != nil { log.Printf("[WARNING] Failed getting new priorities for org %s: %s", org.Id, err) //orgPriorities = []shuffle.Priority{} } else { orgPriorities = newPriorities + + // A way to manage them over time } } @@ -1225,15 +1276,6 @@ func handleContact(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true, "message": "Thanks for reaching out. We will contact you soon!"}`))) } -func verifier() (*shuffle.CodeVerifier, error) { - r := rand.New(rand.NewSource(time.Now().UnixNano())) - b := make([]byte, 32, 32) - for i := 0; i < 32; i++ { - b[i] = byte(r.Intn(255)) - } - return shuffle.CreateCodeVerifierFromBytes(b) -} - func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { cors := shuffle.HandleCors(resp, request) if cors { @@ -1278,51 +1320,7 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { // Should run calculations if len(org.SSOConfig.OpenIdAuthorization) > 0 { - baseSSOUrl = org.SSOConfig.OpenIdAuthorization - - codeChallenge := uuid.NewV4().String() - //h.Write([]byte(v.Value)) - verifier, verifiererr := verifier() - if verifiererr == nil { - codeChallenge = verifier.Value - } - - //log.Printf("[DEBUG] Got challenge value %s (pre state)", codeChallenge) - - // https://192.168.55.222:3443/api/v1/login_openid - //location := strings.Split(request.URL.String(), "/") - //redirectUrl := url.QueryEscape("http://localhost:5001/api/v1/login_openid") - redirectUrl := url.QueryEscape(fmt.Sprintf("http://%s/api/v1/login_openid", request.Host)) - if strings.Contains(request.Host, "shuffle-backend") && !strings.Contains(os.Getenv("BASE_URL"), "shuffle-backend") { - redirectUrl = url.QueryEscape(fmt.Sprintf("%s/api/v1/login_openid", os.Getenv("BASE_URL"))) - } - - if len(os.Getenv("SSO_REDIRECT_URL")) > 0 { - redirectUrl = url.QueryEscape(fmt.Sprintf("%s/api/v1/login_openid", os.Getenv("SSO_REDIRECT_URL"))) - } - - state := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("org=%s&challenge=%s&redirect=%s", org.Id, codeChallenge, redirectUrl))) - - // has to happen after initial value is stored - if verifiererr == nil { - codeChallenge = verifier.CodeChallengeS256() - } - - //log.Printf("[DEBUG] Got challenge value %s (POST state)", codeChallenge) - - if len(org.SSOConfig.OpenIdClientSecret) > 0 { - - //baseSSOUrl += fmt.Sprintf("?client_id=%s&response_type=code&scope=openid&redirect_uri=%s&state=%s&client_secret=%s", org.SSOConfig.OpenIdClientId, redirectUrl, state, org.SSOConfig.OpenIdClientSecret) - state := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("org=%s&redirect=%s&challenge=%s", org.Id, redirectUrl, org.SSOConfig.OpenIdClientSecret))) - log.Printf("URL: %s", redirectUrl) - - baseSSOUrl += fmt.Sprintf("?client_id=%s&response_type=id_token&scope=openid&redirect_uri=%s&state=%s&response_mode=form_post&nonce=%s", org.SSOConfig.OpenIdClientId, redirectUrl, state, state) - //baseSSOUrl += fmt.Sprintf("&client_secret=%s", org.SSOConfig.OpenIdClientSecret) - log.Printf("[DEBUG] Found OpenID url (client secret). Extra redirect check: %s - %s", request.URL.String(), baseSSOUrl) - } else { - log.Printf("[DEBUG] Found OpenID url (PKCE!!). Extra redirect check: %s", request.URL.String()) - baseSSOUrl += fmt.Sprintf("?client_id=%s&response_type=code&scope=openid&redirect_uri=%s&state=%s&code_challenge_method=S256&code_challenge=%s", org.SSOConfig.OpenIdClientId, redirectUrl, state, codeChallenge) - } + baseSSOUrl = shuffle.GetOpenIdUrl(request, *org) break } @@ -1339,156 +1337,6 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "redirect", "sso_url": "%s"}`, baseSSOUrl))) } -func handleLogin(resp http.ResponseWriter, request *http.Request) { - cors := shuffle.HandleCors(resp, request) - if cors { - return - } - - // Gets a struct of Username, password - data, err := shuffle.ParseLoginParameters(resp, request) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - - log.Printf("[INFO] Handling login of %s", data.Username) - - err = checkUsername(data.Username) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - - ctx := context.Background() - log.Printf("[INFO] Login Username: %s", data.Username) - users, err := shuffle.FindUser(ctx, strings.ToLower(strings.TrimSpace(data.Username))) - if err != nil && len(users) == 0 { - log.Printf("[WARNING] Failed getting user %s: %s", data.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) - return - } - - if len(users) != 1 { - log.Printf(`Found multiple or no users with the same username: %s: %d`, data.Username, len(users)) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error: %d users with username %s"}`, len(users), data.Username))) - return - } - - Userdata := users[0] - - err = bcrypt.CompareHashAndPassword([]byte(Userdata.Password), []byte(data.Password)) - if err != nil { - log.Printf("Password for %s is incorrect: %s", data.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) - return - } - - if !Userdata.Active { - log.Printf("%s is not active, but tried to login. Error: %v", data.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "This user is deactivated"}`)) - return - } - - tutorialsFinished := []shuffle.Tutorial{} - for _, tutorial := range Userdata.PersonalInfo.Tutorials { - tutorialsFinished = append(tutorialsFinished, shuffle.Tutorial{ - Name: tutorial, - }) - } - returnValue := shuffle.HandleInfo{ - Success: true, - Tutorials: tutorialsFinished, - } - - loginData := `{"success": true}` - newData, err := json.Marshal(returnValue) - if err == nil { - loginData = string(newData) - } - - if len(Userdata.Session) != 0 { - log.Println("[INFO] User session already exists - resetting it") - expiration := time.Now().Add(3600 * time.Second) - - http.SetCookie(resp, &http.Cookie{ - Name: "session_token", - Value: Userdata.Session, - Expires: expiration, - }) - - returnValue.Cookies = append(returnValue.Cookies, shuffle.SessionCookie{ - Key: "session_token", - Value: Userdata.Session, - Expiration: expiration.Unix(), - }) - - loginData = fmt.Sprintf(`{"success": true, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}]}`, Userdata.Session, expiration.Unix()) - newData, err := json.Marshal(returnValue) - if err == nil { - loginData = string(newData) - } - //log.Printf("SESSION LENGTH MORE THAN 0 IN LOGIN: %s", Userdata.Session) - - err = shuffle.SetSession(ctx, Userdata, Userdata.Session) - if err != nil { - log.Printf("Error adding session to database: %s", err) - } - - resp.WriteHeader(200) - resp.Write([]byte(loginData)) - return - } else { - log.Printf("[INFO] User session is empty - create one!") - - sessionToken := uuid.NewV4().String() - expiration := time.Now().Add(3600 * time.Second) - http.SetCookie(resp, &http.Cookie{ - Name: "session_token", - Value: sessionToken, - Expires: expiration, - }) - - // ADD TO DATABASE - err = shuffle.SetSession(ctx, Userdata, sessionToken) - if err != nil { - log.Printf("Error adding session to database: %s", err) - } - - Userdata.Session = sessionToken - err = shuffle.SetUser(ctx, &Userdata, true) - if err != nil { - log.Printf("Failed updating user when setting session: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false}`)) - return - } - - returnValue.Cookies = append(returnValue.Cookies, shuffle.SessionCookie{ - Key: "session_token", - Value: sessionToken, - Expiration: expiration.Unix(), - }) - - loginData = fmt.Sprintf(`{"success": true, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}]}`, sessionToken, expiration.Unix()) - newData, err := json.Marshal(returnValue) - if err == nil { - loginData = string(newData) - } - } - - log.Printf("[INFO] %s SUCCESSFULLY LOGGED IN with session %s", data.Username, Userdata.Session) - - resp.WriteHeader(200) - resp.Write([]byte(loginData)) -} - func fixOrgUser(ctx context.Context, org *shuffle.Org) *shuffle.Org { //found := false //for _, id := range user.Orgs { @@ -1590,7 +1438,7 @@ func fixUserOrg(ctx context.Context, user *shuffle.User) *shuffle.User { org.Users = append(org.Users, *user) } - err = shuffle.SetOrg(ctx, *org, orgId) + err = shuffle.SetOrg(ctx, *org, org.Id) if err != nil { log.Printf("Failed setting org %s", orgId) } @@ -2211,6 +2059,15 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { } } + // Find user agent header + userAgent := request.Header.Get("User-Agent") + if strings.Contains(strings.ToLower(userAgent), "microsoftpreview") || strings.Contains(strings.ToLower(userAgent), "googlebot") { + log.Printf("[AUDIT] Blocking googlebot and microsoftbot for webhooks. UA: '%s'", userAgent) + resp.WriteHeader(400) + resp.Write([]byte(`{"success": false, "reason": "Google/Microsoft preview bots not allowed. Please change the useragent."}`)) + return + } + // ID: webhook_ if len(hookId) != 44 { log.Printf("[INFO] Couldn't handle hookId. Too short in webhook: %d", len(hookId)) @@ -2296,7 +2153,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { if err == nil { for _, branch := range workflow.Branches { if branch.SourceID == hook.Id { - log.Printf("[INFO] Found ID %s for hook", hook.Id) + log.Printf("[DEBUG] Found ID %s for hook", hook.Id) if branch.DestinationID != hook.Start { newBody.Start = branch.DestinationID break @@ -2308,57 +2165,76 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { b, err := json.Marshal(newBody) if err != nil { - log.Printf("Failed newBody marshaling: %s", err) - resp.WriteHeader(401) + log.Printf("[ERROR] Failed newBody marshaling for webhook: %s", err) + resp.WriteHeader(500) resp.Write([]byte(`{"success": false}`)) return } + // Should wrap the response input Body as well? for _, item := range hook.Workflows { - //log.Printf("Running webhook for workflow %s with startnode %s", item, hook.Start) + log.Printf("[INFO] Running webhook for workflow %s with startnode %s", item, hook.Start) + + // This ID is empty to force it to get the webhook within the execution workflow := shuffle.Workflow{ ID: "", } - //parsedBody := string(body) - //parsedBody = strings.Replace(parsedBody, "\"", "\\\"", -1) - //if len(parsedBody) > 0 { - // if string(parsedBody[0]) == `"` && string(parsedBody[len(parsedBody)-1]) == "\"" { - // parsedBody = parsedBody[1 : len(parsedBody)-1] - // } - //} - - //bodyWrapper := fmt.Sprintf(`{"start": "%s", "execution_source": "webhook", "execution_argument": "%s"}`, hook.Start, string(parsedBody)) - //if len(hook.Start) == 0 { - // log.Printf("No start node for hook %s - running with workflow default.", hook.Id) - // bodyWrapper = string(parsedBody) - //} + if len(hook.Start) == 0 { + log.Printf("[WARNING] No start node for hook %s - running with workflow default.", hook.Id) + //bodyWrapper = string(parsedBody) + } newRequest := &http.Request{ URL: &url.URL{}, Method: "POST", Body: ioutil.NopCloser(bytes.NewReader(b)), } - //start, startok := request.URL.Query()["start"] // OrgId: activeOrgs[0].Id, workflowExecution, executionResp, err := handleExecution(item, workflow, newRequest, hook.OrgId) - if err == nil { - /* - err = increaseStatisticsField(ctx, "total_webhooks_ran", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) - if err != nil { - log.Printf("Failed to increase total apps loaded stats: %s", err) - } - */ + if err == nil { + if hook.Version == "v2" { + timeout := 15 + //if hook.VersionTimeout != 0 { + // timeout = hook.VersionTimeout + //} + + log.Printf("[DEBUG] Waiting for Webhook response from %s for max %d seconds! Checking every 1 second. Hook ID: %s", workflowExecution.ExecutionId, timeout, hook.Id) + // Try every second for 15 seconds + for i := 0; i < timeout; i++ { + time.Sleep(1 * time.Second) + + newExec, err := shuffle.GetWorkflowExecution(ctx, workflowExecution.ExecutionId) + if err != nil { + log.Printf("[ERROR] Failed to get workflow execution: %s", err) + break + } + + if newExec.Status != "EXECUTING" { + log.Printf("[INFO] Got response from webhook v2 of length '%d' <- %s", len(newExec.Result), newExec.ExecutionId) + resp.WriteHeader(200) + resp.Write([]byte(newExec.Result)) + return + } + } + } + + // Fallback resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s"}`, workflowExecution.ExecutionId))) + if len(hook.CustomResponse) > 0 { + resp.Write([]byte(hook.CustomResponse)) + } else { + resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s"}`, workflowExecution.ExecutionId))) + } return } resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, executionResp))) } + } func executeCloudAction(action shuffle.CloudSyncJob, apikey string) error { @@ -2408,6 +2284,8 @@ func executeCloudAction(action shuffle.CloudSyncJob, apikey string) error { return errors.New(fmt.Sprintf("Cloud error from Shuffler: %s", responseData.Reason)) } + log.Printf("[INFO] Cloud action executed successfully for '%s'", action.Action) + return nil } @@ -3688,7 +3566,7 @@ func handleCloudJob(job shuffle.CloudSyncJob) error { } else if job.Type == "schedule" { if job.Action == "execute" { - log.Printf("Should handle schedule for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) + log.Printf("[INFO] Should handle schedule for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) err := handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "schedule", job.ThirdItem) if err != nil { log.Printf("[INFO] Failed executing workflow from cloud schedule: %s", err) @@ -3698,7 +3576,7 @@ func handleCloudJob(job shuffle.CloudSyncJob) error { } } else if job.Type == "email_trigger" { if job.Action == "execute" { - log.Printf("Should handle email for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) + log.Printf("[INFO] Should handle email for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) err := handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "email_trigger", job.ThirdItem) if err != nil { log.Printf("Failed executing workflow from email trigger: %s", err) @@ -3709,7 +3587,7 @@ func handleCloudJob(job shuffle.CloudSyncJob) error { } else if job.Type == "user_input" { if job.Action == "continue" { - log.Printf("Should handle user_input CONTINUE for workflow %s with start node %s and execution ID %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) + log.Printf("[INFO] Should handle user_input CONTINUE for workflow %s with start node %s and execution ID %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) // FIXME: Handle authorization ctx := context.Background() workflowExecution, err := shuffle.GetWorkflowExecution(ctx, job.ThirdItem) @@ -3807,47 +3685,50 @@ func remoteOrgJobController(org shuffle.Org, body []byte) error { if !responseData.Success { log.Printf("[WARNING] Should stop org job controller because no success?") - if strings.Contains(responseData.Reason, "Bad apikey") || strings.Contains(responseData.Reason, "Error getting the organization") || strings.Contains(responseData.Reason, "Organization isn't syncing") { + if strings.Contains(strings.ToLower(responseData.Reason), "bad apikey") || strings.Contains(responseData.Reason, "Error getting the organization") || strings.Contains(responseData.Reason, "Organization isn't syncing") { log.Printf("[WARNING] Remote error; Bad apikey or org error. Stopping sync for org: %s", responseData.Reason) if value, exists := scheduledOrgs[org.Id]; exists { // Looks like this does the trick? Hurr - log.Printf("[WARNING] STOPPING ORG SCHEDULE for: %s", org.Id) - + log.Printf("[INFO] STOPPING ORG SCHEDULE for: %s", org.Id) value.Lock() - org, err := shuffle.GetOrg(ctx, org.Id) - if err != nil { - log.Printf("[WARNING] Failed finding org %s: %s", org.Id, err) - return err - } - - org.SyncConfig.Interval = 0 - org.SyncConfig.Apikey = "" - org.CloudSync = false - - // Just in case - org, err = handleStopCloudSync(syncUrl, *org) - - startDate := time.Now().Unix() - org.SyncFeatures.Webhook = shuffle.SyncData{Active: false, Type: "trigger", Name: "Webhook", StartDate: startDate} - org.SyncFeatures.UserInput = shuffle.SyncData{Active: false, Type: "trigger", Name: "User Input", StartDate: startDate} - org.SyncFeatures.EmailTrigger = shuffle.SyncData{Active: false, Type: "action", Name: "Email Trigger", StartDate: startDate} - org.SyncFeatures.Schedules = shuffle.SyncData{Active: false, Type: "trigger", Name: "Schedule", StartDate: startDate, Limit: 0} - org.SyncFeatures.SendMail = shuffle.SyncData{Active: false, Type: "action", Name: "Send Email", StartDate: startDate, Limit: 0} - org.SyncFeatures.SendSms = shuffle.SyncData{Active: false, Type: "action", Name: "Send SMS", StartDate: startDate, Limit: 0} - org.CloudSyncActive = false - - err = shuffle.SetOrg(ctx, *org, org.Id) - if err != nil { - log.Printf("[WARNING] Failed setting organization when stopping sync: %s", err) - } else { - log.Printf("[INFO] Successfully STOPPED org cloud sync for %s (%s)", org.Name, org.Id) - } - - return errors.New("Stopped schedule for org locally because of bad apikey.") } else { - return errors.New(fmt.Sprintf("Failed finding the schedule for org %s (%s)", org.Name, org.Id)) + log.Printf("[INFO] Failed finding the schedule for org %s (%s)", org.Name, org.Id) } + + org, err := shuffle.GetOrg(ctx, org.Id) + if err != nil { + log.Printf("[WARNING] Failed finding org %s: %s", org.Id, err) + return err + } + + // Just in case + org, err = handleStopCloudSync(syncUrl, *org) + if err != nil { + log.Printf("[ERROR] Failed stopping cloud sync remotely: %s", err) + } + + org.SyncConfig.Interval = 0 + org.CloudSync = false + org.SyncConfig.Apikey = "" + + startDate := time.Now().Unix() + org.SyncFeatures.Webhook = shuffle.SyncData{Active: false, Type: "trigger", Name: "Webhook", StartDate: startDate} + org.SyncFeatures.UserInput = shuffle.SyncData{Active: false, Type: "trigger", Name: "User Input", StartDate: startDate} + org.SyncFeatures.EmailTrigger = shuffle.SyncData{Active: false, Type: "action", Name: "Email Trigger", StartDate: startDate} + org.SyncFeatures.Schedules = shuffle.SyncData{Active: false, Type: "trigger", Name: "Schedule", StartDate: startDate, Limit: 0} + org.SyncFeatures.SendMail = shuffle.SyncData{Active: false, Type: "action", Name: "Send Email", StartDate: startDate, Limit: 0} + org.SyncFeatures.SendSms = shuffle.SyncData{Active: false, Type: "action", Name: "Send SMS", StartDate: startDate, Limit: 0} + org.CloudSyncActive = false + + err = shuffle.SetOrg(ctx, *org, org.Id) + if err != nil { + log.Printf("[WARNING] Failed setting organization when stopping sync: %s", err) + } else { + log.Printf("[INFO] Successfully STOPPED org cloud sync for %s (%s)", org.Name, org.Id) + } + + return nil } return errors.New("[ERROR] Remote job handler issues.") @@ -3916,7 +3797,7 @@ func runInitCloudSetup() { } func runInitEs(ctx context.Context) { - log.Printf("[DEBUG] Starting INIT setup (ES)") + log.Printf("[DEBUG] Starting INIT setup for Elasticsearch/Opensearch") httpProxy := os.Getenv("HTTP_PROXY") if len(httpProxy) > 0 { @@ -3933,7 +3814,7 @@ func runInitEs(ctx context.Context) { log.Printf("[DEBUG] Setting default environment for org to %s", defaultEnv) } - log.Printf("[DEBUG] Getting organizations") + log.Printf("[DEBUG] Getting organizations for Elasticsearch/Opensearch") activeOrgs, err := shuffle.GetAllOrgs(ctx) setUsers := false @@ -4023,6 +3904,11 @@ func runInitEs(ctx context.Context) { } } + if strings.Contains(os.Getenv("SHUFFLE_OPENSEARCH_URL"), "https") { + log.Printf("[INFO] Waiting during init to make sure the opensearch instance is up and running with security features properly") + time.Sleep(30 * time.Second) + } + _ = setUsers schedules, err := shuffle.GetAllSchedules(ctx, "ALL") if err != nil { @@ -4098,7 +3984,7 @@ func runInitEs(ctx context.Context) { CloudSync: false, } - err = shuffle.SetOrg(ctx, newOrg, orgId) + err = shuffle.SetOrg(ctx, newOrg, newOrg.Id) setUsers := false if err != nil { log.Printf("[WARNING] Failed setting organization when creating original user: %s", err) @@ -4150,8 +4036,13 @@ func runInitEs(ctx context.Context) { } for _, org := range activeOrgs { + if len(org.Id) == 0 { + log.Printf("[DEBUG] No ID found for org with name '%s'. Why was it made?", org.Name) + continue + } + if !org.CloudSync { - log.Printf("[INFO] Skipping org syncCheck for %s because sync isn't set (1).", org.Id) + log.Printf("[INFO] Skipping org syncCheck for '%s' because sync isn't set (1).", org.Id) continue } @@ -4301,7 +4192,10 @@ func runInitEs(ctx context.Context) { url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION") if len(url) == 0 { - url = "https://github.com/frikky/shuffle-apps" + log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/shuffle/shuffle-apps") + url = "https://github.com/shuffle/shuffle-apps" + //url = "" + //return } username := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_USERNAME") @@ -4323,10 +4217,9 @@ func runInitEs(ctx context.Context) { cloneOptions.ReferenceName = plumbing.ReferenceName(branch) } - log.Printf("[DEBUG] Getting apps from %s", url) + log.Printf("[DEBUG] Getting apps from url '%s'", url) r, err := git.Clone(storer, fs, cloneOptions) - if err != nil { log.Printf("[WARNING] Failed loading repo into memory (init): %s", err) } @@ -4338,7 +4231,6 @@ func runInitEs(ctx context.Context) { _ = r //iterateAppGithubFolders(fs, dir, "", "testing") - // FIXME: Get all the apps? _, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate) if err != nil { log.Printf("[WARNING] Error from app load in init: %s", err) @@ -4353,7 +4245,7 @@ func runInitEs(ctx context.Context) { } log.Printf("[INFO] Downloading OpenAPI data for search - EXTRA APPS") - apis := "https://github.com/frikky/security-openapis" + apis := "https://github.com/shuffle/security-openapis" // THis gets memory problems hahah //apis := "https://github.com/APIs-guru/openapi-directory" @@ -4388,7 +4280,7 @@ func runInit(ctx context.Context) { //} //log.Printf("[DEBUG] Finalized init statistics update") - log.Printf("[DEBUG] Starting INIT setup") + log.Printf("[DEBUG] Starting INIT setup (NOT Opensearch/Elasticsearch!)") httpProxy := os.Getenv("HTTP_PROXY") if len(httpProxy) > 0 { log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy) @@ -4460,11 +4352,11 @@ func runInit(ctx context.Context) { CloudSync: false, } - err = shuffle.SetOrg(ctx, newOrg, orgId) + err = shuffle.SetOrg(ctx, newOrg, newOrg.Id) if err != nil { - log.Printf("Failed setting organization: %s", err) + log.Printf("[WARNING] Failed setting organization: %s", err) } else { - log.Printf("Successfully created the default org!") + log.Printf("[WARNING] Successfully created the default org!") setUsers = true } } else { @@ -4961,7 +4853,7 @@ func runInit(ctx context.Context) { url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION") if len(url) == 0 { - url = "https://github.com/frikky/shuffle-apps" + url = "https://github.com/shuffle/shuffle-apps" } username := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_USERNAME") @@ -4982,7 +4874,7 @@ func runInit(ctx context.Context) { cloneOptions.ReferenceName = plumbing.ReferenceName(branch) } - log.Printf("[DEBUG] Getting apps from %s", url) + log.Printf("[DEBUG] Getting apps from URL '%s'", url) r, err := git.Clone(storer, fs, cloneOptions) @@ -5012,7 +4904,7 @@ func runInit(ctx context.Context) { } log.Printf("[INFO] Downloading OpenAPI data for search - EXTRA APPS") - apis := "https://github.com/frikky/security-openapis" + apis := "https://github.com/shuffle/security-openapis" // FIXME: This part gets memory problems. Fix in the future to load these apps too. //apis := "https://github.com/APIs-guru/openapi-directory" @@ -5139,7 +5031,7 @@ func handleStopCloudSync(syncUrl string, org shuffle.Org) (*shuffle.Org, error) if err != nil { return &org, err } - log.Printf("Remote disable ret: %s", string(respBody)) + log.Printf("[INFO] Remote disable ret: %s", string(respBody)) responseData := retStruct{} err = json.Unmarshal(respBody, &responseData) @@ -6018,7 +5910,7 @@ func initHandlers() { // General - duplicates and old. r.HandleFunc("/api/v1/getusers", shuffle.HandleGetUsers).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/login", handleLogin).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/login", shuffle.HandleLogin).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/logout", shuffle.HandleLogout).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/register", handleRegister).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/checkusers", checkAdminLogin).Methods("GET", "OPTIONS") @@ -6041,11 +5933,14 @@ func initHandlers() { r.HandleFunc("/api/v1/streams/results", handleGetStreamResults).Methods("POST", "OPTIONS") // Used by orborus - r.HandleFunc("/api/v1/workflows/queue", handleGetWorkflowqueue).Methods("GET") + r.HandleFunc("/api/v1/workflows/queue", handleGetWorkflowqueue).Methods("GET", "POST") r.HandleFunc("/api/v1/workflows/queue/confirm", handleGetWorkflowqueueConfirm).Methods("POST") // App specific // From here down isnt checked for org specific + r.HandleFunc("/api/v1/apps/{key}/execute", executeSingleAction).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/apps/categories", shuffle.GetActiveCategories).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/apps/categories/run", shuffle.RunCategoryAction).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/upload", handleAppZipUpload).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}/activate", activateWorkflowAppDocker).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps/frameworkConfiguration", shuffle.GetFrameworkConfiguration).Methods("GET", "OPTIONS") @@ -6097,6 +5992,12 @@ func initHandlers() { r.HandleFunc("/api/v1/workflows/{key}", deleteWorkflow).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}", shuffle.SaveWorkflow).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}", shuffle.GetSpecificWorkflow).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/workflows/recommend", shuffle.HandleActionRecommendation).Methods("POST", "OPTIONS") + + // New for recommendations in Shuffle + r.HandleFunc("/api/v1/recommendations/get_actions", shuffle.HandleActionRecommendation).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/recommendations/modify", shuffle.HandleRecommendationAction).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/workflows/{key}/revisions", shuffle.GetWorkflowRevisions).Methods("GET", "OPTIONS") // Triggers r.HandleFunc("/api/v1/hooks/new", shuffle.HandleNewHook).Methods("POST", "OPTIONS") @@ -6149,10 +6050,12 @@ func initHandlers() { r.HandleFunc("/api/v1/environments/{key}/rerun", shuffle.HandleRerunExecutions).Methods("GET", "POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/validate_app_values", shuffle.HandleKeyValueCheck).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/orgs/{orgId}/list_cache", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/get_cache", shuffle.HandleGetCacheKey).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/set_cache", shuffle.HandleSetCacheKey).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/orgs/{orgId}/cache/{cache_key}", shuffle.HandleDeleteCacheKey).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/stats", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/apps/{key}/execute", executeSingleAction).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/orgs/{orgId}/revisions", shuffle.GetWorkflowRevisions).Methods("GET", "OPTIONS") // Docker orborus specific - downloads an image r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "OPTIONS") @@ -6182,6 +6085,8 @@ func initHandlers() { r.HandleFunc("/api/v1/users/notifications/clear", shuffle.HandleClearNotifications).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/users/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/conversation", shuffle.RunActionAI).Methods("POST", "OPTIONS") + //r.HandleFunc("/api/v1/users/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/dashboards/{key}/widgets", shuffle.HandleNewWidget).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/dashboards/{key}/widgets/{widget_id}", shuffle.HandleGetWidget).Methods("GET", "OPTIONS") diff --git a/backend/go-app/main_test.go b/backend/go-app/main_test.go old mode 100644 new mode 100755 index e9d358b6..4e39ea0e --- a/backend/go-app/main_test.go +++ b/backend/go-app/main_test.go @@ -24,6 +24,7 @@ type endpoint struct { handler http.HandlerFunc path string method string + body []byte } func init() { @@ -44,7 +45,7 @@ func TestAuthenticationRequired(t *testing.T) { {handler: shuffle.HandleNewOutlookRegister, path: "/functions/outlook/register", method: "GET"}, {handler: shuffle.HandleGetOutlookFolders, path: "/functions/outlook/getFolders", method: "GET"}, {handler: shuffle.HandleApiGeneration, path: "/api/v1/users/generateapikey", method: "GET"}, - {handler: handleLogin, path: "/api/v1/users/login", method: "POST"}, // prob not this one + {handler: shuffle.HandleLogin, path: "/api/v1/users/login", method: "POST"}, // prob not this one // handleRegister generates nil pointer exception. Not necessary for this anyway. //{handler: handleRegister, path: "/api/v1/users/register", method: "POST"}, {handler: shuffle.HandleGetUsers, path: "/api/v1/users/getusers", method: "GET"}, @@ -108,8 +109,8 @@ func TestAuthenticationRequired(t *testing.T) { {handler: verifySwagger, path: "/api/v1/verify_swagger", method: "POST"}, {handler: verifySwagger, path: "/api/v1/verify_openapi", method: "POST"}, - {handler: echoOpenapiData, path: "/api/v1/get_openapi_uri", method: "POST"}, - {handler: echoOpenapiData, path: "/api/v1/validate_openapi", method: "POST"}, + {handler: shuffle.EchoOpenapiData, path: "/api/v1/get_openapi_uri", method: "POST"}, + {handler: shuffle.EchoOpenapiData, path: "/api/v1/validate_openapi", method: "POST"}, {handler: shuffle.ValidateSwagger, path: "/api/v1/validate_openapi", method: "POST"}, {handler: getOpenapi, path: "/api/v1/get_openapi", method: "GET"}, @@ -117,7 +118,7 @@ func TestAuthenticationRequired(t *testing.T) { {handler: handleCloudSetup, path: "/api/v1/cloud/setup", method: "POST"}, {handler: shuffle.HandleGetOrgs, path: "/api/v1/orgs", method: "POST"}, - {handler: shuffle.HandleGetFileContent, path: "/api/v1/files/{fileId}/content", method: "POST"}, + {handler: shuffle.HandleGetFileContent, path: "/api/v1/files/{fileId}/content", method: "POST", body: []byte("hi")}, } var err error @@ -197,10 +198,11 @@ func TestAuthenticationNotRequired(t *testing.T) { // requirements might change after the refactor. func TestCors(t *testing.T) { handlers := []endpoint{ + {handler: handleLogin, path: "/api/v1/users/login", method: "POST"}, // prob not this one + {handler: shuffle.HandleNewOutlookRegister, path: "/functions/outlook/register", method: "GET"}, {handler: shuffle.HandleGetOutlookFolders, path: "/functions/outlook/getFolders", method: "GET"}, {handler: shuffle.HandleApiGeneration, path: "/api/v1/users/generateapikey", method: "GET"}, - {handler: handleLogin, path: "/api/v1/users/login", method: "POST"}, // prob not this one // handleRegister generates nil pointer exception {handler: handleRegister, path: "/api/v1/users/register", method: "POST"}, {handler: shuffle.HandleGetUsers, path: "/api/v1/users/getusers", method: "GET"}, diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go old mode 100644 new mode 100755 index 1a8cba08..2f057c8e --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -11,6 +11,7 @@ import ( "io" "io/ioutil" "log" + "math/rand" "net/http" "net/url" "os" @@ -104,7 +105,7 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode } } - log.Printf("Starting frequency: %d", newfrequency) + log.Printf("[INFO] Starting frequency for execution: %d", newfrequency) jobret, err := newscheduler.Every(newfrequency).Seconds().NotImmediately().Run(job) if err != nil { log.Printf("Failed to schedule workflow: %s", err) @@ -164,7 +165,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque executionRequests, err := shuffle.GetWorkflowQueue(ctx, id, 100) if err != nil { log.Printf("[WARNING] (1) Failed reading body for workflowqueue: %s", err) - resp.WriteHeader(401) + resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Entity parsing error - confirm"}`))) return } @@ -178,8 +179,8 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque body, err := ioutil.ReadAll(request.Body) if err != nil { - log.Println("Failed reading body for stream result queue") - resp.WriteHeader(401) + log.Println("[WARNING] Failed reading body for stream result queue") + resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } @@ -189,16 +190,16 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque var removeExecutionRequests shuffle.ExecutionRequestWrapper err = json.Unmarshal(body, &removeExecutionRequests) if err != nil { - log.Printf("Failed executionrequest in queue unmarshaling: %s", err) - resp.WriteHeader(401) + log.Printf("[WARNING] Failed executionrequest in queue unmarshaling: %s", err) + resp.WriteHeader(400) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } if len(removeExecutionRequests.Data) == 0 { - log.Printf("No requests to fix remove from DB") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Some removal error"}`))) + log.Printf("[WARNING] No requests to fix remove from DB") + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Queue removal error"}`))) return } @@ -251,16 +252,42 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { return } - id := request.Header.Get("Org-Id") - if len(id) == 0 { - log.Printf("[INFO] No org-id header set") + // This is really the environment's name - NOT org-id + orgId := request.Header.Get("Org-Id") + if len(orgId) == 0 { + log.Printf("[AUDIT] No org-id header set") resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org-id header."}`))) return } - ctx := context.Background() - env, err := shuffle.GetEnvironment(ctx, id, "") + environment := request.Header.Get("org") + if len(environment) == 0 { + //log.Printf("[AUDIT] No 'org' header set (get workflow queue). ") + /* + resp.WriteHeader(403) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org header. This can be done by setting the 'ORG' environment variable for Orborus to your Org ID in Shuffle"}`))) + return + */ + } + + orborusLabel := request.Header.Get("x-orborus-label") + + // This section is cloud custom for now + auth := request.Header.Get("Authorization") + if len(auth) == 0 { + //log.Printf("[AUDIT] No Authorization header set. Env: %s, org: %s", orgId, environment) + /* + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the auth header (only applicable for cloud for now)."}`))) + return + */ + } + + //log.Printf("[AUDIT] Get workflow queue for org %s, env %s, orborus label %s", orgId, environment, orborusLabel) + + ctx := shuffle.GetContext(request) + env, err := shuffle.GetEnvironment(ctx, orgId, "") timeNow := time.Now().Unix() if err == nil && len(env.Id) > 0 && len(env.Name) > 0 { if time.Now().Unix() > env.Edited+60 { @@ -273,7 +300,155 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } } - executionRequests, err := shuffle.GetWorkflowQueue(ctx, id, 100) + //log.Printf("Found env: %#v", env) + if len(env.OrgId) > 0 { + environment = env.OrgId + } + + if request.Method == "POST" { + if rand.Intn(1) == 0 { + // Parse out body + body, err := ioutil.ReadAll(request.Body) + if err == nil { + + // Parse out CPU, memory and disk. + + var envData shuffle.OrborusStats + err = json.Unmarshal(body, &envData) + if err == nil && !envData.Swarm && !envData.Kubernetes && (envData.CPU > 0 || envData.Memory > 0 || envData.Disk > 0) { + + // Set the input in memory + envData.OrgId = orgId + envData.Environment = environment + envData.OrborusLabel = orborusLabel + envData.Timestamp = time.Now().Unix() + + if envData.CPU > 0 && envData.MaxCPU > 0 { + envData.CPUPercent = float64(envData.CPU) / float64(envData.MaxCPU) + } + + if envData.Memory > 0 && envData.MaxMemory > 0 { + envData.MemoryPercent = float64(envData.Memory) / float64(envData.MaxMemory) + } + + // Check if CPU percent constantly has stayed above X% for the last Y requests + percentageCheck := 90 + concurrentChecks := 2 + + //if int(envData.CPUPercent) > percentageCheck { + // Get cached data + percentages := []float64{} + cacheKey := fmt.Sprintf("%s_%s_percent", orgId, strings.ToLower(environment)) + + // Marshal float list into []byte + cacheData := []byte{} + cache, err := shuffle.GetCache(ctx, cacheKey) + if err == nil { + // Unmarshal into percentages + cacheData := []byte(cache.([]uint8)) + err = json.Unmarshal(cacheData, &percentages) + if err != nil { + log.Printf("[INFO] error in cache unmarshal for percentages: %s", err) + } + + if len(percentages) > concurrentChecks { + percentages = percentages[:concurrentChecks] + } + + percentages = append(percentages, envData.CPUPercent) + if len(percentages) > concurrentChecks { + //log.Printf("[INFO] Checking percentages: %v", percentages) + + // percentageCheck := 1 + sendAlert := true + for _, p := range percentages { + if int(p) < percentageCheck { + //log.Printf("[AUDIT] CPU percent is below %d: %d", percentageCheck, int(p)) + sendAlert = false + break + } + } + + if sendAlert { + log.Printf("[INFO] CPU percent has been above %d percent for the last 5 requests. Sending alert. Env: %s, org: %s", percentageCheck, environment, orgId) + + // Set notification + alert for organization + err = shuffle.CreateOrgNotification( + ctx, + fmt.Sprintf("CPU percent has been above %d percent", percentageCheck), + fmt.Sprintf("A environment %s has been using more than %d\\% CPU for the last 5 requests.", environment, percentageCheck), + fmt.Sprintf("/admin?tab=environments"), + environment, + true, + ) + + if err != nil { + log.Printf("[ERROR] error creating notification: %s", err) + } + + org, err := shuffle.GetOrg(ctx, environment) + if err == nil { + foundRecommendation := false + for _, recommendation := range org.Priorities { + if strings.Contains(recommendation.Name, "CPU") { + foundRecommendation = true + break + } + } + + if !foundRecommendation { + // Add to start of org.Priorities + org, _ = shuffle.AddPriority(*org, shuffle.Priority{ + Name: fmt.Sprintf("High CPU in environment %s", orgId), + Description: fmt.Sprintf("The environment %s has been using more than %d percent CPU. This indicates you may need to look at scaling.", orgId, percentageCheck), + Type: "scale", + Active: true, + URL: fmt.Sprintf("/admin?tab=environments"), + Severity: 1, + }, false) + + //Make last item the first item + org.Priorities = append([]shuffle.Priority{org.Priorities[len(org.Priorities)-1]}, org.Priorities[:len(org.Priorities)-1]...) + err = shuffle.SetOrg(ctx, *org, org.Id) + if err != nil { + log.Printf("[ERROR] Problem setting org: %s", err) + } + } + } + } + + if len(percentages) > 1 { + percentages = percentages[1:] + } + } + + // Marshal float list into []byte + } else { + //log.Printf("[ERROR] Failed getting cache: %s", err) + percentages = append(percentages, envData.CPUPercent) + } + + if len(percentages) > 0 { + //log.Printf("[DEBUG] Setting cache for %s: %#v", cacheKey, percentages) + cacheData, err = json.Marshal(percentages) + if err != nil { + log.Printf("[INFO] error in cache marshal: %s", err) + } + + // Add the new data + go shuffle.SetCache(ctx, cacheKey, cacheData, 5) + } + } + + //log.Printf("CPU percent: %f", envData.CPUPercent) + //log.Printf("Memory percent: %f", envData.MemoryPercent*100) + + go shuffle.SetenvStats(ctx, envData) + } + } + } + + executionRequests, err := shuffle.GetWorkflowQueue(ctx, orgId, 100) if err != nil { // Skipping as this comes up over and over //log.Printf("(2) Failed reading body for workflowqueue: %s", err) @@ -290,21 +465,21 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { // Try again :) if len(env.Id) == 0 && len(env.Name) == 0 { - orgId := "" + foundId := "" for _, requestData := range executionRequests.Data { execution, err := shuffle.GetWorkflowExecution(ctx, requestData.ExecutionId) if err == nil { if len(execution.ExecutionOrg) > 0 { - orgId = execution.ExecutionOrg + foundId = execution.ExecutionOrg break } } } if len(orgId) > 0 { - env, err := shuffle.GetEnvironment(ctx, id, orgId) + env, err := shuffle.GetEnvironment(ctx, orgId, foundId) if err != nil { - log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", id, err) + log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", orgId, err) //resp.WriteHeader(401) //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No env found matching %s"}`, id))) //return @@ -361,15 +536,15 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { err = json.Unmarshal(body, &actionResult) if err != nil { log.Printf("[WARNING] Failed ActionResult unmarshaling (stream result): %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return + //resp.WriteHeader(401) + //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + //return } ctx := context.Background() workflowExecution, err := shuffle.GetWorkflowExecution(ctx, actionResult.ExecutionId) if err != nil { - log.Printf("[WARNING] Failed getting execution (streamresult) %s: %s", actionResult.ExecutionId, err) + log.Printf("[WARNING][%s] Failed getting execution (streamresult): %s", actionResult.ExecutionId, err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key or execution_id might not exist."}`))) return @@ -386,7 +561,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { } if len(workflowExecution.ExecutionOrg) > 0 && user.ActiveOrg.Id == workflowExecution.ExecutionOrg && user.Role == "admin" { - log.Printf("[DEBUG] Correct org for execution!") + log.Printf("[DEBUG] User %s is in correct org. Allowing org continuation for execution!", user.Username) } else { log.Printf("[WARNING] Bad authorization key when getting stream results %s.", actionResult.ExecutionId) resp.WriteHeader(401) @@ -461,23 +636,26 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { } //log.Printf("Actionresult unmarshal: %s", string(body)) - log.Printf("[DEBUG] Got workflow result from %s of length %d.", request.RemoteAddr, len(body)) - err = shuffle.ValidateNewWorkerExecution(body) + log.Printf("[DEBUG] Got workflow result from %s of length %d", request.RemoteAddr, len(body)) + ctx := context.Background() + err = shuffle.ValidateNewWorkerExecution(ctx, body) if err == nil { resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "success"}`))) return } else { - log.Printf("[DEBUG] Handling other execution variant (subflow): %s", err) + log.Printf("[DEBUG] Handling other execution variant (subflow?): %s", err) } + log.Printf("[DEBUG] Got workflow result from %s of length %d.", request.RemoteAddr, len(body)) + var actionResult shuffle.ActionResult err = json.Unmarshal(body, &actionResult) if err != nil { log.Printf("[WARNING] Failed ActionResult unmarshaling (queue): %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return + //resp.WriteHeader(401) + //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + //return } //log.Printf("Received action: %#v", actionResult) @@ -488,7 +666,6 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { // 4. Push to db // IF FAIL: Set executionstatus: abort or cancel - ctx := context.Background() workflowExecution, err := shuffle.GetWorkflowExecution(ctx, actionResult.ExecutionId) if err != nil { log.Printf("[ERROR] Failed getting execution (workflowqueue) %s: %s", actionResult.ExecutionId, err) @@ -523,66 +700,71 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { } } - if actionResult.Status == "WAITING" && actionResult.Action.AppName == "User Input" { - log.Printf("[INFO] SHOULD WAIT A BIT AND RUN CLOUD STUFF WITH USER INPUT! WAITING!") + /* + // Removed as UserInput is now handled as an app + if actionResult.Status == "WAITING" && actionResult.Action.AppName == "User Input" { + log.Printf("[INFO] SHOULD WAIT A BIT AND RUN USER INPUT! WAITING!") - var trigger shuffle.Trigger - err = json.Unmarshal([]byte(actionResult.Result), &trigger) - if err != nil { - log.Printf("[WARNING] Failed unmarshaling actionresult for user input: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } + var trigger shuffle.Trigger + err = json.Unmarshal([]byte(actionResult.Result), &trigger) + if err != nil { + log.Printf("[WARNING] Failed unmarshaling actionresult for user input: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } - orgId := workflowExecution.ExecutionOrg - if len(workflowExecution.OrgId) == 0 && len(workflowExecution.Workflow.OrgId) > 0 { - orgId = workflowExecution.Workflow.OrgId - } + orgId := workflowExecution.ExecutionOrg + if len(workflowExecution.OrgId) == 0 && len(workflowExecution.Workflow.OrgId) > 0 { + orgId = workflowExecution.Workflow.OrgId + } - err := handleUserInput(trigger, orgId, workflowExecution.Workflow.ID, workflowExecution.ExecutionId) - if err != nil { - log.Printf("[WARNING] Failed userinput handler: %s", err) + err := handleUserInput(trigger, orgId, workflowExecution.Workflow.ID, workflowExecution.ExecutionId) + if err != nil { + log.Printf("[WARNING] Failed userinput handler: %s", err) - actionResult.Result = fmt.Sprintf(`{"success": False, "reason": "%s"}`, err) + actionResult.Result = fmt.Sprintf(`{"success": false, "reason": "%s"}`, err) - workflowExecution.Results = append(workflowExecution.Results, actionResult) - workflowExecution.Status = "ABORTED" - err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) - if err != nil { - log.Printf("[WARNING] Failed to set execution during wait: %s", err) - } else { - log.Printf("[INFO] Successfully set the execution %s to waiting.", workflowExecution.ExecutionId) + workflowExecution.Results = append(workflowExecution.Results, actionResult) + workflowExecution.Status = "ABORTED" + err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) + if err != nil { + log.Printf("[WARNING] Failed to set execution during wait: %s", err) + } else { + log.Printf("[INFO] Successfully set the execution %s to waiting.", workflowExecution.ExecutionId) + } + + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error: %s"}`, err))) + return + } else { + log.Printf("[INFO] Successful userinput handler") + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "CLOUD IS DONE"}`))) + + actionResult.Result = `{"success": True, "reason": "Waiting for user feedback based on configuration"}` + + workflowExecution.Results = append(workflowExecution.Results, actionResult) + workflowExecution.Status = actionResult.Status + err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) + if err != nil { + log.Printf("[WARNING] Failed setting userinput: %s", err) + } else { + log.Printf("[DEBUG] Successfully set the execution to waiting.") + } + } + + return } - - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error: %s"}`, err))) - return - } else { - log.Printf("[INFO] Successful userinput handler") - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "CLOUD IS DONE"}`))) - - actionResult.Result = `{"success": True, "reason": "Waiting for user feedback based on configuration"}` - - workflowExecution.Results = append(workflowExecution.Results, actionResult) - workflowExecution.Status = actionResult.Status - err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) - if err != nil { - log.Printf("[WARNING] Failed setting userinput: %s", err) - } else { - log.Printf("[DEBUG] Successfully set the execution to waiting.") - } - } - - return - } + */ runWorkflowExecutionTransaction(ctx, 0, workflowExecution.ExecutionId, actionResult, resp) } // Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult shuffle.ActionResult, resp http.ResponseWriter) { + log.Printf("[DEBUG] Running workflow execution transaction for %s", workflowExecutionId) + // Should start a tx for the execution here workflowExecution, err := shuffle.GetWorkflowExecution(ctx, workflowExecutionId) if err != nil { @@ -593,7 +775,6 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } //log.Printf("BASE LENGTH: %d", len(workflowExecution.Results)) - workflowExecution, dbSave, err := shuffle.ParsedExecutionResult(ctx, *workflowExecution, actionResult, false, 0) if err != nil { b, suberr := json.Marshal(actionResult) @@ -638,7 +819,7 @@ func JSONCheck(str string) bool { func handleExecutionStatistics(execution shuffle.WorkflowExecution) { // FIXME: CLEAN UP THE JSON THAT'S SAVED. - // https://github.com/frikky/Shuffle/issues/172 + // https://github.com/shuffle/Shuffle/issues/172 appResults := []shuffle.AppExecutionExample{} for _, result := range execution.Results { resultCheck := JSONCheck(result.Result) @@ -872,14 +1053,16 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request workflow = *tmpworkflow } - if len(workflow.ExecutingOrg.Id) == 0 { - if len(orgId) > 0 { - workflow.ExecutingOrg.Id = orgId - } else { - log.Printf("[INFO] Stopped execution because there is no executing org for workflow %s", workflow.ID) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow has no executing org defined"), errors.New("Workflow has no executing org defined") + /* + if len(workflow.ExecutingOrg.Id) == 0 { + if len(orgId) > 0 { + workflow.ExecutingOrg.Id = orgId + } else { + log.Printf("[INFO] Stopped execution because there is no executing org for workflow %s", workflow.ID) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow has no executing org defined"), errors.New("Workflow has no executing org defined") + } } - } + */ if len(workflow.Actions) == 0 { workflow.Actions = []shuffle.Action{} @@ -923,13 +1106,690 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request workflowExecution, execInfo, _, err := shuffle.PrepareWorkflowExecution(ctx, workflow, request, 10) if err != nil { - log.Printf("[WARNING] Failed in prepareExecution for execution Id %s: %s", workflowExecution.ExecutionId, err) - return workflowExecution, fmt.Sprintf("Failed preparration: %s", err), err + if strings.Contains(fmt.Sprintf("%s", err), "User Input") { + // Special for user input callbacks + return workflowExecution, fmt.Sprintf("%s", err), nil + } else { + log.Printf("[WARNING] Failed in prepareExecution: %s", err) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed starting workflow: %s", err), err + } } err = imageCheckBuilder(execInfo.ImageNames) if err != nil { log.Printf("[ERROR] Failed building the required images from %#v: %s", execInfo.ImageNames, err) + return shuffle.WorkflowExecution{}, "Failed unmarshal during execution", err + } + + makeNew := true + start, startok := request.URL.Query()["start"] + if request.Method == "POST" { + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Printf("[ERROR] Failed request POST read: %s", err) + return shuffle.WorkflowExecution{}, "Failed getting body", err + } + + // This one doesn't really matter. + log.Printf("[INFO] Running POST execution with body of length %d for workflow %s", len(string(body)), workflowExecution.Workflow.ID) + + if len(body) >= 4 { + if body[0] == 34 && body[len(body)-1] == 34 { + body = body[1 : len(body)-1] + } + if body[0] == 34 && body[len(body)-1] == 34 { + body = body[1 : len(body)-1] + } + } + + sourceAuth, sourceAuthOk := request.URL.Query()["source_auth"] + if sourceAuthOk { + //log.Printf("\n\n\nSETTING SOURCE WORKFLOW AUTH TO %s!!!\n\n\n", sourceAuth[0]) + workflowExecution.ExecutionSourceAuth = sourceAuth[0] + } else { + //log.Printf("Did NOT get source workflow") + } + + sourceNode, sourceNodeOk := request.URL.Query()["source_node"] + if sourceNodeOk { + //log.Printf("\n\n\nSETTING SOURCE WORKFLOW NODE TO %s!!!\n\n\n", sourceNode[0]) + workflowExecution.ExecutionSourceNode = sourceNode[0] + } else { + //log.Printf("Did NOT get source workflow") + } + + //workflowExecution.ExecutionSource = "default" + sourceWorkflow, sourceWorkflowOk := request.URL.Query()["source_workflow"] + if sourceWorkflowOk { + //log.Printf("Got source workflow %s", sourceWorkflow) + workflowExecution.ExecutionSource = sourceWorkflow[0] + } else { + //log.Printf("Did NOT get source workflow") + } + + sourceExecution, sourceExecutionOk := request.URL.Query()["source_execution"] + if sourceExecutionOk { + //log.Printf("[INFO] Got source execution%s", sourceExecution) + workflowExecution.ExecutionParent = sourceExecution[0] + } else { + //log.Printf("Did NOT get source execution") + } + + if len(string(body)) < 50 { + //log.Println(body) + // String in string + //log.Println(body) + + //if string(body)[0] == "\"" && string(body)[string(body) + log.Printf("[DEBUG] Body: %s", string(body)) + } + + var execution shuffle.ExecutionRequest + err = json.Unmarshal(body, &execution) + if err != nil { + log.Printf("[WARNING] Failed execution POST unmarshaling - continuing anyway: %s", err) + //return shuffle.WorkflowExecution{}, "", err + } + + if execution.Start == "" && len(body) > 0 { + execution.ExecutionArgument = string(body) + } + + // FIXME - this should have "execution_argument" from executeWorkflow frontend + //log.Printf("EXEC: %#v", execution) + if len(execution.ExecutionArgument) > 0 { + workflowExecution.ExecutionArgument = execution.ExecutionArgument + } + + if len(execution.ExecutionSource) > 0 { + workflowExecution.ExecutionSource = execution.ExecutionSource + } + + //log.Printf("Execution data: %#v", execution) + if len(execution.Start) == 36 && len(workflow.Actions) > 0 { + log.Printf("[INFO] Should start execution on node %s", execution.Start) + workflowExecution.Start = execution.Start + + found := false + for _, action := range workflow.Actions { + if action.ID == execution.Start { + found = true + break + } + } + + if !found { + log.Printf("[ERROR] Action %s was NOT found! Exiting execution.", execution.Start) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", workflow.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", workflow.Start)) + } + } else if len(execution.Start) > 0 { + //log.Printf("[INFO] !") + //log.Printf("[ERROR] START ACTION %s IS WRONG ID LENGTH %d!", execution.Start, len(execution.Start)) + //return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", execution.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", execution.Start)) + } + + if len(execution.ExecutionId) == 36 { + workflowExecution.ExecutionId = execution.ExecutionId + } else { + sessionToken := uuid.NewV4() + workflowExecution.ExecutionId = sessionToken.String() + } + } else { + // Check for parameters of start and ExecutionId + // This is mostly used for user input trigger + + answer, answerok := request.URL.Query()["answer"] + referenceId, referenceok := request.URL.Query()["reference_execution"] + if answerok && referenceok { + // If answer is false, reference execution with result + log.Printf("[INFO] Answer is OK AND reference is OK!") + if answer[0] == "false" { + log.Printf("Should update reference and return, no need for further execution!") + + // Get the reference execution + oldExecution, err := shuffle.GetWorkflowExecution(ctx, referenceId[0]) + if err != nil { + log.Printf("Failed getting execution (execution) %s: %s", referenceId[0], err) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err + } + + if oldExecution.Workflow.ID != id { + log.Println("Wrong workflowid!") + return shuffle.WorkflowExecution{}, fmt.Sprintf("Bad ID %s", referenceId), errors.New("Bad ID") + } + + newResults := []shuffle.ActionResult{} + //log.Printf("%#v", oldExecution.Results) + for _, result := range oldExecution.Results { + log.Printf("%s - %s", result.Action.ID, start[0]) + if result.Action.ID == start[0] { + note, noteok := request.URL.Query()["note"] + if noteok { + result.Result = fmt.Sprintf("User note: %s", note[0]) + } else { + result.Result = fmt.Sprintf("User clicked %s", answer[0]) + } + + // Stopping the whole thing + result.CompletedAt = int64(time.Now().Unix()) + result.Status = "ABORTED" + oldExecution.Status = result.Status + oldExecution.Result = result.Result + oldExecution.LastNode = result.Action.ID + } + + newResults = append(newResults, result) + } + + oldExecution.Results = newResults + err = shuffle.SetWorkflowExecution(ctx, *oldExecution, true) + if err != nil { + log.Printf("Error saving workflow execution actionresult setting: %s", err) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed setting workflowexecution actionresult in execution: %s", err), err + } + + return shuffle.WorkflowExecution{}, "", nil + } + } + + if referenceok { + log.Printf("Handling an old execution continuation!") + // Will use the old name, but still continue with NEW ID + oldExecution, err := shuffle.GetWorkflowExecution(ctx, referenceId[0]) + if err != nil { + log.Printf("Failed getting execution (execution) %s: %s", referenceId[0], err) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err + } + + workflowExecution = *oldExecution + } + + if len(workflowExecution.ExecutionId) == 0 { + sessionToken := uuid.NewV4() + workflowExecution.ExecutionId = sessionToken.String() + } else { + log.Printf("Using the same executionId as before: %s", workflowExecution.ExecutionId) + makeNew = false + } + + // Don't override workflow defaults + } + + if startok { + //log.Printf("\n\n[INFO] Setting start to %s based on query!\n\n", start[0]) + //workflowExecution.Workflow.Start = start[0] + workflowExecution.Start = start[0] + } + + // FIXME - regex uuid, and check if already exists? + if len(workflowExecution.ExecutionId) != 36 { + log.Printf("Invalid uuid: %s", workflowExecution.ExecutionId) + return shuffle.WorkflowExecution{}, "Invalid uuid", err + } + + // FIXME - find owner of workflow + // FIXME - get the actual workflow itself and build the request + // MAYBE: Don't send the workflow within the pubsub, as this requires more data to be sent + // Check if a worker already exists for company, else run one with: + // locations, project IDs and subscription names + + // When app is executed: + // Should update with status execution (somewhere), which will trigger the next node + // IF action.type == internal, we need the internal watcher to be running and executing + // This essentially means the WORKER has to be the responsible party for new actions in the INTERNAL landscape + // Results are ALWAYS posted back to cloud@execution_id? + if makeNew { + workflowExecution.Type = "workflow" + //workflowExecution.Stream = "tmp" + //workflowExecution.WorkflowQueue = "tmp" + //workflowExecution.SubscriptionNameNodestream = "testcompany-nodestream" + //workflowExecution.Locations = []string{"europe-west2"} + workflowExecution.ProjectId = gceProject + workflowExecution.WorkflowId = workflow.ID + workflowExecution.StartedAt = int64(time.Now().Unix()) + workflowExecution.CompletedAt = 0 + workflowExecution.Authorization = uuid.NewV4().String() + + // Status for the entire workflow. + workflowExecution.Status = "EXECUTING" + } + + if len(workflowExecution.ExecutionSource) == 0 { + log.Printf("[INFO] No execution source (trigger) specified. Setting to default") + workflowExecution.ExecutionSource = "default" + } else { + log.Printf("[INFO] Execution source is %s for execution ID %s in workflow %s", workflowExecution.ExecutionSource, workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } + + workflowExecution.ExecutionVariables = workflow.ExecutionVariables + if len(workflowExecution.Start) == 0 && len(workflowExecution.Workflow.Start) > 0 { + workflowExecution.Start = workflowExecution.Workflow.Start + } + + startnodeFound := false + newStartnode := "" + for _, item := range workflowExecution.Workflow.Actions { + if item.ID == workflowExecution.Start { + startnodeFound = true + } + + if item.IsStartNode { + newStartnode = item.ID + } + } + + if !startnodeFound { + log.Printf("[INFO] Couldn't find startnode %s. Remapping to %#v", workflowExecution.Start, newStartnode) + + if len(newStartnode) > 0 { + workflowExecution.Start = newStartnode + } else { + return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode couldn't be found"), errors.New("Startnode isn't defined in this workflow..") + } + } + + childNodes := shuffle.FindChildNodes(workflowExecution, workflowExecution.Start, []string{}, []string{}) + + topic := "workflows" + startFound := false + // FIXME - remove this? + newActions := []shuffle.Action{} + defaultResults := []shuffle.ActionResult{} + + allAuths := []shuffle.AppAuthenticationStorage{} + for _, action := range workflowExecution.Workflow.Actions { + //action.LargeImage = "" + if action.ID == workflowExecution.Start { + startFound = true + } + //log.Println(action.Environment) + + if action.Environment == "" { + return shuffle.WorkflowExecution{}, fmt.Sprintf("Environment is not defined for %s", action.Name), errors.New("Environment not defined!") + } + + // FIXME: Authentication parameters + if len(action.AuthenticationId) > 0 { + if len(allAuths) == 0 { + allAuths, err = shuffle.GetAllWorkflowAppAuth(ctx, workflow.ExecutingOrg.Id) + if err != nil { + log.Printf("Api authentication failed in get all app auth: %s", err) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Api authentication failed in get all app auth: %s", err), err + } + } + + curAuth := shuffle.AppAuthenticationStorage{Id: ""} + for _, auth := range allAuths { + if auth.Id == action.AuthenticationId { + curAuth = auth + break + } + } + + if len(curAuth.Id) == 0 { + return shuffle.WorkflowExecution{}, fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId), errors.New(fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId)) + } + + if curAuth.Encrypted { + setField := true + newFields := []shuffle.AuthenticationStore{} + for _, field := range curAuth.Fields { + parsedKey := fmt.Sprintf("%s_%d_%s_%s", curAuth.OrgId, curAuth.Created, curAuth.Label, field.Key) + newValue, err := shuffle.HandleKeyDecryption([]byte(field.Value), parsedKey) + if err != nil { + log.Printf("[WARNING] Failed decryption for %s: %s", field.Key, err) + setField = false + break + } + + field.Value = string(newValue) + newFields = append(newFields, field) + } + + if setField { + curAuth.Fields = newFields + } + } else { + log.Printf("[INFO] AUTH IS NOT ENCRYPTED - attempting encrypting!") + err = shuffle.SetWorkflowAppAuthDatastore(ctx, curAuth, curAuth.Id) + if err != nil { + log.Printf("[WARNING] Failed running encryption during execution: %s", err) + } + } + + newParams := []shuffle.WorkflowAppActionParameter{} + if strings.ToLower(curAuth.Type) == "oauth2" { + log.Printf("[DEBUG] Should replace auth parameters (Oauth2)") + + for _, param := range curAuth.Fields { + if param.Key == "expiration" { + continue + } + + newParams = append(newParams, shuffle.WorkflowAppActionParameter{ + Name: param.Key, + Value: param.Value, + }) + } + + for _, param := range action.Parameters { + //log.Printf("Param: %#v", param) + if param.Configuration { + continue + } + + newParams = append(newParams, param) + } + } else { + // Rebuild params with the right data. This is to prevent issues on the frontend + for _, param := range action.Parameters { + + for _, authparam := range curAuth.Fields { + if param.Name == authparam.Key { + param.Value = authparam.Value + //log.Printf("Name: %s - value: %s", param.Name, param.Value) + //log.Printf("Name: %s - value: %s\n", param.Name, param.Value) + break + } + } + + newParams = append(newParams, param) + } + } + + action.Parameters = newParams + } + + action.LargeImage = "" + if len(action.Label) == 0 { + action.Label = action.ID + } + //log.Printf("LABEL: %s", action.Label) + newActions = append(newActions, action) + + // If the node is NOT found, it's supposed to be set to SKIPPED, + // as it's not a childnode of the startnode + // This is a configuration item for the workflow itself. + if len(workflowExecution.Results) > 0 { + defaultResults = []shuffle.ActionResult{} + for _, result := range workflowExecution.Results { + if result.Status == "WAITING" { + result.Status = "FINISHED" + result.Result = "Continuing" + } + + defaultResults = append(defaultResults, result) + } + } else if len(workflowExecution.Results) == 0 && !workflowExecution.Workflow.Configuration.StartFromTop { + found := false + for _, nodeId := range childNodes { + if nodeId == action.ID { + //log.Printf("Found %s", action.ID) + found = true + } + } + + if !found { + if action.ID == workflowExecution.Start { + continue + } + + //log.Printf("[WARNING] Set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID) + curaction := shuffle.Action{ + AppName: action.AppName, + AppVersion: action.AppVersion, + Label: action.Label, + Name: action.Name, + ID: action.ID, + } + //action + //curaction.Parameters = [] + defaultResults = append(defaultResults, shuffle.ActionResult{ + Action: curaction, + ExecutionId: workflowExecution.ExecutionId, + Authorization: workflowExecution.Authorization, + Result: "Skipped because it's not under the startnode", + StartedAt: 0, + CompletedAt: 0, + Status: "SKIPPED", + }) + } + } + } + + removeTriggers := []string{} + for triggerIndex, trigger := range workflowExecution.Workflow.Triggers { + //log.Printf("[INFO] ID: %s vs %s", trigger.ID, workflowExecution.Start) + if trigger.ID == workflowExecution.Start { + if trigger.AppName == "User Input" { + startFound = true + break + } + } + + if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" { + found := false + for _, node := range childNodes { + if node == trigger.ID { + found = true + break + } + } + + if !found { + //log.Printf("SHOULD SET TRIGGER %s TO BE SKIPPED", trigger.ID) + + curaction := shuffle.Action{ + AppName: "shuffle-subflow", + AppVersion: trigger.AppVersion, + Label: trigger.Label, + Name: trigger.Name, + ID: trigger.ID, + } + + defaultResults = append(defaultResults, shuffle.ActionResult{ + Action: curaction, + ExecutionId: workflowExecution.ExecutionId, + Authorization: workflowExecution.Authorization, + Result: "Skipped because it's not under the startnode", + StartedAt: 0, + CompletedAt: 0, + Status: "SKIPPED", + }) + } else { + // Replaces trigger with the subflow + //if trigger.AppName == "Shuffle Workflow" { + // replaceActions := false + // workflowAction := "" + // for _, param := range trigger.Parameters { + // if param.Name == "argument" && !strings.Contains(param.Value, ".#") { + // replaceActions = true + // } + + // if param.Name == "startnode" { + // workflowAction = param.Value + // } + // } + + // if replaceActions { + // replacementNodes, newBranches, lastnode := shuffle.GetReplacementNodes(ctx, workflowExecution, trigger, trigger.Label) + // log.Printf("REPLACEMENTS: %d, %d", len(replacementNodes), len(newBranches)) + // if len(replacementNodes) > 0 { + // for _, action := range replacementNodes { + // found := false + + // for subActionIndex, subaction := range newActions { + // if subaction.ID == action.ID { + // found = true + // //newActions[subActionIndex].Name = action.Name + // newActions[subActionIndex].Label = action.Label + // break + // } + // } + + // if !found { + // action.SubAction = true + // newActions = append(newActions, action) + // } + + // // Check if it's already set to have a value + // for resultIndex, result := range defaultResults { + // if result.Action.ID == action.ID { + // defaultResults = append(defaultResults[:resultIndex], defaultResults[resultIndex+1:]...) + // break + // } + // } + // } + + // for _, branch := range newBranches { + // workflowExecution.Workflow.Branches = append(workflowExecution.Workflow.Branches, branch) + // } + + // // Append branches: + // // parent -> new inner node (FIRST one) + // for branchIndex, branch := range workflowExecution.Workflow.Branches { + // if branch.DestinationID == trigger.ID { + // log.Printf("REPLACE DESTINATION WITH %s!!", workflowAction) + // workflowExecution.Workflow.Branches[branchIndex].DestinationID = workflowAction + // } + + // if branch.SourceID == trigger.ID { + // log.Printf("REPLACE SOURCE WITH LASTNODE %s!!", lastnode) + // workflowExecution.Workflow.Branches[branchIndex].SourceID = lastnode + // } + // } + + // // Remove the trigger + // removeTriggers = append(removeTriggers, workflowExecution.Workflow.Triggers[triggerIndex].ID) + // } + + // log.Printf("NEW ACTION LENGTH %d, RESULT: %d, Triggers: %d, BRANCHES: %d", len(newActions), len(defaultResults), len(workflowExecution.Workflow.Triggers), len(workflowExecution.Workflow.Branches)) + // } + //} + _ = triggerIndex + } + } + } + + //newTriggers := []shuffle.Trigger{} + //for _, trigger := range workflowExecution.Workflow.Triggers { + // found := false + // for _, triggerId := range removeTriggers { + // if trigger.ID == triggerId { + // found = true + // break + // } + // } + + // if found { + // log.Printf("[WARNING] Removed trigger %s during execution", trigger.ID) + // continue + // } + + // newTriggers = append(newTriggers, trigger) + //} + //workflowExecution.Workflow.Triggers = newTriggers + _ = removeTriggers + + if !startFound { + if len(workflowExecution.Start) == 0 && len(workflowExecution.Workflow.Start) > 0 { + workflowExecution.Start = workflow.Start + } else if len(workflowExecution.Workflow.Actions) > 0 { + workflowExecution.Start = workflowExecution.Workflow.Actions[0].ID + } else { + log.Printf("[ERROR] Startnode %s doesn't exist!!", workflowExecution.Start) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow action %s doesn't exist in workflow", workflowExecution.Start), errors.New(fmt.Sprintf(`Workflow start node "%s" doesn't exist. Exiting!`, workflowExecution.Start)) + } + } + + //log.Printf("EXECUTION START: %s", workflowExecution.Start) + + // Verification for execution environments + workflowExecution.Results = defaultResults + workflowExecution.Workflow.Actions = newActions + onpremExecution := true + environments := []string{} + + if len(workflowExecution.ExecutionOrg) == 0 && len(workflow.ExecutingOrg.Id) > 0 { + workflowExecution.ExecutionOrg = workflow.ExecutingOrg.Id + } + + var allEnvs []shuffle.Environment + if len(workflowExecution.ExecutionOrg) > 0 { + //log.Printf("[INFO] Executing ORG: %s", workflowExecution.ExecutionOrg) + + allEnvironments, err := shuffle.GetEnvironments(ctx, workflowExecution.ExecutionOrg) + if err != nil { + log.Printf("Failed finding environments: %s", err) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow environments not found for this org"), errors.New(fmt.Sprintf("Workflow environments not found for this org")) + } + + for _, curenv := range allEnvironments { + if curenv.Archived { + continue + } + + allEnvs = append(allEnvs, curenv) + } + } else { + log.Printf("[ERROR] No org identified for execution of %s. Returning", workflowExecution.Workflow.ID) + return shuffle.WorkflowExecution{}, "No org identified for execution", errors.New("No org identified for execution") + } + + if len(allEnvs) == 0 { + log.Printf("[ERROR] No active environments found for org: %s", workflowExecution.ExecutionOrg) + return shuffle.WorkflowExecution{}, "No active environments found", errors.New(fmt.Sprintf("No active env found for org %s", workflowExecution.ExecutionOrg)) + } + + // Check if the actions are children of the startnode? + imageNames := []string{} + cloudExec := false + for _, action := range workflowExecution.Workflow.Actions { + // Verify if the action environment exists and append + found := false + for _, env := range allEnvs { + if env.Name == action.Environment { + found = true + + if env.Type == "cloud" { + cloudExec = true + } else if env.Type == "onprem" { + onpremExecution = true + } else { + log.Printf("[ERROR] No handler for environment type %s", env.Type) + return shuffle.WorkflowExecution{}, "No active environments found", errors.New(fmt.Sprintf("No handler for environment type %s", env.Type)) + } + break + } + } + + if !found { + log.Printf("[ERROR] Couldn't find environment %s. Maybe it's inactive?", action.Environment) + return shuffle.WorkflowExecution{}, "Couldn't find the environment", errors.New(fmt.Sprintf("Couldn't find env %s in org %s", action.Environment, workflowExecution.ExecutionOrg)) + } + + found = false + for _, env := range environments { + if env == action.Environment { + + found = true + break + } + } + + // Check if the app exists? + newName := action.AppName + newName = strings.ReplaceAll(newName, " ", "-") + imageNames = append(imageNames, fmt.Sprintf("%s:%s_%s", baseDockerName, newName, action.AppVersion)) + + if !found { + environments = append(environments, action.Environment) + } + } + + err = imageCheckBuilder(imageNames) + if err != nil { + log.Printf("[ERROR] Failed building the required images from %#v: %s", imageNames, err) return shuffle.WorkflowExecution{}, "Failed building missing Docker images", err } @@ -1116,7 +1976,8 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { return } - //memcacheName := fmt.Sprintf("%s_%s", user.Username, fileId) + log.Printf("[INFO] Inside execute workflow for ID %s", fileId) + ctx := context.Background() workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil && workflow.ID == "" { @@ -1133,6 +1994,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { // 1. Parent workflow contains this workflow ID in the source trigger? // 2. Parent workflow's owner is same org? // 3. Parent execution auth is correct + log.Printf("[INFO] Inside execute workflow access validation!") executionAuthValid, newOrgId = shuffle.RunExecuteAccessValidation(request, workflow) if !executionAuthValid { @@ -1161,12 +2023,18 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { } } - log.Printf("[INFO] Starting execution of %s!", fileId) + log.Printf("[AUDIT] Starting execution of workflow '%s' by user %s (%s)!", fileId, user.Username, user.Id) user.ActiveOrg.Users = []shuffle.UserMini{} workflow.ExecutingOrg = user.ActiveOrg workflowExecution, executionResp, err := handleExecution(fileId, *workflow, request, user.ActiveOrg.Id) if err == nil { + if strings.Contains(executionResp, "User Input:") { + resp.WriteHeader(400) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, executionResp))) + return + } + resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization))) return @@ -1627,15 +2495,15 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("Action: %#v", action) + //log.Printf("Starting Cloud schedule Action: %#v", action) err = executeCloudAction(action, org.SyncConfig.Apikey) if err != nil { - log.Printf("Failed cloud action START schedule: %s", err) + log.Printf("[WARNING] Failed cloud action START schedule: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } else { - log.Printf("Successfully set up cloud action schedule") + log.Printf("[INFO] Successfully set up cloud action schedule") resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Done"}`))) return @@ -1923,7 +2791,7 @@ func loadGithubWorkflows(url, username, password, userId, branch, orgId string) storer := memory.NewStorage() r, err := git.Clone(storer, fs, cloneOptions) if err != nil { - log.Printf("Failed loading repo %s into memory (github workflows): %s", url, err) + log.Printf("[INFO] Failed loading repo %s into memory (github workflows): %s", url, err) return err } @@ -2194,7 +3062,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, if !found { err = shuffle.SetWorkflowAppDatastore(ctx, api, api.ID) if err != nil { - log.Printf("[WARNING] Failed setting workflowapp in loop: %s", err) + log.Printf("[WARNING] Failed setting workflowapp %s (%s) in loop: %s", api.Name, api.ID, err) continue } else { appCounter += 1 @@ -2423,7 +3291,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { // Might require reflection into the python code to append the fields as well for index, action := range workflowapp.Actions { if action.AuthNotRequired { - log.Printf("Skipping auth setup: %s", action.Name) + log.Printf("[WARNING] Skipping auth setup for: %s", action.Name) continue } @@ -2471,7 +3339,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { err = shuffle.SetWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err != nil { - log.Printf("Failed setting workflowapp: %s", err) + log.Printf("[WARNING] Failed setting workflowapp: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -2514,7 +3382,7 @@ func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId if len(triggerType) == 0 { log.Printf("[WARNING] No type specified for user input node") - return errors.New("No type specified for user input node") + //return errors.New("No type specified for user input node") } // FIXME: This is not the right time to send them, BUT it's well served for testing. Save -> send email / sms @@ -2597,7 +3465,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { if user.Role == "org-reader" { log.Printf("[WARNING] Org-reader doesn't have access to execute single action: %s (%s)", user.Username, user.Id) - resp.WriteHeader(401) + resp.WriteHeader(403) resp.Write([]byte(`{"success": false, "reason": "Read only user"}`)) return } @@ -2664,10 +3532,14 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { time.Sleep(2 * time.Second) log.Printf("[INFO] Starting validation of execution %s", workflowExecution.ExecutionId) - returnBytes := shuffle.HandleRetValidation(ctx, workflowExecution) + returnBody := shuffle.HandleRetValidation(ctx, workflowExecution, 1) + returnBytes, err := json.Marshal(returnBody) + if err != nil { + log.Printf("[ERROR] Failed to marshal retStruct in single execution: %s", err) + } resp.WriteHeader(200) - resp.Write(returnBytes) + resp.Write([]byte(returnBytes)) } // Onlyname is used to @@ -2696,6 +3568,11 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. // Folder? switch mode := file.Mode(); { case mode.IsDir(): + // Specific folder for skipping + if file.Name() == "unsupported" { + continue + } + tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name()) dir, err := fs.ReadDir(tmpExtra) if err != nil { @@ -2719,7 +3596,8 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. //buildFirst, buildLast, err := IterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate) if !forceUpdate { - return buildLaterFirst, buildLaterList, err + continue + //return buildLaterFirst, buildLaterList, err } } @@ -2737,7 +3615,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. fullPath = fmt.Sprintf("%s%s", extra, "api.yml") fileReader, err = fs.Open(fullPath) if err != nil { - log.Printf("Failed finding api.yaml/yml: %s", err) + log.Printf("[INFO] Failed finding api.yaml/yml for file %s: %s", filename, err) continue } } @@ -2793,7 +3671,8 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. err = gyaml.Unmarshal(appfileData, &workflowapp) if err != nil { log.Printf("[WARNING] Failed building workflowapp %s: %s", extra, err) - return buildLaterFirst, buildLaterList, errors.New(fmt.Sprintf("Failed building %s: %s", extra, err)) + continue + //return buildLaterFirst, buildLaterList, errors.New(fmt.Sprintf("Failed building %s: %s", extra, err)) //continue } @@ -2841,7 +3720,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. } } - workflowapp.ReferenceInfo.GithubUrl = fmt.Sprintf("https://github.com/frikky/shuffle-apps/tree/master/%s/%s", strings.ToLower(newName), workflowapp.AppVersion) + workflowapp.ReferenceInfo.GithubUrl = fmt.Sprintf("https://github.com/shuffle/shuffle-apps/tree/master/%s/%s", strings.ToLower(newName), workflowapp.AppVersion) tags := []string{ fmt.Sprintf("%s:%s_%s", baseDockerName, strings.ToLower(newName), workflowapp.AppVersion), @@ -3008,6 +3887,32 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. cacheKey = fmt.Sprintf("workflowapps-sorted-1000") shuffle.DeleteCache(ctx, cacheKey) + newSortedList := []shuffle.BuildLaterStruct{} + initApps := []string{ + "tools", + "http", + "email", + } + for _, buildLater := range buildLaterFirst { + found := false + for _, appname := range initApps { + for _, tag := range buildLater.Tags { + if strings.Contains(strings.ToLower(tag), appname) { + newSortedList = append(newSortedList, buildLater) + found = true + break + } + } + + if found { + break + } + } + } + + // Prepend newSortedList to buildLaterFirst + buildLaterFirst = append(newSortedList, buildLaterFirst...) + if len(extra) == 0 { log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst)) for _, item := range buildLaterFirst { @@ -3103,8 +4008,8 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { var tmpBody tmpStruct err = json.Unmarshal(body, &tmpBody) if err != nil { - log.Printf("Error with unmarshal tmpBody: %s", err) - resp.WriteHeader(401) + log.Printf("[WARNING] Error with unmarshal app git clone: %s", err) + resp.WriteHeader(500) resp.Write([]byte(`{"success": false}`)) return } @@ -3131,20 +4036,20 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { storer := memory.NewStorage() r, err := git.Clone(storer, fs, cloneOptions) if err != nil { - log.Printf("Failed loading repo %s into memory (github workflows 2): %s", tmpBody.URL, err) - resp.WriteHeader(401) + log.Printf("[WARNING] Failed loading repo %s into memory (github apps 2): %s", tmpBody.URL, err) + resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } dir, err := fs.ReadDir("/") if err != nil { - log.Printf("FAiled reading folder: %s", err) + log.Printf("[WARNING] FAiled reading folder: %s", err) } _ = r if tmpBody.ForceUpdate { - log.Printf("[AUDIT] Running with force update from user %s (%s) for %s!", user.Username, user.Id, tmpBody.URL) + log.Printf("[AUDIT] Running app get with force update from user %s (%s) for %s!", user.Username, user.Id, tmpBody.URL) } else { log.Printf("[AUDIT] Updating apps with updates for user %s (%s) for %s (no force)", user.Username, user.Id, tmpBody.URL) } diff --git a/backend/tests/cache.sh b/backend/tests/cache.sh old mode 100644 new mode 100755 diff --git a/backend/tests/cleanup.sh b/backend/tests/cleanup.sh old mode 100644 new mode 100755 diff --git a/backend/tests/dockerpull.sh b/backend/tests/dockerpull.sh old mode 100644 new mode 100755 diff --git a/backend/tests/execute.sh b/backend/tests/execute.sh old mode 100644 new mode 100755 diff --git a/backend/tests/file_download.sh b/backend/tests/file_download.sh old mode 100644 new mode 100755 diff --git a/backend/tests/files.sh b/backend/tests/files.sh index c92a0ebc..5a699822 100755 --- a/backend/tests/files.sh +++ b/backend/tests/files.sh @@ -16,7 +16,7 @@ #r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS") -#curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer 09627dcb-7e2a-4843-819b-417d268ff840" -d '{"filename": "rule2.yar", "org_id": "11f67b76-6051-4425-b0d6-be23daac6d12", "workflow_id": "global", "namespace": "yara"}' -curl http://localhost:5002/api/v1/files/file_366ee8d2-1af6-4270-8639-213af30b4a29/upload -H "Authorization: Bearer 09627dcb-7e2a-4843-819b-417d268ff840" -F 'shuffle_file=@upload.sh' +curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer 317f5066-395c-414d-aa3d-479cf27f47dd" -d '{"filename": "rule2.yar", "org_id": "292c7e25-40ad-4f05-904f-77d3c7b735e6", "workflow_id": "global", "namespace": "yara"}' +curl http://localhost:5001/api/v1/files/file_eb89e315-eb66-4d76-9df7-530fb003fc84/upload -H "Authorization: Bearer 317f5066-395c-414d-aa3d-479cf27f47dd" -F 'shuffle_file=@upload.sh' #curl http://localhost:5001/api/v1/files/namespaces/yara -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" --output rules.zip diff --git a/backend/tests/forparser.py b/backend/tests/forparser.py old mode 100644 new mode 100755 diff --git a/backend/tests/hooks.sh b/backend/tests/hooks.sh old mode 100644 new mode 100755 diff --git a/backend/tests/hotload.sh b/backend/tests/hotload.sh old mode 100644 new mode 100755 diff --git a/backend/tests/list_files.sh b/backend/tests/list_files.sh old mode 100644 new mode 100755 diff --git a/backend/tests/migrate_db.sh b/backend/tests/migrate_db.sh old mode 100644 new mode 100755 diff --git a/backend/tests/run_function.py b/backend/tests/run_function.py old mode 100644 new mode 100755 diff --git a/backend/tests/scheduleapps.sh b/backend/tests/scheduleapps.sh old mode 100644 new mode 100755 diff --git a/backend/tests/schedules.sh b/backend/tests/schedules.sh old mode 100644 new mode 100755 diff --git a/backend/tests/sendmail.sh b/backend/tests/sendmail.sh old mode 100644 new mode 100755 diff --git a/backend/tests/testWorkflows.sh b/backend/tests/testWorkflows.sh old mode 100644 new mode 100755 diff --git a/backend/tests/test_wrappers.py b/backend/tests/test_wrappers.py old mode 100644 new mode 100755 diff --git a/backend/tests/triggers.sh b/backend/tests/triggers.sh old mode 100644 new mode 100755 diff --git a/backend/tests/users.sh b/backend/tests/users.sh old mode 100644 new mode 100755 diff --git a/backend/tests/validate_app_values.sh b/backend/tests/validate_app_values.sh old mode 100644 new mode 100755 diff --git a/backend/tests/websocket.sh b/backend/tests/websocket.sh old mode 100644 new mode 100755 diff --git a/backend/tests/workflowdata.json b/backend/tests/workflowdata.json old mode 100644 new mode 100755 diff --git a/backend/tests/workflowresults.sh b/backend/tests/workflowresults.sh old mode 100644 new mode 100755 diff --git a/backend/tests/workflows.sh b/backend/tests/workflows.sh old mode 100644 new mode 100755 diff --git a/docker-compose.yml b/docker-compose.yml old mode 100644 new mode 100755 index f64e5899..9e66eb53 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,7 +43,6 @@ services: - /var/run/docker.sock:/var/run/docker.sock environment: #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - - SHUFFLE_WORKER_VERSION=latest - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:5001 - DOCKER_API_VERSION=1.40 @@ -114,4 +113,10 @@ services: networks: shuffle: driver: bridge - #driver: overlay + + # uncomment to set MTU for swarm mode. + # MTU should be whatever is your host's preferred MTU is. + # Refer to this doc to figure out what your host's MTU is: + # https://shuffler.io/docs/troubleshooting#TLS_timeout_error/Timeout_Errors/EOF_Errors + # driver_opts: + # com.docker.network.driver.mtu: 1460 diff --git a/frontend/Dockerfile b/frontend/Dockerfile old mode 100644 new mode 100755 diff --git a/frontend/README.md b/frontend/README.md old mode 100644 new mode 100755 diff --git a/frontend/build.sh b/frontend/build.sh old mode 100644 new mode 100755 diff --git a/frontend/certs/certreq.csr b/frontend/certs/certreq.csr old mode 100644 new mode 100755 diff --git a/frontend/certs/fullchain.pem b/frontend/certs/fullchain.pem old mode 100644 new mode 100755 diff --git a/frontend/certs/privkey.pem b/frontend/certs/privkey.pem old mode 100644 new mode 100755 diff --git a/frontend/confd/conf.d/nginx.conf.toml b/frontend/confd/conf.d/nginx.conf.toml old mode 100644 new mode 100755 diff --git a/frontend/confd/templates/nginx.conf b/frontend/confd/templates/nginx.conf old mode 100644 new mode 100755 diff --git a/frontend/package.json b/frontend/package.json old mode 100644 new mode 100755 index d23724c9..30683143 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,13 +1,14 @@ { "name": "shuffler", "homepage": "https://shuffler.io", - "version": "1.1.0", + "version": "1.2.0", "private": true, "dependencies": { - "@babel/core": "^7.15.8", + "@codemirror/commands": "^6.2.2", "@emotion/is-prop-valid": "^1.1.1", "@emotion/react": "^11.7.0", "@emotion/styled": "^11.6.0", + "@lezer/highlight": "^1.1.3", "@material-ui/core": "^4.5.2", "@material-ui/icons": "^4.5.1", "@material-ui/lab": "^4.0.0-alpha.58", @@ -17,10 +18,10 @@ "@mui/icons-material": "^5.2.1", "@mui/material": "^5.2.3", "@mui/x-data-grid": "^5.17.11", + "@uiw/codemirror-themes": "^4.19.9", "@uiw/react-codemirror": "^3.2.1", "@use-it/interval": "^1.0.0", "algoliasearch": "^4.13.1", - "babel-eslint": "^10.1.0", "class-transformer": "^0.4.0", "create-react-app": "^4.0.3", "cytoscape": "^3.15.1", @@ -58,7 +59,7 @@ "react-draggable": "^3.3.2", "react-driftjs": "^1.2.2", "react-dropzone": "^10.1.10", - "react-ga": "^2.7.0", + "react-ga4": "^2.0.0", "react-iframe": "^1.8.0", "react-instantsearch-dom": "^6.28.0", "react-json-pretty": "^2.2.0", @@ -76,14 +77,12 @@ "shellwords": "^0.1.1", "simplebar": "^4.2.3", "styled-components": "^4.4.0", - "webpack": "^4.44.2", - "websocket": "^1.0.30", "yaml": "^1.7.2", "yamljs": "^0.3.0", "zone.js": "~0.11.4" }, "scripts": { - "start": "react-scripts start", + "start": "HTTPS=false&&PORT=3000 react-scripts --openssl-legacy-provider start", "build": "react-scripts build", "test": "react-scripts test", "eject": "react-scripts eject", @@ -106,6 +105,10 @@ ], "devDependencies": { "prettier": "2.4.1", - "promise-window": "^1.2.1" + "promise-window": "^1.2.1", + "@babel/core": "^7.15.8", + "babel-eslint": "^10.1.0", + "webpack": "^4.44.2", + "@babel/plugin-proposal-private-property-in-object": "^7.21.11" } } diff --git a/frontend/public/favicon.ico b/frontend/public/favicon.ico old mode 100644 new mode 100755 diff --git a/frontend/public/images/Shuffle_logo.png b/frontend/public/images/Shuffle_logo.png old mode 100644 new mode 100755 diff --git a/frontend/public/images/experienced.png b/frontend/public/images/experienced.png new file mode 100644 index 00000000..8e5f3e5a Binary files /dev/null and b/frontend/public/images/experienced.png differ diff --git a/frontend/public/images/logos/orange_logo.svg b/frontend/public/images/logos/orange_logo.svg new file mode 100644 index 00000000..1024dd6a --- /dev/null +++ b/frontend/public/images/logos/orange_logo.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/frontend/public/images/welcome-to-shuffle.png b/frontend/public/images/welcome-to-shuffle.png new file mode 100644 index 00000000..cb976e31 Binary files /dev/null and b/frontend/public/images/welcome-to-shuffle.png differ diff --git a/frontend/public/index.html b/frontend/public/index.html old mode 100644 new mode 100755 diff --git a/frontend/public/manifest.json b/frontend/public/manifest.json old mode 100644 new mode 100755 diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx old mode 100644 new mode 100755 index 49d3566c..78fc0899 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -1,6 +1,5 @@ import React, { useState, useEffect } from "react"; -//import { Route, Routes } from "react-router"; import { Route, Routes, BrowserRouter } from "react-router-dom"; import { CookiesProvider } from "react-cookie"; import { removeCookies, useCookies } from "react-cookie"; @@ -10,7 +9,7 @@ import GettingStarted from "./views/GettingStarted"; import EditWebhook from "./views/EditWebhook"; import AngularWorkflow from "./views/AngularWorkflow"; -import Header from "./components/Header"; +import Header from "./components/Header.jsx"; import theme from "./theme"; import Apps from "./views/Apps"; import AppCreator from "./views/AppCreator"; @@ -25,6 +24,7 @@ import Introduction from "./views/Introduction"; import SetAuthentication from "./views/SetAuthentication"; import SetAuthenticationSSO from "./views/SetAuthenticationSSO"; import Search from "./views/Search.jsx"; +import RunWorkflow from "./views/RunWorkflow.jsx"; import LandingPageNew from "./views/LandingpageNew"; import LoginPage from "./views/LoginPage"; @@ -335,7 +335,9 @@ const App = (message, props) => { userdata={userdata} {...props} /> + {/*
+ */} { globalUrl={globalUrl} setCookie={setCookie} cookies={cookies} + checkLogin={checkLogin} {...props} /> } @@ -499,6 +502,7 @@ const App = (message, props) => { path="/workflows" element={ { /> } /> + } /> + } /> `), + "SIEM": encodeURI(`data:image/svg+xml;utf-8,`), + + "CASES": encodeURI(`data:image/svg+xml;utf-8,`), + "EDR & AV": encodeURI(`data:image/svg+xml;utf-8,`), + + "INTEL": encodeURI(`data:image/svg+xml;utf-8,`), + + "COMMS": encodeURI(`data:image/svg+xml;utf-8,`), + + "NETWORK": encodeURI(`data:image/svg+xml;utf-8,`), + + "INTEL": encodeURI(`data:image/svg+xml;utf-8,`), + + "ASSETS": encodeURI(`data:image/svg+xml;utf-8,`), + + "IAM": encodeURI(`data:image/svg+xml;utf-8,`), } export const usecases = { @@ -99,58 +116,7 @@ export const usecases = { "human": true, }, ]}, - "Ransomware": { - "manual": [], - "automated": [ - { - "source": "BOTTOM_LEFT", - "target": "EDR & AV", - "description": "EDR & AV alert", - "human": false, - }, - { - "source": "EDR & AV", - "target": "SHUFFLE", - "description": "", - "human": false, - }, - { - "source": "SHUFFLE", - "target": "EDR & AV", - "human": false, - "description": "isolate", - }, - { - "source": "SHUFFLE", - "target": "IAM", - "human": false, - "description": "Block access", - }, - { - "source": "SHUFFLE", - "target": "COMMS", - "description": "Notify oncall and affected user", - "human": false, - }, - { - "source": "SHUFFLE", - "target": "CASES", - "description": "Create enriched alert", - "human": false, - }, - { - "source": "SHUFFLE", - "target": "CASES", - "human": false, - }, - { - "source": "CASES", - "target": "EDR & AV", - "description": "Validate alert", - "human": true, - }, - ] - }, + "Exploits": { "manual": [], "automated": [ @@ -203,46 +169,6 @@ export const usecases = { }, ] }, - "AWS S3 honeypots": { - "manual": [], - "automated": [ - { - "source": "TOP_LEFT", - "target": "SIEM", - "description": "S3 logs", - "human": false, - }, - - { - "source": "SIEM", - "target": "SHUFFLE", - "human": false, - }, - { - "source": "SHUFFLE", - "target": "INTEL", - "description": "Add sighting", - "human": false, - }, - { - "source": "INTEL", - "target": "SHUFFLE", - "human": false, - }, - { - "source": "SHUFFLE", - "target": "CASES", - "description": "Create case", - "human": false, - }, - { - "source": "SHUFFLE", - "target": "NETWORK", - "description": "Block IP", - "human": false, - }, - ] - }, "SIEM alerts": { "manual": [], "automated": [ @@ -518,6 +444,58 @@ export const usecases = { }, ] }, + "Ransomware": { + "manual": [], + "automated": [ + { + "source": "BOTTOM_LEFT", + "target": "EDR & AV", + "description": "EDR & AV alert", + "human": false, + }, + { + "source": "EDR & AV", + "target": "SHUFFLE", + "description": "", + "human": false, + }, + { + "source": "SHUFFLE", + "target": "EDR & AV", + "human": false, + "description": "isolate", + }, + { + "source": "SHUFFLE", + "target": "IAM", + "human": false, + "description": "Block access", + }, + { + "source": "SHUFFLE", + "target": "COMMS", + "description": "Notify oncall and affected user", + "human": false, + }, + { + "source": "SHUFFLE", + "target": "CASES", + "description": "Create enriched alert", + "human": false, + }, + { + "source": "SHUFFLE", + "target": "CASES", + "human": false, + }, + { + "source": "CASES", + "target": "EDR & AV", + "description": "Validate alert", + "human": true, + }, + ] + }, "Draw": { } } @@ -541,13 +519,83 @@ const AppFramework = (props) => { const [usecaseType, setUsecaseType] = React.useState(0) const [selectedUsecase, setSelectedUsecase] = React.useState(selectedOption !== undefined ? selectedOption : "Phishing") + const [injectedApps, setInjectedApps] = React.useState([]) + const scale = size === undefined ? 1 : size > 5 ? 3 : size const alert = useAlert() + + const handleLoadNextSuggestion = (frameworkData) => { + console.log("Should check for next apps to load from App suggestion model") + //fetch(globalUrl + "/api/v1/workflows/usecases", { + //credentials: "include", + //cors: "no-cors", + + const max_suggestions = 5 + const max_per_category = 2 + const priorities = { + "edr": 1, + "siem": 2, + "cases": 3, + "intel": 4, + "comms": 5, + } + + // Based on apps recommended from repo https://github.com/Shuffle/app-recommender + //fetch("http://localhost:8080/app_recommendations", { + fetch("https://europe-west2-shuffler.cloudfunctions.net/app_recommendations", { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(frameworkData), + }) + .then((response) => { + //if (response.status !== 200) { + // console.log("Status not 200 for framework!"); + //} + + return response.json(); + }) + .then((responseJson) => { + // Loop the response dict + var suggestions = [] + var suggestion_cnt = 0 + var category_count = {} + for (const [key, value] of Object.entries(responseJson)) { + if (suggestion_cnt >= max_suggestions) { + break + } + + if (category_count[key] === undefined) { + category_count[key] = 1 + } else { + category_count[key] += 1 + } + + if (category_count[key] > max_per_category) { + continue + } + + //console.log("Found: " + key + " with value of len: " + value.length) + + //suggestion_cnt += value.slice(0,1).length + suggestions.push(value.slice(0,1)) + } + + setInjectedApps(suggestions) + }) + .catch((error) => { + console.log("Recommendation error: ", error); + }) + } + const showRecommendations = (changed, frameworkData) => { console.log("Inside recommendation loader") setChangedApp(changed) + handleLoadNextSuggestion(frameworkData) // Alternative changed // This is for secondary values like email = comms @@ -822,6 +870,12 @@ const AppFramework = (props) => { }) } + useEffect(() => { + if (!window.location.pathname.includes("usecases")) { + handleLoadNextSuggestion(frameworkData) + } + }, []) + useEffect(() => { console.log("New selected app: ", newSelectedApp, discoveryData) if (newSelectedApp.objectID === undefined || newSelectedApp.objectID === undefined || newSelectedApp.objectID.length === 0) { @@ -894,9 +948,8 @@ const AppFramework = (props) => { }, [newSelectedApp]) - const isCloud = - window.location.host === "localhost:3002" || - window.location.host === "shuffler.io"; + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + const imgSize = 50; var parsedFrameworkData = frameworkData === undefined ? {} : frameworkData @@ -1206,7 +1259,10 @@ const AppFramework = (props) => { } //setDiscoveryData({}) - setDiscoveryWrapper({}) + if (setDiscoveryWrapper !== undefined) { + setDiscoveryWrapper({}) + } + setSelectionOpen(false) setDefaultSearch("") setPaperTitle("") @@ -1278,7 +1334,10 @@ const AppFramework = (props) => { const baselocationY = 50*scale const shiftmodifier = 3*scale //const svgSize = `${40*scale}px` - const svgSize = `${40}px` + //const svgSize = `${40}px` + + const foundMiddleImage = userdata !== undefined && userdata !== null && userdata.active_org !== undefined && userdata.active_org.image !== undefined && userdata.active_org.image !== null && userdata.active_org.image !== "" ? userdata.active_org.image : '/images/Shuffle_logo.png' + const fontSize = `${12*scale}px` const defaultSize = `${85*scale}px` @@ -1303,9 +1362,8 @@ const AppFramework = (props) => { margin_y: parsedFrameworkData.Cases.large_image === undefined ? `${19*scale}px` : `0px`, width: parsedFrameworkData.Cases.large_image === undefined ? iconSize : defaultSize, height: parsedFrameworkData.Cases.large_image === undefined ? iconSize : defaultSize, - large_image: parsedFrameworkData.Cases.large_image === undefined ? encodeURI(`data:image/svg+xml;utf-8, - - `) : parsedFrameworkData.Cases.large_image, + large_image: parsedFrameworkData.Cases.large_image === undefined ? parsedDatatypeImages["CASES"] : parsedFrameworkData.Cases.large_image, + label: securityFramework[0].text.toUpperCase(), id: securityFramework[0].text.toUpperCase(), animate: true, @@ -1332,9 +1390,8 @@ const AppFramework = (props) => { margin_y: parsedFrameworkData.IAM.large_image === undefined ? `${19*scale}px` : `0px`, width: parsedFrameworkData.IAM.large_image === undefined ? iconSize : defaultSize, height: parsedFrameworkData.IAM.large_image === undefined ? iconSize : defaultSize, - large_image: parsedFrameworkData.IAM.large_image === undefined ? encodeURI(`data:image/svg+xml;utf-8, - , - `) : parsedFrameworkData.IAM.large_image, + large_image: parsedFrameworkData.IAM.large_image === undefined ? parsedDatatypeImages["IAM"] : parsedFrameworkData.IAM.large_image, + label: securityFramework[3].text.toUpperCase(), id: securityFramework[3].text.toUpperCase(), animate: false, @@ -1361,9 +1418,8 @@ const AppFramework = (props) => { margin_y: parsedFrameworkData.Assets.large_image === undefined ? `${19*scale}px` : `0px`, width: parsedFrameworkData.Assets.large_image === undefined ? iconSize : defaultSize, height: parsedFrameworkData.Assets.large_image === undefined ? iconSize : defaultSize, - large_image: parsedFrameworkData.Assets.large_image === undefined ? encodeURI(`data:image/svg+xml;utf-8, - , - `) : parsedFrameworkData.Assets.large_image, + large_image: parsedFrameworkData.Assets.large_image === undefined ? parsedDatatypeImages["ASSETS"] : parsedFrameworkData.Assets.large_image, + label: securityFramework[2].text.toUpperCase(), id: securityFramework[2].text.toUpperCase(), animate: false, @@ -1390,9 +1446,8 @@ const AppFramework = (props) => { margin_y: parsedFrameworkData.Intel.large_image === undefined ? `${19*scale}px` : `0px`, width: parsedFrameworkData.Intel.large_image === undefined ? iconSize : defaultSize, height: parsedFrameworkData.Intel.large_image === undefined ? iconSize : defaultSize, - large_image: parsedFrameworkData.Intel.large_image === undefined ? encodeURI(`data:image/svg+xml;utf-8, - , - `): parsedFrameworkData.Intel.large_image, + large_image: parsedFrameworkData.Intel.large_image === undefined ? parsedDatatypeImages["INTEL"] : parsedFrameworkData.Intel.large_image, + label: securityFramework[4].text.toUpperCase(), id: securityFramework[4].text.toUpperCase(), animate: false, @@ -1419,9 +1474,8 @@ const AppFramework = (props) => { margin_y: parsedFrameworkData.Comms.large_image === undefined ? `${19*scale}px` : `0px`, width: parsedFrameworkData.Comms.large_image === undefined ? iconSize : defaultSize, height: parsedFrameworkData.Comms.large_image === undefined ? iconSize : defaultSize, - large_image: parsedFrameworkData.Comms.large_image === undefined ? encodeURI(`data:image/svg+xml;utf-8, - , - `) : parsedFrameworkData.Comms.large_image, + large_image: parsedFrameworkData.Comms.large_image === undefined ? parsedDatatypeImages["COMMS"] : parsedFrameworkData.Comms.large_image, + label: securityFramework[5].text.toUpperCase(), id: securityFramework[5].text.toUpperCase(), animate: false, @@ -1448,9 +1502,8 @@ const AppFramework = (props) => { margin_y: parsedFrameworkData["EDR & AV"].large_image === undefined ? `${19*scale}px` : `0px`, width: parsedFrameworkData["EDR & AV"].large_image === undefined ? iconSize : defaultSize, height: parsedFrameworkData["EDR & AV"].large_image === undefined ? iconSize : defaultSize, - large_image: parsedFrameworkData["EDR & AV"].large_image === undefined ? encodeURI(`data:image/svg+xml;utf-8, - , - `) : parsedFrameworkData["EDR & AV"].large_image, + large_image: parsedFrameworkData["EDR & AV"].large_image === undefined ? parsedDatatypeImages["EDR & AV"] : parsedFrameworkData["EDR & AV"].large_image, + label: securityFramework[7].text.toUpperCase(), id: securityFramework[7].text.toUpperCase(), animate: false, @@ -1477,9 +1530,7 @@ const AppFramework = (props) => { margin_y: parsedFrameworkData.Network.large_image === undefined ? `${19*scale}px` : `0px`, width: parsedFrameworkData.Network.large_image === undefined ? iconSize : defaultSize, height: parsedFrameworkData.Network.large_image === undefined ? iconSize : defaultSize, - large_image: parsedFrameworkData.Network.large_image === undefined ? encodeURI(`data:image/svg+xml;utf-8, - , - `) : parsedFrameworkData.Network.large_image, + large_image: parsedFrameworkData.Network.large_image === undefined ? parsedDatatypeImages["NETWORK"] : parsedFrameworkData.Network.large_image, label: securityFramework[6].text.toUpperCase(), id: securityFramework[6].text.toUpperCase(), animate: false, @@ -1506,9 +1557,7 @@ const AppFramework = (props) => { margin_y: parsedFrameworkData.SIEM.large_image === undefined ? `${19*scale}px` : `0px`, width: parsedFrameworkData.SIEM.large_image === undefined ? iconSize : defaultSize, height: parsedFrameworkData.SIEM.large_image === undefined ? iconSize : defaultSize, - large_image: parsedFrameworkData.SIEM.large_image === undefined ? encodeURI(`data:image/svg+xml;utf-8, - , - `) : parsedFrameworkData.SIEM.large_image, + large_image: parsedFrameworkData.SIEM.large_image === undefined ? parsedDatatypeImages["SIEM"] : parsedFrameworkData.SIEM.large_image, label: securityFramework[1].text.toUpperCase(), id: securityFramework[1].text.toUpperCase(), animate: false, @@ -1532,6 +1581,7 @@ const AppFramework = (props) => { isValid: true, errors: [], middle_node: true, + large_image: foundMiddleImage, }, renderedPosition: { x: baselocationX, @@ -1643,8 +1693,8 @@ const AppFramework = (props) => { const UsecaseHandler = (props) => { const { data, index, diff } = props - const leftImage = data.left_image !== undefined ? parsedDatatypeImages[data.left_image] : undefined - const rightImage = data.right_image !== undefined ? parsedDatatypeImages[data.right_image] : undefined + const leftImage = data.left_image !== undefined ? parsedDatatypeImages[data.left_image.toUpperCase()] : undefined + const rightImage = data.right_image !== undefined ? parsedDatatypeImages[data.right_image.toUpperCase()] : undefined if (leftImage === undefined) { console.log("LEFT MISSING: ", leftImage) @@ -1660,7 +1710,6 @@ const AppFramework = (props) => { const parsedLeftImage = const parsedLeftText =
{data.left_text}
- const svgSize = 20 var svgIcon = @@ -1797,6 +1846,16 @@ const AppFramework = (props) => { } ] + const getNodeName = (category) => { + if (category === "email") { + category = "COMMS" + } else if (category === "edr") { + category = "EDR & AV" + } + + return category.toUpperCase() + } + //autounselectify={true} var usecasediff = -100 const bgColor = color === undefined || color === null || color.length === 0 ? theme.palette.surfaceColor : color @@ -1816,6 +1875,95 @@ const AppFramework = (props) => { />
+ {injectedApps.map((apps, appindex) => { + var categoryTop = 100 + var categoryLeft = 100 + + const category = apps[0].category + if (category === "edr") { + categoryTop = 355 + categoryLeft = 50 + } else if (category === "siem") { + categoryTop = 95 + categoryLeft = 50 + } else if (category === "cases") { + categoryTop = 65 + categoryLeft = 175 + } else if (category === "intel") { + categoryTop = 355 + categoryLeft = 315 + } else if (category === "comms") { + categoryTop = 415 + categoryLeft = 175 + } else if (category === "iam") { + categoryTop = 95 + categoryLeft = 315 + } else { + return null + } + + return ( +
+ {apps.map((app, appIndex) => { + return ( + } + key={app.id} + label={""} + variant="contained" + style={{}} + onClick={() => { + console.log("Add app to framework and remove category: ", app) + app.objectID = app.id + app.type = app.category + app.large_image = app.image_url + + const nodename = getNodeName(app.category) + + + const foundelement = cy.getElementById(nodename) + console.log("FOUND: ", foundelement) + if (foundelement !== undefined && foundelement !== null) { + foundelement.data("large_image", app.image_url) + foundelement.data("margin_x", "0px") + foundelement.data("margin_y", "0px") + foundelement.data("text_margin_y", `${60*scale}px`) + foundelement.data("width", `${85*scale}px`) + foundelement.data("height", `${85*scale}px`) + } + + if (setFrameworkData !== undefined) { + console.log("Setting frameworkdata") + // Find discoveryData.id + var keys = [] + for (const [key, value] of Object.entries(frameworkData)) { + if (key.toLowerCase() === app.category.toLowerCase()) { + keys.push(key) + } + } + + if (keys.length === 0) { + console.log("Failed to find: ", app.category, " IN ", frameworkData) + } else { + console.log("In else for keys: ", frameworkData, keys) + for (var key in keys) { + frameworkData[keys[key]] = app + } + + setFrameworkData(frameworkData) + showRecommendations(app.category, frameworkData) + } + } + + setFrameworkItem(app) + }} + /> + ) + })} +
+ ) + })} + {showOptions === false ? null :
{Object.keys(usecases).map((data, index) => { @@ -1915,11 +2063,15 @@ const AppFramework = (props) => { setSelectionOpen(true) setDefaultSearch("") + + //handleLoadNextSuggestion(frameworkData) + setInjectedApps([]) const foundelement = cy.getElementById(discoveryData.id) if (foundelement !== undefined && foundelement !== null) { console.log("element: ", foundelement) - foundelement.data("large_image", discoveryData.large_image) + console.log("DISC: ", discoveryData) + foundelement.data("large_image", parsedDatatypeImages[discoveryData.id.toUpperCase()]) foundelement.data("text_margin_y", "14px") foundelement.data("margin_x", "32px") foundelement.data("margin_y", "19x") diff --git a/frontend/src/components/AppGrid.jsx b/frontend/src/components/AppGrid.jsx index 562146c0..af7ddae5 100644 --- a/frontend/src/components/AppGrid.jsx +++ b/frontend/src/components/AppGrid.jsx @@ -1,6 +1,6 @@ import React, {useEffect, useState} from 'react'; -import ReactGA from 'react-ga'; +import ReactGA from 'react-ga4'; import { useTheme } from '@material-ui/core/styles'; import {Link} from 'react-router-dom'; @@ -362,7 +362,7 @@ const AppGrid = props => { : null } - + Search by diff --git a/frontend/src/components/AppGrid1.jsx b/frontend/src/components/AppGrid1.jsx index 8706ef17..b64610c5 100644 --- a/frontend/src/components/AppGrid1.jsx +++ b/frontend/src/components/AppGrid1.jsx @@ -1,6 +1,6 @@ import React, {useEffect, useState} from 'react'; -import ReactGA from 'react-ga'; +import ReactGA from 'react-ga4'; import { useTheme } from '@material-ui/core/styles'; import {Link} from 'react-router-dom'; diff --git a/frontend/src/components/Appsearch.jsx b/frontend/src/components/Appsearch.jsx index 0573883d..9da69a70 100644 --- a/frontend/src/components/Appsearch.jsx +++ b/frontend/src/components/Appsearch.jsx @@ -1,9 +1,8 @@ import React, { useState, useEffect } from 'react'; - -import ReactGA from 'react-ga'; +import ReactGA from 'react-ga4'; import { useTheme } from '@material-ui/core/styles'; import {Link} from 'react-router-dom'; - +import { useAlert } from "react-alert"; import { Search as SearchIcon, CloudQueue as CloudQueueIcon, Code as CodeIcon } from '@material-ui/icons'; //import algoliasearch from 'algoliasearch/lite'; @@ -11,13 +10,12 @@ import algoliasearch from 'algoliasearch'; import { InstantSearch, connectSearchBox, connectHits } from 'react-instantsearch-dom'; import { Grid, Paper, TextField, ButtonBase, InputAdornment, Typography, Button, Tooltip} from '@material-ui/core'; import aa from 'search-insights' - const searchClient = algoliasearch("JNSS5CFDZZ", "db08e40265e2941b9a7d8f644b6e5240") const Appsearch = props => { - const { maxRows, showName, showSuggestion, isMobile, globalUrl, parsedXs, newSelectedApp, setNewSelectedApp, defaultSearch, showSearch, ConfiguredHits, userdata, cy, } = props + const { maxRows, showName, showSuggestion, isMobile, globalUrl, parsedXs, newSelectedApp, setNewSelectedApp, defaultSearch, showSearch, ConfiguredHits, userdata, cy, isCreatorPage, actionImageList, setActionImageList} = props const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; - + const alert = useAlert(); const rowHandler = maxRows === undefined || maxRows === null ? 50 : maxRows const xs = parsedXs === undefined || parsedXs === null ? 12 : parsedXs const theme = useTheme(); @@ -27,13 +25,51 @@ const Appsearch = props => { const [message, setMessage] = React.useState(""); const [formMessage, setFormMessage] = React.useState(""); const [selectedApp, setSelectedApp] = React.useState({}); - const buttonStyle = {borderRadius: 30, height: 50, width: 220, margin: isMobile ? "15px auto 15px auto" : 20, fontSize: 18,} const innerColor = "rgba(255,255,255,0.65)" const borderRadius = 3 window.title = "Shuffle | Apps | Find and integration any app" + const setUserSpecialzedApp = (user, data) => { + // var data = newfields] + console.log("data value", data) + const appData = {"user_id":user,"specialized_apps":[{}]} + console.log("User Check for appdata:", user) + appData["specialized_apps"][0]["name"] = data["name"] + appData["specialized_apps"][0]["image"] = data["image_url"] + appData["specialized_apps"][0]["category"] = data["categories"].toString() + console.log("AppData:",appData) + console.log("setActionImageList",setActionImageList) + console.log("actionImageList",actionImageList) + + const finalData = actionImageList.concat(appData["specialized_apps"]) + appData["specialized_apps"]=finalData + fetch(globalUrl + "/api/v1/users/updateuser", { + method: "PUT", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(appData), + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for set creator :O!"); + } + alert.success("Sucessfully updated specialzed app.") + return response.json(); + }) + .then((responseJson) => { + if (!responseJson.success && responseJson.reason !== undefined) { + alert.error("Failed updating user: " + responseJson.reason); + } + }) + .catch((error) => { + console.log(error); + }); + }; const submitContact = (email, message) => { const data = { "firstname": "", @@ -170,6 +206,16 @@ const Appsearch = props => { }} onMouseOut={() => { setMouseHoverIndex(-1) }} onClick={() => { + if(isCreatorPage === true){ + console.log("data:",data) + console.log("userdata.id",userdata.id) + console.log("is creator", isCreatorPage) + if (setNewSelectedApp !== undefined) { + // setUserSpecialzedApp = data + setUserSpecialzedApp(userdata.id, data) + //setActionImageList(userdata.id, data) + } + } if (setNewSelectedApp !== undefined) { setNewSelectedApp(data) } diff --git a/frontend/src/components/AppsearchPopout.jsx b/frontend/src/components/AppsearchPopout.jsx index 7e15dc67..25ea719c 100644 --- a/frontend/src/components/AppsearchPopout.jsx +++ b/frontend/src/components/AppsearchPopout.jsx @@ -1,6 +1,6 @@ import React, { useState, useEffect } from 'react'; -import theme from '../theme'; +import theme from '../theme.jsx'; import AppSearch from './Appsearch.jsx'; import { @@ -39,8 +39,9 @@ const AppSearchPopout = (props) => { return null } + // return ( - + {paperTitle !== undefined && paperTitle.length > 0 ? diff --git a/frontend/src/components/AuthenticationItem.jsx b/frontend/src/components/AuthenticationItem.jsx index 45879519..eef9806c 100644 --- a/frontend/src/components/AuthenticationItem.jsx +++ b/frontend/src/components/AuthenticationItem.jsx @@ -1,6 +1,6 @@ import React, { useState, useEffect } from "react"; -import theme from '../theme'; +import theme from '../theme.jsx'; import { useAlert } from "react-alert"; import { Tooltip, diff --git a/frontend/src/components/AuthenticationNormal.jsx b/frontend/src/components/AuthenticationNormal.jsx index aae3a128..5d3a4fd3 100644 --- a/frontend/src/components/AuthenticationNormal.jsx +++ b/frontend/src/components/AuthenticationNormal.jsx @@ -1,5 +1,5 @@ import React, { useState, useEffect } from "react"; -import theme from '../theme'; +import theme from '../theme.jsx'; import { v4 as uuidv4 } from "uuid"; diff --git a/frontend/src/components/Billing.jsx b/frontend/src/components/Billing.jsx new file mode 100644 index 00000000..b59b5e4a --- /dev/null +++ b/frontend/src/components/Billing.jsx @@ -0,0 +1,317 @@ +import React, { useState, useEffect } from "react"; +import ReactGA from 'react-ga4'; + +import { useTheme } from "@material-ui/core/styles"; +import { + Paper, + Typography, + Divider, + Button, + Grid, + Card, +} from "@material-ui/core"; + +import { useAlert } from "react-alert"; +import { typecost, typecost_single, } from "../views/HandlePaymentNew.jsx"; + +const Billing = (props) => { + const { globalUrl, userdata, serverside, billingInfo, stripeKey, selectedOrganization, handleGetOrg, } = props; + console.log("Billing: ", billingInfo); + const theme = useTheme(); + const alert = useAlert(); + + const stripe = typeof window === 'undefined' || window.location === undefined ? "" : props.stripeKey === undefined ? "" : window.Stripe ? window.Stripe(props.stripeKey) : "" + console.log("Stripe: ", stripe) + + const paperStyle = { + padding: 20, + height: "100%", + width: "100%", + backgroundColor: theme.palette.surfaceColor, + border: "1px solid rgba(255,255,255,0.3)", + marginRight: 10, + } + + const isCloud = + window.location.host === "localhost:3002" || + window.location.host === "shuffler.io"; + + billingInfo.subscription = { + "active": true, + "name": "Pay as you go", + "price": typecost_single, + "currency": "USD", + "currency_text": "$", + "interval": "app run / month", + "description": "Pay as you go", + "features": [ + "Includes 10.000 app run/month for free. ", + "Pay for what you use with no minimum commitment and cancel anytime.", + ], + "limit": 10000, + } + + + const handleStripeRedirect = () => { + //var priceItem = "price_1MRNF1DzMUgUjxHSfFTUb2Xh" + if (stripe == "") { + console.log("Stripe not loaded") + return + } + + var priceItem = "price_1MROFrDzMUgUjxHShcSxgHO1" + + const successUrl = `${window.location.origin}/admin?admin_tab=billing&payment=success` + const failUrl = `${window.location.origin}/admin?admin_tab=billing&payment=failure` + var checkoutObject = { + lineItems: [ + { + price: priceItem, + quantity: 1 + }, + ], + mode: "subscription", + billingAddressCollection: "auto", + successUrl: successUrl, + cancelUrl: failUrl, + clientReferenceId: props.userdata.active_org.id, + } + //submitType: "donate", + + stripe.redirectToCheckout(checkoutObject) + .then(function (result) { + console.log("SUCCESS STRIPE?: ", result) + + ReactGA.event({ + category: "pricing", + action: "add_card_success", + label: "", + }) + }) + .catch(function(error) { + console.error("STRIPE ERROR: ", error) + + ReactGA.event({ + category: "pricing", + action: "add_card_error", + label: "", + }) + }); + } + + const cancelSubscriptions = (subscription_id) => { + const orgId = selectedOrganization.id; + const data = { + subscription_id: subscription_id, + action: "cancel", + org_id: selectedOrganization.id, + }; + + const url = globalUrl + `/api/v1/orgs/${orgId}/cancel`; + fetch(url, { + mode: "cors", + method: "POST", + body: JSON.stringify(data), + credentials: "include", + crossDomain: true, + withCredentials: true, + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + }) + .then(function (response) { + if (response.status !== 200) { + console.log("Error in response"); + } + + if (handleGetOrg != undefined) { + handleGetOrg(selectedOrganization.id); + } + + return response.json(); + }) + .then(function (responseJson) { + if (responseJson.success !== undefined && responseJson.success) { + alert.success("Successfully stopped subscription!"); + } else { + alert.error("Failed stopping subscription. Please contact us."); + } + }) + .catch(function (error) { + console.log("Error: ", error); + alert.error("Failed stopping subscription. Please contact us."); + }); + }; + + const SubscriptionObject = (props) => { + const { globalUrl, userdata, serverside, billingInfo, stripeKey, selectedOrganization, handleGetOrg, subscription, } = props; + + console.log("Sub: ", subscription) + var top_text = "Base Access" + if (subscription.limit === undefined && subscription.level !== undefined) { + + subscription.name = "Enterprise" + subscription.currency_text = "$" + subscription.price = subscription.level*180 + subscription.limit = subscription.level*100000 + subscription.interval = subscription.recurrence + subscription.features = [ + "Includes " + subscription.limit + " app runs/month. ", + "Multi-Tenancy and Region-Selection", + "And all other features from /pricing", + ] + } + + if (subscription.name === "Enterprise" && subscription.active === true) { + top_text = "Current Plan" + } + + return ( + +
+ + {top_text} + +
+ +
+ + {subscription.name} + +
+ + {subscription.currency_text}{subscription.price} + + + / {subscription.interval} + +
+ + Features + +
    + {subscription.features !== undefined && subscription.features !== null ? + subscription.features.map((feature, index) => { + return ( +
  • + + {feature} + +
    • + ) + }) + : null} +
+
+ {/*subscription.name === "Pay as you go" && subscription.limit <= 10000 ? + + + You are not subscribed to any plan and are using the free plan with max 10,000 apps per month. Activate billing to de-activate this limit. + + + + : null*/} + + ) + } + + + return ( +
+ + Billing + + + We use Stripe to manage subscriptions and do not store any of your billing information. You can manage your subscription and billing information below. + +
+ {billingInfo.subscription !== undefined && billingInfo.subscription !== null ? + + : null} + {isCloud && + selectedOrganization.subscriptions !== undefined && + selectedOrganization.subscriptions !== null && + selectedOrganization.subscriptions.length > 0 ? + + selectedOrganization.subscriptions + .reverse() + .map((sub, index) => { + return ( + + ) + }) + : null} + {/* + + + Quantity: {sub.level} +
+ Recurrence: {sub.recurrence} +
+ {sub.active ? ( +
+ Started:{" "} + {new Date(sub.startdate * 1000).toISOString()} +
+ +
+ ) : ( +
+ Cancelled:{" "} + {new Date( + sub.cancellationdate * 1000 + ).toISOString()} +
+ + Status: Deactivated + +
+ )} + + + */} +
+
+ ) +} + +export default Billing; diff --git a/frontend/src/components/Branding.jsx b/frontend/src/components/Branding.jsx new file mode 100644 index 00000000..40254bbb --- /dev/null +++ b/frontend/src/components/Branding.jsx @@ -0,0 +1,83 @@ +import React, { useState, useEffect } from "react"; +import ReactGA from 'react-ga4'; +import theme from "../theme.jsx"; + +import { useTheme } from "@material-ui/core/styles"; +import { + Paper, + Typography, + Divider, + Button, + Grid, + Card, +} from "@material-ui/core"; + +import { useAlert } from "react-alert"; + +const Branding = (props) => { + const { globalUrl, userdata, serverside, billingInfo, stripeKey, selectedOrganization, handleGetOrg, } = props; + const alert = useAlert(); + const [publishingInfo, setPublishingInfo] = useState(""); + + // Should enable / disable org branding + const handleChangePublishing = () => { + console.log("Handle change publishing"); + } + + const isOrganizationReady = () => { + // A simple checklist to ensure the button shows up properly + if (selectedOrganization.name === selectedOrganization.org) { + return false; + } + + if (selectedOrganization.large_image === "" || selectedOrganization.large_image === theme.palette.defaultImage) { + return false; + } + + return true + } + + return ( +
+ + Branding + + + You can customize your organization's branding by uploading a logo, changing the color scheme and a lot more. + + + +

+ Creator Network +

+
+
+ + + By changing publishing settings, you agree to our Terms of Service, and acknowledge that your organization's non-sensitive data will be turned into a creator account. Support: support@shuffler.io + + + + + {publishingInfo} + + +
+
+
+ ) +} + +export default Branding; diff --git a/frontend/src/components/CacheView.jsx b/frontend/src/components/CacheView.jsx new file mode 100644 index 00000000..873f9f7e --- /dev/null +++ b/frontend/src/components/CacheView.jsx @@ -0,0 +1,427 @@ +import React, { useState, useEffect } from "react"; +import theme from "../theme.jsx"; +import { + Tooltip, + Divider, + TextField, + Button, + Tabs, + Tab, + Grid, + List, + ListItem, + ListItemText, + IconButton, + Dialog, + DialogTitle, + DialogActions, +} from "@material-ui/core"; +import { useAlert } from "react-alert"; + +import { + Edit as EditIcon, + FileCopy as FileCopyIcon, + SelectAll as SelectAllIcon, + OpenInNew as OpenInNewIcon, + CloudDownload as CloudDownloadIcon, + Description as DescriptionIcon, + Polymer as PolymerIcon, + CheckCircle as CheckCircleIcon, + Close as CloseIcon, + Apps as AppsIcon, + Image as ImageIcon, + Delete as DeleteIcon, + Cached as CachedIcon, + AccessibilityNew as AccessibilityNewIcon, + Lock as LockIcon, + Eco as EcoIcon, + Schedule as ScheduleIcon, + Cloud as CloudIcon, + Business as BusinessIcon, + Visibility as VisibilityIcon, + VisibilityOff as VisibilityOffIcon, +} from "@material-ui/icons"; + +const CacheView = (props) => { + const { globalUrl, userdata, serverside, orgId } = props; + const [orgCache, setOrgCache] = React.useState(""); + const [listCache, setListCache] = React.useState([]); + const [addCache, setAddCache] = React.useState(""); + const [modalOpen, setModalOpen] = React.useState(false); + const [key, setKey]= React.useState(""); + const [value, setValue]= React.useState(""); + const [cacheInput, setCacheInput]= React.useState(""); + const [cacheCursor, setCacheCursor]= React.useState(""); + + const alert = useAlert(); + useEffect(() => { + listOrgCache(orgId); + console.log("orgid", orgId); + }, []); + + const listOrgCache = (orgId) => { + fetch(globalUrl + `/api/v1/orgs/${orgId}/list_cache`, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for apps :O!"); + return; + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success === true) { + setListCache(responseJson.keys); + } + + if (responseJson.cursor !== undefined && responseJson.cursor !== null && responseJson.cursor !== "") { + setCacheCursor(responseJson.cursor); + } + }) + .catch((error) => { + alert.error(error.toString()); + }); + }; + + // const getCacheList = (orgId) => { + // fetch(`${globalUrl}/api/v1/orgs/${orgId}/get_cache`, { + // method: "GET", + // headers: { + // "Content-Type": "application/json", + // Accept: "application/json", + // }, + // credentials: "include", + // }) + // .then((response) => { + // if (response.status !== 200) { + // console.log("Status not 200 for WORKFLOW EXECUTION :O!"); + // } + + + // return response.json(); + // }) + // .then((responseJson) => { + // if (responseJson.success !== false) { + // console.log("Found cache: ", responseJson) + // setListCache(responseJson) + // } else { + // console.log("Couldn't find the creator profile (rerun?): ", responseJson) + // // If the current user is any of the Shuffle Creators + // // AND the workflow doesn't have an owner: allow editing. + // // else: Allow suggestions? + // //console.log("User: ", userdata) + // //if (rerun !== true) { + // // getUserProfile(userdata.id, true) + // //} + // } + // }) + // .catch((error) => { + // console.log("Get userprofile error: ", error); + // }) + // } + + + const deleteCache = (orgId, key) => { + alert.info("Attempting to delete Cache"); + fetch(globalUrl + `/api/v1/orgs/${orgId}/cache/${key}`, { + method: "DELETE", + headers: { + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status === 200) { + alert.success("Successfully deleted Cache"); + setTimeout(() => { + listOrgCache(orgId); + }, 1000); + } else { + alert.error("Failed deleting Cache. Does it still exist?"); + } + }) + .catch((error) => { + alert.error(error.toString()); + }); + }; + + const addOrgCache = (orgId) => { + const cache={key:key,value:value}; + setCacheInput([cache]); + console.log("cache input:",cacheInput) + + fetch(globalUrl + `/api/v1/orgs/${orgId}/set_cache`, { + + method: "POST", + body: JSON.stringify(cache), + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for apps :O!"); + return; + } + + return response.json(); + }) + .then((responseJson) => { + setAddCache(responseJson); + alert.success("New Cache Added Successfully!"); + listOrgCache(orgId); + setModalOpen(false); + }) + .catch((error) => { + alert.error(error.toString()); + }); + }; + + const modalView = ( + { + setModalOpen(false); + }} + PaperProps={{ + style: { + backgroundColor: theme.palette.surfaceColor, + color: "white", + minWidth: "800px", + minHeight: "320px", + }, + }} + > + + + Add Cache + + +
+ Key + setKey(e.target.value)} + /> +
+
+ Value + setValue(e.target.value)} + /> +
+ + + + + + ); + + return ( + +
+ {modalView} +
+

Shuffle Datastore

+ + Datastore is a key-value store for storing data that can be used cross-workflow.  + + Learn more + + +
+ + + + + + + + + + + {listCache === undefined || listCache === null + ? null + : listCache.map((data, index) => { + var bgColor = "#27292d"; + if (index % 2 === 0) { + bgColor = "#1f2023"; + } + + return ( + + + + + + {/* + + { + + }} + > + + + + */} + + + { + deleteCache(orgId, data.key); + //deleteFile(orgId); + }} + > + + + + + + /> + + ); + })} + +
+ + ); +} +export default CacheView; diff --git a/frontend/src/components/ConfigureWorkflow.jsx b/frontend/src/components/ConfigureWorkflow.jsx old mode 100644 new mode 100755 index 0e5cc988..b7b9f2b0 --- a/frontend/src/components/ConfigureWorkflow.jsx +++ b/frontend/src/components/ConfigureWorkflow.jsx @@ -55,6 +55,7 @@ const ConfigureWorkflow = (props) => { workflowExecutions, getWorkflowExecution, } = props; + const [requiredActions, setRequiredActions] = React.useState([]); const [requiredVariables, setRequiredVariables] = React.useState([]); const [requiredTriggers, setRequiredTriggers] = React.useState([]); @@ -78,6 +79,17 @@ const ConfigureWorkflow = (props) => { }, }); + // ONLY when component is being unloaded, run stop() function + // This is to prevent the interval from running when the component is not being used + + /* + useEffect(() => { + return () => { + stop() + } + }, []) + */ + // Where is this from? if (workflow === undefined || workflow === null) { return null; @@ -129,7 +141,7 @@ const ConfigureWorkflow = (props) => { setFirstLoad(workflow.id) const newactions = []; - for (var key in workflow.actions) { + for (let [key, keyval] in Object.entries(workflow.actions)) { const action = workflow.actions[key]; var newaction = { large_image: action.large_image, @@ -145,17 +157,18 @@ const ConfigureWorkflow = (props) => { app: {}, steps: [], show_steps: false, - }; + } - const app = apps.find( - (app) => - app.name === action.app_name && - (app.app_version === action.app_version || - (app.loop_versions !== null && - app.loop_versions.includes(action.app_version))) + //console.log("Action: ", key, keyval) + + const app = apps.find((app) => + app.id === action.app_id || + (app.name === action.app_name && + (app.app_version === action.app_version || (app.loop_versions !== null && app.loop_versions.includes(action.app_version)))) ) - //newaction.steps = wazuhSteps + //console.log("FOUND APP: ", app) + if (app === undefined || app === null) { const subapp = apps.find(app => app.name === action.app_name) if (subapp !== undefined && subapp !== null) { @@ -169,13 +182,10 @@ const ConfigureWorkflow = (props) => { "required": true, }) } else { - if ( - action.authentication_id === "" && - app.authentication.required === true - ) { + if (action.authentication_id === "" && app.authentication.required === true && action.parameters !== undefined && action.parameters !== null) { // Check if configuration is filled or not var filled = true; - for (var key in action.parameters) { + for (let [key,keyval] in Object.entries(action.parameters)) { if (action.parameters[key].configuration) { //console.log("Found config: ", action.parameters[key]) if ( @@ -216,7 +226,7 @@ const ConfigureWorkflow = (props) => { if (newaction.must_authenticate) { var authenticationOptions = []; - for (var key in appAuthentication) { + for (let [key,keyval] in Object.entries(appAuthentication)) { const auth = appAuthentication[key]; if (auth.app.name === app.name && auth.active) { //console.log("Found auth: ", auth) @@ -264,20 +274,20 @@ const ConfigureWorkflow = (props) => { } } - for (var key in workflow.workflow_variables) { + if (workflow.workflow_variables !== undefined && workflow.workflow_variables !== null && workflow.workflow_variables.length !== 0) { + for (let [key,keyval] in Object.entries(workflow.workflow_variables)) { const variable = workflow.workflow_variables[key]; - if ( - variable.value === undefined || - variable.value === undefined || - variable.value.length < 2 - ) { + if (variable.value === undefined || variable.value === undefined || variable.value.length === 0) { variable.value = ""; - variable.index = key; requiredVariables.push(variable); } - } - for (var key in workflow.triggers) { + variable.index = key; + } + } + + if (workflow.triggers !== undefined && workflow.triggers !== null && workflow.triggers.length !== 0) { + for (let [key,keyval] in Object.entries(workflow.triggers)) { var trigger = workflow.triggers[key]; trigger.index = key; @@ -299,7 +309,7 @@ const ConfigureWorkflow = (props) => { } ] - for (var subkey in tmpsteps) { + for (let [subkey,subkeyval] in Object.entries(tmpsteps)) { newactions[foundindex].steps.push(tmpsteps[subkey]) } @@ -326,6 +336,7 @@ const ConfigureWorkflow = (props) => { requiredTriggers.push(trigger); } +} if ( requiredTriggers.length === 0 && @@ -340,13 +351,13 @@ const ConfigureWorkflow = (props) => { setRequiredActions(newactions); } - if (appAuthentication.length !== previousAuth.length) { + if (appAuthentication !== undefined && previousAuth !== undefined && appAuthentication.length !== previousAuth.length) { var newactions = [] - for (var actionkey in requiredActions) { + for (let [actionkey, actionkeyval] in Object.entries(requiredActions)) { var newaction = requiredActions[actionkey]; const app = newaction.app; - for (var key in appAuthentication) { + for (let [key,keyval] in Object.entries(appAuthentication)) { const auth = appAuthentication[key]; // Does this account for all the different ones of the same? @@ -690,7 +701,7 @@ const ConfigureWorkflow = (props) => { if (workflow.actions !== null) { //console.log(workflow.actions) alert.info("Setting action to version "+action.update_version) - for (var key in workflow.actions) { + for (let [key,keyval] in Object.entries(workflow.actions)) { if (workflow.actions[key].app_name === action.app_name && workflow.actions[key].app_version === action.app_version) { workflow.actions[key].app_version = action.update_version @@ -1013,7 +1024,7 @@ const ConfigureWorkflow = (props) => { {requiredActions.map((data, index) => { return ( -
+
{data.steps !== undefined && data.steps !== null && data.show_steps === true ? : diff --git a/frontend/src/components/CreatorGrid.jsx b/frontend/src/components/CreatorGrid.jsx index 66d852c5..45e1961f 100644 --- a/frontend/src/components/CreatorGrid.jsx +++ b/frontend/src/components/CreatorGrid.jsx @@ -1,6 +1,6 @@ import React, { useEffect, useState } from 'react'; -import ReactGA from 'react-ga'; +import ReactGA from 'react-ga4'; import { useTheme } from '@material-ui/core/styles'; import {Link} from 'react-router-dom'; @@ -169,7 +169,7 @@ const CreatorGrid = props => { - +
@@ -239,7 +239,7 @@ const CreatorGrid = props => {
- + {showSuggestion === true ?
diff --git a/frontend/src/components/DocsGrid.jsx b/frontend/src/components/DocsGrid.jsx index 706c861a..ea20cd0e 100644 --- a/frontend/src/components/DocsGrid.jsx +++ b/frontend/src/components/DocsGrid.jsx @@ -1,6 +1,6 @@ import React, {useEffect, useState} from 'react'; -import ReactGA from 'react-ga'; +import ReactGA from 'react-ga4'; import { useTheme } from '@material-ui/core/styles'; import {Link} from 'react-router-dom'; diff --git a/frontend/src/components/Dropzone.js b/frontend/src/components/Dropzone.jsx old mode 100644 new mode 100755 similarity index 100% rename from frontend/src/components/Dropzone.js rename to frontend/src/components/Dropzone.jsx diff --git a/frontend/src/components/EditWorkflow.jsx b/frontend/src/components/EditWorkflow.jsx index 57852c5d..22a9d9ea 100644 --- a/frontend/src/components/EditWorkflow.jsx +++ b/frontend/src/components/EditWorkflow.jsx @@ -1,5 +1,5 @@ import React, { useEffect, useContext } from "react"; -import theme from '../theme'; +import theme from '../theme.jsx'; import { isMobile } from "react-device-detect" import ChipInput from "material-ui-chip-input"; import UsecaseSearch from "../components/UsecaseSearch.jsx" @@ -43,6 +43,7 @@ import { ExpandLess as ExpandLessIcon, ExpandMore as ExpandMoreIcon, Publish as PublishIcon, + OpenInNew as OpenInNewIcon, } from "@material-ui/icons"; const EditWorkflow = (props) => { @@ -144,18 +145,41 @@ const EditWorkflow = (props) => { style: { backgroundColor: theme.palette.surfaceColor, color: "white", - minWidth: isMobile ? "90%" : newWorkflow === true ? 1000 : 550, - maxWidth: isMobile ? "90%" : newWorkflow === true ? 1000 : 550, + minWidth: isMobile ? "90%" : 550, + maxWidth: isMobile ? "90%" : 550, minHeight: 400, + //minWidth: isMobile ? "90%" : newWorkflow === true ? 1000 : 550, + //maxWidth: isMobile ? "90%" : newWorkflow === true ? 1000 : 550, }, }} >
- - {newWorkflow ? "New" : "Editing"} workflow - + Workflows can be built from scratch, or from templates. Usecases can help you discover next steps, and you can search for them directly. Learn more @@ -174,7 +198,7 @@ const EditWorkflow = (props) => {
: null}
- {newWorkflow === true ? + {/*newWorkflow === true ?
Use a Template @@ -183,7 +207,7 @@ const EditWorkflow = (props) => { Start your workflow from our templating system. This uses publied workflows from our Creators to generate full Usecases or parts of your Workflow.
- : null} + : null*/}
@@ -416,7 +440,7 @@ const EditWorkflow = (props) => {
- {newWorkflow === true ? + {/*newWorkflow === true ?
{ userdata={userdata} />
- : null} + : null*/} + {/* */} + (upload = ref)} + onChange={(event) => { + //const file = event.target.value + //const fileObject = URL.createObjectURL(actualFile) + //setFile(fileObject) + //const files = event.target.files[0] + uploadFiles(event.target.files); + }} + /> + + + {fileNamespaces !== undefined && + fileNamespaces !== null && + fileNamespaces.length > 1 ? ( + + File Category + + + ) : null} +
+ {renderTextBox ? + + + + + : + + + } + {renderTextBox && { + handleKeyDown(event); + }} + InputProps={{ + style: { + color: "white", + }, + }} + color="primary" + placeholder="File category name" + required + margin="dense" + defaultValue={""} + autoFocus + />}
+ + + + + + + + + + + + + + + + {files === undefined || files === null || files.length === 0 ? null : + files.map((file, index) => { + if (file.namespace === "") { + file.namespace = "default"; + } + + if (file.namespace !== selectedNamespace) { + return null; + } + + var bgColor = "#27292d"; + if (index % 2 === 0) { + bgColor = "#1f2023"; + } + + const filenamesplit = file.filename.split(".") + const iseditable = file.filesize < 2000000 && file.status === "active" && allowedFileTypes.includes(filenamesplit[filenamesplit.length-1]) + + return ( + + + + + + + ) : ( + + + + + + + + + + ) + } + style={{ + minWidth: 100, + maxWidth: 100, + overflow: "hidden", + }} + /> + + + + + + + { + setOpenEditor(true) + setOpenFileId(file.id) + readFileData(file) + }} + > + + + + + + + { + downloadFile(file); + }} + > + + + + + + { + const elementName = "copy_element_shuffle"; + var copyText = + document.getElementById(elementName); + if ( + copyText !== null && + copyText !== undefined + ) { + const clipboard = navigator.clipboard; + if (clipboard === undefined) { + alert.error( + "Can only copy over HTTPS (port 3443)" + ); + return; + } + + navigator.clipboard.writeText(file.id); + copyText.select(); + copyText.setSelectionRange( + 0, + 99999 + ); /* For mobile devices */ + + /* Copy the text inside the text field */ + document.execCommand("copy"); + + alert.info(file.id + " copied to clipboard"); + } + }} + > + + + + + + { + deleteFile(file); + }} + > + + + + + + style={{ + minWidth: 250, + maxWidth: 250, + // overflow: "hidden", + }} + /> + + ); + }) + } + +
+ + ) +} + +export default Files; diff --git a/frontend/src/components/FooterNew.js b/frontend/src/components/FooterNew.js old mode 100644 new mode 100755 diff --git a/frontend/src/components/Header.js b/frontend/src/components/Header.js old mode 100644 new mode 100755 index 6aee8626..0934f2e6 --- a/frontend/src/components/Header.js +++ b/frontend/src/components/Header.js @@ -869,6 +869,7 @@ const Header = (props) => { width: "100%", position: "fixed", minHeight: 60, + maxHeight: 60, top: 0, zIndex: 10000, backgroundColor: "inherit", diff --git a/frontend/src/components/Header.jsx b/frontend/src/components/Header.jsx new file mode 100644 index 00000000..975df6b7 --- /dev/null +++ b/frontend/src/components/Header.jsx @@ -0,0 +1,1028 @@ +import React, {useState} from 'react'; +import {BrowserView, MobileView} from "react-device-detect"; +import { useTheme } from '@material-ui/core/styles'; + +import {Link} from 'react-router-dom'; +import ReactGA from 'react-ga4'; + +import { + Paper, + Typography, + Badge, + Tooltip, + List, + ListItem, + Avatar, + Menu, + MenuItem, + Select, + Button, + Grid, + IconButton, + Divider, + LinearProgress, +} from '@material-ui/core' + +import { + MeetingRoom as MeetingRoomIcon, + HelpOutline as HelpOutlineIcon, + Settings as SettingsIcon, + Notifications as NotificationsIcon, + Home as HomeIcon, + Polymer as PolymerIcon, + Apps as AppsIcon, + Description as DescriptionIcon, + EmojiObjects as EmojiObjectsIcon, + Business as BusinessIcon, +} from '@material-ui/icons'; + +import { + Analytics as AnalyticsIcon, + Lightbulb as LightbulbIcon, +} from "@mui/icons-material"; + +import { useAlert } from "react-alert"; + +import SearchField from '../components/Searchfield.jsx' +const hoverColor = "#f85a3e" +const hoverOutColor = "#e8eaf6" + +const Header = props => { +const { globalUrl, setNotifications, notifications, isLoggedIn, removeCookie, homePage, userdata, serverside, } = props; + const theme = useTheme(); + const alert = useAlert() + + + const [HomeHoverColor, setHomeHoverColor] = useState(hoverOutColor); + const [SoarHoverColor, setSoarHoverColor] = useState(hoverOutColor); + const [LoginHoverColor, setLoginHoverColor] = useState(hoverOutColor); + const [DocsHoverColor, setDocsHoverColor] = useState(hoverOutColor); + const [HelpHoverColor, setHelpHoverColor] = useState(hoverOutColor); + const [anchorEl, setAnchorEl] = React.useState(null); + const [anchorElAvatar, setAnchorElAvatar] = React.useState(null); + const [subAnchorEl, setSubAnchorEl] = React.useState(null); + + + const handleClick = (event) => { + setAnchorEl(event.currentTarget); + }; + + const handleClose = () => { + setAnchorEl(null); + setAnchorElAvatar(null); + }; + + const hrefStyle = { + color: hoverOutColor, + textDecoration: "none", + } + + const isCloud = serverside === true || typeof window === 'undefined' ? true : window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + + const clearNotifications = () => { + // Don't really care about the logout + fetch(`${globalUrl}/api/v1/notifications/clear`, { + credentials: "include", + method: 'GET', + headers: { + 'Content-Type': 'application/json', + }, + }) + .then(function(response) { + if (response.status !== 200) { + console.log("Error in response") + } + + return response.json(); + }).then(function(responseJson) { + if (responseJson.success === true) { + setNotifications([]) + handleClose() + } else { + alert.error("Failed dismissing notifications. Please try again later.") + } + }) + .catch(error => { + console.log("error in notification dismissal: ", error) + //removeCookie("session_token", {path: "/"}) + }) + } + + const dismissNotification = (alert_id) => { + // Don't really care about the logout + fetch(`${globalUrl}/api/v1/notifications/${alert_id}/markasread`, { + credentials: "include", + method: 'GET', + headers: { + 'Content-Type': 'application/json', + }, + }) + .then(function(response) { + if (response.status !== 200) { + console.log("Error in response") + } + + return response.json(); + }).then(function(responseJson) { + if (responseJson.success === true) { + const newNotifications = notifications.filter(data => data.id !== alert_id) + console.log("NEW NOTIFICATIONS: ", newNotifications) + setNotifications(newNotifications) + } else { + alert.error("Failed dismissing notification. Please try again later.") + } + }) + .catch(error => { + console.log("error in notification dismissal: ", error) + //removeCookie("session_token", {path: "/"}) + }) + } + + // DEBUG HERE + const handleClickLogout = () => { + console.log("SHOULD LOG OUT") + + // Don't really care about the logout + fetch(globalUrl+"/api/v1/logout", { + credentials: "include", + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }) + .then(() => { + // Log out anyway + removeCookie("session_token", {path: "/"}) + removeCookie("session_token", {path: "/"}) + removeCookie("session_token", {path: "/"}) + removeCookie("session_token", {path: "/"}) + window.location.pathname = "/" + }) + .catch(error => { + console.log(error) + }); + } + + // Rofl this is weird + const handleDocsHover = () => { + setDocsHoverColor(hoverColor) + } + + const handleDocsHoverOut = () => { + setDocsHoverColor(hoverOutColor) + } + + const handleHomeHover = () => { + setHomeHoverColor(hoverColor) + } + + const handleHelpHover = () => { + setHelpHoverColor(hoverColor) + } + + const handleHelpHoverOut = () => { + setHelpHoverColor(hoverOutColor) + } + + const handleSoarHover = () => { + setSoarHoverColor(hoverColor) + } + + const handleSoarHoverOut = () => { + setSoarHoverColor(hoverOutColor) + } + + const handleHomeHoverOut = () => { + setHomeHoverColor(hoverOutColor) + } + + const handleLoginHover = () => { + setLoginHoverColor(hoverColor) + } + + const handleLoginHoverOut = () => { + setLoginHoverColor(hoverOutColor) + } + + // Should be based on some path + const logoCheck = !homePage ? null : null + + const notificationWidth = 300 + const imagesize = 22; + const boxColor = "#86c142"; + const NotificationItem = (props) => { + const {data} = props + + var image = ""; + var orgName = ""; + var orgId = ""; + if (userdata.orgs !== undefined) { + const foundOrg = userdata.orgs.find((org) => org.id === data["org_id"]); + if (foundOrg !== undefined && foundOrg !== null) { + //position: "absolute", bottom: 5, right: -5, + const imageStyle = { + width: imagesize, + height: imagesize, + pointerEvents: "none", + marginLeft: data.creator_org !== undefined && data.creator_org.length > 0 ? 20 : 0, + borderRadius: 10, + border: foundOrg.id === userdata.active_org.id ? `3px solid ${boxColor}` : null, + cursor: "pointer", + marginRight: 10, + }; + + image = + foundOrg.image === "" ? ( + {foundOrg.name} + ) : ( + {foundOrg.name} {}} + /> + ); + + orgName = foundOrg.name; + orgId = foundOrg.id; + } + } + + return ( + + {/* + {new Date(data.updated_at).toISOString()} + */} + {data.reference_url !== undefined && data.reference_url !== null && data.reference_url.length > 0 ? + + + {data.title} + + + : + + {data.title} + + } + + {data.image !== undefined && data.image !== null && data.image.length > 0 ? + {data.title} + : + null + } + + {data.description} + + {/*data.tags !== undefined && data.tags !== null && data.tags.length > 0 ? + data.tags.map((tag, index) => { + return ( + { + }} + variant="outlined" + color="primary" + /> + ) + }) + : null */} +
+ {data.read === false ? + + : null} + +
{ + }} + > + {image} +
+ +
+ + ) + } + + + + const notificationMenu = + + { + setAnchorEl(event.currentTarget); + }}> + + + + + { + handleClose() + }} + > + +
+ + Your Notifications ({notifications.length}) + + {notifications.length > 1 ? + + : null} +
+ + Notifications are made by Shuffle to help you discover issues or improvements. + + + {notifications.map((data, index) => { + return ( + + ) + })} + + + + const handleClickChangeOrg = (orgId) => { + // Don't really care about the logout + //name: org.name, + //orgId = "asd" + const data = { + org_id: orgId, + } + + localStorage.setItem("globalUrl", "") + localStorage.setItem("getting_started_sidebar", "open"); + + fetch(`${globalUrl}/api/v1/orgs/${orgId}/change`, { + mode: 'cors', + credentials: 'include', + crossDomain: true, + method: 'POST', + body: JSON.stringify(data), + withCredentials: true, + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + }) + .then(function(response) { + if (response.status !== 200) { + console.log("Error in response") + } + + return response.json(); + }).then(function(responseJson) { + if (responseJson.success === true) { + if (responseJson.region_url !== undefined && responseJson.region_url !== null && responseJson.region_url.length > 0) { + console.log("Region Change: ", responseJson.region_url) + localStorage.setItem("globalUrl", responseJson.region_url) + //globalUrl = responseJson.region_url + } + + setTimeout(() => { + window.location.reload() + }, 2000) + alert.success("Successfully changed active organization - refreshing!") + } else { + alert.error("Failed changing org: ", responseJson.reason) + } + }) + .catch(error => { + console.log("error changing: ", error) + //removeCookie("session_token", {path: "/"}) + }) + } + + const supportMenu = + + + + {}}> + Discord Community Join + + + + + + // Should be based on some path + const parsedAvatar = userdata.avatar !== undefined && userdata.avatar !== null && userdata.avatar.length > 0 ? userdata.avatar : "" + + const avatarMenu = + + { + setAnchorElAvatar(event.currentTarget); + }}> + + + { + handleClose() + }} + > + + { + handleClose(); + }} + > + Admin + + + + + { + handleClose(); + }} + > + About + + + {/* + + { + handleClose(); + }} + > + Get Started + + + */} + + { + handleClose(); + }} + > + Use Cases + + + + + { + handleClose() + }}> + Creator page + + + + { + handleClose() + }}> + Settings + + + + { + handleClickLogout() + event.preventDefault() + handleClose() + }}> +  Logout + + + + + const listItemStyle = { + textAlign: "center", + marginTop: "auto", + marginBottom: "auto", + } + + // Handle top bar or something + const loginTextBrowser = !isLoggedIn ? +
+
+ + + + + + + + + + + + + {/* + + + + + + */} + {isCloud ? + + + + + + : null} + {isCloud ? + + + + + + : + + + + + + } + + +
+
+ +
+
+ + + + + + + + {isCloud ? + + + + + + : null} + + {/* + + {supportMenu} + + */} + + {/* + + + + + + */} + +
+
+ : +
+
+
+ + + +
+ + + logo + {/* + + */} + + +
+ + + + +
+ {/* + + */} + Workflows +
+ + + + +
+ {/* + + */} + Apps +
+ + + {/* + + +
Dashboard
+ + + */} + + +
+ {/* + + */} + Docs +
+ + + {/* + + +
+ + Pricing +
+ + + */} + {/* + + +
Configure
+ + + */} + +
+
+ +
+
+ + + {avatarMenu} + {notificationMenu} + {/*supportMenu*/} + {logoCheck} + + + {/* + + + + + + */} + + {/* + + + + + + */} + + {/*userdata.app_execution_limit !== 5000 && userdata.app_execution_limit !== 10000 ? + userdata === undefined || userdata.orgs === undefined || userdata.orgs === null || userdata.orgs.length > 1 ? null : + + + + + + : null*/} + + + + {userdata === undefined || userdata.orgs === undefined || userdata.orgs === null || userdata.orgs.length <= 1 ? + null + : + + + + } + + {userdata === undefined || userdata.app_execution_limit === undefined || userdata.app_execution_usage === undefined || userdata.app_execution_usage < 1000 ? + null + : + +
{ + if (window.drift !== undefined) { + window.drift.api.startInteraction({ interactionId: 326905 }) + } else { + console.log("Couldn't find drift in window.drift and not .drift-open-chat with querySelector: ", window.drift) + } + }}> + + + {(userdata.app_execution_usage/userdata.app_execution_limit*100).toFixed(0)}% + + + +
+ + } + + +
+
+
+ + const loginTextMobile = !isLoggedIn ? +
+ + + +
+ + + logo + {/**/} + + +
+ + + + +
+ About +
+ + + + + + + + + + + + + +
+ : +
+
+ + + +
+ + + + + +
+ + + + + + + + + + + + + +
+
+ + +
+ Logout +
+ + {logoCheck} + + +
+
+ + // + const loadedCheck = +
+ + {loginTextBrowser} + + + {loginTextMobile} + +
+ //
+ return ( +
+ {loadedCheck} +
+ ) +} + +export default Header; diff --git a/frontend/src/components/LandingpageUsecases.jsx b/frontend/src/components/LandingpageUsecases.jsx index 49a4f332..9fd5d016 100644 --- a/frontend/src/components/LandingpageUsecases.jsx +++ b/frontend/src/components/LandingpageUsecases.jsx @@ -2,7 +2,7 @@ import React, { useState, useEffect } from 'react'; import {isMobile} from "react-device-detect"; import AppFramework, { usecases } from "../components/AppFramework.jsx"; import {Link} from 'react-router-dom'; -import ReactGA from 'react-ga'; +import ReactGA from 'react-ga4'; import { Button, LinearProgress, Typography } from '@material-ui/core'; diff --git a/frontend/src/components/LoginPopup.js b/frontend/src/components/LoginPopup.js old mode 100644 new mode 100755 diff --git a/frontend/src/components/NestedMenu.jsx b/frontend/src/components/NestedMenu.jsx old mode 100644 new mode 100755 diff --git a/frontend/src/components/Newsletter.jsx b/frontend/src/components/Newsletter.jsx new file mode 100644 index 00000000..ccadf77b --- /dev/null +++ b/frontend/src/components/Newsletter.jsx @@ -0,0 +1,102 @@ +import React, {useState} from 'react'; +import { useTheme } from '@material-ui/core/styles'; +import {isMobile} from "react-device-detect"; +import ReactGA from 'react-ga4'; + +import {TextField, Typography, Button} from '@material-ui/core'; + +const Newsletter = (props) => { + const { globalUrl, } = props; + + const theme = useTheme(); + const [email, setEmail] = useState(""); + const [msg, setMsg] = useState(""); + const [buttonActive, setButtonActive] = useState(true); + const buttonStyle = {minWidth: 300, borderRadius: 30, height: 60, width: 140, margin: isMobile ? "15px auto 15px auto" : "20px 20px 20px 10px", fontSize: 18,} + + const newsletterSignup = (inemail) => { + if (inemail.length < 4) { + setMsg("Invalid email") + setButtonActive(true) + return + } + + setButtonActive(false) + const data = {"email": inemail} + const url = globalUrl+'/api/v1/functions/newsletter_signup' + fetch(url, { + method: 'POST', + body: JSON.stringify(data), + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + }) + .then(response => + response.json().then(responseJson => { + setButtonActive(true) + setMsg(responseJson["reason"]) + if (responseJson["success"] === false) { + } else { + setEmail("") + } + }), + ) + .catch(error => { + setMsg("Something went wrong: ", error.toString()) + setButtonActive(true) + }); + } + + return ( +
+ + Security Automation Newsletter + + + Defensive security is 99% noise. Join us to sift through it. + +
+ { + setEmail(e.target.value) + }} + placeholder="Your email" + id="standard-required" + margin="normal" + variant="outlined" + /> +
+ +
+ {msg} +
+ ) +} + + +export default Newsletter; diff --git a/frontend/src/components/Oauth2Auth.jsx b/frontend/src/components/Oauth2Auth.jsx old mode 100644 new mode 100755 index fcb03f23..e33a8f1f --- a/frontend/src/components/Oauth2Auth.jsx +++ b/frontend/src/components/Oauth2Auth.jsx @@ -1,6 +1,8 @@ import React, { useRef, useState, useEffect, useLayoutEffect } from "react"; +import { useParams, useNavigate, Link } from "react-router-dom"; import { useTheme } from "@material-ui/core/styles"; -import theme from '../theme'; +import theme from '../theme.jsx'; +import { useAlert } from "react-alert"; import { v4 as uuidv4 } from "uuid"; import { @@ -69,6 +71,13 @@ const registeredApps = [ "todoist", "microsoft_sentinel", "microsoft_365_defender", + "google_sheets", + "google_drive", + "google_disk", + "jira", + "jira_service_desk", + "jira_service_management", + "github", ] const AuthenticationOauth2 = (props) => { @@ -82,11 +91,15 @@ const AuthenticationOauth2 = (props) => { appAuthentication, setSelectedAction, setNewAppAuth, - setAuthenticationModalOpen, isCloud, autoAuth, + authButtonOnly, + isLoggedIn, } = props; + let navigate = useNavigate(); + const alert = useAlert() + //const [update, setUpdate] = React.useState("|") const [defaultConfigSet, setDefaultConfigSet] = React.useState( authenticationType.client_id !== undefined && @@ -107,10 +120,9 @@ const AuthenticationOauth2 = (props) => { const [buttonClicked, setButtonClicked] = React.useState(false); const [offlineAccess, setOfflineAccess] = React.useState(true); - const allscopes = - authenticationType.scope !== undefined ? authenticationType.scope : []; + const allscopes = authenticationType.scope !== undefined ? authenticationType.scope : []; + - console.log("ALLSCOPES: ", allscopes) const [selectedScopes, setSelectedScopes] = React.useState(allscopes.length === 1 ? [allscopes[0]] : []) const [manuallyConfigure, setManuallyConfigure] = React.useState( defaultConfigSet ? false : true @@ -121,15 +133,20 @@ const AuthenticationOauth2 = (props) => { label: "", usage: [ { - workflow_id: workflow.id, + workflow_id: workflow !== undefined ? workflow.id : "", }, ], id: uuidv4(), active: true, }); + useEffect(() => { - console.log("Should automatically click the auto-auth button?") + if (isLoggedIn === false) { + navigate(`/login?view=${window.location.pathname}&message=Log in to authenticate this app`) + } + + console.log("Should automatically click the auto-auth button?: ", autoAuth) if (autoAuth === true && selectedApp !== undefined) { startOauth2Request() } @@ -140,26 +157,31 @@ const AuthenticationOauth2 = (props) => { } const startOauth2Request = (admin_consent) => { - console.log("APP: ", selectedApp) + // Admin consent also means to add refresh tokens + console.log("Inside oauth2 request for app: ", selectedApp.name) + selectedApp.name = selectedApp.name.replace(" ", "_").toLowerCase() + + //console.log("APP: ", selectedApp) if (selectedApp.name.toLowerCase() == "outlook_graph" || selectedApp.name.toLowerCase() == "outlook_office365") { handleOauth2Request( - "efe4c3fe-84a1-4821-a84f-23a6cfe8e72d", "", "https://graph.microsoft.com", - ["Mail.ReadWrite"], + ["Mail.ReadWrite", "Mail.Send", "offline_access"], admin_consent, ); } else if (selectedApp.name.toLowerCase() == "gmail") { handleOauth2Request( - "253565968129-c0a35knic7q1pdk6i6qk9gdkvr07ci49.apps.googleusercontent.com", + "253565968129-6ke8086pkp0at16m8t95rdcsas69ngt1.apps.googleusercontent.com", "", "https://gmail.googleapis.com", ["https://www.googleapis.com/auth/gmail.modify", "https://www.googleapis.com/auth/gmail.send", "https://www.googleapis.com/auth/gmail.insert", - "https://www.googleapis.com/auth/gmail.compose"], + "https://www.googleapis.com/auth/gmail.compose", + ], admin_consent, + "select_account%20consent", ) } else if (selectedApp.name.toLowerCase() == "zoho_desk") { handleOauth2Request( @@ -169,15 +191,16 @@ const AuthenticationOauth2 = (props) => { ["Desk.tickets.READ", "Desk.tickets.UPDATE", "Desk.tickets.DELETE", - "Desk.tickets.CREATE"], + "Desk.tickets.CREATE", + "offline_access"], admin_consent, ) } else if (selectedApp.name.toLowerCase() == "slack") { handleOauth2Request( - "151779186901.2448678750935", + "5155508477298.5168162485601", "", "https://slack.com", - ["admin", "chat:write", "im:read", "im:write", "search:read", "usergroups:read", "usergroups:write"], + ["chat:write:user", "im:read", "im:write", "search:read", "usergroups:read", "usergroups:write",], admin_consent, ) } else if (selectedApp.name.toLowerCase() == "webex") { @@ -193,7 +216,7 @@ const AuthenticationOauth2 = (props) => { "31cb4c84-658e-43d5-ae84-22c9142e967a", "", "https://graph.microsoft.com", - ["ChannelMessage.Edit", "ChannelMessage.Read.All", "ChannelMessage.Send", "Chat.Create", "Chat.ReadWrite", "Chat.Read"], + ["ChannelMessage.Edit", "ChannelMessage.Read.All", "ChannelMessage.Send", "Chat.Create", "Chat.ReadWrite", "Chat.Read", "offline_access"], admin_consent, ) } else if (selectedApp.name.toLowerCase().includes("todoist")) { @@ -201,7 +224,7 @@ const AuthenticationOauth2 = (props) => { "35fa3a384040470db0c8527e90a3c2eb", "", "https://api.todoist.com", - ["task:add"], + ["task:add",], admin_consent, ) } else if (selectedApp.name.toLowerCase().includes("microsoft_sentinel")) { @@ -209,7 +232,7 @@ const AuthenticationOauth2 = (props) => { "4c16e8c4-3d34-4aa1-ac94-262ea170b7f7", "", "https://management.azure.com", - ["https://management.azure.com/user_impersonation"], + ["https://management.azure.com/user_impersonation",], admin_consent, ) } else if (selectedApp.name.toLowerCase().includes("microsoft_365_defender")) { @@ -217,16 +240,53 @@ const AuthenticationOauth2 = (props) => { "4c16e8c4-3d34-4aa1-ac94-262ea170b7f7", "", "https://graph.microsoft.com", - ["SecurityEvents.ReadWrite.All"], + ["SecurityEvents.ReadWrite.All",], admin_consent, ) + } else if (selectedApp.name.toLowerCase().includes("google_sheets")) { + handleOauth2Request( + "253565968129-mppu17aciek8slr3kpgnb37hp86dmvmb.apps.googleusercontent.com", + "", + "https://sheets.googleapis.com", + ["https://www.googleapis.com/auth/spreadsheets"], + admin_consent, + "consent", + ) + } else if (selectedApp.name.toLowerCase().includes("google_drive") || selectedApp.name.toLowerCase().includes("google_disk")) { + handleOauth2Request( + "253565968129-6pij4g6ojim4gpum0h9m9u3bc357qsq7.apps.googleusercontent.com", + "", + "https://www.googleapis.com", + ["https://www.googleapis.com/auth/drive",], + admin_consent, + "consent", + ) + } else if (selectedApp.name.toLowerCase().includes("jira_service_desk") || selectedApp.name.toLowerCase().includes("jira") || selectedApp.name.toLowerCase().includes("jira_service_management")) { + handleOauth2Request( + "AI02egeCQh1Zskm1QAJaaR6dzjR97V2F", + "", + "https://api.atlassian.com", + ["read:jira-work", "write:jira-work", "read:servicedesk:jira-service-management", "write:servicedesk:jira-service-management", "read:request:jira-service-management", "write:request:jira-service-management",], + admin_consent, + ) + } else if (selectedApp.name.toLowerCase().includes("github")) { + handleOauth2Request( + "3d272b1b782b100b1e61", + "", + "https://api.github.com", + ["repo","user","project","notifications",], + admin_consent, + ) + } else { + console.log("No match found for: ", selectedApp.name) } + // write:request:jira-service-management } - const handleOauth2Request = (client_id, client_secret, oauth_url, scopes, admin_consent) => { + const handleOauth2Request = (client_id, client_secret, oauth_url, scopes, admin_consent, prompt) => { setButtonClicked(true); - console.log("SCOPES: ", scopes); + //console.log("SCOPES: ", scopes); client_id = client_id.trim() client_secret = client_secret.trim() @@ -234,8 +294,11 @@ const AuthenticationOauth2 = (props) => { var resources = ""; if (scopes !== undefined && (scopes !== null) & (scopes.length > 0)) { + console.log("IN scope 1") if (offlineAccess === true && !scopes.includes("offline_access")) { - if (authenticationType.redirect_uri.includes("microsoft")) { + + console.log("IN scope 2") + if (!authenticationType.redirect_uri.includes("google")) { console.log("Appending offline access") scopes.push("offline_access") } @@ -249,7 +312,53 @@ const AuthenticationOauth2 = (props) => { //console.log("AUTH: ", authenticationType) //console.log("SCOPES2: ", resources) const redirectUri = `${window.location.protocol}//${window.location.host}/set_authentication`; - var state = `workflow_id%3D${workflow.id}%26reference_action_id%3d${selectedAction.app_id}%26app_name%3d${selectedAction.app_name}%26app_id%3d${selectedAction.app_id}%26app_version%3d${selectedAction.app_version}%26authentication_url%3d${authentication_url}%26scope%3d${resources}%26client_id%3d${client_id}%26client_secret%3d${client_secret}`; + const workflowId = workflow !== undefined ? workflow.id : ""; + var state = `workflow_id%3D${workflowId}%26reference_action_id%3d${selectedAction.app_id}%26app_name%3d${selectedAction.app_name}%26app_id%3d${selectedAction.app_id}%26app_version%3d${selectedAction.app_version}%26authentication_url%3d${authentication_url}%26scope%3d${resources}%26client_id%3d${client_id}%26client_secret%3d${client_secret}`; + + + // This is to make sure authorization can be handled WITHOUT being logged in, + // kind of making it act like an api key + // https://shuffler.io/authorization -> 3rd party integration auth + const urlParams = new URLSearchParams(window.location.search); + const userAuth = urlParams.get("authorization"); + if (userAuth !== undefined && userAuth !== null && userAuth.length > 0) { + console.log("Adding authorization from user side") + state += `%26authorization%3d${userAuth}`; + } + // write:request:jira-service-management + } + + + const handleOauth2Request = (client_id, client_secret, oauth_url, scopes, admin_consent, prompt) => { + setButtonClicked(true); + //console.log("SCOPES: ", scopes); + + client_id = client_id.trim() + client_secret = client_secret.trim() + oauth_url = oauth_url.trim() + + var resources = ""; + if (scopes !== undefined && (scopes !== null) & (scopes.length > 0)) { + console.log("IN scope 1") + if (offlineAccess === true && !scopes.includes("offline_access")) { + + console.log("IN scope 2") + if (!authenticationType.redirect_uri.includes("google")) { + console.log("Appending offline access") + scopes.push("offline_access") + } + } + + resources = scopes.join(" "); + //resources = scopes.join(","); + } + + const authentication_url = authenticationType.token_uri; + //console.log("AUTH: ", authenticationType) + //console.log("SCOPES2: ", resources) + const redirectUri = `${window.location.protocol}//${window.location.host}/set_authentication`; + const workflowId = workflow !== undefined ? workflow.id : ""; + var state = `workflow_id%3D${workflowId}%26reference_action_id%3d${selectedAction.app_id}%26app_name%3d${selectedAction.app_name}%26app_id%3d${selectedAction.app_id}%26app_version%3d${selectedAction.app_version}%26authentication_url%3d${authentication_url}%26scope%3d${resources}%26client_id%3d${client_id}%26client_secret%3d${client_secret}`; if (oauth_url !== undefined && oauth_url !== null && oauth_url.length > 0) { state += `%26oauth_url%3d${oauth_url}`; console.log("ADDING OAUTH2 URL: ", state); @@ -266,13 +375,51 @@ const AuthenticationOauth2 = (props) => { } // No prompt forcing - var url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=login&scope=${resources}&state=${state}&access_type=offline`; + //var url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=login&scope=${resources}&state=${state}&access_type=offline`; + var defaultPrompt = "login" + if (prompt !== undefined && prompt !== null && prompt.length > 0) { + defaultPrompt = prompt + } + // Check for org_id + const orgId = urlParams.get("org_id"); + if (orgId !== undefined && orgId !== null && orgId.length > 0) { + console.log("Adding org_id from user side") + state += `%26org_id%3d${orgId}`; + } + + if (oauth_url !== undefined && oauth_url !== null && oauth_url.length > 0) { + state += `%26oauth_url%3d${oauth_url}`; + console.log("ADDING OAUTH2 URL: ", state); + } + + + if ( + authenticationType.refresh_uri !== undefined && + authenticationType.refresh_uri !== null && + authenticationType.refresh_uri.length > 0 + ) { + state += `%26refresh_uri%3d${authenticationType.refresh_uri}`; + } else { + state += `%26refresh_uri%3d${authentication_url}`; + } + + // No prompt forcing + //var url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=login&scope=${resources}&state=${state}&access_type=offline`; + var defaultPrompt = "login" + if (prompt !== undefined && prompt !== null && prompt.length > 0) { + defaultPrompt = prompt + } + + var url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=${defaultPrompt}&scope=${resources}&state=${state}&access_type=offline`; + if (admin_consent === true) { console.log("Running Oauth2 WITH admin consent") //url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=consent&scope=${resources}&state=${state}&access_type=offline`; url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=admin_consent&scope=${resources}&state=${state}&access_type=offline`; } + console.log("URL: ", url) + // Force new consent //const url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&scope=${resources}&prompt=consent&state=${state}&access_type=offline`; @@ -285,7 +432,7 @@ const AuthenticationOauth2 = (props) => { // How can we get a callback properly realtime? // How can we properly try-catch without breaks on error? try { - var newwin = window.open(url, "", "width=800,height=600"); + var newwin = window.open(url, "", "width=582,height=700"); //console.log(newwin) var open = true; @@ -293,15 +440,15 @@ const AuthenticationOauth2 = (props) => { if (newwin.closed) { console.log("Closing?") - if (setAuthenticationModalOpen !== undefined) { - setAuthenticationModalOpen(false) - } setButtonClicked(false); clearInterval(timer); //alert('"Secure Payment" window closed!'); + // - getAppAuthentication(true, true); + if (getAppAuthentication !== undefined) { + getAppAuthentication(true, true, true); + } } else { console.log("Not closed") } @@ -376,9 +523,7 @@ const AuthenticationOauth2 = (props) => { ] = "false"; } else { alert.info( - "Field " + - selectedApp.authentication.parameters[key].name + - " can't be empty" + "Field " + selectedApp.authentication.parameters[key].name.replace("_basic", "", -1).replace("_", " ", -1) + " can't be empty" ); return; } @@ -446,7 +591,59 @@ const AuthenticationOauth2 = (props) => { authenticationOption.label = selectedApp.name + " authentication"; } - //console.log( + + const autoAuthButton = + + + if (authButtonOnly === true) { + return autoAuthButton + } + return (
@@ -456,11 +653,11 @@ const AuthenticationOauth2 = (props) => { - Oauth2 requires a client ID and secret to authenticate, defined in the remote system. Your redirect URL is https://shuffler.io/set_authentication. + Oauth2 requires a client ID and secret to authenticate, defined in the remote system. Your redirect URL is {window.location.origin}/set_authentication -  {" "} @@ -472,52 +669,8 @@ const AuthenticationOauth2 = (props) => { {isCloud && registeredApps.includes(selectedApp.name.toLowerCase()) ? - + {autoAuthButton} + {buttonClicked ? null : @@ -543,7 +696,8 @@ const AuthenticationOauth2 = (props) => { onClick={() => { // Hardcode some stuff? // This could prolly be added to the app itself with a "default" client ID - startOauth2Request(true) + //startOauth2Request(true) + startOauth2Request() }} color="primary" > diff --git a/frontend/src/components/OrgHeader.js b/frontend/src/components/OrgHeader.js new file mode 100755 index 00000000..65dd6e97 --- /dev/null +++ b/frontend/src/components/OrgHeader.js @@ -0,0 +1,432 @@ +import React, { useEffect} from 'react'; +import { makeStyles } from '@material-ui/styles'; +import { useTheme } from '@material-ui/core/styles'; + +import Tooltip from '@material-ui/core/Tooltip'; +import Grid from '@material-ui/core/Grid'; +import Button from '@material-ui/core/Button'; +import TextField from '@material-ui/core/TextField'; +import Typography from '@material-ui/core/Typography'; +import { useAlert } from "react-alert"; +import IconButton from '@material-ui/core/IconButton'; +import ExpandLessIcon from '@material-ui/icons/ExpandLess'; +import ExpandMoreIcon from '@material-ui/icons/ExpandMore'; +import SaveIcon from '@material-ui/icons/Save'; + +const useStyles = makeStyles({ + notchedOutline: { + borderColor: "#f85a3e !important" + }, +}) + +const defaultImage = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAK4AAACuCAYAAACvDDbuAAAgAElEQVR4Xu19e9CvV1Xe3r/vnJOcBEhBSgMEBaoUK9POCOVmAuP0HwcUCNYZSUsh9xv3hGl1qNippQRE20IFEhQoBJiaKVpEgQRnhD+0QHSmRkAsxE4doBZQTs71u/zezruv6/Ksvffvkn/qd8bBfN/3XvZe+1nPevbaa+/XuxX/Tbe6C/ece8qOd5dNk3um9+7Jk/OPds49zE3uyPy4ST5zCr/y4f+sfxO4j16vHsqfmV7gpgm8Yzm/lP9+km1B9+W2zn/zzk2sDeR55ffy3enn1Lf5J/1e3bb42kX43/Do1nt9fUdpLrSbbFt8vvls+idlm/qsaJPWuNK/TfvOLU44577pnPuSc9PvT95/9szu3p9c/IH/c2oVKDbeyB8z/Yz7noNd9zzn3U+5hX+6m9wjnXM7GqXCHpbR0+PnjudGxEtBkzRo02vFtRB83rkAXPqPGD4MnmWGGa3CDqp9+pp4Bwd2dCwPzIXur6D1xaGRXaQz8nf4Kfix1/3joDXtDm0jHVbYspirbZdkj4Npcv/XO/8F59xdfv/o7zz0A1/9yxEAd4E7/by74OCke5Fb+Bvd5J7unDvGHmyCKiMzX40ByW+XQCxvyoSafgGMBT0fgTayXmA/kykr8PI1kS0JIOMPRiSJ7ctOWfuI+mcw4Uj7lO25Y0TQoyHGbKsjJbqXR5HoKijK6nuz3bPhokEX+f4956Z7nXP/8cz+w//bxR/4H00GbgJ3ep170sHC/WxgWecvUJ5AemrLA4NF2cCAa4BDSCbjYVR2RYN2ZXlAOgxZqcXUDLh9toyvSkybopAJ+KBbOpHADOEjoO1EkVYU6LB0HbPUDtaN8N7TzrmPOnfwry9631/8mcW+JnD3Xu+eu3DubZNzT1Xhu8eyxaXw47lWBIzcAW2fGSpoM1WPgzYNWqSRwqjsnSYokgMyvW5JnzWZNrQpC9tC+ZVZW1Ek9GbBQK8ZuS99mrrW1sPh7XXsTeDmqHrvYvKve+j77/8MAi9E1v6t7vnOu7c7555gaM4SuiHGOsajcV8xmeEUMSDFf4yJQLiUmpYNTg906iVCl7b6ltu4gi5lTMs6Z4RpwLQsVMP+IZaNTtYngRoNI1uOzwdYhJxtEhqKAFsdvoI0XHy/m3ZeedH7v/ZxCV7Vir1b3HP8wr3PBC1HDsog5IkJxSdDHAyBlRqBgyHwAANuI3sg3j4G+shSHASQaRlbZtDW8AmiT7b3TLJD8qACrWrwkclYg2l70qACWow5IZvQ9lHQUmeZ7p+8v/Lh7/3z36NDw6w7a9rljvvw5NwPWykOOuEono5YT8IvXQNBK5yhGnwFVoCgZYPYZgvSBtnGMMkJnR2ZrIh3Fga3mGoR9YgpL2xAjbGlBC2YTEGWJu01J3mkrwYGuFP25EF1XMFj9y6Wiyse9p+/9hXl2il78G7n3T9radomMVphpMdGUG9wAJRLYJ4Wz2CZ73QmUpkxa9jlA2fmKvNz0414Fm+Abw6b+U8D4Km+xfsb0l5E99Z+S4mApIHhaEz2bCAPSmMkaNMzW4Av94ac3vtPHdu96TG3f32evFXhuH+ru8J5f4dzrmYPVGiKN7Bfsx/syVj+C07AA3WQmlYBa4TR+bpWnhYCgjyLtN+OBrbezPdAwNcQCjq4EAsaCEAS8BF4PNIZoBKhWdo9PMO0DX5vbmGTRMi41SgyxrQlTcmsVQB+2jt/3UXv/9qdxVLTre5RBwv/m25yz+Qspe2NQWt5ZEQ5x79kMgzaoTC4DM9SuoqDvcEWSqKsNlnJHYvvQ5oWvRutiInrijdwALGBLZ003sEcMl7DSEdHoGTHyoQ4eqTxaqymzdmYuq40wrS1D5A80i+985/1u0de/LAPf+Vb4Y7917uXO+dud84fRcClIOKdVwPGQZRuNBuj6DujMHakCUCgadUsubeMS1A/vhRLBxZEgR6g8juLbUzAE1tWe8wslv9hh9HyQPknZNpEAsUQaJWPDgpttyCjsDxuTMQMwAOiUoTknNv1frrmovfd/wE/1x4cTP4jbuF+3AItwFe81ACGXm0SAC/5UTYGoSBAdwCwmZE9GAqhInmvBr8JPMmAo0zLB7G5YmdMALkTy/fOz5+NonO0VaJl0Bmyh3jDakzL2xL6JnLF8dGjk8xKDBWPtc3euY+dWT7kJX73VveMhfe/4Zy7ONOcgDpnP+a+iC1WLpZJ7RPhArLCPD7snZSVpnmGkgMZTdIrA5BIwOoHBkDLHWs1eRBMZ/Wr2FWGTcFmGQSMZYhjAInAB1C2WQIlrBGUi1JtUaJ8DHrtVIJtwyjZ6bxKjA3Q1n590y8Wl/uDW9wt08LfFgpmCCgx81FroU40KqhUvNLPYu+0NBSYiBUckjDHxtX0eCJJeqBN7cfh2WazKa/FK4MK+wGmHV/t40wbmyoBj8erYDQ5VGYC1VzFmBZpjUzEjGIjhREO5NS2A+f9v/T7t7q7nPc/qZPbYuhZeJdGL6lIMgFQ12gsJUBxwObLxP1KHujO+8nQZT15ANiOsrRcooZVXjDEyyovAzypbDKNQRifdSZiFXTI9m2JQCNBZcAcv+x7w9iF/zEACwlLZEd6ZZiFmWI7vJvu8gevd/dNk/+hPFDa02rwtXOZdZij8axlUoldwnilAYaRDKa1c5ixizxMVqfAEcUIo122RW2OCwv8/aPSotolLn5Y+nBkRWxk4aQuqGu7GOMxE5nPtQcN0DLggvGGYySjRR3L1L77/P6t/tvOuUc0gdvSZb0qL+hxtbSw/tkGma49SB3J2sl4R0SOBqNm+AYbFW9Hz0KgqIPIgWsCgHhzvaZ9rwYsjgKyzZiJMVmNVoiNsi3oWxi3BlkUwshMW7JO35mBu+dc3LmgwNvUfZVNasfH5AHVX/kOvHNBDzbXfX3jZkxnThliwSwtSswcLQDng7haFVUEWb99umAm6+5iyzA0yDbcntDmhQSwo2U5EvPzI3pWOzzDC/OaLtMm0Pm9GbhVpkD0hheX9hZwpxdWPYQ6iiqMweCgUMizB7nBYtXOeKeo0optXC1Mg8WFagMzAtV62mpKwCiC4miBdcCNSRg2aOnQNSu4Amn5MAi2LGyzoF2aaDtLflfCS2kDtBPRtBVf+cr4Dr+XgMs7Tn9CA86uNkDBr4k/AY0jQ0XRsoIdUqV9NXjLUXrywAJ8Yj2LBYrXovsrqMwI1Mhlhn6xUWq/I9tzTKsT1hNO15dNkqV5VFHAE7bTzpGep/6Afi8jXXWMBnBB2CeGXVUeaNAaYV5kD3IYLP5mFpTwlFSkx9VraWv1m+UYNqB46AL2Q6yf2lgr0HRoDX1PBTk1EliEgkO8dPhcLs+Ba9ybTRsuXmVFLI6Cwgoihh7gBcFx4JohKoli8nAJqGhczbIZQIWsynXCSA2mHQ6Don0YuHYY5HvEWsArMKs6by2Wthml9pkzeWwVilwI8KQPrdJEU/rI+xtsa/SfQwKQ1RRlKL1OLwpph6rAtUEbH5z+Xkm3LyFoOMtXwwkLBO1g8XgeSADa7COlzWqASHhKS0RU0rAwqJwyLrOGCUrD4XFKrtqusDRMefEVsfyaVR0ya17AK7GLrWXmsoxGioOKYXB4p0SjCA7ZyqpFNh3KTRG4LbZIf4vMKXa66hYKyh2ZJfNl3Poe6hhW9iCxDGkjBF6nAJwHi5EQT1hwrrBYYatOjkDM5I3FCxrJxmt9OaDs2oP2ZCqMd2jo6osL7SiJbcww1okCfu+WzNK5mXUjXjZafmAc4Lb2ZSzF1IOlafnzqIcaOw9qc0jnKhD6LMByu8Tbx3KhYiAt0ALDZ9uRqUJiAhm9UPagioTS15ZDkrqIEu0U0WBNW5+fbwDABflXjhMiXSxihDZK76x/I4+t7U3ARR1AtbRCR5mxR4TC1myarYhx7RaWcNkoU8t3KrXsMGOsqInoEJqCnA2F7x5Lz88m9m9FOMpu5LqxsksepfBuaquuAsiXYG5QT6vG3SCf1nVCMpSgGTqKHYqN/t4tCxN+sAjcvjopDvnSRiJcLOOW9fkm6LAnjw0sdryxe9Hyqnhetmyr5hSDNqGarLqxsxWkYxhEkydtyX6mz7P2aaerTVwNtENRT/Rfw6mzcJJu8BC4Bf55JEjnGsDV9bRGI1TdQZUgVVdZUUCyeSsKGJ7bOvfALD6viwsVEONOysOvBTxZTxtTekWDC5aSsixcR5weArcxEVt1qw1lxnVAW/ycSRhgU3Xhwmng1haE7BCuV2XSO/0AKn4QyMWKWKQaOYlrhArmVEL3mQOrwR0v7YX4PBlNxwT1nm+ExiwS2gfEVdCOti0afgTYluyR0my9ZVwbtNl17D1szPk6EsSF1FmYg00KuP0TCTVo1SY3GOo7RyL1QFFop2afMmzDrTgEF6eqbGSE93CBdphcT8tlUx/wehuR9V5UBL7i3rfEtPhQPQu08ffBqQoZrFZ7wHGWbGKNAxnfdeUB1dsMuCZoFTC0hGiHQnz4XGaMuN1jLOVVo8YIeMigRT/lk6Tig3bZIHcKW3pwdxZyxnTK0doDBHoJFM6erD1W9kFJwnFNC8HHcMLbB6NIa9LO5EP+obavAheE4HIvCvkpTKk/WUaqupY4ee6cAkSKrqjzOuzTkIkA1HRIw3iFaUHRi9K4lkRoRhEN2jjuKzpkM7cenpWVChnOFKuK1h/L08J0ngU+I11GMWUeu8pAi5eY/d7rYlahT/scEklbsf1JJNSmx3lv1dLyk1usMBqZsFpesEoFhhqcov1aEgJKmrktqQi8KT9Im0UYDMAeBC1tuLFVR/SNs/mcMsQBsV/WCEoTK2GAvvdDPHI6MfGuIVMcR4XvtRY/AHDJAyDTijBYrCZZU8oDYGATOCTE03BOHawJjDpoOERZuUyUORgJ0/Wa0qyGU2ikVU1rH/ckB9bapmT1TUeq8Xpa0L9evW/zFHZDcil5YGtuv5sYV64mSX6tPyPWMxqS5AFlzAIkU37USZJkWsYsEBi54ECwUiy1EA2XbRY7F0zH6NScNlkayYPVJ2J27QGaIOWRk7JrTB5Qs9UdvzpLwPCSbIBroBtEwAa4OVGc/O7rdjiENKAKfnRDRhcXJNs2JmJQG5FoaTKZZFlgIBOMaHFhtYlYn2lTaWIa4dijUdDSvnhwdBMBJyCEUqheAIWWcG25xh85tkAQMSijd4dpw4uMsknO8EPAZTWVZWJiMd78brAiVjyydboMaZxmaTLi1pLgWgcqI00LDGy2O4GvKV309nG9WGNXaWVp0ZY9DeCx72yMgraSTZ2M2hKOTpJUerSXPSi2a0QBIeQr47ZCt4yy/cxBsnUFQJywII/LwxK3k0BJkn8PU2ZEFrTAAx2NL7HGd1ttxEzDmRbd3znzoHS4x2RygYeYKnh5I3wXpkWM1ulXfk2LqBioQI2xAJ2yMwRudhfcvgHgjoR5fGJiZs32ipHWZYpZWvKAGAUyknnvINOG53Mwjy24EJAQh1w53RWEZXw/Zz7iJIp0eIiOfwY5WgbK/AOYRCvGtLT06KZSKSEa7TP6FoELdVE1TDUYQH/v8LkVWXAcfJxpYxtRiDdYkBJWKWiR90MtHs6ibUsmDdqsaRX5dEoTY8pq6Ms2iSdQBDJAKyJclWc0PYpAZrM0cy7xvQoWTcPybe78yMJHHcfQzt3XsslZ+F02Mg+DABQDp4CbeUbx0T71ThF+CM7Sfw5MxkxtJSdj4xOxvjQIoC0YaO5h6xS4w2382C4MtDzSjTOt5i+bWa0l9vAMk6ykEwwtMUOHlMCtoMjvR40IBwNyeTAM9hz00nOLkUswTE1oMnWs07WPn7dzmSx3GZkWnOaNBmx04+Wgpm2A1pJWbZbX8sDnhRTm9YwtCyj4OCAZojUnjY7ZYtGksI65MmwZ2xHg4igqgIvSMyDU9j4SEhpmTsRYgTaUBsVrbSas7IA6hu5DedoRphWar+VQWUsSpxypQNM7MizbNRiQrO02t9soWl1B0zJNrO8L6b1YeCIC5IimTZgR8oU+iLI5AS7RRqWBRgPsExPjBKKlbZTIG89lhtCpPl7XCk3cGNUmwtM7oa1ffGQ7yur3jjK7ZjU88dMTn0wWsdWD28eFBFD4r2RFkCvsYjJt2yELHEmUSsAFxmqlnhrbbTqnqLCiCNvQmAnVqYlHLnDuyBG1LV6xeJMlyeF0s4XgZNPKsZKBN5gsGt1c0RMn8zSS7wk4/DWAWFQ7ePRjNRTLAzftnSWk1l8RG4+QCLQj0kBEa8HA+ccM3OAl5Rqkv4ayBw15AItBUOcMlk+TuYrBhTvyYz/jdn7wR51bHojQxH+EYyl1n4oEzUcO/tGwR79BHEyDb5NBWr+mtsf7hdv/8/vcubtuc9PuWTsPLB6igTtQzBNsO764kAu45nSK5Sj+3GuP6P2IaM/U2l+24WyFG2ItAWttVO73O+7Yle91O0990eCwHl4mLbD/p59zp3/5ajedPSl0KdKk4XeC4AQ7lhdIQhqti0DRATu/333tkbkxMWuzxopYjHV9pi29hsciGZMkaml5EsshcDf2xP0vf86d/vfjwGXkC8d9BXkAIxxzmFD2iIOTd4Fxqw6TtkgPEpqWd6ABWlF7AAGuDCAmibV1fIHhELgPAnAx02YJUse9tzwdyAx8/jU1OeQy29utcEqQSJ1zryHAlbqPAHZM2+QHoMUBJLo7TNsqmjkE7vaAe+ZU4hQJ3Phzn2nR2M6/G1uxY4G1FATJd/PMg4/ANSZEDLhi2bFRLRU6S9JW0bmwUbT1a9ledUpw72LWuL92qHE3gG+QCkHjEuDW2S+oBBiYiJX74TIumT8C0jKPstLj78+95iiXERGsheYz0+Zbw8+t0kSRPVCgLS6MnUWfvmIAfmbcqw6BuwFuHQcuZzRddjk+gR5jWj03wsVE5b0J9LGdArh6GZezcaMAPDNqcgOmh1i8seVBdAoiN+SoUIc5BO4mmA33RuBeo7IKxt438T40EWvIA2Ns65gT/crkCcUc07iUcSVwx8O7jiujx4RWT1dgz6ZCM9igcX/N7TztMB22LoI1cKuuZHMn40gpLoDXWFxIDcd12gmwoSGa7Py51xyb9NfHI413GTCSdnw9ERyR0/un05Ql3LywRJ9XRkM/P152JAH3heuO29/4+2bgnvrla5wLeVxRy5HHQuWjENOOTMK0NIi4MWp4W9F3GaQCB67MHsQKLBTes7vUBunMQ3vHadMxSg4GvXsRvhcbGfcQuOt6YAZumJyhCfpaoMVkFtuoQM9rm+E19Z7cnPk3ALiV4eCsnjKh6NhK1f1EBkDAmx6fvPsQuOvitdxXgBvSYRJUePLMJeGaK2I5IjP8kMhagAdIa5mW78696li5fZWUVepA4MVVmTZLi+pBoxVi5NyDxSHjborcANxfmidnBLhoPqGZMr0abbw0AE8Ij0fa/AcqCfkzGL5TzYzPwB2bSeIwMF7lxfXqaoCvVVxx08cipMOOHEqFtfEbgXttzCpkxh2aiM2vHNluwyf3OLKiOYwBXHLSZwKuqMlU2obYhnQMZgGgxybAs9NNbLHOR0IXgIduJeAeaty1cev2v/x5zrhxQPPsgmjStD8s/H297IG9YVYCV8uDQoxkQcyffdV5BKbjB3WsxpYo84Bmp8ZELK3E1SGa17oX7ryrfnWzydlcDjmBU6bXx8L6d9KZB30KgVGB1chbWD4L3LCz4/a/+N/dqf9wQ5QKa+9a6JwHkV4NU16S3Y0a8GAaMUwVuCZT8swAZNnsp3lCxewUwdj8hhjeo8THqbw4gTsw7q9uJBX2P/NBt//Hdzvnd8TIGlkUGG2MyJGf2FhlzMerBozBKDeWzSmvYvATs3H5fL9w04nvuP2v/qFzB7KemYd4u2BmbAkYnoBEGw1wU5JK8NO4ziXgdlbESKfXYloCbLyF3GbajF713i0A99ydP+v2Pv0O5xdHe6dVJjvSyJEmisBJNTMm52WHKKM+06S7dggyixaF3yR8s6+kCPBmgBR2nyOXdNogB8TGxpWLwIk9ObbY5lRAlgWwgWXtiZ4/+6rz4bkKVe3wF+vMA2ScWtKWjISFuRVmyHYaybQZKNsA7kfe4Pbv+RXnFkcLVzFi6hQSxUhisaI16UhDM3Y2rdKbdGDtzxw0xqwbCSTbjh7ZBByts/0+NgVMxKztU+Raf/aV5xtTMdn5uBrGVEFrIrYFeWB++G5+71wddtV7NpIK5wJw35mAK8v3DEZMA58nG2nRT1yMQBsHqSCxOPTIZKfBnErLcuCpw5hDA0bqaRv73xRiBPhauGD3oolYm2kLb2Hgoh2kwjtaTKM+Kd+fiMUBHWDa7DlzrcLVmwN3LwGXj//IwBrn05rhr2r9qi7QbmM0mPF3+S8l6iEQgN/1o4hk2SAXmAhCS/v2xgC52qo0s3g2IQ2oaTURAMaNg1ZDEjr+xy5xK/dZIR7+nuhF677S1awDF9sB7t3vdG5nlgrFl9WuYXnuQf5ZhypsF8p69SMo48Dg77EkSA4F8f9nabaSQ5Ybx9tGEbhZygsxLRkTFuoXTgBXG0WHGkvTJb3aK5hRo03ytOYZXgBUs8a9es4qvEB57+gvzn3oDW7v0xS4IzWnKGwnPrRCaJa1pX8UGIjtKr9C0LJBRPcLSYePouJba8qLRqSLiL6pPbimRWIK4Ccv46LzglURV2wfAS7WtKU/JhOSgbS+hlhoWDaa7Etqnm0LALU14L7LuZ35bAbLIanhJSAIyw0fgTrCZhq4kTmlHTDo9WmSqG9Ivo20zXiWkofSmQ09mw/0s3aRM8+tkTkBV4NWV/J0BrZ4HfZGrYdEOqkl2hGo5rLGq+/YjHE/8ga3d/e7yuRMM7UuYuZ6UYQy9gCuac1zvBh7WkzbYPSq6dipjkUi9E6DDG1GB093okjxWS/OcEPhHWMny5m0uFDmrVpPE8Dmpb2zrzwe7s8J8OLZTLwAMOa/q3pK5Mn8d7HB/GuN8L2g1rd0KkzONgRukAoIuDKKWBMxzD5kBFJN81geFIXakvJCMgR8kagMSwyoA3o9AVd5bT/Er6tpS1daEzHm0DJn7mepcLzkcfGBxRZoeceK90jW0TMY8DmmTigjMqX+544776o73JF/tInG/VdR44Y8bp6Pyn41Io2haQNwm3laBCokQyR4CJuRd8txi5UFDVnB0D1amogJSZlgRtN8fHD4Q4NpG4sLBULhGXihpwAXAg/qWq6rasPHmTY0rCUNwt+VHha3bB243NDJ8NDvOkxWl2/HQaHfg0Ar5Yd1mLU8twAAiICCk+34Mm6RI+UBtM0GaAcWFyo+xLluJAL7s69IUmHVCvho6UQufdCqrSGmR9qGK4M7/8diS8Cd87ghHYZDIyO2sp9JOxVTVi1Q2E6biRpuhwK7B4ydKWPAG8vTctaMmjL+DoIWEh1dc7E+jRvsGRWWsJ0ixvQLf+YVxye9ImaHbl4sI/aWQaaMQ1oOVG4yrWaZ2Bu5fy3uOTvv6tu3IBVmjXuEgSWHXrL4l3CJQZG7FM0vWUKEdzXiI0X0KMqNkIUI18X266W8eqfLNORBdMrBxQW9/y05CsGOP/OKC+qP2GPqoKHywrU0LdI/NijCWEvA+53tAPeeOR12lEoXcEJ5Z+8ca9s4KKJTgu8tIOcmYwPly0j2YF15kDSz/owtGUcdQUsE8Uke6HYjohqzHwCuwbbprYoBqb6BFpWrYsZkpxo+BwMSP9LginMVtsG4u/e8y/mdo8kxrLZhRwunby9RPW/HCQsJH6nzGHXSjwQFYe4MdvV+zszxMrmZjLRt/tMcbZpRsO7EXUEe1JVXrmmzr4L5hJ6ItTIqFbjaY9TDY/+6wCtQDl8gJ4CPf2D3hzkorBBS+5zEe7fCuD/ndu95p/M7x0S9MAkjFpNNS7d4zJPdzt9/bu3T7KOwLh2Twf6f/r47+F/3haL4pKdY/MrprCpFqGSa3M6jv98decpzNDbRGJEFgtAav3DLb3/d7f3R7zq3v8+iKm3EWPUb719AZ1oNaxaoZ2UbXggmYooI63v8mZtnqWCAkcyelUhuMi0qlgGgbzgLazNcgNiGVJiB++6kcQVmgvUbNRkH++7YpVe48696q3MLq5JKPFP8eOZDv+B2P/HuWiuBpJp1ntbBgTv67Be7C69/i3M7qKa2/e75r/tf+oI7+bYbnIM7IKwNBCgSVPwwnGgnVuWusZVGbXMjEvgzN18IAjzXHqsxLZcG+Ulzco+ZEoKWh7bmh/22xrgWcO0JaiCKg3139NIr3PFNgHvnL7jdT94RMiT1H9J98XcsVG8FuJ93J992o3Nye3oam6YmDY0BTDs3NLCttB+aTFbQVvLV/VdFTnOMJsCt+kM1qK17uW8v2qtweQigHqahEL0zh9RlYMnzrn73hlkFi3E7Kbm57csDd/TSl2wZuEqjJq0IjrNaLt2xZ7/YXXD9bRsw7ufdyV+8MTJu2eWbVYsBPBQVEgCqRGg7fdXUekVMbEXRGj3PgCDjqu81IG+RjYvyIDhiU+y3S//aEwWiBefNktfM6bCf6MdE44qzd/6c25ulwlxkk/91qvZL17YOXA3aktaksixT0/JgY+Dufenz7tTMuEQqzJo2t4RzC2VCDcxqlzHQsu/NjdieXBNwVoFbG1YAaIh8rImTrl0RtPFd8pwyoIdpYXM6V+H8DYF77s7EuAW4/bLG8g2xgz139LJZKrxlfY175791u5+4vTqOsJ0KStSpZscJGndzxp13+YZV2qGUGp4PBWnQ+E6IzbJ56UFKBPAeUhdTgFuS7ubhutb+MKJTVi1NHP4ehJvcNH/KkhLjwm0HuDNwZo3ZTmGpAvCDvenopVf441dvCbggBFcCSY5MkbwVxv2CO/WLN6ZjRkcKgYxJPNS0gnxK20HBzFSSOlrrV/FbF9YK47ZOzGvSONq5gNhyYP9aaKBhGPSpoa1JhdtxVoHurxNGDyYJk2tz7OsAAB/vSURBVLOXuO0CN/afARZOYn18/49sxrh7X4rAnaXCqlvIWfYgTci4IpMMamQOGDABdkQFXH6vP3PTQ8J/K+8ugM3/oTUte6dmjPhIAMbxw/FQnWjacBiAO0/ONtW4beDmEKc02ZaAe+4TdzifsgqqOi9EgTpnrsCYgXvgjv7I5RtJhQxcfSCI1NvAoUJjVpMHHKMD2YPUYVjuGYBL8ixjZ4jFnQsFq+ZMEzeueE1TWiQJIq4pTfWzVNgUuG+Mk7NQq1AdlEwyZ7fzcCKxJeDufuIONy129A5qsxY5te5BAy4GLZNpiWFZDUkGsrqwSJDQ8Dj2CBeSbWuUVlp/ttbpxLhwwhXuIIdDhJeinQtISAPaR3uKoDzQK0nqPIfFgwFcvn08GszQfvPkbAtSYWZc7jjJbnq0+KpjAO4sFd68QTrsCyEdNp09neBmT5BUc3A9bVwJJUTIC2YGACtIkL+3gjkBdyTdRQYxOT37Liz1tMZEJ1xmMnRhWRIfa5gqr5j/ujXgzlIhTc5KDUB+U6PgYzlr3J/eWOMOAldvbJyzChsCN0iFtybgijwuIrIwbO0KrwhbMidQErTnkOnvWjZxh/ZnbnooOLbK8gyZ8mrk7EgDV9K0XAixmWSEU92efv6179pQ477R7d1N0lEJryXlxZxRhLitAfc9fOVMTcZQfncG0NIdffbl7sIb1mfcCNybuufjlqE0vufMCEVEqQqD7jaihHFAVEBe+NM3PZT7AAvd6XjJ1Jj2woIV4kAtLayN0BOx5jljs8bdMnCzCIOaVkaRWeNeNmcVbtsgj/smFxk3Lfm2zqbN6IjyLWUVtgxc5TTVc+OhlpKokFPpjY2McMojUZSv72CA52QWoo8ALvKKGi5tabBd0DIBb0mLLQF3lzJuBgViWhHi5lqFY5sC94MUuEb0YrGWSJeQDtsicK1PmI7kaMsYEaxIgDL7YdByojIiTWLfClzzXIFYexAKpUyPLNICa9NWo8XWaAXaYhTR2ZBVeKc78vQXGDWxCn3qF2c/9PNu9573kHQU0LQgdIdfHextEbg0q5GbifOgpRNbA+7N9HxcMn5pKmIc86kKX6BE6KS8yNhq6du+NwIXApJQPllNYxNGqjkJLAZPdIT7+bEexmFq5wee5fwjH08OZ7bCTz2SiKJ3ef8fuYNvfCWFwLHK+zxgoTrs2T/ljr/830SpACcdDedZLNzZD93mzt39PlEdJvpqRYEHB7iswaY8KExGL+8WzOBa7snn3WDkYf19c/70jQ9TJs8aL8CgtQQsdJ+tS1AY5Cmv6Opy/1WjdmC+YVq6sAuBnQkb+m9W/nPy3zHOh0XSR0wapsktHn6xWzz2B5PBUR/xlp/c14Nv/E+3/NZfiAHtMG0miwLcf7d2OmyenJ18682yHjcCqdQd0H4hYjDkgULViDwgTtuM0s5h4M5LrIVarQHBQr28r/wHuF+xSAVFeS0I0ZwOeEhlK4MwgtR2FNk4pUNJupqW31suD1tn5AilvliGp3aZdz6UMuU2KGoT03VbAu6pt97sptOnnMulIDMXhNEfmYjNrZJVgQZexNYkKAnNRRcWzmOqmDJu2GqjcpmlIVX/GFVEbKwa8kOsL/OcdQvwbEeGNlBse1sbccdCwG3v5I0mJO+WABU/g3DWnp3nMYISgbRtTseFydlmjHvqLTe7aS4kX+Ro58O2myI/ZX/LH7g0iBaZK8wkCyBbrfE1SYZLAdyyKmbStAWKKHrCbU3gSS1Iwm+hQQEMaDgRtoutbNCGx4+cLsNHrOSRi0Pn9nQZNQ0kG8d+2aSbyEGA9F5JBFsE7vLM6Tj0wymvyLQQ3K0QX8IikITMYREG+OnvgXEZ05bWIMq3j/WpTGaFCg7arPO4g7ZFeQQfB20FFAItZkedp+2UNBZ/bLN5LcgxnE+xEWpfo7yQ3r8/55G3wbivcMszJ/mh2tLZGEJr7UHdrNifTEWUA8IJg9o/LkrWCvvTN/4tHuFb+tDwJh5+EXA7tQeQxfSg4vNX03UtUORPcaDySKPgw1xybEUVYTtuWIMIGNMMgnYOyftzHvlFG0uFk295RZQKQ5qWfCTRlGW5Q9LJUZTsEEF6lC5wD1IhAreGUmDgTUsTxWDzAe3sps0mhTXDuWf2RGLdc7zyE2Nbx5jW3Gbf1X2tXcIGAAJwZ8Z902ZZhRWBq+RBq28getfLB2RTmtNoTkrAfXBWxDTLRpiBjX9wCZgP2Nhp19rbY6cNYMBdqsbEwRogoemVUw4yGYvO5YeGNJuBe+nl7sIbtw1c5KTcfpHpLEnIJ2M4cqH79fNaW4n8qRuyVLBZi0646MCYX8UJHQOdRYPY1X0MSPnqmK5p3FvCCARu47yEpr4jjhFFespzG4PYqz0IjRxZrROyaX7trHEvvdw9ZKvARRmAdfO087hlIxGjhgpndu4UlinWCefpUf7UDQ8Hw6+32kTT1Y41WZrkgWvTpcbphIrQcMurka6Vnj7OtDGixfvbkz09sLgQSAMtRxvtbBS4Y5ovtHcrwL3XBY0753HpsRcFEegAv4GJWJJ1Cp+43JWhOyaR+1EPALc/O69s1piIJX1DNWLpCA4zbLUrhwndeStMcaeK9wk2E5OnAiZJAErTin6mgWX61wzvQr4U3bfaEnO5LdZGuWl/PtfhhZsx7hcTcOmBIAC0tfed3djFLiPRmzt3G1MKk9MgcEd1aUp5FbShIzQbbFRGx2LbwRAPswdtXcXDToNVBPhVuGrJAwIKrWn7bJuLnIJ5Z+BetiFwv3SvO3kbyirIxQUEWDGOsW98S1fTmTm4cQTHhBHezKTCUPZglGUB5ff27TfrIpA8qMbralriFONMa7ElYfciLwZmySZwkbYUAyvD7IMGXJny6kc4Kn/gbmHm3Qg/0XZas9p2qcCNoCJ6A7GsxZYogzBSMENYJtCI8ZGQrD5Vz7g8MPeHgfsMI8X+lz9iTZt/S+ZncpZtHB00lt1QOVWk97cB3CIV8p4zuRo2NvunCwt1REc20/Lns3qTKteSmTngE3DlZAwclQ71YZoACRT0vU47gK1pG2F7marDMjHmdhjfHGPNbNQf11qKaCwO0NGJF2CWhsbO1VjxXZbEEoXkz7ncPeTmDbbuBOC+Mi1AiOyBOd5AvxLD1v8cm8TJoaPRsHUclz91wyPA19NX1LTp7XqyMhY+zexB03gLt/P3nuUWj3qiqsfVbNrKO1K1icJY7yw0cr/Z3ngNGlTK3mzQtAhOD8mctnRHvv8fuvN+9PK1tw7tAeAW3+8dElO8mcum0uyhVOBIsQ0niowxf+r6R5Bxrg9KGx7SL+SA6lpauL0dpz/YKALQVrliHqocP15y/rXvcEef8eMEuNZo///6+zknip1tpMczcB8gjFvm1D3QBS9EgEUTTD4PiVegKjIRyTAJhM8czM8gwK1hkdODMRlT4UHOAC2D5t9bjcfF19Qxwgx0seOOX/ef3NGnP39kjA6vARYowD09H3qX9G1vAl1ChwRuf4IZNVej9LExh4mvjRGcABeB1gKenIiN3is7ZkzEWjpwbnzu3CFwN3bGCtwzjT2FGJDG5Nb8nkQsfB2v16WdK6qEOJU/df338DlL+cmQB0ysjeZpqdbtZg6KzMqarzY8d2feObBwx69/xyHjbgDfCNxX15WzFuMJQuGXdspRy8fJwcQut589kF+HahYKcMt9kPHQ/rCkVZjhOnvEkr7hE5X0gEaICsAtuZKUZzwE7gaQjbcG4N726phVYJkYnBExT3TshPiYo1XPjCnDwkpC45beYUkZgNsG7fyEWgSeZ3XQak1RbyWZBzRtQTrZanMI3O0CNwIrwUjOVxBJjeZ4O9+DMCN8axK3cP5klgoDTFsEsjRZR5fm7e9aFyUvszw2r6aEIxNFwcchcB8M4Or9dDwnzjdSDkymmh/1E7KTqoaQ4VBEWHPN/uR1WeOi8BAuzJE6PXcke0BmnKnaB8sDOzyE64s2AitOM3Cvm9Nhh1mFdRFcpEKpDuMTsUi/YvKd5V5Dk4ZLzOq+fvbBXowqkX/yJ697JCBClKcFIOukTnKJGvWkYuSBe+MbQWndfO9iBu7bD4G7Lmqpxj192ihr7JcX2vn7+cSLxlctgSwpMlQhkhBX+hsGLqmuMjVtL9/XqqfV0qJoK5pBiAlxWZqYPPYQuBtANt4aGfc1sB63eeAgmTixRoTBG015oWiL7tWgDXdqxm1sIS/UiXK86+ZpReojhRnItLRO9hC4WwDuH8asQpYKhFBUuktpPYyBsXMVNGj1uXQzYOf98hK48b0EuPZO3Mh86V+Lac2Om/fy4vFinIHdrmFydigVNkHv3hclcOVqGAFYayKW/ja+L1Bo6QCwsXWD3F8O3NIAROP9ukxW6Kws2ikCL4YZAG1oXlqAeMbzNhm7v9H3BuC+ec7jzmWNqxe8yPmKwnY4WD/pXJH2yj+GSrgBTcuW/CvjysM6UFmjLQ/Ce9MXCWHKC6bLonOklEVKIIxuZ5lv3UmMewjcdb1vBu6JN7/aOQjcNN4dps27eHNMZYeENOTFKhMxtO3fn7z2UbF2wUpvNEBX1IN5jb24QCdh8TnDoI2zVb9wFwSpcAjczYD7mnIgSJWE/ZRVwGRvi7qBKRu0AgcN6RmBW3K1eMWEG0auIxuyIkposB+j5gXjQf3W2bJop3ES5uHR3h37x1e6nSc9DZyYWFsMI4BigtbQo0hDrjdf0LlPvLI4cvl9inrm81GbV3inX7j9//1Vd/Y33++m3V3yMMS0+rm5UgtiwyLBzGOQ6FT2IJlE6u48OYvAxVvBe3WZ4ECN0JEeS5cjkcaZFgt/ckynFdIMbVXb2dDUJaQYgOjaZ5UyPvAOfUhfncwa381lDjCSsmShdhC0LaYtDcA2w6fao5SX4SxpTBLjVhasHlQYLzelLAdGXYruyUrHSJUkULMshZWnVYRinPAnAbv2hk8UbdBhHbbWz85Qm8RCrmIQpd0YGOV7UPQiX0EKl4/pUnMi1tKzaexKmGfjI9+7yvjb8oAHRm5L/8C1fycMNWvzgKfKWR4AvIBeAnprGRcydWlw9JcsQWqvOCAAw/L+dfZClft72Q3ECLmKDYCuGYmSqXofCmGDJPaIhed3nCq2obC2crBGiI9OKckKOTtxIIIALC2a516QI/bpe1IxeQFuk+IZ2o3UyUCVVxMU7WqjYSMn8Ofrc8ubmqwYuKG3ufuDY4NybYX8poE2ugwmwfTLRjpKvXtkN25+S2FDnTPXJJADcRFJNfUzuoVcO23SoqLbTdCGHsOT0ZPzBMYtT+zWD4A0Wbm5X0wcL0XhV3upmcxuhbOWntUTxTqQPZbtMVFiI9W0DIyATJzHLhFEaVZiE/Jg9n0OOjfp2qUCipFATx6UXSfCASm0KQYEPNdg2kju1KnyL4gNI3DbIZoc1oAFdy97UHWIBC3SZMjAeBCZRiRM25cGIJxB8LbbV8ZuKC3Ebcfw0vrMKGHbsQOpcd/GisDB+LZ2+7YcepoZE+3kbDOtiqyl/7xt/oFrMuMaHpVutLeQG3oujWr3fFrGJo0Dz1i4lNKlhkXNeiOatqdnRdilrNLdxo21cHhEAqxyNPb8GqWKmjOjnLBLKnipcon0A4YHfn/zIG1jPMpjIRl25YFm2hTNBJE7/8A1F4Mu0A5IQU4eMTMNt3rNPLCnji4uoHc1WI9IgzHACjYKN422TTNZ6/zWGu/sKBW/I0b/YYek8qACo00YuHjfdnjZEnQKOLtGyLISfQygWcUy7D50OqchXwFwA0OFlTyac1Ne29XDuUkjmtZyDgO0gnrW0sPBwAi0CaCNMBj+1DrzoTJibmlRS6FH6SMh0NnYSNZJWOWHAflCSkohm7dCvLkiJhwFMKpdq4JLE3larvv8aMv0XgFcoi9N3TaWdomdGGWzkYJlwnjJ8Guddg21rGA9hShp1JEMALGlpDOlaS0mlCmvEdnTOenSiJDFX/KsnaVIUfvqPsfqazjXrKKKkIfx7zrKS7MFvihZBSoVClgNBoThgQ9q1rRmPa0AhQYf8jwNWjOP3FvNCr1HTsWYLDElCPOwQF4OrL4vPBDmaVV/p/q1PLptCjGtHPD4s8nkHabNk2x+f5vh87W4ymtuSv7yH0+X1qBpSB4gYPOhiPMd/oFrHl0vWW3XQnKI2jH+Lsq2yhNDu8eq7PHgFPzVsGzWRZRLWlGAi63ctxrq6945khi326bat+I3xCog4oJBs0CbpKwUoYKQrrfbrH0kUsEAbx+ahAlbpTsZBqwon3CZ+8aBS8IvoujxlNlIyqt2ooLdCpc6TGvQosUP9LyR7MEoy7ZBy9o4mO6i0qrYBQJvhGkHGDqFaO0U6Pn8d7V9KLdv1x4Qh4yRv6TMjHrt1H9JwJVxm2wLBpPuww/3rjbRWSmcEYkC70NhhWkmJA2yyxuhqjCCpemlfBGSKXs/BG26l7e7hFXmzEN9Q9KuHeJrFBqRFlj64AnqGhOxVo2FOPC5sO4sFXDFTnsJN8X69P8MYIwUvMBGE6OX2DdUZKO1aRl4EAkUKAD4WqG2OlSRFJxpUXi0IovczTwAPMFG8Q5iJ9Y/g3zUSeDyvei+egAdj84StMiZAVEwsZoB0d506R+4+jHhNjWGAyFqlcWF+I5sWrAOTVi1yM1STs9BW7A8NBFbLRLQd0cYGA5TjKYHJ7bPp7QXHdoUDpkQBSmv3kocfzfW3C2nJJqRj33oS0yGln/1PzNLmSWmtKsEP9mrY5OovaztXDHKtaJrAO5KoE0fr4olZauBYizMa4/nHYdhVqRTwjX6OPtsuB7T9pwWOFkBfH6HtbgA5AH/1YA2Je/v34s0ewMUzKnKvRGz4X87KS9hG9i+YizM5vYnFeq7/YnEuIzyjeR69bh89Xi9KvQ6jWRwYLAd4hArVO2G5Mto+G2cd9UDbSt7oPoL5AGyidLrnAVjnwf6BhcXkHTRNsfFMgAHUB6OMm0FZjUDllYAuI0kN9NDI9mD+NLciLi1jRReIBC0PHY4hIa2ZV+p+B5e7QM6jMVUNLC19oCRgAJd/Kt50Am7WbeD6gI1uAz0oI2t/rfunQtmwuG2vGetZVyTafPOX4O02gxdHaACN4dHFiq0J8cHj6SV8ApOWEk2NFQ1DDvSPw50oy6Cecfgd3vRztFWlRt/B2A8xbTZkJIJQS2tKrkUoAPSpUaw/kdCYqTkJMJY2nDKONaWdLFTXuxxQ5q2UevLQnW1iwYu8vgm06bOKQ8ywhnybK4j9P63XmV/eaZRFyFOHIyjIcGxWlqIRBE3rZCnpWGgAKNhu+gwDWnAUaIK3MOjW5NYk2mj45knJrIP9RrSQmLJkEH1cEN6Q9t5/YmrHwu+uiPYsgkMrEEboSK1jmuXOKDohHNDwCsmGK+LoObJA6sCjXo+b0cBbm8ZV9iO2iW+UzKm1nTr1tKaac6OHi5jAYGW7Fz+xp2q2pHay1pcWD0K5LEDwK0PC40wQWtV9MuVFOQ5PARV+wCQ9lJe4eYx6aJZdmQSRtpKhXMALP+bFX5nPZt7phzaYrz0ewpaBQqTLfO4rTYWNYokDLDnr1lLC48oWC+K0MDsT1z1WOFXtdHRUKOgiIPIoz7SR5hRIOP1QmgIg6PtE4zZ+kq3ofmyt+McrdCDpEMZuNU2I1VeiI0szTkie9S9Kl1YzK1OJWqDlvWLhbNGtBRjq0jAZPT4TAHcCrT6XCN7QAaXhnkOQAlcBdry0eJ8XzwkZORbEtlCxooYawgIZ0O6DxveXFxQepO3DYfudkQaZOiMnWiU1m7cFkuPTMSgZpbRBxEWQfSwZscROHQ2Mi6XB/E06YZm7IGicYJNNGxRKoMrdmhw12daXR2FmMwA7cBELKJI1tKO1THTnQv9KKSjSO5bm0C0/MEZlb48yPKI+cNgLTeTiL3oKivEMnDDfeXMA6QbLS8aKQLnTJvDAjVu/O9+agfX0iaF0tPDvS8agvBU5QHcQp6YTh4XhTTtWN9wcXznKKs0dmZtc4NlC49AsJGJWBgg4SiWxlcgrHUvhpSs8tp4pjzbzp+46pKI2fIyYxm3AQoVzpBGJBetVYeL9SwEDpqE5ZOyNYvJAeHRh8qg/KHoGvSkM4O6gzwQLUZpRCBcoM2lT8jHkE+PMyZDY8FqU8K9ot5hlGll9snQtC1p0LBLQbOqW/BBKuy5yR+J/VsNtNqx2qmdzLRUv+dQAyoqRE7SOqwjhb2WdjM/oEKBZ0mDOWkhC2aUVmeZjbYu5WE6a1JoS0r3xWjCscKNqO2N+UUB7np1B2U+wirR1gCt6p98hlkhtu9PXPW4b7tpekRT0zKkNcIF8h7GtMjIIzNsJF0QYPXzYzgxjFqkifH3/pFIOfIV4Cq2Y4wn2iekCTNfb3m6PHcctKSxkSda8kA8H0bVAjwLtLy/MdLSVrD5jviDPMaUveM7/q+vvOSPvVs8BbKgYXTaibJo0GC8ONlDE6yG0Zk3GsU8LO6DZ/U+WWSyVXw5XhEjg0HkFQcFmg+g/sdnKUD1loDTuJgLEx0mGzpdxgSu7EefaSuMUKQyMJBsC4nATX/iT1x5ya9PbvFPOHA1C+Iw3xggAWRFxpss4w5M4rqrRg0mDm1dhuR9Z39ZjAR9aQAijZglF/t3J5irTGKN6KgGo69p4fg39ojRgvaIJp8OLGQCGy9RJ80OAR/Gxt3lT1z5fbdObnqzc24nGs8GYx98yKMQ23aYdorZ3FW2tzOSaR7rn9+NBrWeeaAjEGZaFZTW3tHRtn3tHxofdG/pXyX0FGH4OKI8eLxFRgK2JG9Fq1aUUwDSUWggihwsJ/cv/Hdf/vhnOr/8qHPu4hgf5ek0cjWsYaTc01KUk79LSJiLe2m6gwLeAm1nwNLIdpmWHLNJwVls2tuNG2yEKrySXTqSiebM6/uxw4PsCGcuTjRCI0rHRPZr11PX4VxNHkA93LJLrv6DbMGwMV/5zQO38yL/zZf+gwuPH/nrDzvnf0Ks1zIaV1oDNYTpkrZ2yTgrnzRkz1uvYGbsXIDarvzKIJUhYC0WHFlcwJqe+DYZprYmDu3s1SIb1VosxFsSofzekBalpfZWm6ipetucgGRKztfbfl9OsHHut/z+4qdDS7/78u99qXP+Pc65Yxz0sSNDoE254AACMRELv1tb044zLawuY6Gbgza2Nf8z9ogpLSCWcY1CccaWBBiQjdQ7BIBSWM6/tccDkYVle1lPq+/lZx5YoENLzOJawzGi5VEtLopA4Xf7y+Xy2ks+8bvvCz+duO5Jj5x2z/5X5/xl5WEEwbbXElAVFEiwDyx1lo6tybRVV/FwaUx0FPHMxhs6gC5OxBjghx3SIAEGWgC8BFrYZnBvNUBiP4tljfcqp8g4MBh/nJCAY8C5iARt/dk7/wd+37/w4k996i/Lb7/7su/7p5N3t3vnLmBbnLmO0uHNMF69cN08rel1LCjEMArYoLPSZ8sDFOLn5wNduwJo+8BDfYjRqzoKsom+rxBNC7Qj42bZlkYpJg9J9Oo45CDgaQXb6YMDd+3jPnnPh1iPv37dYy64cPfYrzjnXtY3MtZk6r6B1I5Re4DBSIuByASQIVmFbhQCE2u2CmaE5t460ypjaYdpzrAb4Xc8T8ulU462SkJpQJQQr1ROpwA/v6OpaUHfls5/cLFz/vWP+djH5s9gcjF64p8/8UnLxXJG9FOZZGANbzMhCzWtwWnJgx6TldoKCUo7zJSIlw6fy8u4ZmgMI1KlQR2gxrI29SCyvq4Y07Jnjm6TONpARj0hy1jfeCERkU60YCZlqFuTqdBhrJlRJVlvgSk0hETGYXKMfb/3YOFe8rjfuvvPBKdXi//VlY9/rp+m9zq3eEL4bRO03GOLPQeAZzItrgnlH97oz7Dh+n3oymD2II64XFxQg5mAgR1GsfTAwgkFhXKqBuCH5IHB0uVe5gHtmmgOPBSBgYQhwBVREnhqtqn/2r7zV37vb3/qM/Qe6FLffdkTnz/56e1uchG81OMZmEX6I4xU6TDxdjmwjVpaajz53sJIwp8kI4msBgPQQD1tbAKo9OpJnzSaYwdN48kKH1Bh+x7wFI3JzIEGUxxO2pb26TJqeMzic/7MzLgQ8AJT5B33T37nlZd8/JMfl3axYoH7q5c94TneTb/knH8qWk0bK020JhNrFIH3wowxqNUIMXOg9JsaCSYPkr2xpmfGFO9XA9Rhy1iW2JM+OQbWsynCYyGTCdACeaGYlhCDciBgf72ShhyDE5mKIsz+DOz3Huy4Wx73sbt/DzmzCdz54u++/Ik/sJzcG72bLnfOX0B2Log1/E7Kiw1ae7WmNrKChXYWAg+AgodplO4iRi73V2nQ1qVigIDTMFA0QFsYT7Fley5RSQp981fWHuhJWPwNjZhGuE+AVc0bmoQlO1kH62HQnnaT//WlX77pkt/+9FdwBMKFnOzasLK2OPmCaXKvnpz/Ye/cUcbspm6JhRWZIzbZiavDGQqhIPTmy1bMHnBjDUzGoESAQNDnma37OabmMmmZiCX/1cDlQ29p1M7ignbIzBfRhKqQSIwRd/hdN/nPueX0Tnfs+G/k7MHawM03PvDSv/uofeee57z/Se+mp03O/W3n/E63AJywmQIE8wAGxmKAsXwfN0h5ZW8ixt4PFhd65YUEsCoSaIrSS+jDs3YOvPCuXi0t6xvhD1R0bk2m4a6KyqIqOrIBbh8TmoB94Jz/lp/856Zp+V+Wx93vXPLRT3/bAiv9fVMqoAcEBvYP/NByWlzm3OJZzk1Pds5f7Nx0kXPuCF7qNHbiio6q9yWDKr0IBkXei2sPLI9HW25GmXbFLfmFiRIALBsA+VG6rYEGvrWAn6/8ySxNRCwtnmmtpqVFk8i4lZC8c3uTcw+45fQN5xdfnvzyD5bTkc8e3Xf3XfypT50aAWy+5v8BUrIHNHvQF7oAAAAASUVORK5CYII=" + +const OrgHeader = (props) => { + const {userdata, selectedOrganization, setSelectedOrganization, globalUrl} = props + + const theme = useTheme() + const alert = useAlert() + const classes = useStyles() + + var upload = "" + const defaultBranch = "master" + const [orgName, setOrgName] = React.useState(selectedOrganization.name) + const [orgDescription, setOrgDescription] = React.useState(selectedOrganization.description) + const [appDownloadUrl, setAppDownloadUrl] = React.useState(selectedOrganization.defaults === undefined ? "https://github.com/frikky/shuffle-apps" : selectedOrganization.defaults.app_download_repo === undefined || selectedOrganization.defaults.app_download_repo.length === 0 ? "https://github.com/frikky/shuffle-apps" : selectedOrganization.defaults.app_download_repo) + const [appDownloadBranch, setAppDownloadBranch] = React.useState(selectedOrganization.defaults === undefined ? defaultBranch : selectedOrganization.defaults.app_download_branch === undefined || selectedOrganization.defaults.app_download_branch.length === 0 ? defaultBranch : selectedOrganization.defaults.app_download_branch) + const [workflowDownloadUrl, setWorkflowDownloadUrl] = React.useState(selectedOrganization.defaults === undefined ? "https://github.com/frikky/shuffle-apps" : selectedOrganization.defaults.workflow_download_repo === undefined || selectedOrganization.defaults.workflow_download_repo.length === 0 ? "https://github.com/frikky/shuffle-workflows" : selectedOrganization.defaults.workflow_download_repo) + const [workflowDownloadBranch, setWorkflowDownloadBranch] = React.useState(selectedOrganization.defaults === undefined ? defaultBranch : selectedOrganization.defaults.workflow_download_branch === undefined || selectedOrganization.defaults.workflow_download_branch.length === 0 ? defaultBranch : selectedOrganization.defaults.workflow_download_branch) + const [ssoEntrypoint, setSsoEntrypoint] = React.useState(selectedOrganization.sso_config === undefined ? "" : selectedOrganization.sso_config.sso_entrypoint === undefined || selectedOrganization.sso_config.sso_entrypoint.length === 0 ? "" : selectedOrganization.sso_config.sso_entrypoint) + const [ssoCertificate, setSsoCertificate] = React.useState(selectedOrganization.sso_config === undefined ? "" : selectedOrganization.sso_config.sso_certificate === undefined || selectedOrganization.sso_config.sso_certificate.length === 0 ? "" : selectedOrganization.sso_config.sso_certificate) + + const [file, setFile] = React.useState("") + const [fileBase64, setFileBase64] = React.useState(selectedOrganization.image) + const [expanded, setExpanded] = React.useState(false) + + if (file !== "") { + const img = document.getElementById('logo') + var canvas = document.createElement('canvas') + canvas.width = 174 + canvas.height = 174 + var ctx = canvas.getContext('2d') + + img.onload = function() { + // img, x, y, width, height + //ctx.drawImage(img, 174, 174) + //console.log("IMG natural: ", img.naturalWidth, img.naturalHeight) + //ctx.drawImage(img, 0, 0, 174, 174) + ctx.drawImage(img, + 0, 0, img.width, img.height, + 0, 0, canvas.width, canvas.height + ) + + const canvasUrl = canvas.toDataURL() + if (canvasUrl !== fileBase64) { + setFileBase64(canvasUrl) + selectedOrganization.image = canvasUrl + setSelectedOrganization(selectedOrganization) + } + } + } + + const handleEditOrg = (name, description, orgId, image, defaults, sso_config) => { + const data = { + "name": name, + "description": description, + "org_id": orgId, + "image": image, + "defaults": defaults, + "sso_config": sso_config, + } + + const url = globalUrl + `/api/v1/orgs/${selectedOrganization.id}`; + fetch(url, { + mode: 'cors', + method: 'POST', + body: JSON.stringify(data), + credentials: 'include', + crossDomain: true, + withCredentials: true, + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + }) + .then(response => + response.json().then(responseJson => { + if (responseJson["success"] === false) { + alert.error("Failed updating org: ", responseJson.reason) + } else { + alert.success("Successfully edited org!") + } + }), + ) + .catch(error => { + alert.error("Err: " + error.toString()) + }); + } + + var image = "" + const editHeaderImage = (event) => { + const file = event.target.value + const actualFile = event.target.files[0] + const fileObject = URL.createObjectURL(actualFile) + setFile(fileObject) + } + + console.log("USER: ", userdata) + const orgSaveButton = + + + + + var imageData = file.length > 0 ? file : fileBase64 + imageData = imageData === undefined || imageData.length === 0 ? defaultImage : imageData + const imageInfo = + return ( +
+
+ +
0 ? null : "1px solid #f85a3e", cursor: "pointer", backgroundColor: imageData !== undefined && imageData.length > 0 ? null : theme.palette.inputColor, maxWidth: 174, maxHeight: 174}} onClick={() => {upload.click()}}> + upload = ref} onChange={editHeaderImage} /> + {imageInfo} +
+ +
+
+ Name + { + const invalid = ["#", ":", "."] + for (var key in invalid) { + if (e.target.value.includes(invalid[key])) { + alert.error("Can't use "+invalid[key]+" in name") + return + } + } + + if (e.target.value.length > 100) { + alert.error("Choose a shorter name.") + return + } + + setOrgName(e.target.value) + }} + color="primary" + InputProps={{ + style:{ + color: "white", + height: "50px", + fontSize: "1em", + }, + classes: { + notchedOutline: classes.notchedOutline, + }, + }} + /> +
+ Description +
+ { + setOrgDescription(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style:{ + color: "white", + }, + }} + /> +
+ {orgSaveButton} +
+
+
+
+
+ { + setExpanded(!expanded) + }}> + {expanded ? + + : + + } + + {expanded ? + + + + + App Download URL + + { + setAppDownloadUrl(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style:{ + color: "white", + }, + }} + /> + + + + + + App Download Branch + + { + setAppDownloadBranch(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style:{ + color: "white", + }, + }} + /> + + + + + + Workflow Download URL + + { + setWorkflowDownloadUrl(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style:{ + color: "white", + }, + }} + /> + + + + + + Workflow Download Branch + + { + setWorkflowDownloadBranch(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style:{ + color: "white", + }, + }} + /> + + + + + + SSO Entrypoint (IdP) + + 0} + id="outlined-with-placeholder" + margin="normal" + variant="outlined" + placeholder="The entrypoint URL from your provider" + value={ssoEntrypoint} + onChange={e => { + setSsoEntrypoint(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style:{ + color: "white", + }, + }} + /> + + + + + + SSO Certificate (X509) + + { + setSsoCertificate(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style:{ + color: "white", + }, + }} + /> + + + {/* + + {expanded ? + + : + + } + + */} + + : + null + } +
+
+ ) +} + +export default OrgHeader diff --git a/frontend/src/components/OrgHeader.jsx b/frontend/src/components/OrgHeader.jsx index 881c303b..2f1b765e 100644 --- a/frontend/src/components/OrgHeader.jsx +++ b/frontend/src/components/OrgHeader.jsx @@ -29,6 +29,8 @@ const OrgHeader = (props) => { setSelectedOrganization, globalUrl, isCloud, + adminTab, + handleEditOrg, } = props; const theme = useTheme(); @@ -41,103 +43,7 @@ const OrgHeader = (props) => { const [orgDescription, setOrgDescription] = React.useState( selectedOrganization.description ); - const [appDownloadUrl, setAppDownloadUrl] = React.useState( - selectedOrganization.defaults === undefined - ? "https://github.com/frikky/shuffle-apps" - : selectedOrganization.defaults.app_download_repo === undefined || - selectedOrganization.defaults.app_download_repo.length === 0 - ? "https://github.com/frikky/shuffle-apps" - : selectedOrganization.defaults.app_download_repo - ); - const [appDownloadBranch, setAppDownloadBranch] = React.useState( - selectedOrganization.defaults === undefined - ? defaultBranch - : selectedOrganization.defaults.app_download_branch === undefined || - selectedOrganization.defaults.app_download_branch.length === 0 - ? defaultBranch - : selectedOrganization.defaults.app_download_branch - ); - const [workflowDownloadUrl, setWorkflowDownloadUrl] = React.useState( - selectedOrganization.defaults === undefined - ? "https://github.com/frikky/shuffle-apps" - : selectedOrganization.defaults.workflow_download_repo === undefined || - selectedOrganization.defaults.workflow_download_repo.length === 0 - ? "https://github.com/frikky/shuffle-workflows" - : selectedOrganization.defaults.workflow_download_repo - ); - const [workflowDownloadBranch, setWorkflowDownloadBranch] = React.useState( - selectedOrganization.defaults === undefined - ? defaultBranch - : selectedOrganization.defaults.workflow_download_branch === undefined || - selectedOrganization.defaults.workflow_download_branch.length === 0 - ? defaultBranch - : selectedOrganization.defaults.workflow_download_branch - ); - const [ssoEntrypoint, setSsoEntrypoint] = React.useState( - selectedOrganization.sso_config === undefined - ? "" - : selectedOrganization.sso_config.sso_entrypoint === undefined || - selectedOrganization.sso_config.sso_entrypoint.length === 0 - ? "" - : selectedOrganization.sso_config.sso_entrypoint - ); - const [ssoCertificate, setSsoCertificate] = React.useState( - selectedOrganization.sso_config === undefined - ? "" - : selectedOrganization.sso_config.sso_certificate === undefined || - selectedOrganization.sso_config.sso_certificate.length === 0 - ? "" - : selectedOrganization.sso_config.sso_certificate - ); - const [notificationWorkflow, setNotificationWorkflow] = React.useState( - selectedOrganization.defaults === undefined - ? "" - : selectedOrganization.defaults.notification_workflow === undefined || - selectedOrganization.defaults.notification_workflow.length === 0 - ? "" - : selectedOrganization.defaults.notification_workflow - ); - const [documentationReference, setDocumentationReference] = React.useState( - selectedOrganization.defaults === undefined - ? "" - : selectedOrganization.defaults.documentation_reference === undefined || - selectedOrganization.defaults.documentation_reference.length === 0 - ? "" - : selectedOrganization.defaults.documentation_reference - ); - const [openidClientId, setOpenidClientId] = React.useState( - selectedOrganization.sso_config === undefined - ? "" - : selectedOrganization.sso_config.client_id === undefined || - selectedOrganization.sso_config.client_id.length === 0 - ? "" - : selectedOrganization.sso_config.client_id - ); - const [openidClientSecret, setOpenidClientSecret] = React.useState( - selectedOrganization.sso_config === undefined - ? "" - : selectedOrganization.sso_config.client_secret === undefined || - selectedOrganization.sso_config.client_secret.length === 0 - ? "" - : selectedOrganization.sso_config.client_secret - ); - const [openidAuthorization, setOpenidAuthorization] = React.useState( - selectedOrganization.sso_config === undefined - ? "" - : selectedOrganization.sso_config.openid_authorization === undefined || - selectedOrganization.sso_config.openid_authorization.length === 0 - ? "" - : selectedOrganization.sso_config.openid_authorization - ); - const [openidToken, setOpenidToken] = React.useState( - selectedOrganization.sso_config === undefined - ? "" - : selectedOrganization.sso_config.openid_token === undefined || - selectedOrganization.sso_config.openid_token.length === 0 - ? "" - : selectedOrganization.sso_config.openid_token - ) const [file, setFile] = React.useState(""); const [fileBase64, setFileBase64] = React.useState( @@ -178,49 +84,6 @@ const OrgHeader = (props) => { }; } - const handleEditOrg = ( - name, - description, - orgId, - image, - defaults, - sso_config - ) => { - - const data = { - name: name, - description: description, - org_id: orgId, - image: image, - defaults: defaults, - sso_config: sso_config, - }; - - const url = globalUrl + `/api/v1/orgs/${selectedOrganization.id}`; - fetch(url, { - mode: "cors", - method: "POST", - body: JSON.stringify(data), - credentials: "include", - crossDomain: true, - withCredentials: true, - headers: { - "Content-Type": "application/json; charset=utf-8", - }, - }) - .then((response) => - response.json().then((responseJson) => { - if (responseJson["success"] === false) { - alert.error("Failed updating org: ", responseJson.reason); - } else { - alert.success("Successfully edited org!"); - } - }) - ) - .catch((error) => { - alert.error("Err: " + error.toString()); - }); - }; var image = ""; const editHeaderImage = (event) => { @@ -238,9 +101,7 @@ const OrgHeader = (props) => { variant="contained" color="primary" disabled={ - userdata === undefined || - userdata === null || - userdata.admin !== "true" + userdata === undefined || userdata === null || userdata.admin !== "true" } onClick={() => handleEditOrg( @@ -248,22 +109,9 @@ const OrgHeader = (props) => { orgDescription, selectedOrganization.id, selectedOrganization.image, - { - app_download_repo: appDownloadUrl, - app_download_branch: appDownloadBranch, - workflow_download_repo: workflowDownloadUrl, - workflow_download_branch: workflowDownloadBranch, - notification_workflow: notificationWorkflow, - documentation_reference: documentationReference, - }, - { - sso_entrypoint: ssoEntrypoint, - sso_certificate: ssoCertificate, - client_id: openidClientId, - client_secret: openidClientSecret, - openid_authorization: openidAuthorization, - openid_token: openidToken, - } + {}, + {}, + [], ) } > @@ -291,6 +139,7 @@ const OrgHeader = (props) => { }} /> ); + return (
{
-
- { - setExpanded(!expanded); - }} - > - {expanded ? : } - - {expanded ? ( - - - - Notification Workflow ID - { - setNotificationWorkflow(e.target.value); - }} - InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, - style: { - color: "white", - }, - }} - /> - - - - - Org Documentation reference - { - setDocumentationReference(e.target.value); - }} - InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, - style: { - color: "white", - }, - }} - /> - - - {isCloud ? null : - - OpenID connect - - - - Client ID - 0 - } - id="outlined-with-placeholder" - margin="normal" - variant="outlined" - placeholder="The OpenID client ID from the identity provider" - value={openidClientId} - onChange={(e) => { - setOpenidClientId(e.target.value); - }} - InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, - style: { - color: "white", - }, - }} - /> - - - - - Client Secret (optional) - 0 - } - id="outlined-with-placeholder" - margin="normal" - variant="outlined" - placeholder="The OpenID client secret - DONT use this if dealing with implicit auth / PKCE" - value={openidClientSecret} - onChange={(e) => { - setOpenidClientSecret(e.target.value); - }} - InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, - style: { - color: "white", - }, - }} - /> - - - - - - - Authorization URL - { - setOpenidAuthorization(e.target.value) - }} - InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, - style: { - color: "white", - }, - }} - /> - - - - - Token URL - { - setOpenidToken(e.target.value) - }} - InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, - style: { - color: "white", - }, - }} - /> - - - - - } - {/*isCloud ? null : */} - - SAML SSO (v1.1) - - - - SSO Entrypoint (IdP) - 0 - } - id="outlined-with-placeholder" - margin="normal" - variant="outlined" - placeholder="The entrypoint URL from your provider" - value={ssoEntrypoint} - onChange={(e) => { - setSsoEntrypoint(e.target.value); - }} - InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, - style: { - color: "white", - }, - }} - /> - - - - - SSO Certificate (X509) - { - setSsoCertificate(e.target.value); - }} - InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, - style: { - color: "white", - }, - }} - /> - - - - {isCloud ? - - IdP URL for Shuffle: https://shuffler.io/api/v1/login_sso - - : null} - - {isCloud ? null : ( - - - App Download URL - { - setAppDownloadUrl(e.target.value); - }} - InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, - style: { - color: "white", - }, - }} - /> - - - )} - {isCloud ? null : ( - - - App Download Branch - { - setAppDownloadBranch(e.target.value); - }} - InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, - style: { - color: "white", - }, - }} - /> - - - )} - {isCloud ? null : ( - - - Workflow Download URL - { - setWorkflowDownloadUrl(e.target.value); - }} - InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, - style: { - color: "white", - }, - }} - /> - - - )} - {isCloud ? null : ( - - - Workflow Download Branch - { - setWorkflowDownloadBranch(e.target.value); - }} - InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, - style: { - color: "white", - }, - }} - /> - - - )} - -
- {orgSaveButton} -
- {/* - - {expanded ? - - : - - } - - */} - - ) : null} -
); }; diff --git a/frontend/src/components/OrgHeaderexpanded.jsx b/frontend/src/components/OrgHeaderexpanded.jsx new file mode 100644 index 00000000..f2691e54 --- /dev/null +++ b/frontend/src/components/OrgHeaderexpanded.jsx @@ -0,0 +1,702 @@ +import React, { useEffect } from "react"; + +import { makeStyles } from "@material-ui/styles"; +import { useTheme } from "@material-ui/core/styles"; +import { useAlert } from "react-alert"; + +import { + FormControl, + InputLabel, + Paper, + OutlinedInput, + Checkbox, + Card, + Tooltip, + FormControlLabel, + Typography, + Switch, + Select, + MenuItem, + Divider, + TextField, + Button, + Tabs, + Tab, + Grid, +} from "@material-ui/core"; + +import IconButton from "@material-ui/core/IconButton"; +import ExpandLessIcon from "@material-ui/icons/ExpandLess"; +import ExpandMoreIcon from "@material-ui/icons/ExpandMore"; +import SaveIcon from "@material-ui/icons/Save"; + +const useStyles = makeStyles({ + notchedOutline: { + borderColor: "#f85a3e !important", + }, +}); + +const OrgHeaderexpanded = (props) => { + const { + userdata, + selectedOrganization, + setSelectedOrganization, + globalUrl, + isCloud, + adminTab, + } = props; + + const theme = useTheme(); + const alert = useAlert(); + const classes = useStyles(); + const defaultBranch = "master"; + + const [orgName, setOrgName] = React.useState(selectedOrganization.name); + const [orgDescription, setOrgDescription] = React.useState( + selectedOrganization.description + ); + + const [appDownloadUrl, setAppDownloadUrl] = React.useState( + selectedOrganization.defaults === undefined + ? "https://github.com/frikky/shuffle-apps" + : selectedOrganization.defaults.app_download_repo === undefined || + selectedOrganization.defaults.app_download_repo.length === 0 + ? "https://github.com/frikky/shuffle-apps" + : selectedOrganization.defaults.app_download_repo + ); + const [appDownloadBranch, setAppDownloadBranch] = React.useState( + selectedOrganization.defaults === undefined + ? defaultBranch + : selectedOrganization.defaults.app_download_branch === undefined || + selectedOrganization.defaults.app_download_branch.length === 0 + ? defaultBranch + : selectedOrganization.defaults.app_download_branch + ); + const [workflowDownloadUrl, setWorkflowDownloadUrl] = React.useState( + selectedOrganization.defaults === undefined + ? "https://github.com/frikky/shuffle-apps" + : selectedOrganization.defaults.workflow_download_repo === undefined || + selectedOrganization.defaults.workflow_download_repo.length === 0 + ? "https://github.com/frikky/shuffle-workflows" + : selectedOrganization.defaults.workflow_download_repo + ); + const [workflowDownloadBranch, setWorkflowDownloadBranch] = React.useState( + selectedOrganization.defaults === undefined + ? defaultBranch + : selectedOrganization.defaults.workflow_download_branch === undefined || + selectedOrganization.defaults.workflow_download_branch.length === 0 + ? defaultBranch + : selectedOrganization.defaults.workflow_download_branch + ); + const [ssoEntrypoint, setSsoEntrypoint] = React.useState( + selectedOrganization.sso_config === undefined + ? "" + : selectedOrganization.sso_config.sso_entrypoint === undefined || + selectedOrganization.sso_config.sso_entrypoint.length === 0 + ? "" + : selectedOrganization.sso_config.sso_entrypoint + ); + const [ssoCertificate, setSsoCertificate] = React.useState( + selectedOrganization.sso_config === undefined + ? "" + : selectedOrganization.sso_config.sso_certificate === undefined || + selectedOrganization.sso_config.sso_certificate.length === 0 + ? "" + : selectedOrganization.sso_config.sso_certificate + ); + const [notificationWorkflow, setNotificationWorkflow] = React.useState( + selectedOrganization.defaults === undefined + ? "" + : selectedOrganization.defaults.notification_workflow === undefined || + selectedOrganization.defaults.notification_workflow.length === 0 + ? "" + : selectedOrganization.defaults.notification_workflow + ); + + const [documentationReference, setDocumentationReference] = React.useState( + selectedOrganization.defaults === undefined + ? "" + : selectedOrganization.defaults.documentation_reference === undefined || + selectedOrganization.defaults.documentation_reference.length === 0 + ? "" + : selectedOrganization.defaults.documentation_reference + ); + const [openidClientId, setOpenidClientId] = React.useState( + selectedOrganization.sso_config === undefined + ? "" + : selectedOrganization.sso_config.client_id === undefined || + selectedOrganization.sso_config.client_id.length === 0 + ? "" + : selectedOrganization.sso_config.client_id + ); + const [openidClientSecret, setOpenidClientSecret] = React.useState( + selectedOrganization.sso_config === undefined + ? "" + : selectedOrganization.sso_config.client_secret === undefined || + selectedOrganization.sso_config.client_secret.length === 0 + ? "" + : selectedOrganization.sso_config.client_secret + ); + const [openidAuthorization, setOpenidAuthorization] = React.useState( + selectedOrganization.sso_config === undefined + ? "" + : selectedOrganization.sso_config.openid_authorization === undefined || + selectedOrganization.sso_config.openid_authorization.length === 0 + ? "" + : selectedOrganization.sso_config.openid_authorization + ); + const [openidToken, setOpenidToken] = React.useState( + selectedOrganization.sso_config === undefined + ? "" + : selectedOrganization.sso_config.openid_token === undefined || + selectedOrganization.sso_config.openid_token.length === 0 + ? "" + : selectedOrganization.sso_config.openid_token + ) + + const handleEditOrg = ( + name, + description, + orgId, + image, + defaults, + sso_config + ) => { + + const data = { + name: name, + description: description, + org_id: orgId, + image: image, + defaults: defaults, + sso_config: sso_config, + }; + + const url = globalUrl + `/api/v1/orgs/${selectedOrganization.id}`; + fetch(url, { + mode: "cors", + method: "POST", + body: JSON.stringify(data), + credentials: "include", + crossDomain: true, + withCredentials: true, + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + }) + .then((response) => + response.json().then((responseJson) => { + if (responseJson["success"] === false) { + alert.error("Failed updating org: ", responseJson.reason); + } else { + alert.success("Successfully edited org!"); + } + }) + ) + .catch((error) => { + alert.error("Err: " + error.toString()); + }); + }; + + const orgSaveButton = ( + + + + ); + + return ( +
+ + + + Notification Workflow ID + { + setNotificationWorkflow(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + Org Documentation reference + { + setDocumentationReference(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + {isCloud ? null : + + OpenID connect + + + + Client ID + 0 + } + id="outlined-with-placeholder" + margin="normal" + variant="outlined" + placeholder="The OpenID client ID from the identity provider" + value={openidClientId} + onChange={(e) => { + setOpenidClientId(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + Client Secret (optional) + 0 + } + id="outlined-with-placeholder" + margin="normal" + variant="outlined" + placeholder="The OpenID client secret - DONT use this if dealing with implicit auth / PKCE" + value={openidClientSecret} + onChange={(e) => { + setOpenidClientSecret(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + + + Authorization URL + { + setOpenidAuthorization(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + Token URL + { + setOpenidToken(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + } + {/*isCloud ? null : */} + + SAML SSO (v1.1) + + + + SSO Entrypoint (IdP) + 0 + } + id="outlined-with-placeholder" + margin="normal" + variant="outlined" + placeholder="The entrypoint URL from your provider" + value={ssoEntrypoint} + onChange={(e) => { + setSsoEntrypoint(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + SSO Certificate (X509) + { + setSsoCertificate(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + {isCloud ? + + IdP URL for Shuffle: https://shuffler.io/api/v1/login_sso + + : null} + + {isCloud ? null : ( + + + App Download URL + { + setAppDownloadUrl(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + )} + {isCloud ? null : ( + + + App Download Branch + { + setAppDownloadBranch(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + )} + {isCloud ? null : ( + + + Workflow Download URL + { + setWorkflowDownloadUrl(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + )} + {isCloud ? null : ( + + + Workflow Download Branch + { + setWorkflowDownloadBranch(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + )} + +
+ {orgSaveButton} +
+ {/* + + {expanded ? + + : + + } + + */} + +
+ ) +} + +export default OrgHeaderexpanded; \ No newline at end of file diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx old mode 100644 new mode 100755 index 455c0e57..1d628133 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -7,7 +7,7 @@ import { sortByKey } from "../views/AngularWorkflow.jsx"; import { useTheme } from "@material-ui/core/styles"; import NestedMenuItem from "material-ui-nested-menu-item"; import { useAlert } from "react-alert"; -import theme from '../theme'; +import theme from '../theme.jsx'; import { ButtonGroup, @@ -45,6 +45,10 @@ import { Fade, } from "@material-ui/core"; +import { + Autocomplete +} from "@material-ui/lab"; + import { HelpOutline as HelpOutlineIcon, Description as DescriptionIcon, @@ -72,7 +76,6 @@ import { Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, - Circle as CircleIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, @@ -82,11 +85,11 @@ import { ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon, AutoFixHigh as AutoFixHighIcon, + Circle as CircleIcon, SquareFoot as SquareFootIcon, } from '@mui/icons-material'; //} from "@material-ui/icons"; -import Autocomplete from "@material-ui/lab/Autocomplete"; //import CodeMirror from "@uiw/react-codemirror"; //import "codemirror/keymap/sublime"; @@ -163,6 +166,8 @@ const ParsedAction = (props) => { lastSaved, setLastSaved, setShowVideo, + toolsAppId, + aiSubmit, //expansionModalOpen, //setExpansionModalOpen, } = props; @@ -194,13 +199,11 @@ const ParsedAction = (props) => { if (paramcheck.id === "TOGGLED"){ setHideBody(false) setActivateHidingBodyButton(false) - console.log("TOGGLED BODY!") } else { setHideBody(true) if (paramcheck.id === "UNTOGGLED") { setActivateHidingBodyButton(false) - console.log("UNTOGGLED!") } } } @@ -277,7 +280,7 @@ const ParsedAction = (props) => { console.log("FOUNDACTION: ", foundAction); if (foundAction !== null && foundAction !== undefined) { var foundparams = []; - for (var paramkey in foundAction.parameters) { + for (let [paramkey,paramkeyval] in Object.entries(foundAction.parameters)) { const param = foundAction.parameters[paramkey]; const foundParam = selectedAction.parameters.find( @@ -401,7 +404,7 @@ const ParsedAction = (props) => { if (actionlist.length === 0) { // FIXME: Have previous execution values in here if (workflowExecutions.length > 0) { - for (var key in workflowExecutions) { + for (let [key,keyval] in Object.entries(workflowExecutions)) { if ( workflowExecutions[key].execution_argument === undefined || workflowExecutions[key].execution_argument === null || @@ -451,7 +454,7 @@ const ParsedAction = (props) => { workflow.workflow_variables !== undefined && workflow.workflow_variables.length > 0 ) { - for (var key in workflow.workflow_variables) { + for (let [key,keyval] in Object.entries(workflow.workflow_variables)) { const item = workflow.workflow_variables[key]; actionlist.push({ type: "workflow_variable", @@ -470,7 +473,7 @@ const ParsedAction = (props) => { workflow.execution_variables !== undefined && workflow.execution_variables.length > 0 ) { - for (var key in workflow.execution_variables) { + for (let [key,keyval] in Object.entries(workflow.execution_variables)) { const item = workflow.execution_variables[key]; actionlist.push({ type: "execution_variable", @@ -488,7 +491,7 @@ const ParsedAction = (props) => { var parents = getParents(selectedAction); if (parents.length > 1) { - for (var key in parents) { + for (let [key,keyval] in Object.entries(parents)) { const item = parents[key]; if (item.label === "Execution Argument") { continue; @@ -500,7 +503,7 @@ const ParsedAction = (props) => { if (workflowExecutions.length > 0) { // Look for the ID const found = false; - for (var key in workflowExecutions) { + for (let [key,keyval] in Object.entries(workflowExecutions)) { if ( workflowExecutions[key].results === undefined || workflowExecutions[key].results === null @@ -566,20 +569,19 @@ const ParsedAction = (props) => { if (found !== null && found !== undefined) { var new_occurences = [] - for (var key in found) { + for (let [key,keyval] in Object.entries(found)) { if (found[key][0] !== "\\") { new_occurences.push(found[key]) } } - console.log("New found: ", new_occurences) found = new_occurences.valueOf() } if (found !== null) { try { // When the found array is empty. - for (var i = 0; i < found.length; i++) { + for (let i = 0; i < found.length; i++) { const variableSplit = found[i].split(".#") if ((variableSplit.length-1) > 1) { //console.log("Larger than 1: ", variableSplit) @@ -589,7 +591,7 @@ const ParsedAction = (props) => { } var foundSlice = false - for (var j = 0; j < actionlist.length; j++) { + for (let j = 0; j < actionlist.length; j++) { //console.log("ACTION: ", found[i], actionlist[j]) //console.log("ACTION :", found[i].split(".")[0].slice(1,).toLowerCase(), actionlist[j].autocomplete.toLowerCase()) if(found[i].split(".")[0].slice(1,).toLowerCase() == actionlist[j].autocomplete.toLowerCase()){ @@ -623,13 +625,12 @@ const ParsedAction = (props) => { return helperText } - const changeActionParameter = (event, count, data) => { + const changeActionParameter = (event, count, data, viewForceUpdate) => { //console.log("Action change: ", selectedAction, data) if (data.name.startsWith("${") && data.name.endsWith("}")) { // PARAM FIX - Gonna use the ID field, even though it's a hack - const paramcheck = selectedAction.parameters.find( - (param) => param.name === "body" - ); + const paramcheck = selectedAction.parameters.find((param) => param.name === "body"); + if (paramcheck !== undefined) { // Escapes all double quotes var toReplace = event.target.value.trim() @@ -705,10 +706,7 @@ const ParsedAction = (props) => { } // bad detection mechanism probably - if ( - event.target.value[event.target.value.length - 1] === "." && - actionlist.length > 0 - ) { + if (event.target.value[event.target.value.length - 1] === "." && actionlist.length > 0) { console.log("GET THE LAST ARGUMENT FOR NODE!"); // THIS IS AN EXAMPLE OF SHOWING IT /* @@ -730,7 +728,7 @@ const ParsedAction = (props) => { var curstring = ""; var record = false; - for (var key in selectedActionParameters[count].value) { + for (let [key,keyval] in Object.entries(selectedActionParameters[count].value)) { const item = selectedActionParameters[count].value[key]; if (record) { curstring += item; @@ -819,8 +817,8 @@ const ParsedAction = (props) => { } setSelectedAction(selectedAction); - if (forceUpdate) { - setUpdate(Math.random()) + if (forceUpdate || viewForceUpdate === true) { + setUpdate(Math.random()) } //setUpdate(event.target.value) }; @@ -911,7 +909,7 @@ const ParsedAction = (props) => { var curstring = "" var record = false - for (var key in selectedActionParameters[count].value) { + for (let [key,keyval] in Object.entries(selectedActionParameters[count].value)) { const item = selectedActionParameters[count].value[key] if (record) { curstring += item @@ -1187,7 +1185,7 @@ const ParsedAction = (props) => { renderInput={(params) => { if (params.inputProps !== undefined && params.inputProps !== null && params.inputProps.value !== undefined && params.inputProps.value !== null) { const prefixes = ["Post", "Put", "Patch"] - for (var key in prefixes) { + for (let [key,keyval] in Object.entries(prefixes)) { if (params.inputProps.value.startsWith(prefixes[key])) { params.inputProps.value = params.inputProps.value.replace(prefixes[key]+" ", "", -1) if (params.inputProps.value.length > 1) { @@ -1346,9 +1344,8 @@ const ParsedAction = (props) => { } if (data.name.startsWith("${") && data.name.endsWith("}")) { - const paramcheck = selectedAction.parameters.find( - (param) => param.name === "body" - ); + const paramcheck = selectedAction.parameters.find((param) => param.name === "body"); + if (paramcheck !== undefined && paramcheck !== null) { if ( @@ -1426,8 +1423,8 @@ const ParsedAction = (props) => { setHideBody(!hideBody); - for (var key in selectedActionParameters) { - var currentItem = selectedActionParameters[key]; + for (let paramkey in Object.entries(selectedActionParameters)) { + var currentItem = selectedActionParameters[paramkey]; if (currentItem.name === "ssl_verify") { } @@ -1462,7 +1459,7 @@ const ParsedAction = (props) => { if (found === null) { setActivateHidingBodyButton(true); } else { - console.log("In found: ", found, hideBody) + //console.log("In found: ", found, hideBody) } } else { //console.log("SHOW BUTTON"); @@ -1472,10 +1469,11 @@ const ParsedAction = (props) => { openApiHelperText = "OpenAPI spec: fill the following fields."; //console.log("SHOULD ADD TO selectedActionParameters!: ", found, selectedActionParameters) var changed = false; - for (var specKey in found) { + for (let specKey in found) { const tmpitem = found[specKey]; var skip = false; - for (var innerkey in selectedActionParameters) { + + for (let innerkey in selectedActionParameters) { if (selectedActionParameters[innerkey].name === tmpitem) { skip = true; break; @@ -1524,6 +1522,8 @@ const ParsedAction = (props) => { const shufflecode = fieldCount !== count ? null : ( { expansionModalOpen={expansionModalOpen} setExpansionModalOpen={setExpansionModalOpen} globalUrl={globalUrl} + + workflowExecutions={workflowExecutions} + getParents={getParents} + selectedAction={selectedAction} + parameterName={data.name} + aiSubmit={aiSubmit} /> ) @@ -1589,6 +1595,7 @@ const ParsedAction = (props) => { maxWidth: "95%", fontSize: "1em", }, + disableUnderline: true, endAdornment: hideExtraTypes ? null : ( @@ -1608,6 +1615,9 @@ const ParsedAction = (props) => { style={{ cursor: "pointer", margin: multiline ? 5 : 0, }} onClick={(event) => { event.preventDefault() + + // Get cursor position + // This makes it so we can put it in the right location? setMenuPosition({ top: event.pageY + 10, left: event.pageX + 10, @@ -1626,12 +1636,13 @@ const ParsedAction = (props) => { helperText={returnHelperText(data.name, data.value)} onClick={() => { console.log("Clicked field: ", clickedFieldId, data.name) - if (data.name === "file_id") { - console.log("show file video?") - if (setShowVideo !== undefined) { - setShowVideo("https://www.youtube.com/embed/DPYowyTbsSk") - } - } + + //if (data.name === "file_id") { + // console.log("show file video?") + // if (setShowVideo !== undefined) { + // setShowVideo("https://www.youtube.com/embed/DPYowyTbsSk") + // } + //} //(data.name.toLowerCase().includes("api") || /* setExpansionModalOpen(false); @@ -1707,7 +1718,7 @@ const ParsedAction = (props) => { var foundnewline = false var allValues = [] - for (var key in splitdata) { + for (let [key,keyval] in Object.entries(splitdata)) { const line = splitdata[key] if (line === "") { foundnewline = true @@ -1777,7 +1788,7 @@ const ParsedAction = (props) => { const tmpsplit = selectedActionParameters[count].value.split("\n") var valsplit = [] var add_empty = false - for (var key in tmpsplit) { + for (let [key,keyval] in Object.entries(tmpsplit)) { if (tmpsplit[key] === "") { add_empty = true continue @@ -1792,7 +1803,7 @@ const ParsedAction = (props) => { console.log("Split: ", valsplit) var newarr = [] - for (var key in valsplit) { + for (let [key,keyval] in Object.entries(valsplit)) { var line = valsplit[key] if (key == index) { @@ -1832,7 +1843,7 @@ const ParsedAction = (props) => { var tmpsplit = selectedActionParameters[count].value.split("\n") var valsplit = [] var add_empty = false - for (var key in tmpsplit) { + for (let [key,keyval] in Object.entries(tmpsplit)) { if (tmpsplit[key] === "") { add_empty = true continue @@ -1847,7 +1858,7 @@ const ParsedAction = (props) => { console.log("Split: ", valsplit) var newarr = [] - for (var key in valsplit) { + for (let [key,keyval] in Object.entries(valsplit)) { var line = valsplit[key] if (key == index) { @@ -1911,7 +1922,7 @@ const ParsedAction = (props) => { // Basic helpertext - if (data.name.toLowerCase() === "file_category") { + if (files !== undefined && files !== null && data.name.toLowerCase() === "file_category") { //selectedActionParameters[count].options.length > 0 console.log("FileS: ", files) if (files.namespaces !== undefined && files.namespaces !== null && files.namespaces.length > 0) { @@ -1941,7 +1952,7 @@ const ParsedAction = (props) => { }, endAdornment: hideExtraTypes ? null : ( - + { @@ -2080,11 +2091,8 @@ const ParsedAction = (props) => { }; const handleItemClick = (values) => { - if ( - values === undefined || - values === null || - values.length === 0 - ) { + console.log("In normal itemclick") + if (values === undefined ||values === null ||values.length === 0) { return; } @@ -2094,7 +2102,7 @@ const ParsedAction = (props) => { : "$" + values[0].autocomplete; toComplete = toComplete.toLowerCase().replaceAll(" ", "_"); - for (var key in values) { + for (let [key,keyval] in Object.entries(values)) { if (key == 0 || values[key].autocomplete.length === 0) { continue; } @@ -2102,6 +2110,8 @@ const ParsedAction = (props) => { toComplete += values[key].autocomplete; } + + // Handles the fields under OpenAPI body to be parsed. if (data.name.startsWith("${") && data.name.endsWith("}")) { console.log("INSIDE VALUE REPLACE: ", data.name, toComplete); @@ -2135,10 +2145,8 @@ const ParsedAction = (props) => { } } - selectedActionParameters[count]["value_replace"] = - paramcheck; - selectedAction.parameters[count]["value_replace"] = - paramcheck; + selectedActionParameters[count]["value_replace"] = paramcheck; + selectedAction.parameters[count]["value_replace"] = paramcheck; setSelectedAction(selectedAction); setUpdate(Math.random()); @@ -2148,13 +2156,15 @@ const ParsedAction = (props) => { } } - selectedActionParameters[count].value += toComplete; - selectedAction.parameters[count].value = - selectedActionParameters[count].value; - setSelectedAction(selectedAction); - setUpdate(Math.random()); - - setShowDropdown(false); + console.log("In nestedclick!!") + var newValue = selectedActionParameters[count].value + toComplete + changeActionParameter({target: {value: newValue}}, count, data, true) + //selectedActionParameters[count].value += toComplete; + //selectedAction.parameters[count].value = selectedActionParameters[count].value; + //setSelectedAction(selectedAction); + //setUpdate(Math.random()); + + setShowDropdown(false); setMenuPosition(null); }; @@ -2205,7 +2215,7 @@ const ParsedAction = (props) => { workflow.triggers !== null && workflow.triggers.length > 0 ) { - for (var key in workflow.triggers) { + for (let [key,keyval] in Object.entries(workflow.triggers)) { const item = workflow.triggers[key]; if (cy !== undefined) { @@ -2608,7 +2618,9 @@ const ParsedAction = (props) => { setUpdate(Math.random()); }} - onClick={() => setShowAutocomplete(true)} + onClick={() => { + setShowAutocomplete(true) + }} fullWidth open={showAutocomplete} style={{ @@ -2618,22 +2630,12 @@ const ParsedAction = (props) => { borderRadius: theme.palette.borderRadius, }} onChange={(e) => { - if ( - selectedActionParameters[count].value[ - selectedActionParameters[count].value.length - 1 - ] === "." - ) { - e.target.value.autocomplete = - e.target.value.autocomplete.slice( - 1, - e.target.value.autocomplete.length - ); + if (selectedActionParameters[count].value[selectedActionParameters[count].value.length - 1] === ".") { + e.target.value.autocomplete = e.target.value.autocomplete.slice(1,e.target.value.autocomplete.length); } - selectedActionParameters[count].value += - e.target.value.autocomplete; - selectedAction.parameters[count].value = - selectedActionParameters[count].value; + selectedActionParameters[count].value += e.target.value.autocomplete; + selectedAction.parameters[count].value = selectedActionParameters[count].value; setSelectedAction(selectedAction); setUpdate(Math.random()); @@ -2727,14 +2729,16 @@ const ParsedAction = (props) => { if (workflowExecutions.length > 0) { // Look for the ID const found = false; - for (var key in workflowExecutions) { - if ( - workflowExecutions[key].results === undefined || - workflowExecutions[key].results === null - ) { + for (let [key,keyval] in Object.entries(workflowExecutions)) { + if (workflowExecutions[key].results === undefined || workflowExecutions[key].results === null) { continue; } + // Enforces it to show at least one + //if (workflowExecutions[key].execution_argument.includes("too large") && key !== workflowExecutions.length - 1) { + // continue + //} + var foundResult = workflowExecutions[key].results.find( (result) => result.action.id === selectedAction.id ) @@ -2955,6 +2959,7 @@ const ParsedAction = (props) => { style={theme.palette.textFieldStyle} InputProps={{ style: theme.palette.innerTextfieldStyle, + disableUnderline: true, }} fullWidth color="primary" @@ -2971,121 +2976,123 @@ const ParsedAction = (props) => { // // Should make it a function lol if (workflow.branches !== undefined && workflow.branches !== null) { - for (var key in workflow.branches) { - for (var subkey in workflow.branches[key].conditions) { - const condition = workflow.branches[key].conditions[subkey] - const sourceparam = condition.source - const destinationparam = condition.destination + for (let [key,keyval] in Object.entries(workflow.branches)) { + if (workflow.branches[key].conditions !== undefined && workflow.branches[key].conditions !== null) { + for (let [subkey,subkeyval] in Object.entries(workflow.branches[key].conditions)) { + const condition = workflow.branches[key].conditions[subkey] + const sourceparam = condition.source + const destinationparam = condition.destination - // Should have a smarter way of discovering node names - // Finding index(es) and replacing at the location - if (sourceparam.value.includes("$")) { - try { - var cnt = -1 - var previous = 0 - while (true) { - cnt += 1 - // Need to make sure e.g. changing the first here doesn't change the 2nd - // $change_me - // $change_me_2 - - const foundindex = sourceparam.value.toLowerCase().indexOf(parsedBaseLabel, previous) - if (foundindex === previous && foundindex !== 0) { - break - } - - if (foundindex >= 0) { - previous = foundindex+newname.length - // Need to add diff of length to word - - // Check location: - // If it's a-zA-Z_ then don't replace - if (sourceparam.value.length > foundindex+parsedBaseLabel.length) { - const regex = /[a-zA-Z0-9_]/g; - const match = sourceparam.value[foundindex+parsedBaseLabel.length].match(regex); - if (match !== null) { - continue - } - } + // Should have a smarter way of discovering node names + // Finding index(es) and replacing at the location + if (sourceparam.value.includes("$")) { + try { + var cnt = -1 + var previous = 0 + while (true) { + cnt += 1 + // Need to make sure e.g. changing the first here doesn't change the 2nd + // $change_me + // $change_me_2 - console.log("Old found: ", workflow.branches[key].conditions[subkey].source.value) - const extralength = newname.length-parsedBaseLabel.length - sourceparam.value = sourceparam.value.substring(0, foundindex) + newname + sourceparam.value.substring(foundindex-extralength+newname.length, sourceparam.value.length) + const foundindex = sourceparam.value.toLowerCase().indexOf(parsedBaseLabel, previous) + if (foundindex === previous && foundindex !== 0) { + break + } + + if (foundindex >= 0) { + previous = foundindex+newname.length + // Need to add diff of length to word + + // Check location: + // If it's a-zA-Z_ then don't replace + if (sourceparam.value.length > foundindex+parsedBaseLabel.length) { + const regex = /[a-zA-Z0-9_]/g; + const match = sourceparam.value[foundindex+parsedBaseLabel.length].match(regex); + if (match !== null) { + continue + } + } + + console.log("Old found: ", workflow.branches[key].conditions[subkey].source.value) + const extralength = newname.length-parsedBaseLabel.length + sourceparam.value = sourceparam.value.substring(0, foundindex) + newname + sourceparam.value.substring(foundindex-extralength+newname.length, sourceparam.value.length) - console.log("New: ", workflow.branches[key].conditions[subkey].source.value) - } else { - break - } + console.log("New: ", workflow.branches[key].conditions[subkey].source.value) + } else { + break + } - // Break no matter what after 5 replaces. May need to increase - if (cnt >= 5) { - break - } + // Break no matter what after 5 replaces. May need to increase + if (cnt >= 5) { + break + } + } + } catch (e) { + console.log("Failed value replacement based on index: ", e) } - } catch (e) { - console.log("Failed value replacement based on index: ", e) } - } - if (destinationparam.value.includes("$")) { - try { - var cnt = -1 - var previous = 0 - while (true) { - cnt += 1 - // Need to make sure e.g. changing the first here doesn't change the 2nd - // $change_me - // $change_me_2 - - const foundindex = destinationparam.value.toLowerCase().indexOf(parsedBaseLabel, previous) - if (foundindex === previous && foundindex !== 0) { - break - } - - if (foundindex >= 0) { - previous = foundindex+newname.length - // Need to add diff of length to word - - // Check location: - // If it's a-zA-Z_ then don't replace - if (destinationparam.value.length > foundindex+parsedBaseLabel.length) { - const regex = /[a-zA-Z0-9_]/g; - const match = destinationparam.value[foundindex+parsedBaseLabel.length].match(regex); - if (match !== null) { - continue - } - } + if (destinationparam.value.includes("$")) { + try { + var cnt = -1 + var previous = 0 + while (true) { + cnt += 1 + // Need to make sure e.g. changing the first here doesn't change the 2nd + // $change_me + // $change_me_2 - console.log("Old found: ", workflow.branches[key].conditions[subkey].destination.value) - const extralength = newname.length-parsedBaseLabel.length - destinationparam.value = destinationparam.value.substring(0, foundindex) + newname + destinationparam.value.substring(foundindex-extralength+newname.length, destinationparam.value.length) + const foundindex = destinationparam.value.toLowerCase().indexOf(parsedBaseLabel, previous) + if (foundindex === previous && foundindex !== 0) { + break + } + + if (foundindex >= 0) { + previous = foundindex+newname.length + // Need to add diff of length to word + + // Check location: + // If it's a-zA-Z_ then don't replace + if (destinationparam.value.length > foundindex+parsedBaseLabel.length) { + const regex = /[a-zA-Z0-9_]/g; + const match = destinationparam.value[foundindex+parsedBaseLabel.length].match(regex); + if (match !== null) { + continue + } + } + + console.log("Old found: ", workflow.branches[key].conditions[subkey].destination.value) + const extralength = newname.length-parsedBaseLabel.length + destinationparam.value = destinationparam.value.substring(0, foundindex) + newname + destinationparam.value.substring(foundindex-extralength+newname.length, destinationparam.value.length) - console.log("New: ", workflow.branches[key].conditions[subkey].destination.value) - } else { - break - } + console.log("New: ", workflow.branches[key].conditions[subkey].destination.value) + } else { + break + } - // Break no matter what after 5 replaces. May need to increase - if (cnt >= 5) { - break - } + // Break no matter what after 5 replaces. May need to increase + if (cnt >= 5) { + break + } + } + } catch (e) { + console.log("Failed value replacement based on index: ", e) } - } catch (e) { - console.log("Failed value replacement based on index: ", e) } } } } } - for (var key in workflow.actions) { + for (let [key,keyval] in Object.entries(workflow.actions)) { if (workflow.actions[key].id === selectedAction.id) { continue } - for (var subkey in workflow.actions[key].parameters) { + for (let [subkey, subkeyval] in Object.entries(workflow.actions[key].parameters)) { const param = workflow.actions[key].parameters[subkey]; if (!param.value.includes("$")) { continue @@ -3172,6 +3179,7 @@ const ParsedAction = (props) => { }} InputProps={{ style: theme.palette.innerTextfieldStyle, + disableUnderline: true, }} placeholder={selectedAction.execution_delay} defaultValue={selectedAction.execution_delay} @@ -3221,6 +3229,7 @@ const ParsedAction = (props) => {
) : null} + {selectedAction.authentication !== undefined && selectedAction.authentication !== null && selectedAction.authentication.length > 0 ? ( @@ -3251,7 +3260,7 @@ const ParsedAction = (props) => { selectedAction.selectedAuthentication = {}; selectedAction.authentication_id = ""; - for (var key in selectedAction.parameters) { + for (let [key,keyval] in Object.entries(selectedAction.parameters)) { //console.log(selectedAction.parameters[key]) if (selectedAction.parameters[key].configuration) { selectedAction.parameters[key].value = ""; @@ -3472,6 +3481,21 @@ const ParsedAction = (props) => { autoHighlight value={selectedAction} classes={{ inputRoot: classes.inputRoot }} + groupBy={(option) => { + // Most popular + // Is categorized + // Uncategorized + return option.category_label !== undefined && option.category_label !== null && option.category_label.length > 0 ? "Most used" : "All Actions"; + }} + renderGroup={(params) => { + return ( +
  • + {params.group} + {params.children} +
    • + ) + }} + options={selectedApp.actions === undefined || selectedApp.actions === null ? [] : selectedApp.actions.filter((a) => a.category_label !== undefined && a.category_label !== null && a.category_label.length > 0).concat(sortByKey(selectedApp.actions, "label"))} ListboxProps={{ style: { backgroundColor: theme.palette.inputColor, @@ -3486,12 +3510,7 @@ const ParsedAction = (props) => { return options }} getOptionLabel={(option) => { - if ( - option === undefined || - option === null || - option.name === undefined || - option.name === null - ) { + if (option === undefined || option === null || option.name === undefined || option.name === null ) { return null; } @@ -3501,7 +3520,6 @@ const ParsedAction = (props) => { return newname; }} - options={sortByKey(selectedApp.actions, "label")} fullWidth style={{ backgroundColor: theme.palette.inputColor, @@ -3520,19 +3538,13 @@ const ParsedAction = (props) => { }} renderOption={(data) => { var newActionname = data.name; - if ( - data.label !== undefined && - data.label !== null && - data.label.length > 0 - ) { + if (data.label !== undefined && data.label !== null && data.label.length > 0) { newActionname = data.label; } var newActiondescription = data.description; //console.log("DESC: ", newActiondescription) - if ( - data.description === undefined || data.description === null - ) { + if (data.description === undefined || data.description === null) { newActiondescription = "Description: No description defined for this action" } else { newActiondescription = "Description: "+newActiondescription @@ -3568,21 +3580,26 @@ const ParsedAction = (props) => { if (method.length > 0 && data.description !== undefined && data.description !== null && data.description.includes("http")) { var extraUrl = "" const descSplit = data.description.split("\n") - for (var line in descSplit) { - if (descSplit[line].includes("http") && descSplit[line].includes("://")) { - const urlsplit = descSplit[line].split("/") - try { - extraUrl = "/"+urlsplit.slice(3, urlsplit.length).join("/") - } catch (e) { - console.log("Failed - running with -1") - extraUrl = "/"+urlsplit.slice(3, urlsplit.length-1).join("/") - } + // Last line of descSplit + if (descSplit.length > 0) { + extraUrl = descSplit[descSplit.length-1] + } + + //for (let [line,lineval] in Object.entries(descSplit)) { + // if (descSplit[line].includes("http") && descSplit[line].includes("://")) { + // const urlsplit = descSplit[line].split("/") + // try { + // extraUrl = "/"+urlsplit.slice(3, urlsplit.length).join("/") + // } catch (e) { + // //console.log("Failed - running with -1") + // extraUrl = "/"+urlsplit.slice(3, urlsplit.length-1).join("/") + // } - //console.log("NO BASEURL TOO!! Why missing last one in certain scenarios (sevco)?", extraUrl, urlsplit, descSplit[line]) - break - } - } + // //console.log("NO BASEURL TOO!! Why missing last one in certain scenarios (sevco)?", extraUrl, urlsplit, descSplit[line]) + // //break + // } + //} if (extraUrl.length > 0) { if (extraUrl.includes(" ")) { @@ -3592,9 +3609,10 @@ const ParsedAction = (props) => { if (extraUrl.includes("#")) { extraUrl = extraUrl.split("#")[0] } + extraDescription = `${method} ${extraUrl}` } else { - console.log("No url found. Check again :)") + //console.log("No url found. Check again :)") } } @@ -3615,10 +3633,10 @@ const ParsedAction = (props) => { > {useIcon} - {newActionname} + {newActionname}
    {extraDescription.length > 0 ? - + {extraDescription} : null} @@ -3629,7 +3647,7 @@ const ParsedAction = (props) => { renderInput={(params) => { if (params.inputProps !== undefined && params.inputProps !== null && params.inputProps.value !== undefined && params.inputProps.value !== null) { const prefixes = ["Post", "Put", "Patch"] - for (var key in prefixes) { + for (let [key,keyval] in Object.entries(prefixes)) { if (params.inputProps.value.startsWith(prefixes[key])) { params.inputProps.value = params.inputProps.value.replace(prefixes[key]+" ", "", -1) if (params.inputProps.value.length > 1) { diff --git a/frontend/src/components/Priorities.jsx b/frontend/src/components/Priorities.jsx new file mode 100644 index 00000000..88bea907 --- /dev/null +++ b/frontend/src/components/Priorities.jsx @@ -0,0 +1,91 @@ +import React, { useState, useEffect } from "react"; + +import theme from "../theme.jsx"; +import { + Paper, + Typography, + Divider, + Button, + Grid, + Card, + Switch, +} from "@material-ui/core"; + +import Priority from "../components/Priority.jsx"; +import { useAlert } from "react-alert"; + +const Priorities = (props) => { + const { globalUrl, userdata, serverside, billingInfo, stripeKey, checkLogin, } = props; + const [showDismissed, setShowDismissed] = React.useState(false); + const [showRead, setShowRead] = React.useState(false); + + if (userdata === undefined || userdata === null) { + return + } + + return ( +
    +

    Suggestions

    + + Suggestions are tasks identified by Shuffle to help you discover ways to protect your and customers' company. These range from simple configurations in Shuffle to Usecases you may have missed.  +     + Learn more + + +
    + { + setShowDismissed(!showDismissed); + }} + />  Show dismissed + {userdata.priorities === null || userdata.priorities === undefined || userdata.priorities.length === 0 ? + + No Suggestions found + + : + userdata.priorities.map((priority, index) => { + if (showDismissed === false && priority.active === false) { + return null + } + + return ( + + ) + }) + } + +

    Notifications

    + + Notifications help you find potential problems with your workflows and apps.  + + Learn more + + +
    + { + setShowRead(!showRead); + }} + />  Show read +
    + ) +} + +export default Priorities; diff --git a/frontend/src/components/Priority.jsx b/frontend/src/components/Priority.jsx new file mode 100644 index 00000000..0123b37d --- /dev/null +++ b/frontend/src/components/Priority.jsx @@ -0,0 +1,128 @@ +import React, { useState, useEffect } from "react"; + +import theme from "../theme.jsx"; +import { useNavigate, Link } from "react-router-dom"; +import { + Paper, + Typography, + Divider, + Button, + Grid, + Card, +} from "@material-ui/core"; + +// import magic wand icon from material ui icons +import { + AutoFixHigh as AutoFixHighIcon, + ArrowForward as ArrowForwardIcon, +} from '@mui/icons-material'; +import { useAlert } from "react-alert"; + +const Priority = (props) => { + const { globalUrl, userdata, serverside, priority, checkLogin, } = props; + + + let navigate = useNavigate(); + const changeRecommendation = (recommendation, action) => { + const data = { + action: action, + name: recommendation.name, + }; + + fetch(`${globalUrl}/api/v1/recommendations/modify`, { + mode: "cors", + method: "POST", + body: JSON.stringify(data), + credentials: "include", + crossDomain: true, + withCredentials: true, + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + }) + .then((response) => { + if (response.status === 200) { + } else { + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success === true) { + if (checkLogin !== undefined) { + checkLogin() + } + } else { + if (responseJson.success === false && responseJson.reason !== undefined) { + alert.error("Failed change recommendation: ", responseJson.reason) + } else { + alert.error("Failed change recommendation"); + } + } + }) + .catch((error) => { + alert.info("Failed dismissing alert. Please contact support@shuffler.io if this persists."); + }); + } + + + return ( +
    +
    + + {priority.type === "usecase" || priority.type == "apps" ? : null} + + {priority.name} + + + {priority.type === "usecase" && priority.description.includes("&") ? + + {priority.name} + + {priority.description.split("&")[0]} + + + {priority.description.split("&").length > 3 ? + + + {priority.name+"2"} + + {priority.description.split("&")[2]} + + + : null} + + + : + + {priority.description} + + } +
    +
    + + {priority.active === true ? + + : null } +
    +
    + ) +} + +export default Priority; diff --git a/frontend/src/components/RenderCytoscape.js b/frontend/src/components/RenderCytoscape.js old mode 100644 new mode 100755 diff --git a/frontend/src/components/RenderCytoscape.jsx b/frontend/src/components/RenderCytoscape.jsx new file mode 100644 index 00000000..c112b448 --- /dev/null +++ b/frontend/src/components/RenderCytoscape.jsx @@ -0,0 +1,157 @@ +import React, { useState, useEffect, useLayoutEffect } from "react"; +import * as cytoscape from "cytoscape"; +import CytoscapeComponent from "react-cytoscapejs"; +import cystyle from "../defaultCytoscapeStyle.jsx"; + +const surfaceColor = "#27292D"; +const CytoscapeWrapper = (props) => { + const { globalUrl, inworkflow } = props; + + const [elements, setElements] = useState([]); + const [workflow, setWorkflow] = useState(inworkflow); + const [cy, setCy] = React.useState(); + const bodyWidth = 200; + const bodyHeight = 150; + + const setupGraph = () => { + const actions = workflow.actions.map((action) => { + const node = {}; + node.position = action.position; + node.data = action; + + node.data._id = action["id"]; + node.data.type = "ACTION"; + node.isStartNode = action["id"] === workflow.start; + + var example = ""; + if ( + action.example !== undefined && + action.example !== null && + action.example.length > 0 + ) { + example = action.example; + } + + node.data.example = example; + return node; + }); + + const triggers = workflow.triggers.map((trigger) => { + const node = {}; + node.position = trigger.position; + node.data = trigger; + + node.data._id = trigger["id"]; + node.data.type = "TRIGGER"; + + return node; + }); + + // FIXME - tmp branch update + var insertedNodes = [].concat(actions, triggers); + const edges = workflow.branches.map((branch, index) => { + //workflow.branches[index].conditions = [{ + + const edge = {}; + var conditions = workflow.branches[index].conditions; + if (conditions === undefined || conditions === null) { + conditions = []; + } + + var label = ""; + if (conditions.length === 1) { + label = conditions.length + " condition"; + } else if (conditions.length > 1) { + label = conditions.length + " conditions"; + } + + edge.data = { + id: branch.id, + _id: branch.id, + source: branch.source_id, + target: branch.destination_id, + label: label, + conditions: conditions, + hasErrors: branch.has_errors, + }; + + // This is an attempt at prettier edges. The numbers are weird to work with. + /* + //http://manual.graphspace.org/projects/graphspace-python/en/latest/demos/edge-types.html + const sourcenode = actions.find(node => node.data._id === branch.source_id) + const destinationnode = actions.find(node => node.data._id === branch.destination_id) + if (sourcenode !== undefined && destinationnode !== undefined && branch.source_id !== branch.destination_id) { + //node.data._id = action["id"] + console.log("SOURCE: ", sourcenode.position) + console.log("DESTINATIONNODE: ", destinationnode.position) + + var opposite = true + if (sourcenode.position.x > destinationnode.position.x) { + opposite = false + } else { + opposite = true + } + + edge.style = { + 'control-point-distance': opposite ? ["25%", "-75%"] : ["-10%", "90%"], + 'control-point-weight': ['0.3', '0.7'], + } + } + */ + + return edge; + }); + + setWorkflow(workflow); + + // Verifies if a branch is valid and skips others + var newedges = []; + for (var key in edges) { + var item = edges[key]; + + const sourcecheck = insertedNodes.find( + (data) => data.data.id === item.data.source + ); + const destcheck = insertedNodes.find( + (data) => data.data.id === item.data.target + ); + if (sourcecheck === undefined || destcheck === undefined) { + continue; + } + + newedges.push(item); + } + + insertedNodes = insertedNodes.concat(newedges); + setElements(insertedNodes); + }; + + if (elements.length === 0) { + setupGraph(); + } + + return ( + { + // FIXME: There's something specific loading when + // you do the first hover of a node. Why is this different? + //console.log("CY: ", incy) + setCy(incy); + }} + /> + ); +}; + +export default CytoscapeWrapper; diff --git a/frontend/src/components/ScrollToTop.jsx b/frontend/src/components/ScrollToTop.jsx old mode 100644 new mode 100755 diff --git a/frontend/src/components/Searchfield.js b/frontend/src/components/Searchfield.js index a15ff5f3..43fa7934 100644 --- a/frontend/src/components/Searchfield.js +++ b/frontend/src/components/Searchfield.js @@ -625,7 +625,7 @@ const SearchField = props => { const CustomDocHits = connectHits(DocHits) return ( -
    +
    { console.log("CLICKED") }}> diff --git a/frontend/src/components/Searchfield.jsx b/frontend/src/components/Searchfield.jsx new file mode 100644 index 00000000..37e44bd5 --- /dev/null +++ b/frontend/src/components/Searchfield.jsx @@ -0,0 +1,652 @@ +import React, {useState, useEffect, useRef} from 'react'; + +import { useNavigate, Link, useParams } from "react-router-dom"; +import { useTheme } from '@material-ui/core/styles'; +import SearchIcon from '@material-ui/icons/Search'; + +import { + Chip, + IconButton, + TextField, + InputAdornment, + List, + Card, + ListItem, + ListItemAvatar, + ListItemText, + Avatar, + Typography, + Tooltip, +} from '@material-ui/core'; + +import { + AvatarGroup, +} from "@mui/material" + +import {Close as CloseIcon, Folder as FolderIcon, Polymer as PolymerIcon, LibraryBooks as LibraryBooksIcon} from '@material-ui/icons' + +import algoliasearch from 'algoliasearch/lite'; +import aa from 'search-insights' +import { InstantSearch, Configure, connectSearchBox, connectHits, Index } from 'react-instantsearch-dom'; +//import { InstantSearch, SearchBox, Hits, connectSearchBox, connectHits, Index } from 'react-instantsearch-dom'; + +// https://www.algolia.com/doc/api-reference/widgets/search-box/react/ +const chipStyle = { + backgroundColor: "#3d3f43", height: 30, marginRight: 5, paddingLeft: 5, paddingRight: 5, height: 28, cursor: "pointer", borderColor: "#3d3f43", color: "white", +} + +const searchClient = algoliasearch("JNSS5CFDZZ", "db08e40265e2941b9a7d8f644b6e5240") +const SearchField = props => { + const { serverside, userdata } = props + + const theme = useTheme(); + let navigate = useNavigate(); + const borderRadius = 3 + const node = useRef() + const [searchOpen, setSearchOpen] = useState(false) + const [oldPath, setOldPath] = useState("") + + if (serverside === true) { + return null + } + + if (window !== undefined && window.location !== undefined && window.location.pathname === "/search") { + return null + } + + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + + if (window.location.pathname !== oldPath) { + setSearchOpen(false) + setOldPath(window.location.pathname) + } + + //useEffect(() => { + // if (searchOpen) { + // var tarfield = document.getElementById("shuffle_search_field") + // tarfield.focus() + // } + //}, searchOpen) + + const SearchBox = ({currentRefinement, refine, isSearchStalled, } ) => { + + /* + endAdornment: ( + { + event.preventDefault() + }}> + { + setSearchOpen(false) + }} /> + + ), + */ + + return ( +