diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml deleted file mode 100644 index b3d1927e..00000000 --- a/.github/workflows/release-please.yml +++ /dev/null @@ -1,13 +0,0 @@ -on: - push: - branches: - - launch -name: release-please -jobs: - release-please: - runs-on: ubuntu-latest - steps: - - uses: google-github-actions/release-please-action@v3 - with: - release-type: node - package-name: release-please-action diff --git a/.github/workflows/upload_sdk.yml b/.github/workflows/upload_sdk.yml deleted file mode 100644 index 9faa33e4..00000000 --- a/.github/workflows/upload_sdk.yml +++ /dev/null @@ -1,51 +0,0 @@ -# This is a basic workflow to help you get started with Actions - -name: App SDK upload - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the main branch - push: - branches: [ master, launch ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - # This workflow contains a single job called "build" - build: - # The type of runner that the job will run on - runs-on: ubuntu-latest - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v3 - - - id: 'auth' - name: 'Authenticate to Google Cloud' - uses: 'google-github-actions/auth@v0' - with: - credentials_json: '${{ secrets.SANDBOX_CREDENTIALS }}' - - - id: 'upload_sdk' - name: Cloud Storage Uploader - uses: google-github-actions/upload-cloud-storage@v0.9.0 - with: - path: 'backend/app_sdk/app_base.py' - destination: 'shuffle-sandbox-337810.appspot.com/generated_apps/baseline' - - - id: 'upload_requirement' - name: Cloud Storage Uploader - uses: google-github-actions/upload-cloud-storage@v0.9.0 - with: - path: 'backend/app_sdk/requirements.txt' - destination: 'shuffle-sandbox-337810.appspot.com/generated_apps/baseline' - - - id: 'upload_Dockerfile' - name: Cloud Storage Uploader - uses: google-github-actions/upload-cloud-storage@v0.9.0 - with: - path: 'backend/app_sdk/Dockerfile' - destination: 'shuffle-sandbox-337810.appspot.com/generated_apps/baseline' diff --git a/README.md b/README.md index 08362e25..687f89e7 100755 --- a/README.md +++ b/README.md @@ -4,10 +4,13 @@ Shuffle Automation +<<<<<<< HEAD [![CodeQL](https://github.com/Shuffle/Shuffle/actions/workflows/codeql-analysis.yml/badge.svg?branch=launch)](https://github.com/Shuffle/Shuffle/actions/workflows/codeql-analysis.yml) [![Autobuild](https://github.com/Shuffle/Shuffle/actions/workflows/dockerbuild.yaml/badge.svg?branch=launch)](https://github.com/Shuffle/Shuffle/actions/workflows/dockerbuild.yaml) [![Deploy to AWS](https://d1.awsstatic.com/cloudformation-deploy-to-aws-button.png)](https://console.aws.amazon.com/cloudformation/home?#/stacks/new?stackName=Shuffle-Instance&templateURL=https://raw.githubusercontent.com/Shuffle/Shuffle/refs/heads/2.0.0/template.yaml) +======= +>>>>>>> 8883de54d10855428866f598b70e16674df6d7c1

[Shuffle](https://shuffler.io) is an automation platform for and by the community, focusing on accessibility for anyone to automate. Security operations is complex, but it doesn't have to be. diff --git a/app.json b/app.json deleted file mode 100644 index e8cd75ca..00000000 --- a/app.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "Shuffle", - "description": "Security Automation Platform", - "repository": "https://github.com/0x0elliot/Shuffle", - "ref": "2.0.0", - "scripts": { - "postclone": "chmod +x startup.sh && ./startup.sh" - } - } \ No newline at end of file diff --git a/backend/app_sdk/Dockerfile b/backend/app_sdk/Dockerfile deleted file mode 100755 index d3709032..00000000 --- a/backend/app_sdk/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -#FROM python:3.9.1-alpine as base -FROM python:3.10.0-alpine as base -#FROM python:3.11.3-alpine as base - -FROM base as builder -RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev tzdata coreutils - -RUN mkdir /install -WORKDIR /install - -FROM base - -#--no-cache -RUN apk update && apk add --update tzdata libmagic alpine-sdk libffi libffi-dev musl-dev openssl-dev coreutils - -COPY --from=builder /install /usr/local -COPY requirements.txt /requirements.txt -RUN pip3 install -r /requirements.txt - -COPY __init__.py /app/walkoff_app_sdk/__init__.py -COPY app_base.py /app/walkoff_app_sdk/app_base.py diff --git a/backend/app_sdk/Dockerfile_alpine_grpc b/backend/app_sdk/Dockerfile_alpine_grpc deleted file mode 100644 index dedc95b1..00000000 --- a/backend/app_sdk/Dockerfile_alpine_grpc +++ /dev/null @@ -1,42 +0,0 @@ -FROM python:3.10.0-alpine as base - -FROM base as builder -RUN apk --no-cache add --update \ - alpine-sdk \ - build-base \ - g++ \ - gcc \ - libffi \ - libffi-dev \ - libstdc++ \ - linux-headers \ - musl-dev \ - openssl-dev \ - tzdata \ - coreutils - -RUN pip install --upgrade pip && \ - pip install --prefix="/install" --no-cache-dir grpcio grpcio-tools && \ - apk del --purge \ - g++ \ - gcc \ - musl-dev \ - libffi-dev \ - libstdc++ \ - build-base \ - linux-headers - -RUN mkdir -p /install -WORKDIR /install - -FROM base - -#--no-cache -RUN apk update && apk add --update tzdata libmagic alpine-sdk libffi libffi-dev musl-dev openssl-dev coreutils - -COPY --from=builder /install /usr/local -COPY requirements.txt /requirements.txt -RUN pip3 install -r /requirements.txt - -COPY __init__.py /app/walkoff_app_sdk/__init__.py -COPY app_base.py /app/walkoff_app_sdk/app_base.py diff --git a/backend/app_sdk/Dockerfile_blackarch b/backend/app_sdk/Dockerfile_blackarch deleted file mode 100755 index e7469166..00000000 --- a/backend/app_sdk/Dockerfile_blackarch +++ /dev/null @@ -1,19 +0,0 @@ -FROM blackarchlinux/blackarch as base - -FROM base as builder - -RUN /bin/pacman -Syu --noconfirm - -RUN /bin/pacman -Sy --noconfirm base-devel libffi musl openssl python python-pip -y - -RUN mkdir /install -WORKDIR /install - -COPY requirements.txt /requirements.txt -RUN pip install --prefix="/install" -r /requirements.txt - -FROM base - -COPY --from=builder /install /usr/local -COPY __init__.py /app/walkoff_app_sdk/__init__.py -COPY app_base.py /app/walkoff_app_sdk/app_base.py diff --git a/backend/app_sdk/Dockerfile_kali b/backend/app_sdk/Dockerfile_kali deleted file mode 100755 index af7fd24f..00000000 --- a/backend/app_sdk/Dockerfile_kali +++ /dev/null @@ -1,19 +0,0 @@ -FROM kalilinux/kali-rolling as base - -FROM base as builder - -RUN apt-get update -RUN apt-get dist-upgrade -y -RUN apt install build-essential libffi-dev musl-dev openssl python3 python3-pip -y - -RUN mkdir /install -WORKDIR /install - -COPY requirements.txt /requirements.txt -RUN pip install --prefix="/install" -r /requirements.txt - -FROM base - -COPY --from=builder /install /usr/local -COPY __init__.py /app/walkoff_app_sdk/__init__.py -COPY app_base.py /app/walkoff_app_sdk/app_base.py diff --git a/backend/app_sdk/Dockerfile_ubuntu b/backend/app_sdk/Dockerfile_ubuntu deleted file mode 100644 index 3f34d4bd..00000000 --- a/backend/app_sdk/Dockerfile_ubuntu +++ /dev/null @@ -1,22 +0,0 @@ -FROM ubuntu as base - -FROM base as builder - -RUN apt-get update -RUN apt-get dist-upgrade -y -RUN apt install build-essential libffi-dev musl-dev openssl python3 python3-pip -y - -RUN mkdir /install -WORKDIR /install - -COPY requirements.txt /requirements.txt -RUN pip install --prefix="/install" -r /requirements.txt - -FROM base -RUN apt-get update -RUN apt-get dist-upgrade -y -RUN apt install build-essential libffi-dev musl-dev openssl python3 python3-pip -y - -COPY --from=builder /install /usr/local -COPY __init__.py /app/walkoff_app_sdk/__init__.py -COPY app_base.py /app/walkoff_app_sdk/app_base.py diff --git a/backend/app_sdk/LICENSE b/backend/app_sdk/LICENSE deleted file mode 100755 index ce11f6f3..00000000 --- a/backend/app_sdk/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2020 Frikkylikeme - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/backend/app_sdk/README.md b/backend/app_sdk/README.md old mode 100755 new mode 100644 index 478fb394..1e4dc72e --- a/backend/app_sdk/README.md +++ b/backend/app_sdk/README.md @@ -1,22 +1,2 @@ -# app_sdk.py -This is the SDK used for apps to behave like they should. - -## If you want to update apps.. PS: downloads from docker hub do overrides.. :) -1. Write your code & check if runtime works -2. Build app_base image -3. docker rm $(docker ps -aq) # Remove all stopped containers -4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...) -5. Rebuild the Docker image (click load in GUI?) - -## Cloud updates -1. Go to shuffle cloud on GCP -2. Go to Cloud Storage -3. Find shuffler.appspot.com -4. Navigate to generated_apps/baseline -5. Update SDK there. This will make all new apps run with the new SDK - -## Cloud app force-updates -1. Run the "stitcher.go" program in the public shuffle-shared repository. - -# LICENSE -Everything in here is MIT, not AGPLv3 as indicated by the license. +## CHANGES +In November 2024, we moved this to its own repistory: https://github.com/shuffle/app_sdk diff --git a/backend/app_sdk/__init__.py b/backend/app_sdk/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py deleted file mode 100755 index 680cfb0e..00000000 --- a/backend/app_sdk/app_base.py +++ /dev/null @@ -1,4219 +0,0 @@ -import os -import ast -import sys -import re -import copy -import time -import base64 -import json -import random -import liquid -import logging -import urllib3 -import hashlib -import zipfile -import asyncio -import requests -import http.client -import urllib.parse -import jinja2 -import datetime -import dateutil -import threading -import concurrent.futures - -from io import StringIO as StringBuffer, BytesIO -from liquid import Liquid, defaults -from requests.packages.urllib3.exceptions import InsecureRequestWarning - -# Suppress the warning -requests.packages.urllib3.disable_warnings(InsecureRequestWarning) - - -runtime = os.getenv("SHUFFLE_SWARM_CONFIG", "") - -### -### -### -#### Filters for liquidpy -### -### -### - -defaults.MODE = 'wild' -defaults.FROM_FILE = False -from liquid.filters.manager import FilterManager -from liquid.filters.standard import standard_filter_manager - -shuffle_filters = FilterManager() -for key, value in standard_filter_manager.filters.items(): - shuffle_filters.filters[key] = value - -#@shuffle_filters.register -#def plus(a, b): -# try: -# a = int(a) -# except: -# a = 0 -# -# try: -# b = int(b) -# except: -# b = 0 -# -# return standard_filter_manager.filters["plus"](a, b) -# -#@shuffle_filters.register -#def minus(a, b): -# a = int(a) -# b = int(b) -# return standard_filter_manager.filters["minus"](a, b) -# -#@shuffle_filters.register -#def multiply(a, b): -# a = int(a) -# b = int(b) -# return standard_filter_manager.filters["multiply"](a, b) -# -#@shuffle_filters.register -#def divide(a, b): -# a = int(a) -# b = int(b) -# return standard_filter_manager.filters["divide"](a, b) - -@shuffle_filters.register -def md5(a): - a = str(a) - return hashlib.md5(a.encode('utf-8')).hexdigest() - -@shuffle_filters.register -def sha256(a): - a = str(a) - return hashlib.sha256(str(a).encode("utf-8")).hexdigest() - -@shuffle_filters.register -def md5_base64(a): - a = str(a) - foundhash = hashlib.md5(a.encode('utf-8')).hexdigest() - return base64.b64encode(foundhash.encode('utf-8')) - - -@shuffle_filters.register -def base64_encode(a): - a = str(a) - - try: - return base64.b64encode(a.encode('utf-8')).decode() - except: - return base64.b64encode(a).decode() - -@shuffle_filters.register -def random_element(a): - # Choose a random item from an array - a = list(a) - - if len(a) == 0: - return "" - - return random.choice(a) - - -@shuffle_filters.register -def base64_decode(a): - a = str(a) - - if "-" in a: - a = a.replace("-", "+", -1) - - if "_" in a: - a = a.replace("_", "/", -1) - - # Fix padding - if len(a) % 4 != 0: - a += "=" * (4 - len(a) % 4) - - try: - return base64.b64decode(a).decode("unicode_escape") - except: - try: - return base64.b64decode(a).decode() - except: - return base64.b64decode(a) - -@shuffle_filters.register -def json_parse(a): - return json.loads(str(a)) - -@shuffle_filters.register -def as_object(a): - return json.loads(str(a)) - -@shuffle_filters.register -def ast_eval(a): - return ast.literal_eval(str(a)) - -@shuffle_filters.register -def escape_string(a): - a = str(a) - return a.replace("\\\'", "\'", -1).replace("\\\"", "\"", -1).replace("'", "\\\'", -1).replace("\"", "\\\"", -1) - -@shuffle_filters.register -def json_escape(a): - a = str(a) - return a.replace("\\\'", "\'", -1).replace("\\\"", "\"", -1).replace("'", "\\\\\'", -1).replace("\"", "\\\\\"", -1) - -@shuffle_filters.register -def escape_json(a): - a = str(a) - return a.replace("\\\'", "\'", -1).replace("\\\"", "\"", -1).replace("'", "\\\\\'", -1).replace("\"", "\\\\\"", -1) - -# By default using json escape to add all backslashes -@shuffle_filters.register -def escape(a): - a = str(a) - return json_escape(a) - - -@shuffle_filters.register -def neat_json(a): - try: - a = json.loads(a) - except: - pass - - return json.dumps(a, indent=4, sort_keys=True) - -@shuffle_filters.register -def flatten(a): - a = list(a) - - flat_list = [a for xs in a for a in xs] - return flat_list - -@shuffle_filters.register -def last(a): - try: - a = json.loads(a) - except: - pass - - if len(a) == 0: - return "" - - return a[-1] - -@shuffle_filters.register -def first(a): - try: - a = json.loads(a) - except: - pass - - if len(a) == 0: - return "" - - return a[0] - - -@shuffle_filters.register -def csv_parse(a): - a = str(a) - splitdata = a.split("\n") - columns = [] - if len(splitdata) > 1: - columns = splitdata[0].split(",") - else: - return a.split("\n") - - allitems = [] - cnt = -1 - for item in splitdata[1:]: - cnt += 1 - commasplit = item.split(",") - - fullitem = {} - fullitem["unparsed"] = item - fullitem["index"] = cnt - fullitem["parsed"] = {} - if len(columns) != len(commasplit): - - if len(commasplit) > len(columns): - diff = len(commasplit)-len(columns) - - try: - commasplit = commasplit[0:len(commasplit)-diff] - except: - pass - else: - for item in range(0, len(columns)-len(commasplit)): - commasplit.append("") - - for key in range(len(columns)): - try: - fullitem["parsed"][columns[key]] = commasplit[key] - except: - continue - - allitems.append(fullitem) - - try: - return json.dumps(allitems) - except: - return allitems - -@shuffle_filters.register -def parse_csv(a): - return csv_parse(a) - -@shuffle_filters.register -def format_csv(a): - return csv_parse(a) - -@shuffle_filters.register -def csv_format(a): - return csv_parse(a)@standard_filter_manager.register - -@shuffle_filters.register -def split(base, sep): - if not sep: - try: - return json.dumps(list(base)) - except: - return list(base) - - try: - return json.dumps(base.split(sep)) - except: - return base.split(sep) - - -@shuffle_filters.register -def url_encode(base): - """url-encode a string""" - try: - from urllib import quote - except ImportError: - from urllib.parse import quote - - return quote(base) - -@shuffle_filters.register -def url_decode(base): - """url-decode a string""" - try: - from urllib import unquote_plus - except ImportError: - from urllib.parse import unquote_plus - - return unquote_plus(base) - - -### -### -### -### -### -### -### - - -class AppBase: - __version__ = None - app_name = None - - def __init__(self, redis=None, logger=None, console_logger=None):#, docker_client=None): - self.logger = logger if logger is not None else logging.getLogger("AppBaseLogger") - - if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true": - self.log_capture_string = StringBuffer() - ch = logging.StreamHandler(self.log_capture_string) - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - ch.setFormatter(formatter) - logger.addHandler(ch) - - self.redis=redis - self.console_logger = logger if logger is not None else logging.getLogger("AppBaseLogger") - - # apikey is for the user / org - # authorization is for the specific workflow - - self.url = os.getenv("CALLBACK_URL", "https://shuffler.io") - self.base_url = os.getenv("BASE_URL", "https://shuffler.io") - self.action = os.getenv("ACTION", "") - self.original_action = os.getenv("ACTION", "") - self.authorization = os.getenv("AUTHORIZATION", "") - self.current_execution_id = os.getenv("EXECUTIONID", "") - self.full_execution = os.getenv("FULL_EXECUTION", "") - self.result_wrapper_count = 0 - - # Make start time with milliseconds - self.start_time = int(time.time_ns()) - - self.action_result = { - "action": self.action, - "authorization": self.authorization, - "execution_id": self.current_execution_id, - "result": f"", - "started_at": self.start_time, - "status": "", - "completed_at": int(time.time_ns()), - } - - self.proxy_config = { - "http": os.getenv("HTTP_PROXY", ""), - "https": os.getenv("HTTPS_PROXY", ""), - "no_proxy": os.getenv("NO_PROXY", ""), - } - - if len(os.getenv("SHUFFLE_INTERNAL_HTTP_PROXY", "")) > 0: - self.proxy_config["http"] = os.getenv("SHUFFLE_INTERNAL_HTTP_PROXY", "") - - if len(os.getenv("SHUFFLE_INTERNAL_HTTPS_PROXY", "")) > 0: - self.proxy_config["https"] = os.getenv("SHUFFLE_INTERNAL_HTTP_PROXY", "") - - if len(os.getenv("SHUFFLE_INTERNAL_NO_PROXY", "")) > 0: - self.proxy_config["no_proxy"] = os.getenv("SHUFFLE_INTERNAL_NO_PROXY", "") - - try: - if self.proxy_config["http"].lower() == "noproxy": - self.proxy_config["http"] = "" - if self.proxy_config["https"].lower() == "noproxy": - self.proxy_config["https"] = "" - except Exception as e: - self.logger.info(f"[WARNING] Failed setting proxy config: {e}. NOT important if running apps with webserver. This is NOT critical.") - - - if isinstance(self.action, str): - try: - self.action = json.loads(self.action) - self.original_action = json.loads(self.action) - except Exception as e: - pass - - if len(self.base_url) == 0: - self.base_url = self.url - - - self.local_storage = [] - - # Checks output for whether it should be automatically parsed or not - def run_magic_parser(self, input_data): - if not isinstance(input_data, str): - return input_data - - # Don't touch existing JSON/lists - if (input_data.startswith("[") and input_data.endswith("]")) or (input_data.startswith("{") and input_data.endswith("}")): - return input_data - - if len(input_data) < 3: - return input_data - - # Don't touch large data. - if len(input_data) > 100000: - return input_data - - if not "\n" in input_data and not "," in input_data: - return input_data - - new_input = input_data - try: - #new_input.strip() - new_input = input_data.split() - new_return = [] - - index = 0 - for item in new_input: - splititem = "," - if ", " in item: - splititem = ", " - elif "," in item: - splititem = "," - else: - new_return.append(item) - - index += 1 - continue - - for subitem in item.split(splititem): - new_return.insert(index, subitem) - - index += 1 - - # Prevent large data or infinite loops - if index > 10000: - #self.logger.info(f"[DEBUG] Infinite loop. Returning default data.") - return input_data - - fixed_return = [] - for item in new_return: - if not item: - continue - - if not isinstance(item, str): - fixed_return.append(item) - continue - - if item.endswith(","): - item = item[0:-1] - - fixed_return.append(item) - - new_input = fixed_return - except Exception as e: - # Not used anymore - #self.logger.info(f"[ERROR] Failed to run magic parser (2): {e}") - return input_data - - try: - new_input = input_data.split() - except Exception as e: - self.logger.info(f"[ERROR] Failed to run parser during split (1): {e}") - return input_data - - # Won't ever touch this one? - if isinstance(new_input, list) or isinstance(new_input, object): - try: - return json.dumps(new_input) - except Exception as e: - self.logger.info(f"[ERROR] Failed to run magic parser (3): {e}") - - return new_input - - def prepare_response(self, request): - try: - parsedheaders = {} - for key, value in request.headers.items(): - parsedheaders[key] = value - - cookies = {} - if request.cookies: - for key, value in request.cookies.items(): - cookies[key] = value - - - jsondata = request.text - try: - jsondata = json.loads(jsondata) - except: - pass - - return json.dumps({ - "success": True, - "status": request.status_code, - "url": request.url, - "body": jsondata, - "headers": parsedheaders, - "cookies":cookies, - }) - except Exception as e: - return request.text - - # Fixes pattern issues in json/liquid based on input and supplied patterns - def patternfix_string(self, liquiddata, patterns, regex_patterns, inputtype="liquid"): - if not inputtype or inputtype == "liquid": - if "{{" not in liquiddata or "}}" not in liquiddata: - return liquiddata - elif inputtype == "json": - liquiddata = liquiddata.strip() - - # Validating if it looks like json or not - if liquiddata[0] == "{" and liquiddata[len(liquiddata)-1] == "}": - pass - else: - if liquiddata[0] == "[" and liquiddata[len(liquiddata)-1] == "]": - pass - else: - return liquiddata - - # If it's already json, don't touch it - try: - json.loads(liquiddata) - return liquiddata - except Exception as e: - pass - else: - print("No replace handler for %s" % inputtype) - return liquiddata - - skipkeys = [" "] - newoutput = liquiddata[:] - for pattern in patterns: - keylocations = [] - parsedvalue = "" - record = False - index = -1 - for key in liquiddata: - - # Return instant if possible - if inputtype == "json": - try: - json.loads(newoutput) - return newoutput - except: - pass - - index += 1 - if not key: - if record: - keylocations.append(index) - parsedvalue += key - - continue - - if key in skipkeys: - if record: - keylocations.append(index) - parsedvalue += key - - continue - - if key == pattern[0] and not record: - record = True - - if key not in pattern: - keylocations = [] - parsedvalue = "" - record = False - - if record: - keylocations.append(index) - parsedvalue += key - - if len(parsedvalue) == 0: - continue - - evaluated_value = parsedvalue[:] - for skipkey in skipkeys: - evaluated_value = "".join(evaluated_value.split(skipkey)) - - if evaluated_value == pattern: - #print("Found matching: %s (%s)" % (parsedvalue, keylocations)) - #print("Should replace with: %s" % patterns[pattern]) - - newoutput = newoutput.replace(parsedvalue, patterns[pattern], -1) - - # Return instant if possible - if inputtype == "json": - try: - json.loads(newoutput) - return newoutput - except: - pass - - - for pattern in regex_patterns: - newlines = [] - for line in newoutput.split("\n"): - replaced_line = re.sub(pattern, regex_patterns[pattern], line) - newlines.append(replaced_line) - - newoutput = "\n".join(newlines) - - # Return instant if possible - if inputtype == "json": - try: - json.loads(newoutput) - return newoutput - except: - pass - - # Dont return json properly unless actually json - if inputtype == "json": - try: - json.loads(newoutput) - return newoutput - except: - # Returns original if json fixing didn't work - return liquiddata - - return newoutput - - # FIXME: Add more info like logs in here. - # Docker logs: https://forums.docker.com/t/docker-logs-inside-the-docker-container/68190/2 - def send_result(self, action_result, headers, stream_path): - if action_result["status"] == "EXECUTING": - action_result["status"] = "FAILURE" - - try: - if self.action["run_magic_output"] == True: - action_result["result"] = self.run_magic_parser(action_result["result"]) - except KeyError as e: - pass - except Exception as e: - pass - - # Try it with some magic - - action_result["completed_at"] = int(time.time_ns()) - #if isinstance(action_result, - - # FIXME: Add cleanup of parameters to not send to frontend here - params = {} - - # I wonder if this actually works - url = "%s%s" % (self.base_url, stream_path) - self.logger.info(f"[DEBUG][%s] Sending result to %s" % (self.current_execution_id, url)) - - try: - log_contents = "disabled: add env SHUFFLE_LOGS_DISABLED=true to Orborus to re-enable logs for apps. Can not be enabled natively in Cloud except in Hybrid mode." - if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true": - log_contents = self.log_capture_string.getvalue() - - if len(action_result["action"]["parameters"]) == 0: - action_result["action"]["parameters"] = [] - - param_found = False - for param in action_result["action"]["parameters"]: - if param["name"] == "shuffle_action_logs": - param_found = True - break - - if not param_found: - action_result["action"]["parameters"].append({ - "name": "shuffle_action_logs", - "value": log_contents, - }) - - except Exception as e: - pass - - # Check if type of headers is right - if not isinstance(headers, dict): - headers = {} - - if not "User-Agent" in headers: - headers["User-Agent"] = "Shuffle App" - - self.logger.info(f"[DEBUG][{self.current_execution_id}] Starting to send result to {url}") - try: - finished = False - ret = {} - for i in range (0, 10): - # Random sleeptime between 0 and 1 second, with 0.1 increments - sleeptime = float(random.randint(0, 10) / 10) - - try: - ret = requests.post( - url, - headers=headers, - json=action_result, - timeout=10, - verify=False, - proxies=self.proxy_config, - ) - - if ret.status_code == 200 or ret.status_code == 201: - self.logger.info(f"[DEBUG][{self.current_execution_id}] Successful send_result request: Status= {ret.status_code} & Action status: {action_result['status']}. Response= {ret.text}") - finished = True - break - else: - # FIXME: Add a checker for 403, and Proxy logs failing - headerauth = "" - if "Authorization" in headers: - headerauth = headers["Authorization"] - - try: - self.logger.info(f"[ERROR] Bad resp ({ret.status_code}) in send_result for url '{url}'. Execution ID: %d, Authorization: %d, Header Auth: %d" % (len(action_result["execution_id"]), len(action_result["authorization"]), len(headerauth))) - - except Exception as e: - self.logger.info(f"[ERROR] Bad resp ({ret.status_code}) in send_result for url '{url}' (no detail)") - - time.sleep(sleeptime) - - - # Proxyerrror - except requests.exceptions.ProxyError as e: - self.logger.info(f"[ERROR][{self.current_execution_id}] Proxy error in send_result for url '{url}': {e}") - - self.proxy_config = {} - continue - - except requests.exceptions.RequestException as e: - self.logger.info(f"[ERROR][{self.current_execution_id}] Request error in send_result for url '{url}': {e}") - - # Check if we have a read timeout. If we do, exit as we most likely sent the result without getting a good result - if "Read timed out" in str(e): - self.logger.warning(f"[WARNING] Read timed out: {e}") - finished = True - break - - if "Max retries exceeded with url" in str(e): - self.logger.warning(f"[WARNING] Max retries exceeded with url: {e}") - finished = True - break - - time.sleep(sleeptime) - - #time.sleep(5) - continue - except TimeoutError as e: - self.logger.info(f"[ERROR][{self.current_execution_id}] Timeout error in send_result for url '{url}': {e}") - - time.sleep(sleeptime) - - #time.sleep(5) - continue - except requests.exceptions.ConnectionError as e: - self.logger.info(f"[ERROR][{self.current_execution_id}] Connection error in send_result for url '{url}': {e}") - - time.sleep(sleeptime) - - #time.sleep(5) - continue - except http.client.RemoteDisconnected as e: - self.logger.info(f"[ERROR][{self.current_execution_id}] RemoteDisconnected error in send_result for url '{url}': {e}") - - time.sleep(sleeptime) - - #time.sleep(5) - continue - except urllib3.exceptions.ProtocolError as e: - self.logger.info(f"[ERROR][{self.current_execution_id}] ProtocolError error in send_result for url '{url}': {e}") - - time.sleep(0.1) - - #time.sleep(5) - continue - - #time.sleep(5) - - if not finished: - # Not sure why this would work tho :) - action_result["status"] = "FAILURE" - action_result["result"] = json.dumps({"success": False, "reason": "POST error: Failed connecting to %s over 10 retries to the backend" % url}) - self.send_result(action_result, {"Content-Type": "application/json", "Authorization": "Bearer %s" % self.authorization}, "/api/v1/streams") - return - - except requests.exceptions.ConnectionError as e: - #self.logger.info(f"[DEBUG] Unexpected ConnectionError happened: {e}") - pass - except TypeError as e: - action_result["status"] = "FAILURE" - action_result["result"] = json.dumps({"success": False, "reason": "Typeerror when sending to backend URL %s" % url}) - - ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False, proxies=self.proxy_config) - #self.logger.info(f"[DEBUG] Result: {ret.status_code}") - #if ret.status_code != 200: - # pr - - #self.logger.info(f"[DEBUG] TypeError request: Status= {ret.status_code} & Response= {ret.text}") - except http.client.RemoteDisconnected as e: - self.logger.info(f"[DEBUG] Expected Remotedisconnect happened: {e}") - except urllib3.exceptions.ProtocolError as e: - self.logger.info(f"[DEBUG] Expected ProtocolError happened: {e}") - - - # FIXME: Re-enable data flushing otherwise we'll overload it all - # Or nah? - if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true": - try: - self.log_capture_string.flush() - #self.log_capture_string.close() - #pass - except Exception as e: - pass - - #async def cartesian_product(self, L): - def cartesian_product(self, L): - if L: - #return {(a, ) + b for a in L[0] for b in await self.cartesian_product(L[1:])} - return {(a, ) + b for a in L[0] for b in self.cartesian_product(L[1:])} - else: - return {()} - - # Handles unique fields by negoiating with the backend - def validate_unique_fields(self, params): - #self.logger.info("IN THE UNIQUE FIELDS PLACE!") - - newlist = [params] - if isinstance(params, list): - #self.logger.info("ITS A LIST!") - newlist = params - - # FIXME: Also handle MULTI PARAM - values = [] - param_names = [] - all_values = {} - index = 0 - for outerparam in newlist: - - #self.logger.info(f"INNERTYPE: {type(outerparam)}") - #self.logger.info(f"HANDLING PARAM {key}") - param_value = "" - for key, value in outerparam.items(): - #self.logger.info("KEY: %s" % key) - #value = params[key] - for param in self.action["parameters"]: - try: - if param["name"] == key and param["unique_toggled"]: - self.logger.info(f"[DEBUG] FOUND: {key} with param {param}!") - if isinstance(value, dict) or isinstance(value, list): - try: - value = json.dumps(value) - except json.decoder.JSONDecodeError as e: - self.logger.info(f"[WARNING] Error in json decode for param {value}: {e}") - continue - elif isinstance(value, int) or isinstance(value, float): - value = str(value) - elif value == False: - value = "False" - elif value == True: - value = "True" - - self.logger.info(f"[DEBUG] VALUE APPEND: {value}") - param_value += value - if param["name"] not in param_names: - param_names.append(param["name"]) - - except (KeyError, NameError) as e: - self.logger.info(f"""Key/NameError in param handler for {param["name"]}: {e}""") - - #self.logger.info(f"[DEBUG] OUTER VALUE: {param_value}") - if len(param_value) > 0: - md5 = hashlib.md5(param_value.encode('utf-8')).hexdigest() - values.append(md5) - all_values[md5] = { - "index": index, - } - - index += 1 - - # When in here, it means it should be unique - # Should this be done by the backend? E.g. ask it if the value is valid? - # 1. Check if it's unique towards key:value store in org for action - # 2. Check if COMBINATION is unique towards key:value store of action for org - # 3. Have a workflow configuration for unique ID's in unison or per field? E.g. if toggled, then send a hash of all fields together alphabetically, but if not, send one field at a time - - # org_id = full_execution["workflow"]["execution_org"]["id"] - - # USE ARRAY? - - new_params = [] - if len(values) > 0: - org_id = self.full_execution["workflow"]["execution_org"]["id"] - data = { - "append": True, - "workflow_check": False, - "authorization": self.authorization, - "execution_ref": self.current_execution_id, - "org_id": org_id, - "values": [{ - "app": self.action["app_name"], - "action": self.action["name"], - "parameternames": param_names, - "parametervalues": values, - }] - } - - #self.logger.info(f"DATA: {data}") - # 1594869a676630b397bc34f7dc0951a3 - - url = f"{self.url}/api/v1/orgs/{org_id}/validate_app_values" - ret = requests.post(url, json=data, verify=False, proxies=self.proxy_config) - if ret.status_code == 200: - json_value = ret.json() - if len(json_value["found"]) > 0: - modifier = 0 - for item in json_value["found"]: - self.logger.info(f"Should remove {item}") - - try: - self.logger.info(f"FOUND: {all_values[item]}") - self.logger.info(f"SHOULD REMOVE INDEX: {all_values[item]['index']}") - - try: - newlist.pop(all_values[item]["index"]-modifier) - modifier += 1 - except IndexError as e: - self.logger.info(f"Error popping value from array: {e}") - except (NameError, KeyError) as e: - self.logger.info(f"Failed removal: {e}") - - - #return False - else: - self.logger.info("None of the items were found!") - return newlist - else: - self.logger.info(f"[WARNING] Failed checking values with status code {ret.status_code}!") - - #return True - return newlist - - # Returns a list of all the executions to be done in the inner loop - # FIXME: Doesn't take into account whether you actually WANT to loop or not - # Check if the last part of the value is #? - #async def get_param_multipliers(self, baseparams): - def get_param_multipliers(self, baseparams): - # Example: - # {'call': ['hello', 'hello4'], 'call2': ['hello2', 'hello3'], 'call3': '1'} - # - # Should become this because of pairs (all same-length arrays, PROBABLY indicates same source node's values. - # [ - # {'call': 'hello', 'call2': 'hello2', 'call3': '1'}, - # {'call': 'hello4', 'call2': 'hello3', 'call3': '1'} - # ] - # - # ---------------------------------------------------------------------- - # Example2: - # {'call': ['hello'], 'call2': ['hello2', 'hello3'], 'call3': '1'} - # - # Should become this because NOT pairs/triplets: - # [ - # {'call': 'hello', 'call2': 'hello2', 'call3': '1'}, - # {'call': 'hello', 'call2': 'hello3', 'call3': '1'} - # ] - # - # ---------------------------------------------------------------------- - # Example3: - # {'call': ['hello', 'hello2'], 'call2': ['hello3', 'hello4', 'hello5'], 'call3': '1'} - # - # Should become this because arrays are not same length, aka no pairs/triplets. This is the multiplier effect. 2x3 arrays = 6 iterations - # [ - # {'call': 'hello', 'call2': 'hello3', 'call3': '1'}, - # {'call': 'hello', 'call2': 'hello4', 'call3': '1'}, - # {'call': 'hello', 'call2': 'hello5', 'call3': '1'}, - # {'call': 'hello2', 'call2': 'hello3', 'call3': '1'}, - # {'call': 'hello2', 'call2': 'hello4', 'call3': '1'}, - # {'call': 'hello2', 'call2': 'hello5', 'call3': '1'} - # ] - # To achieve this, we'll do this: - # 1. For the first array, take the total amount(y) (2x3=6) and divide it by the current array (x): 2. x/y = 3. This means do 3 of each value - # 2. For the second array, take the total amount(y) (2x3=6) and divide it by the current array (x): 3. x/y = 2. - # 3. What does the 3rd array do? Same, but ehhh? - # - # Example4: - # What if there are multiple loops inside a single item? - # - # - - paramlist = [] - listitems = [] - listlengths = [] - all_lists = [] - all_list_keys = [] - - #check_value = "$Filter_list_testing.wrapper.#.tmp" - #self.action = action - - loopnames = [] - self.logger.info(f"Baseparams to check: {baseparams}") - for key, value in baseparams.items(): - check_value = "" - for param in self.original_action["parameters"]: - if param["name"] == key: - check_value = param["value"] - # self.result_wrapper_count = 0 - - octothorpe_count = param["value"].count(".#") - if octothorpe_count > self.result_wrapper_count: - self.result_wrapper_count = octothorpe_count - self.logger.info("[INFO] NEW OCTOTHORPE WRAPPER: %d" % octothorpe_count) - - - # This whole thing is hard. - # item = [{"data": "1.2.3.4", "dataType": "ip"}] - # $item = DONT loop items. - # $item.# = Loop items - # $item.#.data = Loop items - # With a single item, this is fine. - - # item = [{"list": [{"data": "1.2.3.4", "dataType": "ip"}]}] - # $item = DONT loop items - # $item.# = Loop items - # $item.#.list = DONT loop items - # $item.#.list.# = Loop items - # $item.#.list.#.data = Loop items - # If the item itself is a list.. hmm - - # FIXME: Check the above, and fix so that nested looped items can be - # Skipped if wanted - - #self.logger.info("\nCHECK: %s" % check_value) - #try: - # values = parameter["value_replace"] - # if values != None: - # self.logger.info(values) - # for val in values: - # self.logger.info(val) - #except: - # pass - - should_merge = False - if "#" in check_value: - should_merge = True - - # Specific for OpenAPI body replacement - #self.logger.info("\n\n\nDOING STUFF BELOW HERE") - if not should_merge: - for parameter in self.original_action["parameters"]: - if parameter["name"] == key: - #self.logger.info("CHECKING BODY FOR VALUE REPLACE DATA!") - try: - values = parameter["value_replace"] - if values != None: - self.logger.info(values) - for val in values: - if "#" in val["value"]: - should_merge = True - break - except: - pass - - #self.logger.info(f"VALUE LENGTH: {len(value)}") - if isinstance(value, list): - #subvalue = [] - # Override for single vs multi items - #if len(value) > 0: - # if isinstance(value[0], list) and len(value[0]) == 1: - # subvalue = value[0] - - # subvalue = value[0] - - - if len(value) <= 1: - # FIXME: This broke some shit for a single item fml - # Necessary as override again :( - if len(value) == 1: - baseparams[key] = value[0] - - #if "#" in check_value: - # should_merge = True - else: - #if len(value) > 1: - if not should_merge: - self.logger.info("[DEBUG] Adding WITHOUT looping list") - else: - if len(value) not in listlengths: - listlengths.append(len(value)) - #listlength - - listitems.append( - { - key: len(value) - } - ) - - all_list_keys.append(key) - all_lists.append(baseparams[key]) - else: - #self.logger.info(f"{value} is not a list") - pass - - self.logger.info("[DEBUG] Listlengths: %s - listitems: %d" % (listlengths, len(listitems))) - #if len(listitems) == 0: - if len(listlengths) == 0: - self.logger.info("[DEBUG] NO multiplier. Running a single iteration.") - paramlist.append(baseparams) - - #elif len(listitems) == 1: - elif len(listlengths) == 1: - self.logger.info("All subitems are the same length") - - for item in listitems: - # This loops should always be length 1 - for key, value in item.items(): - if not isinstance(value, int): - continue - - if len(paramlist) == value: - for subloop in range(value): - baseitem = copy.deepcopy(baseparams) - paramlist[subloop][key] = baseparams[key][subloop] - else: - for subloop in range(value): - baseitem = copy.deepcopy(baseparams) - baseitem[key] = baseparams[key][subloop] - paramlist.append(baseitem) - - else: - newlength = 1 - for item in listitems: - for key, value in item.items(): - newlength = newlength * value - - self.logger.info("[DEBUG] Newlength of array: %d. Lists: %s" % (newlength, all_lists)) - # Get the cartesian product of the arrays - #cartesian = await self.cartesian_product(all_lists) - try: - cartesian = self.cartesian_product(all_lists) - newlist = [] - for item in cartesian: - newlist.append(list(item)) - except Exception as e: - self.logger.info(f"[ERROR] Error in cartesian product: {e}") - newlist = [] - - newobject = {} - for subitem in range(len(newlist)): - baseitem = copy.deepcopy(baseparams) - for key in range(len(newlist[subitem])): - baseitem[all_list_keys[key]] = newlist[subitem][key] - - paramlist.append(baseitem) - - self.logger.info("CARTESIAN PARAMLIST: %s" % paramlist) - - #newlist[subitem[0]] - #if len(newlist) > 0: - # itemlength = len(newlist[0]) - - # How do we get it back, ordered? - #for item in cartesian: - #self.logger.info("Listlengths: %s" % listlengths) - #paramlist = [baseparams] - - #self.logger.info("[INFO] Return paramlist (1): %s" % paramlist) - return paramlist - - - # Runs recursed versions with inner loops and such - #async def run_recursed_items(self, func, baseparams, loop_wrapper): - def run_recursed_items(self, func, baseparams, loop_wrapper): - self.logger.info(f"PRE RECURSED ITEMS: {baseparams}") - has_loop = False - - newparams = {} - for key, value in baseparams.items(): - if isinstance(value, list) and len(value) > 0: - self.logger.info(f"[DEBUG] In list check for {key}") - - for value_index in range(len(value)): - try: - # Added skip for body (OpenAPI) which uses data= in requests - # Can be screwed up if they name theirs body too - if key != "body": - value[value_index] = json.loads(value[value_index]) - except json.decoder.JSONDecodeError as e: - pass - except TypeError as e: - pass - - try: - #if isinstance(value, list) and len(value) == 1 and isinstance(value[0], list): - # try: - # loop_wrapper[key] += 1 - # except Exception as e: - # self.logger.info(f"[WARNING] Exception in loop wrapper: {e}") - # loop_wrapper[key] = 1 - - # newparams[key] = value[0] - # has_loop = True - #else: - #self.logger.info(f"Key {key} is NOT a list within a list. Value: {value}") - newparams[key] = value - except Exception as e: - self.logger.info(f"[WARNING] Error in baseparams list: {e}") - newparams[key] = value - - results = [] - if has_loop: - #self.logger.info(f"[DEBUG] Should run inner loop: {newparams}") - self.logger.info(f"[DEBUG] Should run inner loop") - #ret = await self.run_recursed_items(func, newparams, loop_wrapper) - ret = self.run_recursed_items(func, newparams, loop_wrapper) - else: - self.logger.info(f"[DEBUG] Should run multiplier check with params (inner): {newparams}") - #self.logger.info(f"[DEBUG] Should run multiplier check with params (inner)") - - # 1. Find the loops that are required and create new multipliers - # If here: check for multipliers within this scope. - ret = [] - param_multiplier = self.get_param_multipliers(newparams) - - #self.logger.info("PARAM MULTIPLIER: %s" % param_multiplier) - - # FIXME: This does a deduplication of the data - new_params = self.validate_unique_fields(param_multiplier) - if len(new_params) == 0: - self.logger.info("[WARNING] SHOULD STOP MULTI-EXECUTION BECAUSE FIELDS AREN'T UNIQUE") - self.action_result = { - "action": self.action, - "authorization": self.authorization, - "execution_id": self.current_execution_id, - "result": f"All {len(param_multiplier)} values were non-unique", - "started_at": self.start_time, - "status": "SKIPPED", - "completed_at": int(time.time_ns()), - } - - self.send_result(self.action_result, {"Content-Type": "application/json", "Authorization": "Bearer %s" % self.authorization}, "/api/v1/streams") - if runtime != "run": - exit() - else: - return - else: - #subparams = new_params - param_multiplier = new_params - - #self.logger.info(f"NEW PARAM MULTIPLIER: {param_multiplier}") - - #if isinstance(new_params, list) and len(new_params) == 1: - # params = new_params[0] - #else: - # self.logger.info("[WARNING] SHOULD STOP EXECUTION BECAUSE FIELDS AREN'T UNIQUE") - # action_result["status"] = "SKIPPED" - # action_result["result"] = f"A non-unique value was found" - # action_result["completed_at"] = int(time.time()) - # self.send_result(action_result, headers, stream_path) - # return - - self.logger.info(f"[DEBUG][{self.current_execution_id}] Pre function with {len(param_multiplier)} multipliers") - for subparams in param_multiplier: - #self.logger.info(f"SUBPARAMS IN MULTI: {subparams}") - tmp = "" - try: - - while True: - try: - tmp = func(**subparams) - break - except TypeError as e: - self.logger.info("BASE TYPEERROR: %s" % e) - errorstring = "%s" % e - if "got an unexpected keyword argument" in errorstring: - fieldsplit = errorstring.split("'") - if len(fieldsplit) > 1: - field = fieldsplit[1] - - try: - del subparams[field] - self.logger.info("Removed invalid field %s (1)" % field) - except KeyError: - break - else: - raise Exception(json.dumps({ - "success": False, - "exception": f"TypeError: {e}", - "reason": "You may be running an old version of this action. Please delete and remake the node.", - })) - break - - - except: - e = "" - try: - e = sys.exc_info()[1] - except: - self.logger.info("Exec check fail: %s" % e) - pass - - tmp = json.dumps({ - "success": False, - "reason": f"An error occured during the App Function Run (not Shuffle)", - "details": f"{e}", - }) - - - # An attempt at decomposing coroutine results - # Backwards compatibility - try: - if asyncio.iscoroutine(tmp): - self.logger.info("[DEBUG] In coroutine (2)") - async def parse_value(tmp): - value = await asyncio.gather( - tmp - ) - - return value[0] - - - tmp = asyncio.run(parse_value(tmp)) - else: - #self.logger.info("[DEBUG] Not in coroutine (2)") - pass - except Exception as e: - self.logger.warning("[ERROR] Failed to parse coroutine value for old app: {e}") - - new_value = tmp - if tmp == None: - new_value = "" - elif isinstance(tmp, dict): - new_value = json.dumps(tmp) - elif isinstance(tmp, list): - new_value = json.dumps(tmp) - #else: - #tmp = tmp.replace("\"", "\\\"", -1) - - try: - new_value = json.loads(new_value) - except json.decoder.JSONDecodeError as e: - pass - except TypeError as e: - pass - except: - pass - #self.logger.info("Json: %s" % e) - #ret.append(tmp) - - #if self.result_wrapper_count > 0: - # ret.append("["*(self.result_wrapper_count-1)+new_value+"]"*(self.result_wrapper_count-1)) - #else: - ret.append(new_value) - - self.logger.info("[INFO] Function return length: %d" % len(ret)) - if len(ret) == 1: - #ret = ret[0] - self.logger.info("[DEBUG] DONT make list of 1 into 0!!") - - self.logger.info(f"[DEBUG][%s] Done with execution recursion %d times" % (self.current_execution_id, len(param_multiplier))) - - #self.logger.info("Return from execution: %s" % ret) - if ret == None: - results.append("") - json_object = False - elif isinstance(ret, dict): - results.append(ret) - json_object = True - elif isinstance(ret, list): - results = ret - json_object = True - else: - ret = ret.replace("\"", "\\\"", -1) - - try: - results.append(json.loads(ret)) - json_object = True - except json.decoder.JSONDecodeError as e: - #self.logger.info("Json: %s" % e) - results.append(ret) - except TypeError as e: - results.append(ret) - except: - results.append(ret) - - #if len(results) == 1: - # #results = results[0] - # #self.logger.info("DONT MAKE LIST FROM 1 TO 0!!") - # pass - - #self.logger.info("\nLOOP: %s\nRESULTS: %s" % (loop_wrapper, results)) - return results - - # Downloads all files from a namespace - # Currently only working on local version of Shuffle (2023) - def get_file_category_ids(self, category): - org_id = self.full_execution["workflow"]["execution_org"]["id"] - - get_path = "/api/v1/files/namespaces/%s?execution_id=%s&ids=true" % (category, self.full_execution["execution_id"]) - headers = { - "Authorization": "Bearer %s" % self.authorization, - "User-Agent": "Shuffle 1.1.0", - } - - ret = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False, proxies=self.proxy_config) - return ret.json() - #if ret1.status_code != 200: - # return { - # "success": False, - # "reason": "Status code is %d from backend for category %s" % category, - # "list": [], - # } - - #return { - # "success": True, - # "ids": ret1.json(), - #} - - - # Downloads all files from a namespace - # Currently only working on local version of Shuffle - def get_file_namespace(self, namespace): - org_id = self.full_execution["workflow"]["execution_org"]["id"] - - get_path = "/api/v1/files/namespaces/%s?execution_id=%s" % (namespace, self.full_execution["execution_id"]) - headers = { - "Authorization": "Bearer %s" % self.authorization, - "User-Agent": "Shuffle 1.1.0", - } - - ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False, proxies=self.proxy_config) - if ret1.status_code != 200: - return None - - filebytes = BytesIO(ret1.content) - myzipfile = zipfile.ZipFile(filebytes) - - # Unzip and build here! - #for member in files.namelist(): - # filename = os.path.basename(member) - # if not filename: - # continue - - # self.logger.info("File: %s" % member) - # source = files.open(member) - # with open("%s/%s" % (basedir, source.name), "wb+") as tmp: - # filedata = source.read() - # self.logger.info("Filedata (%s): %s" % (source.name, filedata)) - # tmp.write(filedata) - - return myzipfile - - def get_file_namespace_ids(self, namespace): - return self.get_file_category_ids(self, namespace) - - def get_file_category(self, category): - return self.get_file_namespace(self, category) - - # Things to consider for files: - # - How can you download / stream a file? - # - Can you decide if you want a stream or the files directly? - def get_file(self, value): - full_execution = self.full_execution - org_id = full_execution["workflow"]["execution_org"]["id"] - - if isinstance(value, list): - self.logger.info("IS LIST!") - #if len(value) == 1: - # value = value[0] - else: - value = [value] - - returns = [] - for item in value: - self.logger.info("FILE VALUE: %s" % item) - # Check if item is a dict, and if it is, check if it has the key "id" - if isinstance(item, dict): - if "file_id" in item: - item = item["file_id"] - elif "id" in item: - item = item["id"] - - if len(item) != 36 and not item.startswith("file_"): - self.logger.info("Bad length for file value: '%s'" % item) - continue - #return { - # "filename": "", - # "data": "", - # "success": False, - #} - - get_path = "/api/v1/files/%s?execution_id=%s" % (item, full_execution["execution_id"]) - headers = { - "Content-Type": "application/json", - "Authorization": "Bearer %s" % self.authorization, - "User-Agent": "Shuffle 1.1.0", - } - - ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False, proxies=self.proxy_config) - if ret1.status_code != 200: - returns.append({ - "filename": "", - "data": "", - "success": False, - }) - continue - - content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"]) - ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers, verify=False, proxies=self.proxy_config) - if ret2.status_code == 200: - tmpdata = ret1.json() - returndata = { - "success": True, - "filename": tmpdata["filename"], - "data": ret2.content, - } - returns.append(returndata) - - if len(returns) == 0: - return { - "success": False, - "filename": "", - "data": b"", - } - elif len(returns) == 1: - return returns[0] - else: - return returns - - def delete_cache(self, key): - org_id = self.full_execution["workflow"]["execution_org"]["id"] - url = "%s/api/v1/orgs/%s/delete_cache" % (self.url, org_id) - - data = { - "workflow_id": self.full_execution["workflow"]["id"], - "execution_id": self.current_execution_id, - "authorization": self.authorization, - "org_id": org_id, - "key": key, - } - - try: - newstorage = [] - for item in self.local_storage: - if item["execution_id"] == self.current_execution_id and item["key"] == key: - continue - - newstorage.append(item) - - self.local_storage = newstorage - - except Exception as e: - print("[ERROR] Failed DELETING current execution id local storage: %s" % e) - - response = requests.post(url, json=data, verify=False, proxies=self.proxy_config) - try: - allvalues = response.json() - return json.dumps(allvalues) - except Exception as e: - self.logger.info("[ERROR} Failed to parse response from delete_cache: %s" % e) - #return response.json() - return json.dumps({"success": False, "reason": f"Failed to delete cache for key '{key}'"}) - - def set_cache(self, key, value): - org_id = self.full_execution["workflow"]["execution_org"]["id"] - url = "%s/api/v1/orgs/%s/set_cache" % (self.url, org_id) - data = { - "workflow_id": self.full_execution["workflow"]["id"], - "execution_id": self.current_execution_id, - "authorization": self.authorization, - "org_id": org_id, - "key": key, - "value": str(value), - } - - try: - newstorage = [] - for item in self.local_storage: - if item["execution_id"] == self.current_execution_id and item["key"] == key: - continue - - newstorage.append(item) - - self.local_storage = newstorage - - except Exception as e: - print("[ERROR] Failed SETTING current execution id local storage: %s" % e) - - response = requests.post(url, json=data, verify=False, proxies=self.proxy_config) - try: - allvalues = response.json() - allvalues["key"] = key - allvalues["value"] = str(value) - return allvalues - except Exception as e: - self.logger.info("[ERROR} Failed to parse response from set cache: %s" % e) - #return response.json() - return {"success": False} - - def get_cache(self, key): - org_id = self.full_execution["workflow"]["execution_org"]["id"] - url = "%s/api/v1/orgs/%s/get_cache" % (self.url, org_id) - data = { - "workflow_id": self.full_execution["workflow"]["id"], - "execution_id": self.current_execution_id, - "authorization": self.authorization, - "org_id": org_id, - "key": key, - } - - # Makes it so that loops for the same action doesn't re-ask the db unless necessary - try: - for item in self.local_storage: - if item["execution_id"] == self.current_execution_id and item["key"] == key: - # Max keeping the local cache properly for 5 seconds due to workflow continuations - elapsed_time = time.time() - item["time_set"] - if elapsed_time > 5: - break - - return item["data"] - except Exception as e: - print("[ERROR] Failed getting current execution id local storage: %s" % e) - - value = requests.post(url, json=data, verify=False, proxies=self.proxy_config) - try: - allvalues = value.json() - allvalues["key"] = key - - try: - parsedvalue = json.loads(allvalues["value"]) - allvalues["value"] = parsedvalue - except: - self.logger.info("Parsing of value as JSON failed. Continue anyway!") - - try: - newdata = json.loads(json.dumps(data)) - newdata["time_set"] = time.time() - newdata["data"] = allvalues - self.local_storage.append(newdata) - except Exception as e: - print("[ERROR] Failed in local storage append: %s" % e) - - return allvalues - except: - self.logger.info("Value couldn't be parsed, or json dump of value failed") - #return value.json() - return {"success": False} - - def update_file(self, file_id, content): - full_execution = self.full_execution - workflow_id = full_execution["workflow"]["id"] - org_id = full_execution["workflow"]["execution_org"]["id"] - - new_headers = { - "Authorization": f"Bearer {self.authorization}", - "User-Agent": "Shuffle 1.1.0", - } - - upload_path = "/api/v1/files/%s/edit?execution_id=%s" % (file_id, full_execution["execution_id"]) - - files={"shuffle_file": ("filename", content)} - - ret = requests.put("%s%s" % (self.url, upload_path), files=files, headers=new_headers, verify=False, proxies=self.proxy_config) - print(ret.status_code) - return ret.json() - - - # Wrapper for set_files - def set_file(self, infiles): - return self.set_files(infiles) - - # Sets files in the backend - def set_files(self, infiles): - full_execution = self.full_execution - workflow_id = full_execution["workflow"]["id"] - org_id = full_execution["workflow"]["execution_org"]["id"] - headers = { - "Content-Type": "application/json", - "Authorization": "Bearer %s" % self.authorization, - "User-Agent": "Shuffle 1.1.0", - } - - if not isinstance(infiles, list): - infiles = [infiles] - - create_path = "/api/v1/files/create?execution_id=%s" % full_execution["execution_id"] - file_ids = [] - for curfile in infiles: - filename = "unspecified" - data = { - "filename": filename, - "workflow_id": workflow_id, - "org_id": org_id, - } - - try: - data["filename"] = curfile["filename"] - filename = curfile["filename"] - except KeyError as e: - self.logger.info(f"KeyError in file setup: {e}") - pass - - ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data, verify=False, proxies=self.proxy_config) - #self.logger.info(f"Ret CREATE: {ret.text}") - cur_id = "" - if ret.status_code == 200: - ret_json = ret.json() - if not ret_json["success"]: - self.logger.info("Not success in file upload creation.") - continue - - self.logger.info("Should handle ID %s" % ret_json["id"]) - file_ids.append(ret_json["id"]) - cur_id = ret_json["id"] - else: - self.logger.info("Bad status code: %d" % ret.status_code) - continue - - if len(cur_id) == 0: - self.logger.info("No file ID specified from backend") - continue - - new_headers = { - "Authorization": f"Bearer {self.authorization}", - "User-Agent": "Shuffle 1.1.0", - } - - upload_path = "/api/v1/files/%s/upload?execution_id=%s" % (cur_id, full_execution["execution_id"]) - - files={"shuffle_file": (filename, curfile["data"])} - #open(filename,'rb')} - - ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers, verify=False, proxies=self.proxy_config) - - return file_ids - - def validate_condition(self, sourcevalue, check, destinationvalue): - if check == "=" or check == "==" or check.lower() == "equals": - if str(sourcevalue).lower() == str(destinationvalue).lower(): - return True - elif check == "!=" or check.lower() == "does not equal": - if str(sourcevalue).lower() != str(destinationvalue).lower(): - return True - elif check.lower() == "startswith": - if str(sourcevalue).lower().startswith(str(destinationvalue).lower()): - return True - elif check.lower() == "endswith": - if str(sourcevalue).lower().endswith(str(destinationvalue).lower()): - return True - elif check.lower() == "contains": - if destinationvalue.lower() in sourcevalue.lower(): - return True - - elif check.lower() == "is empty" or check.lower() == "is_empty": - try: - if len(json.loads(sourcevalue)) == 0: - return True - except Exception as e: - self.logger.info(f"[ERROR] Failed to check if empty as list: {e}") - - if len(str(sourcevalue)) == 0: - return True - - elif check.lower() == "contains_any_of": - newvalue = [destinationvalue.lower()] - if ", " in destinationvalue: - newvalue = destinationvalue.split(", ") - elif "," in destinationvalue: - newvalue = destinationvalue.split(",") - - for item in newvalue: - if not item: - continue - - if item.strip() in sourcevalue: - return True - - - # FIXME: This will be buggy if using > and >= operators in the future. - elif check.lower() == "larger than" or check.lower() == "bigger than" or check == ">" or check == ">=": - try: - if str(sourcevalue).isdigit() and str(destinationvalue).isdigit(): - if int(sourcevalue) > int(destinationvalue): - return True - - except AttributeError as e: - self.logger.info("[WARNING] Condition larger than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) - - try: - destinationvalue = len(json.loads(destinationvalue)) - except Exception as e: - self.logger.info(f"[WARNING] Failed to convert destination to list: {e}") - try: - # Check if it's a list in autocast and if so, check the length - if len(json.loads(sourcevalue)) > int(destinationvalue): - return True - except Exception as e: - self.logger.info(f"[WARNING] Failed to check if larger than as list: {e}") - - - # FIXME: This will be buggy if using < and <= operators in the future. - elif check.lower() == "smaller than" or check.lower() == "less than" or check == "<" or check == "<=": - self.logger.info("In smaller than check: %s %s" % (sourcevalue, destinationvalue)) - - try: - if str(sourcevalue).isdigit() and str(destinationvalue).isdigit(): - if int(sourcevalue) < int(destinationvalue): - return True - - except AttributeError as e: - pass - - try: - destinationvalue = len(json.loads(destinationvalue)) - except Exception as e: - self.logger.info(f"[WARNING] Failed to convert destination to list: {e}") - - try: - # Check if it's a list in autocast and if so, check the length - if len(json.loads(sourcevalue)) < int(destinationvalue): - return True - except Exception as e: - self.logger.info(f"[WARNING] Failed to check if smaller than as list: {e}") - - elif check.lower() == "re" or check.lower() == "matches regex": - try: - found = re.search(str(destinationvalue), str(sourcevalue)) - except re.error as e: - return False - except Exception as e: - return False - - if found == None: - return False - - return True - else: - self.logger.error("[DEBUG] Condition: can't handle %s yet. Setting to true" % check) - - return False - - - #async def execute_action(self, action): - def execute_action(self, action): - # !!! Let this line stay - its used for some horrible codegeneration / stitching !!! # - #STARTCOPY - stream_path = "/api/v1/streams" - self.action_result = { - "action": action, - "authorization": self.authorization, - "execution_id": self.current_execution_id, - "result": "", - "started_at": int(time.time_ns()), - "status": "EXECUTING" - } - - # Simple validation of parameters in general - replace_params = False - try: - tmp_parameters = action["parameters"] - for param in tmp_parameters: - if param["value"] == "SHUFFLE_AUTO_REMOVED": - replace_params = True - except KeyError: - action["parameters"] = [] - except TypeError: - pass - - self.action = copy.deepcopy(action) - - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {self.authorization}", - "User-Agent": "Shuffle 1.1.0", - } - - if len(self.action) == 0: - self.logger.info("[WARNING] ACTION env not defined") - self.action_result["result"] = "Error in setup ENV: ACTION not defined" - self.send_result(self.action_result, headers, stream_path) - return - - if len(self.authorization) == 0: - self.logger.info("[WARING] AUTHORIZATION env not defined") - self.action_result["result"] = "Error in setup ENV: AUTHORIZATION not defined" - self.send_result(self.action_result, headers, stream_path) - return - - if len(self.current_execution_id) == 0: - self.logger.info("[WARNING] EXECUTIONID env not defined") - self.action_result["result"] = "Error in setup ENV: EXECUTIONID not defined" - self.send_result(self.action_result, headers, stream_path) - return - - # Verify whether there are any parameters with ACTION_RESULT required - # If found, we get the full results list from backend - - # Forcing this to run due to potential self.full_execution loading issues in cloud run - fullexecution = {} - if True or (isinstance(self.full_execution, str) and len(self.full_execution) == 0): - #self.logger.info("[DEBUG] NO EXECUTION - LOADING!") - try: - failed = False - rettext = "" - for i in range(0, 5): - tmpdata = { - "authorization": self.authorization, - "execution_id": self.current_execution_id - } - - resultsurl = "%s/api/v1/streams/results" % (self.base_url) - ret = requests.post( - resultsurl, - headers=headers, - json=tmpdata, - verify=False, - proxies=self.proxy_config, - ) - - if ret.status_code == 200: - fullexecution = ret.json() - failed = False - break - - #elif ret.status_code == 500 or ret.status_code == 400: - elif ret.status_code >= 400: - self.logger.info("[ERROR] (fails: %d) Error in app with status code %d for results (1). RETRYING because results can't be handled" % (i+1, ret.status_code)) - - rettext = ret.text - failed = True - time.sleep(8) - continue - - else: - self.logger.info("[ERROR] (fails: %d) Error in app with status code %d for results (2). Crashing because results can't be handled. Details: %s" % (i+1, ret.status_code, ret.text)) - - rettext = ret.text - failed = True - time.sleep(8) - break - - if failed: - self.action_result["result"] = json.dumps({ - "success": False, - "reason": f"Bad result from backend during startup of app: {ret.status_code}", - "extended_reason": f"{rettext}" - }) - - self.send_result(self.action_result, headers, stream_path) - return - - except requests.exceptions.ConnectionError as e: - self.logger.info("[ERROR] FullExec Connectionerror: %s" % e) - self.action_result["result"] = json.dumps({ - "success": False, - "reason": f"Connection error during startup (connection error): {e}" - }) - - self.send_result(self.action_result, headers, stream_path) - return - except Exception as e: - self.logger.info("[ERROR] FullExec Exception outer: %s" % e) - self.action_result["result"] = json.dumps({ - "success": False, - "reason": f"Exception during startup of app (general error): {e}" - }) - - self.send_result(self.action_result, headers, stream_path) - return - - else: - self.logger.info(f"[DEBUG] Setting execution to default value with type {type(self.full_execution)}") - try: - fullexecution = json.loads(self.full_execution) - except json.decoder.JSONDecodeError as e: - self.logger.info("[ERROR] Json decode execution error: %s" % e) - self.action_result["result"] = "Json error during startup: %s" % e - self.send_result(self.action_result, headers, stream_path) - return - - self.logger.info("") - - - self.full_execution = fullexecution - - found_id = "" - try: - if "execution_id" in self.full_execution and len(self.full_execution["execution_id"]) > 0: - found_id = self.full_execution["execution_id"] - elif len(self.current_execution_id) > 0: - found_id = self.current_execution_id - except Exception as e: - print("[ERROR] Failed in get full exec") - - try: - contains_body = False - parameter_count = 0 - - if "parameters" in self.action: - parameter_count = len(self.action["parameters"]) - for param in self.action["parameters"]: - if param["name"] == "body": - contains_body = True - - print("[DEBUG][%s] Action name: %s, Params: %d, Has Body: %s" % (self.current_execution_id, self.action["name"], parameter_count, str(contains_body))) - except Exception as e: - print("[ERROR] Failed in init print handler: %s" % e) - - try: - if replace_params == True: - for inner_action in self.full_execution["workflow"]["actions"]: - self.logger.info("[DEBUG] ID: %s vs %s" % (inner_action["id"], self.action["id"])) - - # In case of some kind of magic, we're just doing params - if inner_action["id"] != self.action["id"]: - continue - self.logger.info("FOUND!") - - if isinstance(self.action, str): - self.logger.info("Params is in string object for self.action?") - else: - self.action["parameters"] = inner_action["parameters"] - self.action_result["action"]["parameters"] = inner_action["parameters"] - - if isinstance(self.original_action, str): - self.logger.info("Params for original actions is in string object?") - else: - self.original_action["parameters"] = inner_action["parameters"] - - break - - except Exception as e: - self.logger.info(f"[WARNING] Failed in replace params action parsing: {e}") - - # Gets the value at the parenthesis level you want - def parse_nested_param(string, level): - """ - Generate strings contained in nested (), indexing i = level - """ - if len(re.findall('(', string)) == len(re.findall(')', string)): - LeftRightIndex = [x for x in zip( - [Left.start()+1 for Left in re.finditer('(', string)], - reversed([Right.start() for Right in re.finditer(')', string)]))] - - elif len(re.findall('(', string)) > len(re.findall(')', string)): - return parse_nested_param(string + ')', level) - elif len(re.findall('(', string)) < len(re.findall('"', string)): - return parse_nested_param('(' + string, level) - else: - return 'Failed to parse params' - - try: - return [string[LeftRightIndex[level][0]:LeftRightIndex[level][1]]] - except IndexError: - return [string[LeftRightIndex[level+1][0]:LeftRightIndex[level+1][1]]] - - # Finds the deepest level parenthesis in a string - def maxDepth(S): - current_max = 0 - max = 0 - n = len(S) - - # Traverse the input string - for i in range(n): - if S[i] == '(': - current_max += 1 - - if current_max > max: - max = current_max - elif S[i] == ')': - if current_max > 0: - current_max -= 1 - else: - return -1 - - # finally check for unbalanced string - if current_max != 0: - return -1 - - return max-1 - - # Specific type parsing - def parse_type(data, thistype): - if data == None: - return "Empty" - - if "int" in thistype or "number" in thistype: - try: - return int(data) - except ValueError: - return data - - if "lower" in thistype: - return data.lower() - if "upper" in thistype: - return data.upper() - if "trim" in thistype: - return data.strip() - if "strip" in thistype: - return data.strip() - if "split" in thistype: - return data.split() - if "replace" in thistype: - splitvalues = data.split(",") - - if len(splitvalues) > 2: - for i in range(len(splitvalues)): - if i != 0: - if splitvalues[i] == " ": - splitvalues[i] = " " - continue - - splitvalues[i] = splitvalues[i].strip() - - if splitvalues[i] == "\"\"": - splitvalues[i] = "" - if splitvalues[i] == "\" \"": - splitvalues[i] = " " - if len(splitvalues[i]) > 2: - if splitvalues[i][0] == "\"" and splitvalues[i][len(splitvalues[i])-1] == "\"": - splitvalues[i] = splitvalues[i][1:-1] - if splitvalues[i][0] == "'" and splitvalues[i][len(splitvalues[i])-1] == "'": - splitvalues[i] = splitvalues[i][1:-1] - - - replacementvalue = splitvalues[0] - return replacementvalue.replace(splitvalues[1], splitvalues[2], -1) - else: - return f"replace({data})" - if "join" in thistype: - try: - splitvalues = data.split(",") - if "," not in data: - return f"join({data})" - - if len(splitvalues) >= 2: - - # 1. Take the list and parse it from string - # 2. Take all the items and join them - # 3. Parse them back as string and return - values = ",".join(splitvalues[0:-1]) - tmp = json.loads(values) - try: - newvalues = splitvalues[-1].join(str(item).strip() for item in tmp) - except TypeError: - newvalues = splitvalues[-1].join(json.dumps(item).strip() for item in tmp) - - return newvalues - else: - return f"join({data})" - - except (KeyError, IndexError) as e: - pass - except json.decoder.JSONDecodeError as e: - pass - - if "len" in thistype or "length" in thistype or "lenght" in thistype: - #self.logger.info(f"Trying to length-parse: {data}") - try: - tmp_len = json.loads(data, parse_float=str, parse_int=str, parse_constant=str) - except (NameError, KeyError, TypeError, json.decoder.JSONDecodeError) as e: - try: - #self.logger.info(f"[WARNING] INITIAL Parsing bug for length in app sdk: {e}") - # data = data.replace("\'", "\"") - data = data.replace("True", "true", -1) - data = data.replace("False", "false", -1) - data = data.replace("None", "null", -1) - data = data.replace("\"", "\\\"", -1) - data = data.replace("'", "\"", -1) - - tmp_len = json.loads(data, parse_float=str, parse_int=str, parse_constant=str) - except (NameError, KeyError, TypeError, json.decoder.JSONDecodeError) as e: - tmp_len = str(data) - - return str(len(tmp_len)) - - if "parse" in thistype: - splitvalues = [] - default_error = """Error. Expected syntax: parse(["hello","test1"],0:1)""" - if "," in data: - splitvalues = data.split(",") - - for item in range(len(splitvalues)): - splitvalues[item] = splitvalues[item].strip() - else: - return default_error - - lastsplit = [] - if ":" in splitvalues[-1]: - lastsplit = splitvalues[-1].split(":") - else: - try: - lastsplit = [int(splitvalues[-1])] - except ValueError: - return default_error - - try: - parsedlist = ",".join(splitvalues[0:-1]) - if len(lastsplit) > 1: - tmp = json.loads(parsedlist)[int(lastsplit[0]):int(lastsplit[1])] - else: - tmp = json.loads(parsedlist)[lastsplit[0]] - - return tmp - except IndexError as e: - return default_error - - # Parses the INNER value and recurses until everything is done - # Looks for a way to use e.g. int() or number() as a value - def parse_wrapper(data): - try: - if "(" not in data or ")" not in data: - return data, False - except TypeError: - return data, False - - # Because liquid can handle ALL of this now. - # Implemented for >0.9.25 - #self.logger.info("[DEBUG] Skipping parser because use of its been deprecated >0.9.25 due to Liquid implementation") - return data, False - - wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length", "lenght", "join", "replace"] - - if not any(wrapper in data for wrapper in wrappers): - return data, False - - # Do stuff here. - inner_value = parse_nested_param(data, maxDepth(data) - 0) - outer_value = parse_nested_param(data, maxDepth(data) - 1) - - wrapper_group = "|".join(wrappers) - parse_string = data - max_depth = maxDepth(parse_string) - - if outer_value != inner_value: - for casting_items in reversed(range(max_depth + 1)): - c_parentheses = parse_nested_param(parse_string, casting_items)[0] - match_string = re.escape(c_parentheses) - custom_casting = re.findall(fr"({wrapper_group})\({match_string}", parse_string) - - # no matching ; go next group - if len(custom_casting) == 0: - continue - - inner_result = parse_type(c_parentheses, custom_casting[0]) - - # if result is a string then parse else return - if isinstance(inner_result, str): - parse_string = parse_string.replace(f"{custom_casting[0]}({c_parentheses})", inner_result, 1) - elif isinstance(inner_result, list): - parse_string = parse_string.replace(f"{custom_casting[0]}({c_parentheses})", json.dumps(inner_result), 1) - else: - parse_string = inner_result - break - else: - c_parentheses = parse_nested_param(parse_string, 0)[0] - match_string = re.escape(c_parentheses) - custom_casting = re.findall(fr"({wrapper_group})\({match_string}", parse_string) - # check if a wrapper was found - if len(custom_casting) != 0: - inner_result = parse_type(c_parentheses, custom_casting[0]) - if isinstance(inner_result, str): - parse_string = parse_string.replace(f"{custom_casting[0]}({c_parentheses})", inner_result) - elif isinstance(inner_result, list): - parse_string = parse_string.replace(f"{custom_casting[0]}({c_parentheses})", - json.dumps(inner_result)) - else: - parse_string = inner_result - - return parse_string, True - - # Looks for parantheses to grab special cases within a string, e.g: - # int(1) lower(HELLO) or length(what's the length) - # FIXME: - # There is an issue in here where it returns data wrong. Example: - # Authorization=Bearer authkey - # = - # Authorization=Bearer authkey - # ^ Double space. - def parse_wrapper_start(data, self): - try: - data = parse_liquid(data, self) - except: - pass - - if "(" not in data or ")" not in data: - return data - - if isinstance(data, str) and len(data) > 4: - if (data[0] == "{" or data[0] == "[") and (data[len(data)-1] == "]" or data[len(data)-1] == "}"): - self.logger.info("[DEBUG] Skipping parser because use of {[ and ]}") - return data - - newdata = [] - newstring = "" - record = True - paranCnt = 0 - charcnt = 0 - for char in data: - if char == "(": - charskip = False - if charcnt > 0: - if data[charcnt-1] == " ": - charskip = True - - if not charskip: - paranCnt += 1 - - if not record: - record = True - - if record: - newstring += char - - if paranCnt == 0 and char == " ": - newdata.append(newstring) - newstring = "" - record = True - - if char == ")": - paranCnt -= 1 - - if paranCnt == 0: - record = False - - charcnt += 1 - - if len(newstring) > 0: - newdata.append(newstring) - - parsedlist = [] - non_string = False - parsed = False - for item in newdata: - ret = parse_wrapper(item) - if not isinstance(ret[0], str): - non_string = True - - parsedlist.append(ret[0]) - if ret[1]: - parsed = True - - if not parsed: - return data - - if len(parsedlist) > 0 and not non_string: - #self.logger.info("Returning parsed list: ", parsedlist) - return " ".join(parsedlist) - elif len(parsedlist) == 1 and non_string: - return parsedlist[0] - else: - #self.logger.info("Casting back to string because multi: ", parsedlist) - newlist = [] - for item in parsedlist: - try: - newlist.append(str(item)) - except ValueError: - newlist.append("parsing_error") - - # Does this create the issue? - return " ".join(newlist) - - # Parses JSON loops and such down to the item you're looking for - # Check recurse_test.py for examples and tests of this function - # $nodename.#.id - # $nodename.data.#min-max.info.id - # $nodename.data.#1-max.info.id - # $nodename.data.#min-1.info.id - def recurse_json(basejson, parsersplit): - match = "#([0-9a-z]+):?-?([0-9a-z]+)?#?" - try: - outercnt = 0 - - # Loops over split values - splitcnt = -1 - for value in parsersplit: - splitcnt += 1 - #if " " in value: - # value = value.replace(" ", "_", -1) - - actualitem = re.findall(match, value, re.MULTILINE) - # Goes here if loop - if value == "#": - newvalue = [] - - if basejson == None: - return "", False - - for innervalue in basejson: - # 1. Check the next item (message) - # 2. Call this function again - - try: - ret, is_loop = recurse_json(innervalue, parsersplit[outercnt+1:]) - except IndexError: - # Only in here if it's the last loop without anything in it? - ret, is_loop = recurse_json(innervalue, parsersplit[outercnt:]) - - newvalue.append(ret) - - # Magical way of returning which makes app sdk identify - # it as multi execution - return newvalue, True - - # Checks specific regex like #1-2 for index 1-2 in a loop - elif len(actualitem) > 0: - - is_loop = True - newvalue = [] - firstitem = actualitem[0][0] - seconditem = actualitem[0][1] - if isinstance(firstitem, int): - firstitem = str(firstitem) - if isinstance(seconditem, int): - seconditem = str(seconditem) - - # Means it's a single item -> continue - if seconditem == "": - if str(firstitem).lower() == "max" or str(firstitem).lower() == "last" or str(firstitem).lower() == "end": - firstitem = len(basejson)-1 - elif str(firstitem).lower() == "min" or str(firstitem).lower() == "first": - firstitem = 0 - else: - firstitem = int(firstitem) - - tmpitem = basejson[int(firstitem)] - try: - newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:]) - except IndexError: - newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:]) - else: - if isinstance(firstitem, str): - if firstitem.lower() == "max" or firstitem.lower() == "last" or firstitem.lower() == "end": - firstitem = len(basejson)-1 - elif firstitem.lower() == "min" or firstitem.lower() == "first": - firstitem = 0 - else: - firstitem = int(firstitem) - else: - firstitem = int(firstitem) - - if isinstance(seconditem, str): - if str(seconditem).lower() == "max" or str(seconditem).lower() == "last" or str(firstitem).lower() == "end": - seconditem = len(basejson)-1 - elif str(seconditem).lower() == "min" or str(seconditem).lower() == "first": - seconditem = 0 - else: - seconditem = int(seconditem) - else: - seconditem = int(seconditem) - - newvalue = [] - if int(seconditem) > len(basejson): - seconditem = len(basejson) - - for i in range(int(firstitem), int(seconditem)+1): - # 1. Check the next item (message) - # 2. Call this function again - - try: - ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt+1:]) - except IndexError: - #ret = innervalue - ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt:]) - - newvalue.append(ret) - - return newvalue, is_loop - - else: - if len(value) == 0: - return basejson, False - - try: - if isinstance(basejson, list): - return basejson, False - elif isinstance(basejson, bool): - return basejson, False - elif isinstance(basejson, int): - return basejson, False - elif isinstance(basejson[value], str): - try: - if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")): - basejson = json.loads(basejson[value]) - else: - # Should we sanitize here? - # Check if we are on the last item? - if outercnt == len(parsersplit)-1: - return str(basejson[value]), False - else: - pass - - except json.decoder.JSONDecodeError as e: - return str(basejson[value]), False - else: - basejson = basejson[value] - except KeyError as e: - if "_" in value: - value = value.replace("_", " ", -1) - elif " " in value: - value = value.replace(" ", "_", -1) - - try: - if isinstance(basejson, list): - return basejson, False - elif isinstance(basejson, bool): - return basejson, False - elif isinstance(basejson, int): - return basejson, False - elif isinstance(basejson[value], str): - try: - if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")): - basejson = json.loads(basejson[value]) - else: - - if outercnt == len(parsersplit)-1: - return str(basejson[value]), False - else: - pass - - except json.decoder.JSONDecodeError as e: - return str(basejson[value]), False - else: - basejson = basejson[value] - except KeyError as e: - # Check if previous key was handled or not - previouskey = parsersplit[outercnt-1] - - tmpval = previouskey + "." + value - if tmpval in basejson: - return basejson[tmpval], False - - try: - currentsplitcnt = splitcnt - - recursed_value = value - handled = False - - #tmpbase = basejson - previouskey = value - while True: - newvalue = parsersplit[currentsplitcnt+1] - if newvalue == "#" or newvalue == "": - break - - recursed_value += "." + newvalue - - found = False - for key, value in basejson.items(): - if recursed_value.lower() in key.lower(): - found = True - - if found == False: - # Check if we are on the last key or not - return "", False - - if recursed_value in basejson: - basejson = basejson[recursed_value] - - # Whether to dig deeper or not - if isinstance(basejson, bool) or isinstance(basejson, int) or isinstance(basejson, str): - handled = False - else: - handled = True - - break - - currentsplitcnt += 1 - - if handled: - continue - - break - except IndexError as e: - return "", False - - outercnt += 1 - - except KeyError as e: - return "", False - except Exception as e: - return "", False - - return basejson, False - - # Takes a workflow execution as argument - # Returns a string if the result is single, or a list if it's a list - def get_json_value(execution_data, input_data): - parsersplit = input_data.split(".") - actionname_lower = parsersplit[0][1:].lower() - - #Actionname: Start_node - - # 1. Find the action - baseresult = "" - - appendresult = "" - if (actionname_lower.startswith("exec ") or actionname_lower.startswith("webhook ") or actionname_lower.startswith("schedule ") or actionname_lower.startswith("userinput ") or actionname_lower.startswith("email_trigger ") or actionname_lower.startswith("trigger ")) and len(parsersplit) == 1: - record = False - for char in actionname_lower: - if char == " ": - record = True - - if record: - appendresult += char - - actionname_lower = "exec" - elif actionname_lower.startswith("shuffle_cache ") or actionname_lower.startswith("shuffle_db "): - actionname_lower = "shuffle_cache" - - actionname_lower = actionname_lower.replace(" ", "_", -1) - - try: - if actionname_lower == "exec" or actionname_lower == "webhook" or actionname_lower == "schedule" or actionname_lower == "userinput" or actionname_lower == "email_trigger" or actionname_lower == "trigger": - baseresult = execution_data["execution_argument"] - elif actionname_lower == "shuffle_cache": - if len(parsersplit) > 1: - actual_key = parsersplit[1] - cachedata = self.get_cache(actual_key) - parsersplit.pop(1) - try: - baseresult = json.dumps(cachedata) - except json.decoder.JSONDecodeError as e: - pass - - - else: - if execution_data["results"] != None: - for result in execution_data["results"]: - resultlabel = result["action"]["label"].replace(" ", "_", -1).lower() - if resultlabel.lower() == actionname_lower: - baseresult = result["result"] - break - else: - baseresult = "$" + parsersplit[0][1:] - - if len(baseresult) == 0: - try: - for variable in execution_data["workflow"]["workflow_variables"]: - variablename = variable["name"].replace(" ", "_", -1).lower() - - if variablename.lower() == actionname_lower: - baseresult = variable["value"] - break - - except KeyError as e: - pass - except TypeError as e: - pass - - if len(baseresult) == 0: - try: - for variable in execution_data["execution_variables"]: - variablename = variable["name"].replace(" ", "_", -1).lower() - if variablename.lower() == actionname_lower: - baseresult = variable["value"] - break - except KeyError as e: - pass - except TypeError as e: - pass - - except KeyError as error: - pass - - # 2. Find the JSON data - # Returns if there isn't any JSON in the base ($nodename) - if len(baseresult) == 0: - return ""+appendresult, False - - # Returns if the result is JUST something like $nodename, not $nodename.value - if len(parsersplit) == 1: - returndata = str(baseresult)+str(appendresult) - return returndata, False - - baseresult = baseresult.replace(" True,", " true,") - baseresult = baseresult.replace(" False", " false,") - - # Tries to actually read it as JSON with some stupid formatting - basejson = {} - try: - basejson = json.loads(baseresult) - except json.decoder.JSONDecodeError as e: - try: - baseresult = baseresult.replace("\'", "\"") - basejson = json.loads(baseresult) - except json.decoder.JSONDecodeError as e: - return str(baseresult)+str(appendresult), False - - # Finds the ACTUAL value which is in the $nodename.value.test - focusing on value.test - data, is_loop = recurse_json(basejson, parsersplit[1:]) - parseditem = data - - if isinstance(parseditem, dict) or isinstance(parseditem, list): - try: - parseditem = json.dumps(parseditem) - except json.decoder.JSONDecodeError as e: - pass - - if is_loop: - if parsersplit[-1] == "#": - parseditem = "${SHUFFLE_NO_SPLITTER%s}$" % json.dumps(data) - else: - # Return value: ${id[12345, 45678]}$ - parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data)) - - - returndata = str(parseditem)+str(appendresult) - - # New in 0.8.97: Don't return items without lists - #return returndata, is_loop - - # 0.9.70: - # The {} and [] checks are required because e.g. 7e7 is valid JSON for some reason... - # This breaks EVERYTHING - try: - if (returndata.endswith("}") and returndata.endswith("}")) or (returndata.startswith("[") and returndata.endswith("]")): - return json.dumps(json.loads(returndata)), is_loop - else: - return returndata, is_loop - except json.decoder.JSONDecodeError as e: - return returndata, is_loop - - - # Sending self as it's not a normal function - def parse_liquid(template, self): - - errors = False - error_msg = "" - try: - if len(template) > 10000000: - self.logger.info("[DEBUG] Skipping liquid - size too big (%d)" % len(template)) - return template - - if "${" in template and "}$" in template: - #self.logger.info("[DEBUG] Shuffle loop shouldn't run in liquid. Data length: %d" % len(template)) - return template - - - # New pattern fixer to help with bad liquid formats - try: - newoutput = self.patternfix_string(template, - { - "{{|": '{{ "" |', - }, - { - r'\{\{\s*\$[^|}]+\s*\|': '{{ "" |', - } - , - inputtype="liquid" - ) - - template = newoutput - except Exception as e: - print("[ERROR] Failed liquid parsing fix: %s" % e) - - all_globals = globals() - all_globals["self"] = self - run = Liquid(template, mode="wild", from_file=False, filters=shuffle_filters.filters, globals=all_globals) - - # Add locals that are missing to globals - ret = run.render() - return ret - except jinja2.exceptions.TemplateNotFound as e: - self.logger.info(f"[ERROR] Liquid Template error: {e}") - error = True - error_msg = e - - self.action["parameters"].append({ - "name": "liquid_template_error", - "value": f"There was a Liquid input error (1). Details: {e}", - }) - - self.action_result["action"] = self.action - except SyntaxError as e: - self.logger.info(f"[ERROR] Liquid Syntax error: {e}") - error = True - error_msg = e - - self.action["parameters"].append({ - "name": "liquid_python_syntax_error", - "value": f"There was a syntax error in your Liquid input (2). Details: {e}", - }) - - self.action_result["action"] = self.action - except IndentationError as e: - self.logger.info(f"[ERROR] Liquid IndentationError: {e}") - error = True - error_msg = e - - self.action["parameters"].append({ - "name": "liquid_indentiation_error", - "value": f"There was an indentation error in your Liquid input (2). Details: {e}", - }) - - self.action_result["action"] = self.action - except jinja2.exceptions.TemplateSyntaxError as e: - self.logger.info(f"[ERROR] Liquid Syntax error: {e}") - error = True - error_msg = e - - self.action["parameters"].append({ - "name": "liquid_syntax_error", - "value": f"There was a syntax error in your Liquid input (2). Details: {e}", - }) - - self.action_result["action"] = self.action - except json.decoder.JSONDecodeError as e: - self.logger.info(f"[ERROR] Liquid JSON Syntax error: {e}") - - replace = False - skip_next = False - newlines = [] - thisline = [] - for line in template.split("\n"): - if "\"\"\"" in line or "\'\'\'" in line: - if replace: - skip_next = True - else: - replace = not replace - - if replace == True: - thisline.append(line) - if skip_next == True: - if len(thisline) > 0: - newlines.append(" ".join(thisline)) - thisline = [] - - replace = False - else: - newlines.append(line) - - new_template = "\n".join(newlines) - if new_template != template: - #check_template(new_template) - return parse_liquid(new_template, self) - else: - error = True - error_msg = e - - self.action["parameters"].append({ - "name": "liquid_json_error", - "value": f"There was a syntax error in your input JSON(2). This is typically an issue with escaping newlines. Details: {e}", - }) - - self.action_result["action"] = self.action - except TypeError as e: - try: - if "string as left operand" in f"{e}": - split_left = template.split("|") - if len(split_left) < 2: - return template - - splititem = split_left[0] - additem = "{{" - if "{{" in splititem: - splititem = splititem.replace("{{", "", -1) - - if "{%" in splititem: - splititem = splititem.replace("{%", "", -1) - additem = "{%" - - splititem = "%s \"%s\"" % (additem, splititem.strip()) - parsed_template = template.replace(split_left[0], splititem) - run = Liquid(parsed_template, mode="wild", from_file=False) - return run.render(**globals()) - - except Exception as e: - self.action["parameters"].append({ - "name": "liquid_general_error", - "value": f"There was general error Liquid input (2). Details: {e}", - }) - - self.action_result["action"] = self.action - #return template - - self.logger.info(f"[ERROR] Liquid TypeError error: {e}") - error = True - error_msg = e - - except Exception as e: - self.logger.info(f"[ERROR] General exception for liquid: {e}") - error = True - error_msg = e - - self.action["parameters"].append({ - "name": "liquid_general_exception", - "value": f"There was general exception Liquid input (2). Details: {e}", - }) - - self.action_result["action"] = self.action - - if "fmt" in error_msg and "liquid_date" in error_msg: - return template - - self.logger.info("Done in liquid") - if error == True: - self.action_result["status"] = "FAILURE" - data = { - "success": False, - "reason": f"Failed to parse LiquidPy: {error_msg}", - "input": template, - } - - try: - self.action_result["result"] = json.dumps(data) - except Exception as e: - self.action_result["result"] = f"Failed to parse LiquidPy: {error_msg}" - - self.action_result["completed_at"] = int(time.time_ns()) - self.send_result(self.action_result, headers, stream_path) - - self.logger.info(f"[ERROR] Sent FAILURE response to backend due to : {e}") - - if runtime == "run": - return template - else: - os.exit() - - return template - - # Suboptimal cleanup script for BOdy parsing of OpenAPI - # Should have a regex which looks for the value, then goes out and cleans up the key - def recurse_cleanup_script(data): - deletekeys = [] - newvalue = data - try: - if not isinstance(data, dict): - newvalue = json.loads(data) - else: - newvalue = data - - for key, value in newvalue.items(): - if isinstance(value, str) and len(value) == 0: - deletekeys.append(key) - continue - - if isinstance(value, list): - try: - value = json.dumps(value) - except: - pass - - if value == "${%s}" % key: - deletekeys.append(key) - continue - elif "${" in value and "}" in value: - deletekeys.append(key) - continue - - if isinstance(value, dict): - newvalue[key] = recurse_cleanup_script(value) - - except json.decoder.JSONDecodeError as e: - # Since here the data isn't at all JSON compatible..? - # Seems to happen with newlines in variables being parsed in as strings? - pass - except Exception as e: - pass - - try: - for deletekey in deletekeys: - try: - del newvalue[deletekey] - except: - pass - except Exception as e: - return data - - try: - for key, value in newvalue.items(): - if isinstance(value, bool): - continue - elif isinstance(value, dict) and not bool(value): - continue - - try: - value = json.loads(value) - newvalue[key] = value - except json.decoder.JSONDecodeError as e: - continue - except Exception as e: - continue - - try: - data = json.dumps(newvalue) - except json.decoder.JSONDecodeError as e: - data = newvalue - - except json.decoder.JSONDecodeError as e: - pass - except Exception as e: - pass - - return data - - # Makes JSON string values into valid strings in JSON - # Mainly by removing newlines and such - def fix_json_string_value(value): - try: - value = value.replace("\r\n", "\\r\\n") - value = value.replace("\n", "\\n") - value = value.replace("\r", "\\r") - - # Fix quotes in the string - value = value.replace("\\\"", "\"") - value = value.replace("\"", "\\\"") - - value = value.replace("\\\'", "\'") - value = value.replace("\'", "\\\'") - except Exception as e: - pass - - return value - - - - # Parses parameters sent to it and returns whether it did it successfully with the values found - def parse_params(action, fullexecution, parameter, self): - # Skip if it starts with $? - jsonparsevalue = "$." - is_loop = False - - # Matches with space in the first part, but not in subsequent parts. - # JSON / yaml etc shouldn't have spaces in their fields anyway. - #match = ".*?([$]{1}([a-zA-Z0-9 _-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})[$/, ]?" - #match = ".*?([$]{1}([a-zA-Z0-9 _-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})" - - #match = ".*?([$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})" # Removed space - no longer ok. Force underscore. - #match = "([$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})" # Removed .*? to make it work with large amounts of data - match = "([$]{1}([a-zA-Z0-9_@-]+\.?){1}([a-zA-Z0-9#_@-]+\.?){0,})" # Added @ to the regex - - # Extra replacements for certain scenarios - escaped_dollar = "\\$" - escape_replacement = "\\%\\%\\%\\%\\%" - end_variable = "^_^" - - #self.logger.info("Input value: %s" % parameter["value"]) - try: - parameter["value"] = parameter["value"].replace(escaped_dollar, escape_replacement, -1) - except: - self.logger.info("Error in initial replacement of escaped dollar!") - - paramname = "" - try: - paramname = parameter["name"] - except: - pass - - # Basic fix in case variant isn't set - # Variant is ALWAYS STATIC_VALUE from mid 2021~ - try: - parameter["variant"] = parameter["variant"] - except: - parameter["variant"] = "STATIC_VALUE" - - # Regex to find all the things - # Should just go in here if data is ... not so big - #if parameter["variant"] == "STATIC_VALUE" and len(parameter["value"]) < 1000000: - #if parameter["variant"] == "STATIC_VALUE" and len(parameter["value"]) < 5000000: - if parameter["variant"] == "STATIC_VALUE": - data = parameter["value"] - actualitem = re.findall(match, data, re.MULTILINE) - #self.logger.debug(f"\n\nHandle static data with JSON: {data}\n\n") - #self.logger.info("STATIC PARSED: %s" % actualitem) - #self.logger.info("[INFO] Done with regex matching") - if len(actualitem) > 0: - for replace in actualitem: - try: - to_be_replaced = replace[0] - except IndexError: - continue - - # Handles for loops etc. - # FIXME: Should it dump to string here? Doesn't that defeat the purpose? - # Trying without string dumping. - #self.logger.info("TO BE REPLACED: %s" % to_be_replaced) - value, is_loop = get_json_value(fullexecution, to_be_replaced) - - #self.logger.info(f"\n\nType of value: {type(value)}") - if isinstance(value, str): - # Could we take it here? - #self.logger.info(f"[DEBUG] Got value %s for parameter {paramname}" % value) - # Should check if there is are quotes infront of and after the to_be_replaced - # If there are, then we need to sanitize the value - # 1. Look for the to_be_replaced in the data - # 2. Check if there is a quote infront of it and also if there are {} in the data to validate JSON - # 3. If there are, sanitize! - #if data.find(f'"{to_be_replaced}"') != -1 and data.find("{") != -1 and data.find("}") != -1: - # returnvalue = fix_json_string_value(value) - # value = returnvalue - - parameter["value"] = parameter["value"].replace(to_be_replaced, value, 1) - elif isinstance(value, dict) or isinstance(value, list): - # Changed from JSON dump to str() 28.05.2021 - # This makes it so the parameters gets lists and dicts straight up - parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value), 1) - - #try: - # parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) - #except: - # parameter["value"] = parameter["value"].replace(to_be_replaced, str(value)) - # self.logger.info("Failed parsing value as string?") - else: - self.logger.error("[ERROR] Unknown type %s" % type(value)) - try: - parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value), 1) - except json.decoder.JSONDecodeError as e: - parameter["value"] = parameter["value"].replace(to_be_replaced, value, 1) - - else: - #self.logger.info(f"[ERROR] Not running static variant regex parsing (slow) on value with length {len(parameter['value'])}. Max is 5Mb~.") - pass - - if parameter["variant"] == "WORKFLOW_VARIABLE": - self.logger.info("[DEBUG] Handling workflow variable") - found = False - try: - for item in fullexecution["workflow"]["workflow_variables"]: - if parameter["action_field"] == item["name"]: - found = True - parameter["value"] = item["value"] - break - except KeyError as e: - self.logger.info("KeyError WF variable 1: %s" % e) - pass - except TypeError as e: - self.logger.info("TypeError WF variables 1: %s" % e) - pass - - if not found: - try: - for item in fullexecution["execution_variables"]: - if parameter["action_field"] == item["name"]: - parameter["value"] = item["value"] - break - except KeyError as e: - self.logger.info("KeyError WF variable 2: %s" % e) - pass - except TypeError as e: - self.logger.info("TypeError WF variables 2: %s" % e) - pass - - elif parameter["variant"] == "ACTION_RESULT": - # FIXME - calculate value based on action_field and $if prominent - # FIND THE RIGHT LABEL - # GET THE LABEL'S RESULT - - tmpvalue = "" - self.logger.info("ACTION FIELD: %s" % parameter["action_field"]) - - fullname = "$" - if parameter["action_field"] == "Execution Argument": - tmpvalue = fullexecution["execution_argument"] - fullname += "exec" - else: - fullname += parameter["action_field"] - - self.logger.info("PRE Fullname: %s" % fullname) - - if parameter["value"].startswith(jsonparsevalue): - fullname += parameter["value"][1:] - #else: - # fullname = "$%s" % parameter["action_field"] - - self.logger.info("Fullname: %s" % fullname) - actualitem = re.findall(match, fullname, re.MULTILINE) - self.logger.info("ACTION PARSED: %s" % actualitem) - if len(actualitem) > 0: - for replace in actualitem: - try: - to_be_replaced = replace[0] - except IndexError: - self.logger.info("Nothing to replace?: " % e) - continue - - # This will never be a loop aka multi argument - parameter["value"] = to_be_replaced - - value, is_loop = get_json_value(fullexecution, to_be_replaced) - self.logger.info("Loop: %s" % is_loop) - if isinstance(value, str): - parameter["value"] = parameter["value"].replace(to_be_replaced, value) - elif isinstance(value, dict): - parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) - else: - self.logger.info("Unknown type %s" % type(value)) - try: - parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) - except json.decoder.JSONDecodeError as e: - parameter["value"] = parameter["value"].replace(to_be_replaced, value) - - #self.logger.info("PRE Replaced data: %s" % parameter["value"]) - - try: - parameter["value"] = parameter["value"].replace(end_variable, "", -1) - parameter["value"] = parameter["value"].replace(escape_replacement, "$", -1) - except: - self.logger.info(f"[ERROR] Problem in datareplacement: {e}") - - # Just here in case it breaks - # Implemented 02.08.2021 - #self.logger.info("Pre liquid: %s" % parameter["value"]) - try: - parameter["value"] = parse_liquid(parameter["value"], self) - except: - pass - - return "", parameter["value"], is_loop - - - def check_branch_conditions(action, fullexecution, self): - # relevantbranches = workflow.branches where destination = action - try: - if fullexecution["workflow"]["branches"] == None or len(fullexecution["workflow"]["branches"]) == 0: - return True, "" - except KeyError: - return True, "" - - # Startnode should always run - no need to check incoming - # Removed November 2023 due to people wanting startnode to also check - # This is to make it possible ot - try: - if action["id"] == fullexecution["start"]: - return True, "" - - except Exception as error: - self.logger.info(f"[WARNING] Failed checking startnode: {error}") - #return True, "" - #return True, "" - - available_checks = [ - "=", - "equals", - "!=", - "does not equal", - ">", - "larger than", - "<", - "less than", - ">=", - "<=", - "startswith", - "endswith", - "contains", - "contains_any_of", - "re", - "matches regex", - "is empty", - "is_empty", - ] - - relevantbranches = [] - correct_branches = 0 - matching_branches = 0 - for branch in fullexecution["workflow"]["branches"]: - if branch["destination_id"] != action["id"]: - continue - - matching_branches += 1 - - # Find if previous is skipped or failed. Skipped != correct branch - try: - should_skip = False - for res in fullexecution["results"]: - if res["action"]["id"] == branch["source_id"]: - if res["status"] == "FAILURE" or res["status"] == "SKIPPED": - should_skip = True - - break - - if should_skip: - continue - except Exception as e: - self.logger.info("[WARNING] Failed handling check of if parent is skipped") - - - # Remove anything without a condition - try: - if (branch["conditions"]) == 0 or branch["conditions"] == None: - correct_branches += 1 - continue - except KeyError: - correct_branches += 1 - continue - - successful_conditions = [] - failed_conditions = [] - successful_conditions = 0 - total_conditions = len(branch["conditions"]) - for condition in branch["conditions"]: - # Parse all values first here - sourcevalue = condition["source"]["value"] - check, sourcevalue, is_loop = parse_params(action, fullexecution, condition["source"], self) - if check: - continue - - sourcevalue = parse_wrapper_start(sourcevalue, self) - destinationvalue = condition["destination"]["value"] - - check, destinationvalue, is_loop = parse_params(action, fullexecution, condition["destination"], self) - if check: - continue - - destinationvalue = parse_wrapper_start(destinationvalue, self) - - if not condition["condition"]["value"] in available_checks: - self.logger.error("[ERROR] Skipping '%s' -> %s -> '%s' because %s is invalid." % (sourcevalue, condition["condition"]["value"], destinationvalue, condition["condition"]["value"])) - continue - - validation = self.validate_condition(sourcevalue, condition["condition"]["value"], destinationvalue) - try: - if condition["condition"]["configuration"]: - validation = not validation - except KeyError: - pass - - if validation == True: - successful_conditions += 1 - - if total_conditions == successful_conditions: - correct_branches += 1 - - if matching_branches == 0: - return True, "" - - if matching_branches > 0 and correct_branches > 0: - return True, "" - - #self.logger.info("[DEBUG] Correct branches vs matching branches: %d vs %d" % (correct_branches, matching_branches)) - return False, {"success": False, "reason": "Minimum of one branch's conditions must be correct to continue. Total: %d of %d" % (correct_branches, matching_branches)} - - - # - # - # - # - # CONT - # CONT - # CONT - # CONT - # CONT - # CONT - # CONT - # CONT - # CONT - # CONT - # CONT - # CONT - # CONT - # - # - # - # - - # THE START IS ACTUALLY RIGHT HERE :O - # Checks whether conditions are met, otherwise set - branchcheck, tmpresult = check_branch_conditions(action, fullexecution, self) - if isinstance(tmpresult, object) or isinstance(tmpresult, list) or isinstance(tmpresult, dict): - #self.logger.info("[DEBUG] Fixing branch return as object -> string") - try: - #tmpresult = tmpresult.replace("'", "\"") - tmpresult = json.dumps(tmpresult) - except json.decoder.JSONDecodeError as e: - pass - - - # IF branches fail: Exit! - if not branchcheck: - self.action_result["result"] = tmpresult - self.action_result["status"] = "SKIPPED" - self.action_result["completed_at"] = int(time.time_ns()) - - self.send_result(self.action_result, headers, stream_path) - return - - # Replace name cus there might be issues - # Not doing lower() as there might be user-made functions - actionname = action["name"] - if " " in actionname: - actionname.replace(" ", "_", -1) - - #if action.generated: - # actionname = actionname.lower() - - # Runs the actual functions - try: - func = getattr(self, actionname, None) - if func == None: - self.logger.debug(f"[DEBUG] Failed executing {actionname} because func is None (no function specified).") - self.action_result["status"] = "FAILURE" - self.action_result["result"] = json.dumps({ - "success": False, - "reason": f"Function {actionname} doesn't exist, or the App is out of date.", - "details": "If this persists, please delete the Docker image locally, then restart your Orborus instance before trying again. This will force-download the latest version. Contact support@shuffler.io with this data if the issue persists.", - }) - elif callable(func): - try: - if len(action["parameters"]) < 1: - result = func() - else: - # Potentially parse JSON here - # FIXME - add potential authentication as first parameter(s) here - # params[parameter["name"]] = parameter["value"] - #self.logger.info(fullexecution["authentication"] - # What variables are necessary here tho hmm - - params = {} - - # Fixes OpenAPI body parameters for later. - newparams = [] - counter = -1 - bodyindex = -1 - for parameter in action["parameters"]: - counter += 1 - - # Hack for key:value in options using || - try: - if parameter["options"] != None and len(parameter["options"]) > 0: - #self.logger.info(f'OPTIONS: {parameter["options"]}') - #self.logger.info(f'OPTIONS VAL: {parameter}') - if "||" in parameter["value"]: - splitvalue = parameter["value"].split("||") - if len(splitvalue) > 1: - #self.logger.info(f'[INFO] Parsed split || options of actions["parameters"]["name"]') - action["parameters"][counter]["value"] = splitvalue[1] - - except (IndexError, KeyError, TypeError) as e: - self.logger.info("[WARNING] Options err: {e}") - - # This part is purely for OpenAPI accessibility. - # It replaces the data back into the main item - # Earlier, we handled each of the items and did later string replacement, - # but this has changed to do lists within items and such - if parameter["name"] == "body": - bodyindex = counter - - try: - values = parameter["value_replace"] - if values != None: - added = 0 - for val in values: - replace_value = val["value"] - replace_key = val["key"] - - if (val["value"].startswith("{") and val["value"].endswith("}")) or (val["value"].startswith("[") and val["value"].endswith("]")): - self.logger.info(f"""Trying to parse as JSON: {val["value"]}""") - try: - newval = val["value"] - - # If it gets here, remove the "" infront and behind the key as well - # since this is preventing the JSON from being loaded - tmpvalue = json.loads(newval) - replace_key = f"\"{replace_key}\"" - except json.decoder.JSONDecodeError as e: - self.logger.info("[WARNING] Failed JSON replacement for OpenAPI %s", val["key"]) - - elif val["value"].lower() == "true" or val["value"].lower() == "false": - replace_key = f"\"{replace_key}\"" - else: - if "\"" in replace_value and not "\\\"" in replace_value: - replace_value = replace_value.replace("\"", "\\\"", -1) - - action["parameters"][counter]["value"] = action["parameters"][counter]["value"].replace(replace_key, replace_value, 1) - - self.logger.info(f'[INFO] Added param {val["key"]} for body (using OpenAPI)') - added += 1 - - #action["parameters"]["body"] - - self.logger.info("ADDED %d parameters for body" % added) - except KeyError as e: - self.logger.info("KeyError body OpenAPI: %s" % e) - pass - - - action["parameters"][counter]["value"] = recurse_cleanup_script(action["parameters"][counter]["value"]) - - #self.logger.info(action["parameters"]) - - # This seems redundant now - for parameter in newparams: - action["parameters"].append(parameter) - - self.action = action - - # Setting due to them being overwritten, but still later useful - try: - self.original_action = json.loads(json.dumps(action)) - except Exception as e: - pass - - # calltimes is used to handle forloops in the app itself. - # 2 kinds of loop - one in gui with one app each, and one like this, - # which is super fast, but has a bad overview (potentially good tho) - calltimes = 1 - result = "" - - all_executions = [] - - # Multi_parameter has the data for each. variable - minlength = 0 - multi_parameters = json.loads(json.dumps(params)) - multiexecution = False - multi_execution_lists = [] - remove_params = [] - for parameter in action["parameters"]: - check, value, is_loop = parse_params(action, fullexecution, parameter, self) - if check: - raise Exception(json.dumps({ - "success": False, - "exception": f"Value Error: {check}", - "reason": "Parameter {parameter} has an issue", - })) - - #if parameter["name"] == "body": - # #self.logger.info(f"[INFO] Should debug field with liquid and other checks as it's BODY: {value}") - - # Custom format for ${name[0,1,2,...]}$ - #submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" - #self.logger.info(f"Returnedvalue: {value}") - # OLD: Used until 13.03.2021: submatch = "([${]{2}#?([0-9a-zA-Z_-]+)#?(\[.*\])[}$]{2})" - # \${[0-9a-zA-Z_-]+#?(\[.*?]}\$) - submatch = "([${]{2}#?([0-9a-zA-Z_-]+)#?(\[.*?]}\$))" - actualitem = re.findall(submatch, value, re.MULTILINE) - try: - if action["skip_multicheck"]: - self.logger.info("Skipping multicheck") - actualitem = [] - except KeyError: - pass - - actionname = action["name"] - - # Loops in general goes in here to be parsed out as one->multi - if len(actualitem) > 0: - self.logger.info(f"[INFO] Found {len(actualitem)} items in {parameter['name']}. MULTI EXEC.") - multiexecution = True - - handled = False - - # Has a loop without a variable used inside - - # This is here to handle for loops within variables.. kindof - # 1. Find the length of the longest array - # 2. Build an array with the base values based on parameter["value"] - # 3. Get the n'th value of the generated list from values - # 4. Execute all n answers - replacements = {} - curminlength = 0 - for replace in actualitem: - try: - to_be_replaced = replace[0] - actualitem = replace[2] - if actualitem.endswith("}$"): - actualitem = actualitem[:-2] - - except IndexError: - self.logger.info("[WARNING] Indexerror") - continue - - try: - itemlist = json.loads(actualitem) - if len(itemlist) > minlength: - minlength = len(itemlist) - - if len(itemlist) > curminlength: - curminlength = len(itemlist) - - except json.decoder.JSONDecodeError as e: - self.logger.info("JSON Error (replace): %s in %s" % (e, actualitem)) - - replacements[to_be_replaced] = actualitem - - - # Parses the data as string with length, split etc. before moving on. - #self.logger.info("In second part of else: %s" % (len(itemlist))) - # This is a result array for JUST this value.. - # What if there are more? - resultarray = [] - for i in range(0, curminlength): - tmpitem = json.loads(json.dumps(parameter["value"])) - for key, value in replacements.items(): - replacement = value - try: - replacement = json.dumps(json.loads(value)[i]) - except IndexError as e: - self.logger.info(f"[ERROR] Failed handling value parsing with index: {e}") - pass - - if replacement.startswith("\"") and replacement.endswith("\""): - replacement = replacement[1:len(replacement)-1] - - #except json.decoder.JSONDecodeError as e: - - #self.logger.info("REPLACING %s with %s" % (key, replacement)) - #replacement = parse_wrapper_start(replacement) - tmpitem = tmpitem.replace(key, replacement, -1) - try: - tmpitem = parse_liquid(tmpitem, self) - except Exception as e: - self.logger.info(f"[WARNING] Failed liquid parsing in loop (2): {e}") - - - # This code handles files. - isfile = False - try: - if parameter["schema"]["type"] == "file" and len(value) > 0: - self.logger.info("(2) SHOULD HANDLE FILE IN MULTI. Get based on value %s" % parameter["value"]) - - for tmp_file_split in json.loads(parameter["value"]): - file_value = self.get_file(tmp_file_split) - resultarray.append(file_value) - - - isfile = True - except KeyError as e: - self.logger.info("(2) SCHEMA ERROR IN FILE HANDLING: %s" % e) - except json.decoder.JSONDecodeError as e: - self.logger.info("(2) JSON ERROR IN FILE HANDLING: %s" % e) - - if not isfile: - #tmpitem = tmpitem.replace("\\\\", "\\", -1) - resultarray.append(tmpitem) - - # With this parameter ready, add it to... a greater list of parameters. Rofl - if len(resultarray) == 0: - self.logger.info("[WARNING] Returning empty array because the array length to be looped is 0 (0)") - self.action_result["status"] = "SUCCESS" - self.action_result["result"] = "[]" - self.send_result(self.action_result, headers, stream_path) - return - - #self.logger.info("RESULTARRAY: %s" % resultarray) - if resultarray not in multi_execution_lists: - multi_execution_lists.append(resultarray) - - multi_parameters[parameter["name"]] = resultarray - else: - # Parses things like int(value) - #self.logger.info("[DEBUG] Normal parsing (not looping)")#with data %s" % value) - # This part has fucked over so many random JSON usages because of weird paranthesis parsing - - value = parse_wrapper_start(value, self) - - try: - if str(value).startswith("b'") and str(value).endswith("'"): - value = value[2:-1] - except Exception as e: - pass - - params[parameter["name"]] = value - multi_parameters[parameter["name"]] = value - - # This code handles files. - try: - if parameter["schema"]["type"] == "file" and len(value) > 0: - self.logger.info("\n SHOULD HANDLE FILE. Get based on value %s. <--- is this a valid ID?" % parameter["value"]) - file_value = self.get_file(value) - self.logger.info("FILE VALUE: %s \n" % file_value) - - params[parameter["name"]] = file_value - multi_parameters[parameter["name"]] = file_value - except KeyError as e: - self.logger.info("SCHEMA ERROR IN FILE HANDLING: %s" % e) - - - # Fix lists here - # FIXME: This doesn't really do anything anymore - #self.logger.info("[DEBUG] CHECKING multi execution list: %d!" % len(multi_execution_lists)) - if len(multi_execution_lists) > 0: - filteredlist = [] - for listitem in multi_execution_lists: - if listitem in filteredlist: - continue - - # FIXME: Subsub required?. Recursion! - # Basically multiply what we have with the outer loop? - # - #if isinstance(listitem, list): - # for subitem in listitem: - # filteredlist.append(subitem) - #else: - # filteredlist.append(listitem) - - #self.logger.info("New list length: %d" % len(filteredlist)) - if len(filteredlist) > 1: - self.logger.info(f"Calculating new multi-loop length with {len(filteredlist)} lists") - tmplength = 1 - for innerlist in filteredlist: - tmplength = len(innerlist)*tmplength - self.logger.info("List length: %d. %d*%d" % (tmplength, len(innerlist), tmplength)) - - minlength = tmplength - - self.logger.info("New multi execution length: %d\n" % tmplength) - - # Cleaning up extra list params - for subparam in remove_params: - #self.logger.info(f"DELETING {subparam}") - try: - del params[subparam] - except: - pass - #self.logger.info(f"Error with subparam deletion of {subparam} in {params}") - try: - del multi_parameters[subparam] - except: - #self.logger.info(f"Error with subparam deletion of {subparam} in {multi_parameters} (2)") - pass - - #self.logger.info() - #self.logger.info(f"Param: {params}") - #self.logger.info(f"Multiparams: {multi_parameters}") - #self.logger.info() - - if not multiexecution: - #self.logger.info("NOT MULTI EXEC") - # Runs a single iteration here - new_params = self.validate_unique_fields(params) - if isinstance(new_params, list) and len(new_params) == 1: - params = new_params[0] - #params = new_params - else: - #self.logger.info("[WARNING] SHOULD STOP EXECUTION BECAUSE FIELDS AREN'T UNIQUE") - self.action_result["status"] = "SKIPPED" - self.action_result["result"] = f"A non-unique value was found" - self.action_result["completed_at"] = int(time.time_ns()) - self.send_result(self.action_result, headers, stream_path) - return - - #self.logger.info("[INFO] Running normal execution (not loop)\n\n") - - # Added literal evaluation of anything resembling a string - # The goal is to parse objects that e.g. use single quotes and the like - # FIXME: add this to Multi exec as well. - try: - for key, value in params.items(): - if "-" in key: - try: - newkey = key.replace("-", "_", -1).lower() - params[newkey] = params[key] - except Exception as e: - self.logger.info("[DEBUG] Failed updating key with dash in it: %s" % e) - - try: - if isinstance(value, str) and ((value.startswith("{") and value.endswith("}")) or (value.startswith("[") and value.endswith("]"))): - params[key] = json.loads(value) - except Exception as e: - try: - if isinstance(value, str) and ((value.startswith("{") and value.endswith("}")) or (value.startswith("[") and value.endswith("]"))): - params[key] = ast.literal_eval(value) - except Exception as e: - self.logger.info(f"[DEBUG] Failed parsing value with ast and json.loads - noncritical. Trying next: {e}") - continue - except Exception as e: - self.logger.info("[DEBUG] Failed looping objects. Non critical: {e}") - - # Uncomment below to get the param input - # self.logger.info(f"[DEBUG] PARAMS: {params}") - - #newres = "" - iteration_count = 0 - found_error = "" - while True: - iteration_count += 1 - if iteration_count >= 10: - newres = { - "success": False, - "reason": "Iteration count more than 10. This happens if the input to the action is wrong. Try remaking the action, and contact support@shuffler.io if this persists.", - "details": f"{found_error}", - } - break - - try: - #try: - # Individual functions shouldn't take longer than this - # This is an attempt to make timeouts occur less, incentivizing users to make use efficient API's - # PS: Not implemented for lists - only single actions as of May 2023 - timeout = 30 - - # Check if current app is Shuffle Tools, then set to 55 due to certain actions being slow (ioc parser..) - # In general, this should be disabled for onprem - if self.action["app_name"].lower() == "shuffle tools": - timeout = 55 - - timeout_env = os.getenv("SHUFFLE_APP_SDK_TIMEOUT", timeout) - try: - timeout = int(timeout_env) - #self.logger.info(f"[DEBUG] Timeout set to {timeout} seconds") - except Exception as e: - self.logger.info(f"[ERROR] Failed parsing timeout to int: {e}") - - #timeout = 30 - self.logger.info("[DEBUG][%s] Running function '%s' with timeout %d" % (self.current_execution_id, action["name"], timeout)) - - try: - executor = concurrent.futures.ThreadPoolExecutor() - future = executor.submit(func, **params) - newres = future.result(timeout) - - if not future.done(): - # The future is still running, so we need to cancel it - future.cancel() - newres = json.dumps({ - "success": False, - "exception": str(e), - "reason": "Timeout error within %d seconds (1). This happens if we can't reach or use the API you're trying to use within the time limit. Configure SHUFFLE_APP_SDK_TIMEOUT=100 in Orborus to increase it to 100 seconds. Not changeable for cloud." % timeout, - }) - - else: - # The future is done, so we can just get the result from newres :) - #newres = future.result() - pass - - except concurrent.futures.TimeoutError as e: - newres = json.dumps({ - "success": False, - "reason": "Timeout error (2) within %d seconds (2). This happens if we can't reach or use the API you're trying to use within the time limit. Configure SHUFFLE_APP_SDK_TIMEOUT=100 in Orborus to increase it to 100 seconds. Not changeable for cloud." % timeout, - }) - - break - except TypeError as e: - newres = "" - self.logger.info(f"[ERROR] Got function exec type error: {e}") - try: - e = json.loads(f"{e}") - except: - e = f"{e}" - - found_error = e - errorstring = f"{e}" - - if "the JSON object must be" in errorstring: - self.logger.info("[ERROR] Something is wrong with the input for this function. Are lists and JSON data handled parsed properly (0)? the JSON object must be in...") - - newres = json.dumps({ - "success": False, - "exception": f"{type(e).__name__} - {e}", - "reason": "An exception occurred while running this function (1). See exception for more details and contact support if this persists (support@shuffler.io)", - }) - break - elif "got an unexpected keyword argument" in errorstring: - fieldsplit = errorstring.split("'") - if len(fieldsplit) > 1: - field = fieldsplit[1] - - try: - del params[field] - self.logger.info("[WARNING] Removed invalid field %s (2)" % field) - except KeyError: - break - else: - newres = json.dumps({ - "success": False, - "exception": f"TypeError: {e}", - "reason": "You may be running an old version of this action. Try remaking the node, then contact us at support@shuffler.io if it doesn't work with all these details.", - }) - break - except Exception as e: - self.logger.info(f"[ERROR] Something is wrong with the input for this function. Are lists and JSON data handled parsed properly (1)? err: {e}") - - #try: - # e = json.loads(f"{e}") - #except: - # e = f"{e}" - - newres = json.dumps({ - "success": False, - "exception": f"{type(e).__name__} - {e}", - "reason": "An exception occurred while running this function (2). See exception for more details and contact support if this persists (support@shuffler.io)", - - }) - break - - # Forcing async wait in case of old apps that use async (backwards compatibility) - try: - if asyncio.iscoroutine(newres): - self.logger.info("[DEBUG] In coroutine (1)") - async def parse_value(newres): - value = await asyncio.gather( - newres - ) - - return value[0] - - newres = asyncio.run(parse_value(newres)) - else: - #self.logger.info("[DEBUG] Not in coroutine (1)") - pass - except Exception as e: - self.logger.warning("[ERROR] Failed to parse coroutine value for old app: {e}") - - #self.logger.info("\n\n\n[INFO] Returned from execution with type(s) %s" % type(newres)) - #self.logger.info("\n[INFO] Returned from execution with %s of types %s" % (newres, type(newres)))#, newres) - if isinstance(newres, tuple): - #self.logger.info(f"[INFO] Handling return as tuple: {newres}") - # Handles files. - filedata = "" - file_ids = [] - if isinstance(newres[1], list): - self.logger.info("[INFO] HANDLING LIST FROM RET") - file_ids = self.set_files(newres[1]) - elif isinstance(newres[1], object): - self.logger.info("[INFO] Handling JSON from ret") - file_ids = self.set_files([newres[1]]) - elif isinstance(newres[1], str): - self.logger.info("[INFO] Handling STRING from ret") - file_ids = self.set_files([newres[1]]) - else: - self.logger.info("[INFO] NO FILES TO HANDLE") - - tmp_result = { - "success": True, - "result": newres[0], - "file_ids": file_ids - } - - result = json.dumps(tmp_result) - elif isinstance(newres, str): - #self.logger.info("[INFO] Handling return as string of length %d" % len(newres)) - result += newres - elif isinstance(newres, dict) or isinstance(newres, list): - try: - result += json.dumps(newres, indent=4) - except json.JSONDecodeError as e: - self.logger.info("[WARNING] Failed decoding result: %s" % e) - try: - result += str(newres) - except ValueError: - result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) - self.logger.info("[ERROR] Can't handle type %s value from function" % (type(newres))) - except Exception as e: - self.logger.info("[ERROR] Failed to json dump. Returning as string.") - result += str(newres) - else: - try: - result += str(newres) - except ValueError: - result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) - self.logger.info("Can't handle type %s value from function" % (type(newres))) - - else: - #self.logger.info("[INFO] APP_SDK DONE: Starting MULTI execution (length: %d) with values %s" % (minlength, multi_parameters)) - # 1. Use number of executions based on the arrays being similar - # 2. Find the right value from the parsed multi_params - - #self.logger.info("[INFO] Running WITH loop. MULTI: %s", multi_parameters) - self.logger.info("[INFO] Running WITH loop") - json_object = False - #results = await self.run_recursed_items(func, multi_parameters, {}) - results = self.run_recursed_items(func, multi_parameters, {}) - if isinstance(results, dict) or isinstance(results, list): - json_object = True - - # Dump the result as a string of a list - #self.logger.info("RESULTS: %s" % results) - if isinstance(results, list) or isinstance(results, dict): - - # This part is weird lol - if json_object: - try: - result = json.dumps(results) - except json.JSONDecodeError as e: - self.logger.info(f"Failed to decode: {e}") - result = results - else: - result = "[" - for item in results: - try: - json.loads(item) - result += item - except json.decoder.JSONDecodeError as e: - # Common nested issue which puts " around everything - self.logger.info("Decodingerror: %s" % e) - try: - tmpitem = item.replace("\\\"", "\"", -1) - json.loads(tmpitem) - result += tmpitem - - except: - result += "\"%s\"" % item - - result += ", " - - result = result[:-2] - result += "]" - else: - self.logger.info("Normal result - no list?") - result = results - - self.action_result["status"] = "SUCCESS" - self.action_result["result"] = str(result) - if self.action_result["result"] == "": - self.action_result["result"] = result - - #self.logger.debug(f"[DEBUG] Executed {action['label']}-{action['id']}")#with result: {result}") - #self.logger.debug(f"Data: %s" % action_result) - except TypeError as e: - self.logger.info("[ERROR] TypeError issue: %s" % e) - self.action_result["status"] = "FAILURE" - self.action_result["result"] = json.dumps({ - "success": False, - "reason": f"Typeerror. Most likely due to a list that should've been a string. See details for more info.", - "details": f"{e}", - }) - #self.action_result["result"] = "TypeError: %s" % str(e) - else: - self.logger.info("[DEBUG] Function %s doesn't exist?" % action["name"]) - self.logger.error(f"[ERROR] App {self.__class__.__name__}.{action['name']} is not callable") - self.action_result["status"] = "FAILURE" - #self.action_result["result"] = "Function %s is not callable." % actionname - - self.action_result["result"] = json.dumps({ - "success": False, - "reason": f"Function %s doesn't exist." % actionname, - }) - - # https://ptb.discord.com/channels/747075026288902237/882017498550112286/882043773138382890 - except (requests.exceptions.RequestException, TimeoutError) as e: - self.logger.info(f"[ERROR] Failed to execute request (requests): {e}") - self.logger.exception(f"[ERROR] Failed to execute {e}-{action['id']}") - self.action_result["status"] = "SUCCESS" - try: - e = json.loads(f"{e}") - except: - e = f"{e}" - - try: - self.action_result["result"] = json.dumps({ - "success": False, - "reason": f"Request error - failing silently. Details in detail section", - "details": f"{e}", - }) - except json.decoder.JSONDecodeError as e: - self.action_result["result"] = f"Request error: {e}" - - except Exception as e: - self.logger.info(f"[ERROR] Failed to execute: {e}") - self.logger.exception(f"[ERROR] Failed to execute {e}-{action['id']}") - self.action_result["status"] = "FAILURE" - try: - e = json.loads(f"{e}") - except: - e = f"{e}" - - self.action_result["result"] = json.dumps({ - "success": False, - "reason": f"General exception in the app. See shuffle action logs for more details.", - "details": f"{e}", - }) - - # Send the result :) - self.action_result["completed_at"] = int(time.time_ns()) - self.send_result(self.action_result, headers, stream_path) - - #try: - # try: - # self.log_capture_string.flush() - # except Exception as e: - # print(f"[WARNING] Failed to flush logs (2): {e}") - # pass - - # self.log_capture_string.close() - #except: - # print(f"[WARNING] Failed to close logs (2): {e}") - - return - - @classmethod - def run(cls, action=""): - logging.basicConfig(format="{asctime} - {name} - {levelname}:{message}", style='{') - logger = logging.getLogger(f"{cls.__name__}") - logger.setLevel(logging.DEBUG) - - #logger.info("[DEBUG] Normal execution.") - - ############################################## - - exposed_port = os.getenv("SHUFFLE_APP_EXPOSED_PORT", "") - #logger.info(f"[DEBUG] \"{runtime}\" - run indicates microservices. Port: \"{exposed_port}\"") - if runtime == "run" and exposed_port != "": - # Base port is 33334. Exposed port may differ based on discovery from Worker - port = int(exposed_port) - #logger.info(f"[DEBUG] Starting webserver on port {port} (same as exposed port)") - from flask import Flask, request - from waitress import serve - - flask_app = Flask(__name__) - #flask_app.config['PERMANENT_SESSION_LIFETIME'] = datetime.timedelta(minutes=5) - - #async def execute(): - @flask_app.route("/api/v1/health", methods=["GET", "POST"]) - def check_health(): - return "OK" - - @flask_app.route("/api/v1/run", methods=["POST"]) - def execute(): - if request.method == "POST": - requestdata = {} - try: - requestdata = json.loads(request.data) - except Exception as e: - return { - "success": False, - "reason": f"Invalid Action data {e}", - } - - # Remaking class for each request - - app = cls(redis=None, logger=logger, console_logger=logger) - extra_info = "" - try: - #asyncio.run(AppBase.run(action=requestdata), debug=True) - #value = json.dumps(value) - try: - app.full_execution = json.dumps(requestdata["workflow_execution"]) - except Exception as e: - extra_info += f"\n{e}" - - try: - app.action = requestdata["action"] - except Exception as e: - extra_info += f"\n{e}" - - try: - app.authorization = requestdata["authorization"] - app.current_execution_id = requestdata["execution_id"] - except Exception as e: - extra_info += f"\n{e}" - - # BASE URL (backend) - try: - app.url = requestdata["url"] - except Exception as e: - extra_info += f"\n{e}" - - # URL (worker) - try: - app.base_url = requestdata["base_url"] - except Exception as e: - extra_info += f"\n{e}" - - #await - app.execute_action(app.action) - except Exception as e: - return { - "success": False, - "reason": f"Problem in execution {e}", - "execution_issues": extra_info, - } - - return { - "success": True, - "reason": "App successfully finished", - "execution_issues": extra_info, - } - else: - return { - "success": False, - "reason": f"HTTP method {request.method} not allowed", - } - - logger.info(f"[DEBUG] Serving on port {port}") - - #flask_app.run( - # host="0.0.0.0", - # port=port, - # threaded=True, - # processes=1, - # debug=False, - #) - - serve( - flask_app, - host="0.0.0.0", - port=port, - threads=8, - channel_timeout=30, - expose_tracebacks=True, - asyncore_use_poll=True, - ) - ####################### - else: - # Has to start like this due to imports in other apps - # Move it outside everything? - app = cls(redis=None, logger=logger, console_logger=logger) - - if isinstance(action, str): - #logger.info("[DEBUG] Normal execution (env var). Action is a string.") - pass - elif isinstance(action, object): - #logger.info("[DEBUG] OBJECT execution (cloud). Action is NOT a string.") - app.action = action - - try: - app.authorization = action["authorization"] - app.current_execution_id = action["execution_id"] - except: - pass - - # BASE URL (worker) - try: - app.url = action["url"] - except: - pass - - # Callback URL (backend) - try: - app.base_url = action["base_url"] - except: - pass - else: - #self.logger.info("ACTION TYPE (unhandled): %s" % type(action)) - pass - - app.execute_action(app.action) - -if __name__ == "__main__": - AppBase.run() diff --git a/backend/app_sdk/autocorrect_test.py b/backend/app_sdk/autocorrect_test.py deleted file mode 100644 index a7624c76..00000000 --- a/backend/app_sdk/autocorrect_test.py +++ /dev/null @@ -1,187 +0,0 @@ -import re -import json - -input_data = """{ - "test4": $test, - "test5": , - "test6": "what" - } -""" - -input_data = """{ - "test0": {{ '' | default: [] }}, - "test": {{ | default: [] }}, - "test2": {{ $test.asd | default: [] }}, - "test3": {{ {"key": "val} | default: [] }}, - "test4": $test, - "test5": , - "test6": "what" - } -""" - - -liquiddata = "{{ $test.asd | some other stuff {{ $test.xyz | more stuff" -pattern = r'\{\{\s*\$[^|}]+\s*\|' - -replaced_data = re.sub(pattern, "{{ '' |", liquiddata) -print(replaced_data) - - -def patternfix_string(liquiddata, patterns, regex_patterns, inputtype="liquid"): - if not inputtype or inputtype == "liquid": - if "{{" not in liquiddata or "}}" not in liquiddata: - return liquiddata - elif inputtype == "json": - liquiddata = liquiddata.strip() - - # Validating if it looks like json or not - if liquiddata[0] == "{" and liquiddata[len(liquiddata)-1] == "}": - pass - else: - if liquiddata[0] == "[" and liquiddata[len(liquiddata)-1] == "]": - pass - else: - return liquiddata - - # If it's already json, don't touch it - try: - json.loads(liquiddata) - return liquiddata - except Exception as e: - pass - else: - print("No replace handler for %s" % inputtype) - return liquiddata - - skipkeys = [" "] - newoutput = liquiddata[:] - for pattern in patterns: - keylocations = [] - parsedvalue = "" - record = False - index = -1 - for key in liquiddata: - - # Return instant if possible - if inputtype == "json": - try: - json.loads(newoutput) - return newoutput - except: - pass - - index += 1 - if not key: - if record: - keylocations.append(index) - parsedvalue += key - - continue - - if key in skipkeys: - if record: - keylocations.append(index) - parsedvalue += key - - continue - - if key == pattern[0] and not record: - record = True - - if key not in pattern: - keylocations = [] - parsedvalue = "" - record = False - - if record: - keylocations.append(index) - parsedvalue += key - - if len(parsedvalue) == 0: - continue - - evaluated_value = parsedvalue[:] - for skipkey in skipkeys: - evaluated_value = "".join(evaluated_value.split(skipkey)) - - if evaluated_value == pattern: - #print("Found matching: %s (%s)" % (parsedvalue, keylocations)) - #print("Should replace with: %s" % patterns[pattern]) - - newoutput = newoutput.replace(parsedvalue, patterns[pattern], -1) - - # Return instant if possible - if inputtype == "json": - try: - json.loads(newoutput) - return newoutput - except: - pass - - - for pattern in regex_patterns: - newlines = [] - for line in newoutput.split("\n"): - replaced_line = re.sub(pattern, regex_patterns[pattern], line) - newlines.append(replaced_line) - - newoutput = "\n".join(newlines) - - # Return instant if possible - if inputtype == "json": - try: - json.loads(newoutput) - return newoutput - except: - pass - - # Dont return json properly unless actually json - if inputtype == "json": - try: - json.loads(newoutput) - return newoutput - except: - # Returns original if json fixing didn't work - return liquiddata - - return newoutput - -print("Start:\n%s" % input_data) - -try: - newinput = patternfix_string(input_data, - { - "{{|": '{{ "" |', - }, - { - #r'\{\{\s*|': "{{ '' |", - r'\{\{\s*\$[^|}]+\s*\|': '{{ "" |', - } - , - inputtype="liquid" - ) -except Exception as e: - print("[ERROR} Failed liquid parsing fix: %s" % e) - newinput = input_data - -try: - newinput = patternfix_string(newinput, - { - }, - { - r'\"\s*\:\s*,': '\": "",', - r'\"\s*\:\s*\$[^,]+\w*\,': '\": "",', - } - , - inputtype="json" - ) - - try: - json.loads(newinput) - print("It's json! Override.") - except Exception as e: - print("Bad json. DONT use the value at all: %s" % e) -except Exception as e: - print("[ERROR} Failed json parsing fix: %s" % e) - -print("\nEnd:\n%s" % newinput) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh deleted file mode 100755 index dd540b94..00000000 --- a/backend/app_sdk/build.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -### DEFAULT -NAME=shuffle-app_sdk -VERSION=1.2.0 - -docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force -docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly -t shuffle/shuffle:app_sdk -t shuffle/$NAME:$VERSION -t docker.pkg.github.com/shuffle/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:nightly - -docker push frikky/shuffle:app_sdk -docker push ghcr.io/frikky/$NAME:$VERSION -docker push ghcr.io/frikky/$NAME:nightly -docker push ghcr.io/frikky/$NAME:latest - -docker push shuffle/shuffle:app_sdk -docker push ghcr.io/shuffle/$NAME:$VERSION -docker push ghcr.io/shuffle/$NAME:nightly -docker push ghcr.io/shuffle/$NAME:latest - - - - -#### UBUNTU -NAME=shuffle-app_sdk_ubuntu -docker build . -f Dockerfile_ubuntu -t frikky/shuffle:app_sdk_ubuntu -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -docker push frikky/shuffle:app_sdk_ubuntu -docker push ghcr.io/frikky/$NAME:$VERSION - -#### Alpine GRPC -NAME=shuffle-app_sdk_grpc -docker build . -f Dockerfile_alpine_grpc -t frikky/shuffle:app_sdk_grpc -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -docker push frikky/shuffle:app_sdk_grpc -docker push ghcr.io/frikky/$NAME:$VERSION - - - -#### KALI ### -#NAME=shuffle-app_sdk_kali -#docker build . -f Dockerfile_kali -t frikky/shuffle:app_sdk_kali -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -# -#docker push frikky/shuffle:app_sdk_kali -#docker push ghcr.io/frikky/$NAME:$VERSION -#docker push ghcr.io/frikky/$NAME:nightly - -### BLACKARCH ### -#NAME=shuffle-app_sdk_blackarch -#docker build . -f Dockerfile_blackarch -t frikky/shuffle:app_sdk_blackarch -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -# -#docker push frikky/shuffle:app_sdk_blackarch -#docker push ghcr.io/frikky/$NAME:$VERSION -#docker push ghcr.io/frikky/$NAME:nightly diff --git a/backend/app_sdk/recurse_test.py b/backend/app_sdk/recurse_test.py deleted file mode 100644 index 1cf2bfea..00000000 --- a/backend/app_sdk/recurse_test.py +++ /dev/null @@ -1,326 +0,0 @@ -## A test script of the recurse_json function -## to validate that it can handle the different types of data -## and follow the dot formation format - - -import re -import json - -def recurse_json(basejson, parsersplit): - match = "#([0-9a-z]+):?-?([0-9a-z]+)?#?" - try: - outercnt = 0 - - # Loops over split values - splitcnt = -1 - for value in parsersplit: - splitcnt += 1 - #if " " in value: - # value = value.replace(" ", "_", -1) - - actualitem = re.findall(match, value, re.MULTILINE) - # Goes here if loop - if value == "#": - newvalue = [] - - if basejson == None: - return "", False - - for innervalue in basejson: - # 1. Check the next item (message) - # 2. Call this function again - - try: - ret, is_loop = recurse_json(innervalue, parsersplit[outercnt+1:]) - except IndexError: - # Only in here if it's the last loop without anything in it? - ret, is_loop = recurse_json(innervalue, parsersplit[outercnt:]) - - newvalue.append(ret) - - # Magical way of returning which makes app sdk identify - # it as multi execution - return newvalue, True - - # Checks specific regex like #1-2 for index 1-2 in a loop - elif len(actualitem) > 0: - - is_loop = True - newvalue = [] - firstitem = actualitem[0][0] - seconditem = actualitem[0][1] - if isinstance(firstitem, int): - firstitem = str(firstitem) - if isinstance(seconditem, int): - seconditem = str(seconditem) - - #print("[DEBUG] ACTUAL PARSED: %s" % actualitem) - - # Means it's a single item -> continue - if seconditem == "": - #print("[INFO] In first - handling %s. Len: %d" % (firstitem, len(basejson))) - if str(firstitem).lower() == "max" or str(firstitem).lower() == "last" or str(firstitem).lower() == "end": - firstitem = len(basejson)-1 - elif str(firstitem).lower() == "min" or str(firstitem).lower() == "first": - firstitem = 0 - else: - firstitem = int(firstitem) - - #print(f"[DEBUG] Post lower checks with item {firstitem}") - tmpitem = basejson[int(firstitem)] - try: - newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:]) - except IndexError: - newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:]) - else: - #print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem)) - if isinstance(firstitem, str): - if firstitem.lower() == "max" or firstitem.lower() == "last" or firstitem.lower() == "end": - firstitem = len(basejson)-1 - elif firstitem.lower() == "min" or firstitem.lower() == "first": - firstitem = 0 - else: - firstitem = int(firstitem) - else: - firstitem = int(firstitem) - - if isinstance(seconditem, str): - if str(seconditem).lower() == "max" or str(seconditem).lower() == "last" or str(firstitem).lower() == "end": - seconditem = len(basejson)-1 - elif str(seconditem).lower() == "min" or str(seconditem).lower() == "first": - seconditem = 0 - else: - seconditem = int(seconditem) - else: - seconditem = int(seconditem) - - #print(f"[DEBUG] Post lower checks 2: {firstitem} AND {seconditem}") - newvalue = [] - if int(seconditem) > len(basejson): - seconditem = len(basejson) - - for i in range(int(firstitem), int(seconditem)+1): - # 1. Check the next item (message) - # 2. Call this function again - - try: - ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt+1:]) - except IndexError: - #print("[DEBUG] INDEXERROR (1): ", parsersplit[outercnt]) - #ret = innervalue - ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt:]) - - newvalue.append(ret) - - return newvalue, is_loop - - else: - if len(value) == 0: - return basejson, False - - try: - if isinstance(basejson, list): - #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) - return basejson, False - elif isinstance(basejson, bool): - #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value) - return basejson, False - elif isinstance(basejson, int): - #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value) - return basejson, False - elif isinstance(basejson[value], str): - try: - if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")): - basejson = json.loads(basejson[value]) - else: - # Should we sanitize here? - #print("[DEBUG] VALUE TO SANITIZE FOR KEY '%s'?: %s" % (value, basejson[value])) - - # Check if we are on the last item? - if outercnt == len(parsersplit)-1: - #print("[DEBUG] LAST KEY") - return str(basejson[value]), False - else: - #print("[DEBUG] NOT LAST KEY") - pass - - except json.decoder.JSONDecodeError as e: - return str(basejson[value]), False - else: - basejson = basejson[value] - except KeyError as e: - print("[WARNING] Running secondary value check with replacement of underscore in %s: %s" % (value, e)) - if "_" in value: - value = value.replace("_", " ", -1) - elif " " in value: - value = value.replace(" ", "_", -1) - - try: - if isinstance(basejson, list): - #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) - return basejson, False - elif isinstance(basejson, bool): - #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value) - return basejson, False - elif isinstance(basejson, int): - #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value) - return basejson, False - elif isinstance(basejson[value], str): - #print(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value]) - try: - #print("[DEBUG] BASEJSON: %s" % basejson) - if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")): - basejson = json.loads(basejson[value]) - else: - - if outercnt == len(parsersplit)-1: - #print("LAST KEY (2)") - return str(basejson[value]), False - else: - #print("NOT LAST KEY (2)") - pass - - except json.decoder.JSONDecodeError as e: - #print("[DEBUG] RETURNING BECAUSE '%s' IS A NORMAL STRING (1)" % basejson[value]) - return str(basejson[value]), False - else: - basejson = basejson[value] - except KeyError as e: - # Check if previous key was handled or not - previouskey = parsersplit[outercnt-1] - #print("[DEBUG] PREVIOUS KEY: ", previouskey) - - tmpval = previouskey + "." + value - #print("\n\n[WARNING] Running third dot notation fix '%s' on data %s: %s" % (value, basejson, e)) - if tmpval in basejson: - return basejson[tmpval], False - - try: - currentsplitcnt = splitcnt - - recursed_value = value - handled = False - - #tmpbase = basejson - previouskey = value - while True: - #print("\n\n[DEBUG] CURRENTSPLITCNT: ", currentsplitcnt) - newvalue = parsersplit[currentsplitcnt+1] - if newvalue == "#" or newvalue == "": - break - - recursed_value += "." + newvalue - #print("\n\nRECURSED: ", recursed_value) - - found = False - for key, value in basejson.items(): - if recursed_value.lower() in key.lower(): - found = True - - if found == False: - #print("[INFO] DIDN'T FIND similar VALUE: ", recursed_value) - - # Check if we are on the last key or not - return "", False - #if outercnt == len(parsersplit)-1: - # print("[DEBUG] LAST KEY (3)") - # break - #else: - # print("[DEBUG] NOT LAST KEY (3)") - # return "", False - - if recursed_value in basejson: - #print("[INFO] FOUND RECURSED VALUE: ", recursed_value) - basejson = basejson[recursed_value] - - # Whether to dig deeper or not - if isinstance(basejson, bool) or isinstance(basejson, int) or isinstance(basejson, str): - handled = False - else: - handled = True - - break - - currentsplitcnt += 1 - - if handled: - continue - - break - except IndexError as e: - print("[DEBUG] INDEXERROR (2):", parsersplit[outercnt]) - return "", False - - outercnt += 1 - - except KeyError as e: - print("[INFO] Lower keyerror: %s" % e) - return "", False - except Exception as e: - print("[WARNING] Exception: %s" % e) - return "", False - - return basejson, False - -print("[INFO] Starting") - -#input_data = "test" -#input_data = "test2.data" - - - -# Matchwith -basejson = { - "test": "hello", - "test2": { - "test3": "hello2", - "test3.data": "hello3", - "test4.data.testing": { - "value": "hello4" - }, - "test5.data.hello": "wut", - }, - "test3": ["hello", "hello2", "hello3"], - "test4": [{ - "id": "1", - }] -} - -# Inputexamples (ALL should be True) -inputs = { - #"": "", - "badkey": "", - "test": "hello", - "test2.badkey": "", - "test2.test3": "hello2", - "test2.test3.data": "hello3", - "test2.test4.data.testing": "{'value': 'hello4'}", # FIXME: Doesn't work due to break vs return "", False in last exception - "test2.test4.data.testing.value": "hello4", # FIXME: Doesn't work due to break vs return "", False in last exception. Not fixed as we didn't find one of these yet. - "test2.test5.data.hello": "wut", - "test2.test5.data.badkey": "", - "test3.#1": "hello2", - "test4.#0.id": "1", - "test4.#1.id": "", -} - -outputs = [] -for key, value in inputs.items(): - parsersplit = key.split(".") - ret, is_loop = recurse_json(basejson, parsersplit) - print("\n\nOUTPUT RET (%s): %s" % (key, ret)) - - outputs.append("[%s]: %s = '%s' vs '%s'" % (str(ret) == str(value), key, ret, value)) - -print("\n\n%s" % "\n".join(outputs)) - -#input_data = "" -#input_data = "badkey" -#input_data = "test" -#input_data = "test2.data" -#input_data = "test2.test3.data" -#input_data = "test2.test4.data.testing.value.as" -#input_data = "test2.test5.data.hello" - - - - diff --git a/backend/app_sdk/requirements.txt b/backend/app_sdk/requirements.txt deleted file mode 100644 index 8951ecbe..00000000 --- a/backend/app_sdk/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -urllib3==1.26.18 -requests==2.31.0 -MarkupSafe==2.0.1 -liquidpy==0.8.1 -flask[async]==2.0.2 -waitress==2.1.0 -#flask==1.1.2 -python-dateutil==2.8.1 -PyJWT==2.9.0 diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 6b3ba779..1e8dfddf 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -35,8 +35,8 @@ import ( "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" + gitProxy "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/storage/memory" - gitProxy "github.com/go-git/go-git/v5/plumbing/transport" // Random xj "github.com/basgys/goxml2json" @@ -257,7 +257,6 @@ type Hook struct { Environment string `json:"environment" datastore:"environment"` } - func GetUsersHandler(w http.ResponseWriter, r *http.Request) { data := map[string]interface{}{ "id": "12345", @@ -398,49 +397,49 @@ func checkUsername(Username string) error { } func isGitNoProxy(rawURL string) bool { - noProxy := os.Getenv("NO_PROXY") - if noProxy == "" { - return false - } - - if noProxy == "*" { - return true - } + noProxy := os.Getenv("NO_PROXY") + if noProxy == "" { + return false + } - noProxyList := strings.Split(noProxy, ",") - parsedURL, err := url.Parse(rawURL) - if err != nil { - return false - } - host := parsedURL.Hostname() + if noProxy == "*" { + return true + } - for _,value := range noProxyList { - value = strings.TrimSpace(value) + noProxyList := strings.Split(noProxy, ",") + parsedURL, err := url.Parse(rawURL) + if err != nil { + return false + } + host := parsedURL.Hostname() - if host == value { - return true - } - if strings.HasPrefix(value, "*.") && strings.HasSuffix(host, value[2:]){ - return true - } - } - return false + for _, value := range noProxyList { + value = strings.TrimSpace(value) + + if host == value { + return true + } + if strings.HasPrefix(value, "*.") && strings.HasSuffix(host, value[2:]) { + return true + } + } + return false } func checkGitProxy(cloneOptions *git.CloneOptions) *git.CloneOptions { - if os.Getenv("HTTP_PROXY") != "" && !isGitNoProxy(cloneOptions.URL){ - cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ - URL: os.Getenv("HTTP_PROXY"), - } - } + if os.Getenv("HTTP_PROXY") != "" && !isGitNoProxy(cloneOptions.URL) { + cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ + URL: os.Getenv("HTTP_PROXY"), + } + } - if os.Getenv("HTTPS_PROXY") != "" && !isGitNoProxy(cloneOptions.URL) { - cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ - URL: os.Getenv("HTTPS_PROXY"), - } - } + if os.Getenv("HTTPS_PROXY") != "" && !isGitNoProxy(cloneOptions.URL) { + cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ + URL: os.Getenv("HTTPS_PROXY"), + } + } - return cloneOptions + return cloneOptions } func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) error { @@ -559,7 +558,6 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) } } - return nil } @@ -663,7 +661,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { Name: newOrg.Name, } - user.ActiveOrg = currentOrg + user.ActiveOrg = currentOrg } } } @@ -932,7 +930,6 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { log.Printf("[DEBUG] Failed to get org during getinfo: %s", err) } - //if err == nil { if len(org.Id) > 0 { if userInfo.Role == "" { @@ -1069,9 +1066,9 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { ChatDisabled: chatDisabled, Tutorials: tutorialsFinished, - Interests: orgInterests, - Priorities: orgPriorities, - Licensed: licensed, + Interests: orgInterests, + Priorities: orgPriorities, + Licensed: licensed, } returnData, err := json.Marshal(returnValue) @@ -1092,7 +1089,6 @@ type passwordReset struct { Reference string `json:"reference"` } - func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { cors := shuffle.HandleCors(resp, request) if cors { @@ -3338,7 +3334,6 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User, s } } - log.Printf("[DEBUG] Successfully built app %s (%s)", api.Name, api.ID) if len(user.Id) > 0 { resp.WriteHeader(200) @@ -3379,8 +3374,6 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { buildSwaggerApp(resp, body, user, false) } - - // Hotloads new apps from a folder func handleAppHotload(ctx context.Context, location string, forceUpdate bool) error { @@ -3727,11 +3720,10 @@ func remoteOrgJobController(org shuffle.Org, body []byte) error { return nil } - func remoteOrgJobHandler(org shuffle.Org, interval int) error { // Check if it's 1 in 10 (10% chance random) - backupJob := shuffle.BackupJob{} + backupJob := shuffle.BackupJob{} // Check if workflow backup is active // Check if app backup is active @@ -3777,7 +3769,6 @@ func remoteOrgJobHandler(org shuffle.Org, interval int) error { backupJobData = []byte{} } - syncUrl := fmt.Sprintf("%s/api/v1/cloud/sync", syncUrl) client := shuffle.GetExternalClient(syncUrl) req, err := http.NewRequest( @@ -3983,7 +3974,7 @@ func runInitEs(ctx context.Context) { } // FIXME: Add a randomized timer to avoid all schedules running at the same time - // Many are at 5 minutes / 1 hour. The point is to spread these out + // Many are at 5 minutes / 1 hour. The point is to spread these out // a bit instead of all of them starting at the exact same time //log.Printf("Schedule: %#v", schedule) @@ -4275,7 +4266,7 @@ func runInitEs(ctx context.Context) { } } - cloneOptions = checkGitProxy(cloneOptions) + cloneOptions = checkGitProxy(cloneOptions) branch := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_BRANCH") if len(branch) > 0 && branch != "master" && branch != "main" { @@ -4322,7 +4313,7 @@ func runInitEs(ctx context.Context) { URL: apis, } - cloneOptions = checkGitProxy(cloneOptions) + cloneOptions = checkGitProxy(cloneOptions) _, err = git.Clone(storer, fs, cloneOptions) if err != nil { @@ -4340,17 +4331,16 @@ func runInitEs(ctx context.Context) { log.Printf("[INFO] Skipping download of extra API samples as %d were found", len(workflowapps)) } - if os.Getenv("SHUFFLE_HEALTHCHECK_DISABLED") != "true" { - healthcheckInterval := 30 + healthcheckInterval := 30 log.Printf("[INFO] Starting healthcheck job every %d minute. Stats available on /api/v1/health/stats. Disable with SHUFFLE_HEALTHCHECK_DISABLED=true", healthcheckInterval) job := func() { - // Prepare a fake http.responsewriter + // Prepare a fake http.responsewriter resp := httptest.NewRecorder() request := http.Request{} // Add the "force=true" query to the fake request - request.URL, err = url.Parse("/api/v1/health/stats?force=true") + request.URL, err = url.Parse("/api/v1/health/stats?force=true") if err != nil { log.Printf("[ERROR] Failed to parse test url for healthstats: %s", err) } @@ -4369,7 +4359,6 @@ func runInitEs(ctx context.Context) { log.Printf("[INFO] Finished INIT (ES)") } - func handleVerifyCloudsync(orgId string) (shuffle.SyncFeatures, error) { ctx := context.Background() org, err := shuffle.GetOrg(ctx, orgId) @@ -4948,8 +4937,6 @@ func makeWorkflowPublic(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } - - func handleAppZipUpload(resp http.ResponseWriter, request *http.Request) { cors := shuffle.HandleCors(resp, request) if cors { @@ -5008,8 +4995,6 @@ func handleAppZipUpload(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte("OK")) } - - func initHandlers() { var err error ctx := context.Background() @@ -5056,7 +5041,7 @@ func initHandlers() { r.HandleFunc("/api/v1/users/register", handleRegister).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/users/checkusers", checkAdminLogin).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/users/getinfo", handleInfo).Methods("GET", "OPTIONS") - + r.HandleFunc("/api/v1/users/{userId}/apps", shuffle.HandleGetUserApps).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/users/apps", shuffle.HandleGetUserApps).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/users/generateapikey", shuffle.HandleApiGeneration).Methods("GET", "POST", "OPTIONS") r.HandleFunc("/api/v1/users/logout", shuffle.HandleLogout).Methods("POST", "OPTIONS") @@ -5192,7 +5177,7 @@ func initHandlers() { r.HandleFunc("/api/v1/triggers/gmail/register", shuffle.HandleNewGmailRegister).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/triggers/gmail/getFolders", shuffle.HandleGetGmailFolders).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/triggers/pipeline", shuffle.HandleNewPipelineRegister).Methods("POST", "OPTIONS") - //r.HandleFunc("/api/v1/triggers/pipeline/save", shuffle.HandleSavePipelineInfo).Methods("PUT", "OPTIONS") + //r.HandleFunc("/api/v1/triggers/pipeline/save", shuffle.HandleSavePipelineInfo).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/pipelines/{key}", handlePipelineCallback).Methods("POST", "GET", "PATCH", "PUT", "DELETE", "OPTIONS") r.HandleFunc("/api/v1/triggers", shuffle.HandleGetTriggers).Methods("GET", "OPTIONS") //r.HandleFunc("/api/v1/triggers/gmail/routing", handleGmailRouting).Methods("POST", "OPTIONS") @@ -5220,7 +5205,7 @@ func initHandlers() { r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleDeleteOrg).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/suborgs", shuffle.HandleGetSubOrgs).Methods("GET", "OPTIONS") - + // This is a new API that validates if a key has been seen before. // Not sure what the best course of action is for it. r.HandleFunc("/api/v1/environments/{key}/stop", shuffle.HandleStopExecutions).Methods("GET", "POST", "OPTIONS") @@ -5244,7 +5229,6 @@ func initHandlers() { r.HandleFunc("/api/v1/orgs/{orgId}/datastore", shuffle.HandleSetCacheKey).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/datastore/{cache_key}", shuffle.HandleDeleteCacheKey).Methods("DELETE", "OPTIONS") - // Docker orborus specific - downloads an image r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/login_sso", shuffle.HandleSSO).Methods("GET", "POST", "OPTIONS") @@ -5266,15 +5250,14 @@ func initHandlers() { // This structure is horrendous. Needs fixing after we got the prototype up r.HandleFunc("/api/v1/detections/{detectionType}/connect", shuffle.HandleDetectionAutoConnect).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/detections/{detection_type}", shuffle.HandleGetDetectionRules).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/detections/{triggerId}/selected_rules", shuffle.HandleGetSelectedRules).Methods("GET","OPTIONS") - r.HandleFunc("/api/v1/detections/{triggerId}/selected_rules/save", shuffle.HandleSaveSelectedRules).Methods("POST","OPTIONS") + r.HandleFunc("/api/v1/detections/{triggerId}/selected_rules", shuffle.HandleGetSelectedRules).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/detections/{triggerId}/selected_rules/save", shuffle.HandleSaveSelectedRules).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/detections/{action}", shuffle.HandleFolderToggle).Methods("PUT", "OPTIONS") // This is weird. r.HandleFunc("/api/v1/detections/{fileId}/{action}", shuffle.HandleToggleRule).Methods("PUT", "OPTIONS") //r.HandleFunc("/api/v1/detections/siem/node_health", shuffle.HandleTenzirHealthUpdate).Methods("POST","OPTIONS") - // Introduced in 0.9.21 to handle notifications for e.g. failed Workflow r.HandleFunc("/api/v1/notifications", shuffle.HandleCreateNotification).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/notifications", shuffle.HandleGetNotifications).Methods("GET", "OPTIONS") diff --git a/frontend/package.json b/frontend/package.json index 41c0790d..f71c766d 100755 --- a/frontend/package.json +++ b/frontend/package.json @@ -21,6 +21,7 @@ "@uiw/codemirror-themes": "^4.21.9", "@uiw/react-codemirror": "^4.21.21", "algoliasearch": "^4.8.3", + "chart.js": "^3.0.0", "class-transformer": "^0.2.0", "codemirror": "^6.0.1", "cpx": "^1.5.0", @@ -60,7 +61,7 @@ "react-alice-carousel": "^2.6.4", "react-avatar-editor": "^11.1.0", "react-beforeunload": "^2.2.1", - "react-chartjs-2": "^5.2.0", + "react-chartjs-2": "^5.0.0", "react-cookie": "^4.0.1", "react-cytoscapejs": "^2.0.0", "react-device-detect": "^2.2.3", diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index a2aec50c..04cdd17c 100755 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -14,6 +14,7 @@ import HealthPage from "./components/HealthPage.jsx"; //import Header from "./components/Header.jsx"; import theme from "./theme"; import Apps from "./views/Apps"; +import Apps2 from "./views/Apps2.jsx"; import AppCreator from "./views/AppCreator"; import DetectionDashBoard from "./views/DetectionDashboard.jsx"; @@ -48,7 +49,7 @@ import 'react-toastify/dist/ReactToastify.css'; import Drift from "react-driftjs"; -import { AppContext } from './context/contextApi.jsx'; +import { AppContext } from './context/ContextApi.jsx'; // Production - backend proxy forwarding in nginx var globalUrl = window.location.origin; @@ -408,6 +409,23 @@ const App = (message, props) => { {...props} /> } + /> + + } /> { {data.name}
-
+
{parsedLeftImage} {parsedLeftText}
-
+
{svgIcon}
-
+
{parsedRightImage} {parsedRightText}
diff --git a/frontend/src/components/AppGrid.jsx b/frontend/src/components/AppGrid.jsx index dcc7310a..b0577937 100644 --- a/frontend/src/components/AppGrid.jsx +++ b/frontend/src/components/AppGrid.jsx @@ -464,8 +464,17 @@ const AppGrid = (props) => { }} > {data.name} { + // Replace the image with the default image + const foundImage = document.getElementById(`image_${index}`) + if (foundImage !== undefined && foundImage !== null) { + foundImage.src = theme.palette.defaultImage + data.image_url = theme.palette.defaultImage + } + }} style={{ width: 80, height: 80, @@ -995,7 +1004,7 @@ const AppGrid = (props) => { }} autoComplete="off" color="primary" - placeholder="Search your Activated or Self-built apps" + placeholder="Search your Activated or self-built apps" id="shuffle_search_field" onChange={(event) => { setSearchQuery(event.currentTarget.value); diff --git a/frontend/src/components/AppSelection.jsx b/frontend/src/components/AppSelection.jsx index 5bbb5ff0..df78af8e 100644 --- a/frontend/src/components/AppSelection.jsx +++ b/frontend/src/components/AppSelection.jsx @@ -47,6 +47,7 @@ const AppSelection = props => { defaultSearch, setDefaultSearch, checkLogin, + isAppPage=false } = props; const [discoveryData, setDiscoveryData] = React.useState({}) @@ -507,26 +508,31 @@ const AppSelection = props => { })}
- {!moreButton ? ( -
- { + { + !isAppPage && ( + <> + {!moreButton ? ( +
+ { setMoreButton(true) - setTimeout(() => { navigate("/welcome?tab=2") }, 250) }} >See More Apps -
) : ""} +
) : ""} -
- -
+
+ +
+ + ) + }
) diff --git a/frontend/src/components/Appsearch.jsx b/frontend/src/components/Appsearch.jsx index b5871f2c..dedacfe6 100644 --- a/frontend/src/components/Appsearch.jsx +++ b/frontend/src/components/Appsearch.jsx @@ -27,9 +27,6 @@ const Appsearch = props => { const isCloud = (window.location.host === "localhost:3002" || window.location.host === "shuffler.io") ? true : (process.env.IS_SSR === "true"); const rowHandler = maxRows === undefined || maxRows === null ? 50 : maxRows const xs = parsedXs === undefined || parsedXs === null ? 12 : parsedXs - //const theme = useTheme(); - //const [apps, setApps] = React.useState([]); - //const [filteredApps, setFilteredApps] = React.useState([]); const [formMail, setFormMail] = React.useState(""); const [message, setMessage] = React.useState(""); const [formMessage, setFormMessage] = React.useState(""); diff --git a/frontend/src/components/AuthenticationItem.jsx b/frontend/src/components/AuthenticationItem.jsx index eae08e69..604418b2 100644 --- a/frontend/src/components/AuthenticationItem.jsx +++ b/frontend/src/components/AuthenticationItem.jsx @@ -45,21 +45,21 @@ const AuthenticationItem = (props) => { data.fields = [ { key: "url", - value: "Secret. Replaced during app execution!", + value: "URL Secret. Replaced during runtime", }, { key: "client_id", - value: "Secret. Replaced during app execution!", + value: "ClientID Secret. Replaced during runtime.", }, { key: "client_secret", - value: "Secret. Replaced during app execution!", + value: "Client Secret. Replaced during runtime.", }, { key: "scope", - value: "Secret. Replaced during app execution!", + value: "Scope Secret. Replaced during runtime.", }, - ]; + ] } const deleteAuthentication = (data) => { @@ -152,7 +152,7 @@ const AuthenticationItem = (props) => { src={data.app.large_image} style={{ maxWidth: 50, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} /> style={{ minWidth: 75, maxWidth: 75 }} diff --git a/frontend/src/components/AuthenticationNormal.jsx b/frontend/src/components/AuthenticationNormal.jsx index de0f5cfe..96569b41 100644 --- a/frontend/src/components/AuthenticationNormal.jsx +++ b/frontend/src/components/AuthenticationNormal.jsx @@ -222,7 +222,7 @@ const AuthenticationData = (props) => { { { { const authenticationButtons = } + + {detectionInfo?.category === "SIGMA" || detectionInfo?.category === "SIEM" ? + + + 0 ? green : red}} /> + + + : null} +
+ {filteredRules?.length > 0 ? { - const { globalUrl, workflow, setWorkflow, modalOpen, setModalOpen, showUpload, usecases, setNewWorkflow, appFramework, isEditing, userdata, apps, saveWorkflow, expanded, scrollTo, setRealtimeMarkdown, } = props + const { globalUrl, workflow, setWorkflow, modalOpen, setModalOpen, showUpload, usecases, setNewWorkflow, appFramework, isEditing, userdata, apps, saveWorkflow, expanded, scrollTo, setRealtimeMarkdown, boxWidth, setBoxWidth, } = props const [_, setUpdate] = React.useState(""); // Used for rendering, don't remove @@ -82,12 +83,19 @@ const EditWorkflow = (props) => { const [dueDate, setDueDate] = React.useState(workflow.due_date !== undefined && workflow.due_date !== null && workflow.due_date !== 0 ? dayjs(workflow.due_date*1000) : dayjs().subtract(1, 'day')) const [inputQuestions, setInputQuestions] = React.useState(workflow.input_questions !== undefined && workflow.input_questions !== null ? JSON.parse(JSON.stringify(workflow.input_questions)) : []) - const [inputMarkdown, setInputMarkdown] = React.useState(workflow.input_markdown !== undefined && workflow.input_markdown !== null ? workflow.input_markdown : "") + const [inputMarkdown, setInputMarkdown] = React.useState(workflow?.form_control?.input_markdown !== undefined && workflow?.form_control?.input_markdown !== null ? workflow?.form_control?.input_markdown : "") const [scrollDone, setScrollDone] = React.useState(false) - const [selectedYieldActions, setSelectedYieldActions] = React.useState(workflow.output_yields !== undefined && workflow.output_yields !== null ? JSON.parse(JSON.stringify(workflow.output_yields)) : []) + const [selectedYieldActions, setSelectedYieldActions] = React.useState(workflow?.form_control?.output_yields !== undefined && workflow?.form_control?.output_yields !== null ? JSON.parse(JSON.stringify(workflow?.form_control?.output_yields)) : []) + const [formWidth, setFormWidth] = React.useState(boxWidth === undefined || boxWidth === null ? 500 : boxWidth) const classes = useStyles(); + useEffect(() => { + if (setBoxWidth !== undefined && boxWidth !== formWidth) { + setBoxWidth(formWidth) + } + }, [formWidth]) + if (scrollTo !== undefined && scrollTo !== null && scrollTo.length > 0 && scrollDone === false) { setTimeout(() => { const foundScroll = document.getElementById(scrollTo) @@ -308,9 +316,14 @@ const EditWorkflow = (props) => { } innerWorkflow.input_questions = validfields - innerWorkflow.input_markdown = inputMarkdown - innerWorkflow.output_yields = selectedYieldActions + if (innerWorkflow.form_control === undefined || innerWorkflow.form_control === null) { + innerWorkflow.form_control = {} + } + + innerWorkflow.form_control.input_markdown = inputMarkdown + innerWorkflow.form_control.output_yields = selectedYieldActions + innerWorkflow.form_control.form_width = formWidth innerWorkflow.name = name innerWorkflow.description = description @@ -1049,13 +1062,34 @@ const EditWorkflow = (props) => { } setInputMarkdown(e.target.value) - workflow.input_markdown = e.target.value + workflow.form_control.input_markdown = e.target.value setWorkflow(workflow) setUpdate(Math.random()) }} />
+
+ + Form Size + + + Control the width of the form. It will grow vertically as needed. + + { + setFormWidth(value) + }} + /> +
+
Output Control ({selectedYieldActions.length === 0 ? "No Returns" : selectedYieldActions.length === 1 ? "Returning 1 node" : `Returning ${selectedYieldActions.length} nodes`}) @@ -1117,7 +1151,7 @@ const EditWorkflow = (props) => { + + + {!leftSideBarOpenByClick && setExpandLeftNav(true);}} onMouseLeave={()=>{!leftSideBarOpenByClick && setExpandLeftNav(false);setOpenAutocomplete(false);}}> + + {expandLeftNav ? ( + + + ), + disableUnderline: true, + }} + onClick={()=>{setSearchBarModalOpen(true)}} + onChange={()=> {setSearchBarModalOpen(true)}} + /> + + ):( + <> + + + + )} + + + + + + + + + { + setOpenautomateTab((prev) => !prev); + setOpenSecurityTab(false); + }} + sx={{ + marginLeft: 0.625, + }} + > + {openautomatetab ? ( + + ) : ( + + )} + + + + + + + + + + + + + + + + + + + + { + setOpenSecurityTab((prev) => !prev); + setOpenautomateTab(false); + }} + sx={{ + marginLeft: 0.625, + }} + > + {openSecurityTab ? ( + + ) : ( + + )} + + + + + + + + + + + + + + + + + + + + Recent Workflows + + + {recentworkflows?.slice(0, 2).map((workflow, index) => { + return ( + + ) + }) } + + + + + + orgOptions, [orgOptions])} + getOptionLabel={(option) => option.name} + PopperComponent={CustomPopper} + open={openAutocomplete} + onOpen={() => setOpenAutocomplete(true)} + renderOption={(props, option, { selected, index }) => ( +
  • { + e.currentTarget.style.backgroundColor = "#444444"; + }} + onMouseOut={(e) => { + e.currentTarget.style.backgroundColor = option.name === selectedOrg ? "#696969" : "transparent"; + }} + onClick={(e) => { + if (option.id !== userdata?.active_org?.id) { + setSelectedOrg(option.name); + handleClickChangeOrg(option.id); + } else { + setSelectedOrg(userdata?.active_org?.name); + } + setOpenAutocomplete(false); + }} + > + + {option.region_url} + + {option.name} + + {option.name} + +
    • + )} + onChange={(event, newValue) => { + if (newValue) { + if (userdata?.active_org?.id !== newValue.id) { + setSelectedOrg(newValue.name); + handleClickChangeOrg(newValue.id); + } else { + setSelectedOrg(userdata?.active_org?.name); + } + setOpenAutocomplete(false); + } + }} + onInputChange={(event, newInputValue) => { + setAutocompleteValue(newInputValue); + }} + filterOptions={(options, params) => { + return options.filter((option) => + option.name + .toLowerCase() + .includes(params.inputValue.toLowerCase()) + ); + }} + value={userOrgs} + renderInput={(params) => ( + + {expandLeftNav ? ( + + ) : ( + + + setOpenAutocomplete((prev) => !prev)} + style={{ + cursor: "pointer", + fontSize: "24px", + opacity: expandLeftNav ? 0 : 1, + transition: "opacity 0.3s ease", + }} + /> + + )} + + )} + /> +     + {event.currentTarget.style.backgroundColor = "#2f2f2f";setHoverOnAvatar(true)}} + onMouseLeave ={(event)=>{event.currentTarget.style.backgroundColor = "transparent";setHoverOnAvatar(false)}} + > + {expandLeftNav ? ( + <> + + + ) : ( + <> + + + )} + +     +     +
    + ); +}; + +export default LeftSideBar; diff --git a/frontend/src/components/LicencePopup.jsx b/frontend/src/components/LicencePopup.jsx index bd2a1cba..883f156d 100644 --- a/frontend/src/components/LicencePopup.jsx +++ b/frontend/src/components/LicencePopup.jsx @@ -87,7 +87,7 @@ const LicencePopup = (props) => { const paperStyle = { padding: 20, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, height: "100%", } @@ -193,7 +193,7 @@ const LicencePopup = (props) => { return (
    @@ -319,7 +319,7 @@ const LicencePopup = (props) => { margin: "auto", width: 100, backgroundColor: "white", - // borderRadius: theme.palette.borderRadius, + // borderRadius: theme.palette?.borderRadius, }} /> : null} @@ -389,7 +389,7 @@ const LicencePopup = (props) => { value={feature.split("Worker License: ")[1]} style={{ // backgroundColor: theme.palette.inputColor, - // borderRadius: theme.palette.borderRadius, + // borderRadius: theme.palette?.borderRadius, }} id={fieldId} onClick={() => { }} @@ -823,7 +823,7 @@ const LicencePopup = (props) => { {errorMessage.length > 0 ? Error: {errorMessage} : null}
    diff --git a/frontend/src/components/NewHeader.jsx b/frontend/src/components/NewHeader.jsx index 7432caf3..abdbcafd 100644 --- a/frontend/src/components/NewHeader.jsx +++ b/frontend/src/components/NewHeader.jsx @@ -65,7 +65,6 @@ const useStyles = makeStyles((theme) => ({ }, }, dropdownMenu: { - marginTop: theme.spacing(1), borderRadius: "12px !important", zIndex: 10, "& .MuiPaper-root": { @@ -88,7 +87,6 @@ const useStyles = makeStyles((theme) => ({ }, }, dropdownMenuItem: { - padding: theme.spacing(2, 3), fontSize: "16px", fontWeight: 400, color: "#fff", @@ -503,7 +501,7 @@ const Header = (props) => { minHeight: 370, padding: 20, backgroundColor: "rgba(0, 0, 0, 1)", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }, }} > @@ -928,7 +926,7 @@ const Header = (props) => { }, }} style={{ - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, backgroundColor: theme.palette.surfaceColor, marginRight: 15, color: "white", @@ -1056,7 +1054,7 @@ const Header = (props) => { { padding: 8, textAlign: "center", cursor: "pointer", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, marginTop: 5, backgroundColor: theme.palette.surfaceColor, minWidth: 60, diff --git a/frontend/src/components/Newsletter.jsx b/frontend/src/components/Newsletter.jsx index fa3b0fa9..402f687c 100644 --- a/frontend/src/components/Newsletter.jsx +++ b/frontend/src/components/Newsletter.jsx @@ -1,7 +1,7 @@ import React, {useState} from 'react'; -import { useTheme } from '@mui/styles'; import {isMobile} from "react-device-detect"; import ReactGA from 'react-ga4'; +import theme from '../theme.jsx'; import { TextField, @@ -12,7 +12,6 @@ import { const Newsletter = (props) => { const { globalUrl, } = props; - const theme = useTheme(); const [email, setEmail] = useState(""); const [msg, setMsg] = useState(""); const [buttonActive, setButtonActive] = useState(true); diff --git a/frontend/src/components/Oauth2Auth.jsx b/frontend/src/components/Oauth2Auth.jsx index 61d5cda0..98f64dcc 100755 --- a/frontend/src/components/Oauth2Auth.jsx +++ b/frontend/src/components/Oauth2Auth.jsx @@ -96,7 +96,7 @@ const AuthenticationOauth2 = (props) => { autoAuth, authButtonOnly, isLoggedIn, - + org_id, setFinalized, } = props; @@ -148,7 +148,6 @@ const AuthenticationOauth2 = (props) => { navigate(`/login?view=${window.location.pathname}&message=Log in to authenticate this app`) } - console.log("Should automatically click the auto-auth button?: ", autoAuth) if (autoAuth === true && selectedApp !== undefined) { startOauth2Request() } @@ -452,6 +451,8 @@ const AuthenticationOauth2 = (props) => { if (orgId !== undefined && orgId !== null && orgId.length > 0) { console.log("Adding org_id from user side") state += `%26org_id%3d${orgId}`; + }else{ + state += `%26org_id%3d${org_id}` } if (oauth_url !== undefined && oauth_url !== null && oauth_url.length > 0) { @@ -466,7 +467,7 @@ const AuthenticationOauth2 = (props) => { state += `%26refresh_uri%3d${authentication_url}` } - if (workflow.org_id !== undefined && workflow.org_id !== null && workflow.org_id.length > 0) { + if (workflow?.org_id !== undefined && workflow?.org_id !== null && workflow?.org_id.length > 0) { state += `%26org_id%3d${workflow.org_id}` } @@ -696,7 +697,7 @@ const AuthenticationOauth2 = (props) => { justifyContent: "flex-start", backgroundColor: "#ffffff", color: "#2f2f2f", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, minWidth: 300, maxWidth: 300, maxHeight: 50, @@ -721,7 +722,7 @@ const AuthenticationOauth2 = (props) => { {selectedAction.app_name} @@ -803,7 +804,7 @@ const AuthenticationOauth2 = (props) => { : null} {/* { setOauthUrl(data.value); } - const defaultValue = data.name === "url" && authenticationType.token_uri !== undefined && authenticationType.token_uri !== null && authenticationType.token_uri.length > 0 && (authenticationType.authorizationUrl === undefined || authenticationType.authorizationUrl === null || authenticationType.authorizationUrl.length === 0) && authenticationType.type === "oauth2-app" ? authenticationType.token_uri : data.value === undefined || data.value === null ? "" : data.value + const isNormalOauth = authenticationType.redirect_uri !== undefined && authenticationType.redirect_uri !== null && authenticationType.redirect_uri.length > 0 + const defaultValue = !isNormalOauth && data.name === "url" && authenticationType.token_uri !== undefined && authenticationType.token_uri !== null && authenticationType.token_uri.length > 0 && (authenticationType.authorizationUrl === undefined || authenticationType.authorizationUrl === null || authenticationType.authorizationUrl.length === 0) && authenticationType.type === "oauth2-app" ? authenticationType.token_uri : data.value === undefined || data.value === null ? "" : data.value - const fieldname = data.name === "url" && authenticationType.grant_type !== undefined && authenticationType.grant_type !== null && authenticationType.grant_type.length > 0 && authenticationType.type === "oauth2-app" ? "Token URL" : data.name + const fieldname = !isNormalOauth && data.name === "url" && authenticationType.grant_type !== undefined && authenticationType.grant_type !== null && authenticationType.grant_type.length > 0 && authenticationType.type === "oauth2-app" ? "Token URL" : data.name return ( -
    +
    {fieldname} @@ -893,7 +895,7 @@ const AuthenticationOauth2 = (props) => { { style={{ marginTop: 20, backgroundColor: theme.palette.inputColor, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} InputProps={{ style: { @@ -942,7 +944,7 @@ const AuthenticationOauth2 = (props) => { { { { style={{ marginBottom: 40, marginTop: 20, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} disabled={ clientSecret.length === 0 || clientId.length === 0 || buttonClicked || (allscopes.length !== 0 && selectedScopes.length === 0) @@ -1092,7 +1094,7 @@ const AuthenticationOauth2 = (props) => {
    + ) : null*/} + {workflow.execution_variables !== undefined && workflow.execution_variables !== null && workflow.execution_variables.length > 0 ? ( @@ -2201,7 +2245,7 @@ const ParsedAction = (props) => { backgroundColor: theme.palette.inputColor, color: "white", height: "50px", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} > { style={{ backgroundColor: theme.palette.inputColor, height: 50, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} onChange={(event, newValue) => { // Workaround with event lol @@ -2476,7 +2520,7 @@ const ParsedAction = (props) => { variant="body1" style={{ backgroundColor: theme.palette.inputColor, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} label={isIntegration ? "Choose a category" : "Find Actions"} variant="outlined" @@ -2496,7 +2540,7 @@ const ParsedAction = (props) => { value={selectedAction.name} fullWidth onChange={setNewSelectedAction} - style={{backgroundColor: theme.palette.inputColor, color: "white", height: 50, borderRadius: theme.palette.borderRadius,}} + style={{backgroundColor: theme.palette.inputColor, color: "white", height: 50, borderRadius: theme.palette?.borderRadius,}} SelectDisplayProps={{ style: { marginLeft: 10, @@ -2724,7 +2768,7 @@ const ParsedAction = (props) => { style={{ backgroundColor: theme.palette.inputColor, height: 50, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} onChange={(event, newValue) => { console.log("SELECT: ", event, newValue) @@ -2786,7 +2830,7 @@ const ParsedAction = (props) => { {
    { if (selectedAction.parameters === undefined || selectedAction.parameters === null || selectedAction.parameters.length !== selectedActionParameters.length) { //selectedAction.parameters = selectedActionParameters - console.log("PARAM BUG: ", selectedAction) + //console.log("PARAM BUG - length change(?): ", selectedAction) } //!selectedAction.auth_not_required && @@ -3008,7 +3052,7 @@ const ParsedAction = (props) => { marginTop: 50, border: "1px solid rgba(255,255,255,0.7)", borderTop: "1px solid rgba(255,255,255,0.7)", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, alignItems: "center", textAlign: "center", }} @@ -3284,11 +3328,13 @@ const ParsedAction = (props) => { }} > { "field_number": count, "actionlist": actionlist, "field_id": clickedFieldId, + + "example": selectedActionParameters[count].example, }) }} /> @@ -3646,7 +3694,7 @@ const ParsedAction = (props) => { { backgroundColor: theme.palette.surfaceColor, color: "white", height: "50px", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} > {parsedoptions.map( @@ -4307,7 +4355,7 @@ const ParsedAction = (props) => { color: "white", height: 50, marginTop: 2, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} onChange={(e) => { console.log("SELECT ONCHANGE DONE") diff --git a/frontend/src/components/Priorities.jsx b/frontend/src/components/Priorities.jsx index e66afe8f..b65aef93 100644 --- a/frontend/src/components/Priorities.jsx +++ b/frontend/src/components/Priorities.jsx @@ -286,7 +286,7 @@ const Priorities = (props) => { borderBottom: "1px solid rgba(255,255,255,0.4)", marginBottom: 20, border: highlighted ? "2px solid #f85a3e" : null, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} >
    diff --git a/frontend/src/components/Priority.jsx b/frontend/src/components/Priority.jsx index 9294967c..4d37142f 100644 --- a/frontend/src/components/Priority.jsx +++ b/frontend/src/components/Priority.jsx @@ -121,7 +121,7 @@ const Priority = (props) => { const srcSize = realignedSrc ? 35 : 30 const dstSize = realignedDst ? 35 : 30 return ( -
    +
    {priority.type === "usecase" || priority.type == "apps" ? : null} @@ -131,7 +131,7 @@ const Priority = (props) => { {priority.type === "usecase" && priority.description.includes("&") ? - {priority.name} + {priority.name} {newdescription.split("&")[0]} @@ -139,7 +139,7 @@ const Priority = (props) => { {newdescription.split("&").length > 3 ? - {priority.name+"2"} + {priority.name+"2"} {newdescription.split("&")[2]} diff --git a/frontend/src/components/RecentWorkflow.jsx b/frontend/src/components/RecentWorkflow.jsx index 3bd804b4..f61492e2 100644 --- a/frontend/src/components/RecentWorkflow.jsx +++ b/frontend/src/components/RecentWorkflow.jsx @@ -1,5 +1,6 @@ import React from "react" +import { Link } from "react-router-dom"; import { Avatar, Box, @@ -20,7 +21,6 @@ const RecentWorkflow = ({ workflow, onclickHandler, leftNavOpen, currentWorkflow const navigate = useNavigate(); const [hovered, setHovered] = React.useState(false) - if (workflow === undefined || workflow === null) { console.log("No workflow") return null @@ -39,12 +39,11 @@ const RecentWorkflow = ({ workflow, onclickHandler, leftNavOpen, currentWorkflow // Check if workflow.input_markdown has an image in it // If it does, show it as the main thing - // var relevantImageUrl = "" - if (workflow.input_markdown !== undefined && workflow.input_markdown !== null && workflow.input_markdown !== "") { + if (workflow?.form_control?.input_markdown !== undefined && workflow?.form_control?.input_markdown !== null && workflow?.form_control?.input_markdown !== "") { // Look for tag or ![alt](src) markdown // html > markdown - const imgTag = workflow.input_markdown.match(/]+>/g) + const imgTag = workflow?.form_control?.input_markdown.match(/]+>/g) if (imgTag !== null) { const src = imgTag[0].match(/src="([^"]+)"/) @@ -52,7 +51,7 @@ const RecentWorkflow = ({ workflow, onclickHandler, leftNavOpen, currentWorkflow relevantImageUrl = src[1] } } else { - const markdownTag = workflow.input_markdown.match(/!\[.*\]\(.*\)/g) + const markdownTag = workflow?.form_control?.input_markdown.match(/!\[.*\]\(.*\)/g) if (markdownTag !== null) { const src = markdownTag[0].match(/\(([^)]+)\)/) @@ -68,15 +67,13 @@ const RecentWorkflow = ({ workflow, onclickHandler, leftNavOpen, currentWorkflow onMouseEnter={() => setHovered(true)} onMouseLeave={() => setHovered(false)} > + +
    ) } diff --git a/frontend/src/components/RuntimeDebugger.jsx b/frontend/src/components/RuntimeDebugger.jsx index 9a91eb3e..83345bbc 100644 --- a/frontend/src/components/RuntimeDebugger.jsx +++ b/frontend/src/components/RuntimeDebugger.jsx @@ -35,6 +35,7 @@ import { PlayArrow as PlayArrowIcon, Insights as InsightsIcon, Replay as ReplayIcon, + EditNote as EditNoteIcon, } from '@mui/icons-material'; import { DataGrid, GridColDef, GridValueGetterParams } from '@mui/x-data-grid' @@ -76,6 +77,10 @@ const RuntimeDebugger = (props) => { {"id": "", "name": "All Workflows",} ]) + if (document != undefined) { + document.title = "Workflow Run Debugger" + } + // Shitty workflow search on purpose :) const handleWorkflowUsageCount = (workflows) => { if (workflows === undefined || workflows === null || workflows.length === 0) { @@ -219,7 +224,7 @@ const RuntimeDebugger = (props) => { } - const getAvailableWorkflows = () => { + const getAvailableWorkflows = (workflowId) => { fetch(globalUrl + "/api/v1/workflows", { method: "GET", headers: { @@ -240,6 +245,15 @@ const RuntimeDebugger = (props) => { var foundWorkflows = [{"id": "", "name": "All Workflows",}] foundWorkflows.push(...responseJson) setWorkflows(foundWorkflows) + + if (workflowId !== undefined && workflowId !== null && workflowId !== "" && workflowId.length === 36) { + for (var key in responseJson) { + if (responseJson[key].id === workflowId) { + setWorkflow(responseJson[key]) + break + } + } + } } }) .catch((error) => { @@ -248,7 +262,6 @@ const RuntimeDebugger = (props) => { } useEffect(() => { - getAvailableWorkflows() // Find workflow_id in url query const urlParams = new URLSearchParams(window.location.search); @@ -265,6 +278,8 @@ const RuntimeDebugger = (props) => { } } + getAvailableWorkflows(workflowId) + const foundStatus = urlParams.get('status'); if (foundStatus !== undefined && foundStatus !== null && foundStatus !== "") { setStatus(foundStatus) @@ -314,15 +329,17 @@ const RuntimeDebugger = (props) => { var source = params.row.execution_source if (source === "schedule") { - foundSource = schedule + foundSource = schedule } else if (source === "webhook") { - foundSource = webhook + foundSource = webhook } else if (source === "subflow" || source.length === 36) { - foundSource = subflow + foundSource = subflow source = "subflow" } else if (source === "rerun" || source.length === 36) { foundSource = source = "rerun of a previous run" + } else if (source === "form") { + foundSource = } else { source = "manual" } @@ -888,13 +905,13 @@ const RuntimeDebugger = (props) => { {userdata.support === true ? : null}
    @@ -942,11 +959,8 @@ const RuntimeDebugger = (props) => { }, }} getOptionLabel={(option) => { - if ( - option === undefined || - option === null || - option.name === undefined || - option.name === null + if (option === undefined || option === null || + option.name === undefined || option.name === null ) { return "No Workflow Selected"; } @@ -961,7 +975,7 @@ const RuntimeDebugger = (props) => { style={{ backgroundColor: theme.palette.inputColor, height: 50, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, marginTop: 5, marginLeft: 5, }} @@ -995,13 +1009,13 @@ const RuntimeDebugger = (props) => { {data.image !== undefined && data.image !== null && data.image.length > 0 ? - {data.name} + {data.name} : null} Choose {data.name} - } placement="bottom"> + }> { { - const { serverside, globalUrl, userdata, searchBarModalOpen, setSearchBarModalOpen } = props + const { serverside, globalUrl, userdata } = props let navigate = useNavigate(); + const { searchBarModalOpen, setSearchBarModalOpen } = useContext(Context); const borderRadius = 3 const node = useRef() const [searchOpen, setSearchOpen] = useState(false) @@ -547,8 +550,6 @@ const SearchData = props => { e.preventDefault() e.stopPropagation() - console.log("OBJECT CHANGE: ", hit.objectID) - // This does nothing rofl if (userdata.active_apps === undefined || userdata.active_apps === null) { activateApp(hit.name, hit.objectID, "activate") @@ -873,10 +874,11 @@ const SearchData = props => { - + + + ) : null diff --git a/frontend/src/components/Searchfield.jsx b/frontend/src/components/Searchfield.jsx index b3a89011..ec257cc6 100644 --- a/frontend/src/components/Searchfield.jsx +++ b/frontend/src/components/Searchfield.jsx @@ -4,7 +4,7 @@ import theme from '../theme.jsx'; import { useNavigate, Link, useParams } from "react-router-dom"; import SearchBox from "../components/SearchData.jsx"; -import { Context } from '../context/contextApi.jsx'; +import { Context } from '../context/ContextApi.jsx'; import { Chip, @@ -101,7 +101,7 @@ const SearchField = props => {
    : null} - + @@ -131,7 +131,7 @@ const SearchField = props => {
    {modalView} { setActiveDialog, fieldname, contentLoading, + editorData, + + setAiQueryModalOpen, } = props const [localcodedata, setlocalcodedata] = React.useState(codedata === undefined || codedata === null || codedata.length === 0 ? "" : codedata); @@ -163,6 +168,11 @@ const CodeEditor = (props) => { setMenuPosition(null); } + useEffect(() => { + highlight_variables(localcodedata) + expectedOutput(localcodedata) + }, [localcodedata]) + let navigate = useNavigate(); useEffect(() => { @@ -811,11 +821,7 @@ const CodeEditor = (props) => { } const handleItemClick = (values) => { - if ( - values === undefined || - values === null || - values.length === 0 - ) { + if (values === undefined || values === null || values.length === 0) { return; } @@ -830,7 +836,11 @@ const CodeEditor = (props) => { toComplete += values[key].autocomplete; } - setlocalcodedata(localcodedata+toComplete) + + handleClick({ + "value": toComplete + }) + //setlocalcodedata(localcodedata+toComplete) setMenuPosition(null) } @@ -839,10 +849,37 @@ const CodeEditor = (props) => { return } - if (!item.value.includes("{%") && !item.value.includes("{{")) { - setlocalcodedata(localcodedata+" | "+item.value+" }}") - } else { - setlocalcodedata(localcodedata+item.value) + // Injects it in the right spot instead of random + var edited = false + if (currentCharacter !== undefined && currentCharacter !== null && currentCharacter !== -1 && currentLine !== undefined && currentLine !== null && currentLine !== -1) { + // Input at the right spot + var codedatasplit = localcodedata.split('\n') + if (codedatasplit.length > currentLine) { + var currentLineData = codedatasplit[currentLine] + + // Remove newlines from item.value + if (item.value.includes("% python %")) { + item.value = item.value.replaceAll("\n", ";") + item.value = item.value.replaceAll("python %};", "python %}") + } else { + item.value = item.value.replaceAll("\n", "") + } + + currentLineData = currentLineData.slice(0, currentCharacter) + item.value + currentLineData.slice(currentCharacter) + codedatasplit[currentLine] = currentLineData + + setlocalcodedata(codedatasplit.join('\n')) + + edited = true + } + } + + if (edited === false) { + if (!item.value.includes("{%") && !item.value.includes("{{")) { + setlocalcodedata(localcodedata+" | "+item.value+" }}") + } else { + setlocalcodedata(localcodedata+item.value) + } } setAnchorEl(null) @@ -946,14 +983,19 @@ const CodeEditor = (props) => { // Define a custom completer for the Ace Editor - const customVariables = availableVariables const customCompleter = { getCompletions: function(editor, session, pos, prefix, callback) { - callback(null, customVariables.map((variable) => ({ - caption: variable, - value: variable, - meta: 'custom', - }))); + console.log("CUSTOM COMPLETER: ", prefix) + + callback(null, availableVariables.map((variable) => { + console.log("CUSTOM VAR: ", variable) + + return ({ + caption: variable, + value: variable, + meta: 'custom', + }) + })) } } @@ -1338,6 +1380,8 @@ const CodeEditor = (props) => { onClick={() => { console.log("CLICKED: ", innerdata); console.log(innerdata.example) + + //const handleClick = (item) => { handleItemClick([innerdata]); }} > @@ -1482,14 +1526,37 @@ const CodeEditor = (props) => { width: 50, marginLeft: 100, }} - disabled={isAiLoading} + disabled={editorData === undefined || editorData.example === undefined || editorData.example === null || editorData.example.length === 0} onClick={() => { - autoFormat(localcodedata) + setlocalcodedata(editorData.example) + }} + color="secondary" + > + + + + + { + if (setAiQueryModalOpen !== undefined) { + setAiQueryModalOpen(true) + } else { + autoFormat(localcodedata) + } }} > {isAiLoading ? @@ -1504,7 +1571,7 @@ const CodeEditor = (props) => { }
    { }}> {(availableVariables !== undefined && availableVariables !== null && availableVariables.length > 0) || isFileEditor ? ( { highlight_variables(localcodedata) }} onCursorChange={(cursorPosition, editor, value) => { - setCurrentCharacter(cursorPosition.column) - setCurrentLine(cursorPosition.row) + setCurrentCharacter(cursorPosition.cursor.column) + setCurrentLine(cursorPosition.cursor.row) findIndex(cursorPosition.row, cursorPosition.column) - //highlight_variables(localcodedata) - //console.log("VALUE CURSOR: ", value) }} onChange={(value, editor) => { // setlocalcodedata(value) @@ -1666,7 +1733,7 @@ const CodeEditor = (props) => { padding: 10, marginTop: -2, border: `2px solid ${theme.palette.inputColor}`, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, maxHeight: 450, minHeight: 450, overflow: "auto", diff --git a/frontend/src/components/SuggestedWorkflows.jsx b/frontend/src/components/SuggestedWorkflows.jsx index 1f01529a..7780b1a9 100644 --- a/frontend/src/components/SuggestedWorkflows.jsx +++ b/frontend/src/components/SuggestedWorkflows.jsx @@ -102,7 +102,7 @@ const SuggestedWorkflows = (props) => { placement="top" style={{ zIndex: 10011 }} > -
    { +
    { setHovering(true) }} onMouseOut={() => { setHovering(false) @@ -140,7 +140,7 @@ const SuggestedWorkflows = (props) => { // return ( - + 0 && usecaseSearchType.length > 0} onClose={() => { @@ -193,7 +193,7 @@ const SuggestedWorkflows = (props) => { apps={apps} /> -
    +
    Suggested Workflows ({finishedUsecases.length}/{usecaseSuggestions.length}) diff --git a/frontend/src/components/UsecaseSearch.jsx b/frontend/src/components/UsecaseSearch.jsx index 19aa2e53..2460876d 100644 --- a/frontend/src/components/UsecaseSearch.jsx +++ b/frontend/src/components/UsecaseSearch.jsx @@ -594,7 +594,7 @@ const UsecaseSearch = (props) => { width: 30, height: 30, border: "2px solid rgba(255,255,255,0.6)", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, maxHeight: 30, maxWidth: 30, overflow: "hidden", @@ -616,7 +616,7 @@ const UsecaseSearch = (props) => { width: 30, height: 30, border: "2px solid rgba(255,255,255,0.6)", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, maxWidth: 30, maxHeight: 30, overflow: "hidden", @@ -1266,9 +1266,11 @@ const UsecaseSearch = (props) => { .then((responseJson) => { if (responseJson.success === false) { var msgString = "Failed to activate the app" + if (responseJson.reason !== undefined) { msgString += ": " + responseJson.reason } + toast(msgString) } else { //toast("App activated for your organization! Refresh the page to use the app.") @@ -1388,7 +1390,7 @@ const UsecaseSearch = (props) => { {startText} -
    +
    {selectionOpen === true ? { // {defaultSearch}: {allusecases[usecaseIndex].name} //console.log("UseCase: ", usecases) return ( -
    +
    {configureWorkflowModal} {authenticationModal} {showTitle !== false && defaultSearch !== undefined ? diff --git a/frontend/src/components/WelcomeForm2.jsx b/frontend/src/components/WelcomeForm2.jsx index 8dc43d5f..3e11cfc2 100644 --- a/frontend/src/components/WelcomeForm2.jsx +++ b/frontend/src/components/WelcomeForm2.jsx @@ -648,7 +648,7 @@ const WelcomeForm = (props) => {
    -
    +
    { {/*errorMessage === "" && configurationFinished === true && workflow.id !== undefined && workflowLoading === false ? { // Open in new tab window.open("/workflows/" + workflow.id, "_blank") diff --git a/frontend/src/components/WorkflowTemplatePopup2.jsx b/frontend/src/components/WorkflowTemplatePopup2.jsx index ddd0a47f..2e666637 100644 --- a/frontend/src/components/WorkflowTemplatePopup2.jsx +++ b/frontend/src/components/WorkflowTemplatePopup2.jsx @@ -759,7 +759,7 @@ const WorkflowTemplatePopup = (props) => { const borderStyle = isHomePage ? null : isHovered && isActive ? errorMessage !== "" ? "1px solid red" : `2px solid ${theme.palette.green}` : isHovered ? `1px solid ${highlightColor}` : "1px solid rgba(33, 33, 33, 1)" return ( -
    diff --git a/frontend/src/components/WorkflowValidationTimeline.jsx b/frontend/src/components/WorkflowValidationTimeline.jsx index 5e20c8d9..5ef7ba70 100644 --- a/frontend/src/components/WorkflowValidationTimeline.jsx +++ b/frontend/src/components/WorkflowValidationTimeline.jsx @@ -319,7 +319,7 @@ const WorkflowValidationTimeline = (props) => {
    { + + // Left side bar global states + const [searchBarModalOpen, setSearchBarModalOpen] = useState(false); + const [leftSideBarOpenByClick, setLeftSideBarOpenByClick] = useState(false); + const [windowWidth, setWindowWidth] = useState(window.innerWidth); + + + //Calculate window width + useEffect(() => { + const handleResize = () => { + setWindowWidth(window?.innerWidth); + }; + + window.addEventListener('resize', handleResize); + + return () => { + window.removeEventListener('resize', handleResize); + }; + }, []); + + + return ( + + {props.children} + + ) +} diff --git a/frontend/src/context/contextApi.jsx b/frontend/src/context/contextApi.jsx deleted file mode 100644 index b5506a49..00000000 --- a/frontend/src/context/contextApi.jsx +++ /dev/null @@ -1,23 +0,0 @@ - -import { createContext, useState } from 'react'; - -export const Context = createContext(); - -export const AppContext =(props) => { - - // Left side bar global states - const [searchBarModalOpen, setSearchBarModalOpen] = useState(false); - const [leftSideBarOpenByClick, setLeftSideBarOpenByClick] = useState(false); - - - return ( - - {props.children} - - ) -} diff --git a/frontend/src/defaultCytoscapeStyle.jsx b/frontend/src/defaultCytoscapeStyle.jsx index 101d8c44..cfeeff62 100644 --- a/frontend/src/defaultCytoscapeStyle.jsx +++ b/frontend/src/defaultCytoscapeStyle.jsx @@ -13,6 +13,30 @@ const data = [ return elementname }, "text-valign": "center", + "text-margin-x": function(element) { + // Attempt at bottom-positioning + // Required text-valign: bottom + // FIXME: Disabled for now. + return "15px" + + + + const name = element.data("label") + console.log("Name: ", name) + if (name === null || name === undefined || name == "" || document=== undefined || document === null) { + return "0px" + } + + const canvas = document.createElement('canvas'); + const context = canvas.getContext('2d') + + context.font = '18px Segoe UI, Tahoma, Geneva, Verdana, sans-serif, sans-serif' + + const textWidth = context.measureText(name).width + return textWidth + "px" + //return -1*(textWidth) + "px" + }, + "font-family": "Segoe UI, Tahoma, Geneva, Verdana, sans-serif, sans-serif", "font-weight": "lighter", "font-size": "18px", @@ -23,7 +47,6 @@ const data = [ padding: "10px", margin: "5px", "border-width": "1px", - "text-margin-x": "10px", "z-index": 5001, }, }, @@ -263,7 +286,9 @@ const data = [ { selector: "node[?isStartNode]", css: { - shape: "ellipse", + shape: function(element) { + return "ellipse" + }, "border-color": "#80deea", width: "80px", height: "80px", diff --git a/frontend/src/theme.jsx b/frontend/src/theme.jsx index d276c446..54e7a790 100644 --- a/frontend/src/theme.jsx +++ b/frontend/src/theme.jsx @@ -1,7 +1,6 @@ import React from "react"; import { createTheme, adaptV4Theme } from "@mui/material/styles"; -//const theme = createTheme({ const theme = createTheme(adaptV4Theme({ palette: { theme: "dark", diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index ed520498..a13519a0 100755 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -6,6 +6,7 @@ import { makeStyles } from "@mui/styles"; import { useNavigate, Link } from "react-router-dom"; import countries from "../components/Countries.jsx"; import CacheView from "../components/CacheView.jsx"; +import { CopyToClipboard } from "../views/Docs.jsx"; import { FormControl, @@ -77,6 +78,9 @@ import { Flag as FlagIcon, FmdGood as FmdGoodIcon, Warning as WarningIcon, + + ExpandLess as ExpandLessIcon, + ExpandMore as ExpandMoreIcon, } from "@mui/icons-material"; //import { useAlert @@ -215,6 +219,13 @@ const Admin = (props) => { const [pipelines, setPipelines] = React.useState([]); const [, forceUpdate] = React.useState(); const [MFARequired, setMFARequired] = React.useState(selectedOrganization.mfa_required === undefined ? false : selectedOrganization.mfa_required); + const [listItemExpanded, setListItemExpanded] = React.useState(-1); + const [installationTab, setInstallationTab] = React.useState(0); + const [commandController, setCommandController] = React.useState({ + pipelines: false, + proxies: false, + }) + const [, setUpdate] = React.useState(0); useEffect(() => { if (selectedOrganization.mfa_required !== undefined) { @@ -234,6 +245,13 @@ const Admin = (props) => { setTimeout(() => { const urlSearchParams = new URLSearchParams(window.location.search); const params = Object.fromEntries(urlSearchParams.entries()); + + const foundOrgID = params["org_id"] + if (foundOrgID !== null && foundOrgID !== undefined) { + handleClickChangeOrg(foundOrgID) + } + + const foundTab = params["admin_tab"] if (foundTab !== null && foundTab !== undefined) { if (adminTab === 3) { @@ -862,7 +880,7 @@ If you're interested, please let me know a time that works for you, or set up a .then((responseJson) => { setWebHooks(responseJson.webhooks || []); // Handling the case where the result is null or undefined setAllSchedules(responseJson.schedules || []); - // setPipelines(responseJson.pipelines || []); + setPipelines(responseJson.pipelines || []); }) .catch((error) => { // toast(error.toString()); @@ -1048,14 +1066,14 @@ If you're interested, please let me know a time that works for you, or set up a const data = { name: pipeline.name, + id: pipeline.id, type: state, + command: pipeline.definition, environment: pipeline.environment, - workflow_id: pipeline.workflow_id, - trigger_id: pipeline.trigger_id, }; if (state === "start") toast("starting the pipeline"); - else toast("stopping the pipeline"); + else toast.info("Stopping the pipeline. This may take a few minutes to propagate.") const url = `${globalUrl}/api/v1/triggers/pipeline`; fetch(url, { @@ -1077,10 +1095,26 @@ If you're interested, please let me know a time that works for you, or set up a }) .then((responseJson) => { if (!responseJson.success) { - toast("Failed to update the pipeline: " + responseJson.reason); + toast.error("Failed to update the pipeline: " + responseJson.reason); } else { - if (state === "start") toast("Successfully created pipeline"); - else toast("Sucessfully stopped the pipeline"); + setTimeout(() => { + handleGetAllTriggers() + }, 5000) + + setTimeout(() => { + handleGetAllTriggers() + }, 20000) + + setTimeout(() => { + handleGetAllTriggers() + }, 120000) + /* + if (state === "start") { + toast("Successfully created pipeline"); + } else { + toast("Sucessfully stopped the pipeline"); + } + */ } }) .catch((error) => { @@ -1255,10 +1289,10 @@ If you're interested, please let me know a time that works for you, or set up a } selectedOrganization.cloud_sync = !selectedOrganization.cloud_sync; - setSelectedOrganization(selectedOrganization); - setCloudSyncApikey(""); + setSelectedOrganization(selectedOrganization) + setCloudSyncApikey("") - handleGetOrg(userdata.active_org.id); + handleGetOrg(userdata.active_org.id) } }) .catch((error) => { @@ -1510,23 +1544,17 @@ If you're interested, please let me know a time that works for you, or set up a }; const handleGetOrg = (orgId) => { - if ( - serverside !== true && - window.location.search !== undefined && - window.location.search !== null - ) { + if (serverside !== true && window.location.search !== undefined && window.location.search !== null) { const urlSearchParams = new URLSearchParams(window.location.search); const params = Object.fromEntries(urlSearchParams.entries()); - const foundorgid = params["org_id"]; - if (foundorgid !== undefined && foundorgid !== null) { - orgId = foundorgid; + const foundorgid = params["org_id"] + if (foundorgid !== undefined && foundorgid !== null && foundorgid.length === 36) { + orgId = foundorgid } } if (orgId.length === 0) { - toast( - "Organization ID not defined. Please contact us on https://shuffler.io if this persists logout.", - ); + toast("Organization ID not defined. Please contact us on https://shuffler.io if this persists logout.") return; } @@ -1569,7 +1597,6 @@ If you're interested, please let me know a time that works for you, or set up a responseJson.lead_info !== null ) { var leads = []; - console.log("LEADINFO: ", responseJson.lead_info) if (responseJson.lead_info.testing_shuffle) { leads.push("testing shuffle"); } @@ -1730,21 +1757,23 @@ If you're interested, please let me know a time that works for you, or set up a }) .then(function (responseJson) { if (responseJson.success === true) { - if ( - responseJson.region_url !== undefined && - responseJson.region_url !== null && - responseJson.region_url.length > 0 - ) { - localStorage.setItem("globalUrl", responseJson.region_url); + if (responseJson.region_url !== undefined && responseJson.region_url !== null && responseJson.region_url.length > 0) { + localStorage.setItem("globalUrl", responseJson.region_url) //globalUrl = responseJson.region_url } setTimeout(() => { - window.location.reload(); + window.location.reload() }, 2000); toast("Successfully changed active organization - refreshing!"); } else { - toast("Failed changing org: " + responseJson.reason); + if (responseJson.reason !== undefined && responseJson.reason !== null) { + if (!responseJson.reason.includes("already")) { + toast("Failed changing org: " + responseJson.reason); + } + } else { + toast("Failed changing org") + } } }) .catch((error) => { @@ -1919,7 +1948,7 @@ If you're interested, please let me know a time that works for you, or set up a }; const rerunCloudWorkflows = (environment) => { - toast("Starting execution reruns. This can run in the background."); + toast("Starting execution reruns. This runs in the background. Check the /debug view to see the progress."); fetch(`${globalUrl}/api/v1/environments/${environment.id}/rerun`, { method: "GET", credentials: "include", @@ -2238,11 +2267,11 @@ If you're interested, please let me know a time that works for you, or set up a setEnvironments(responseJson); // Helper info for users in case they have a large queue and don't know about queue flushing - if ( - responseJson !== undefined && - responseJson !== null && - responseJson.length > 0 - ) { + if (responseJson !== undefined && responseJson !== null && responseJson.length > 0) { + if (responseJson.length === 1 && responseJson[0].Type !== "cloud") { + setListItemExpanded(0) + } + for (var i = 0; i < responseJson.length; i++) { const env = responseJson[i]; @@ -2351,8 +2380,8 @@ If you're interested, please let me know a time that works for you, or set up a 2: "app_auth", 3: "files", 4: "cache", - 5: "schedules", - 6: "environments", + 5: "triggers", + 6: "locations", 7: "suborgs", }; @@ -2446,14 +2475,9 @@ If you're interested, please let me know a time that works for you, or set up a } } - if ( - selectedOrganization.id === undefined && - userdata !== undefined && - userdata.active_org !== undefined && - orgRequest - ) { + if ( selectedOrganization.id === undefined && userdata !== undefined && userdata.active_org !== undefined && orgRequest) { setOrgRequest(false); - handleGetOrg(userdata.active_org.id); + handleGetOrg(userdata.active_org.id) } const paperStyle = { @@ -2802,10 +2826,11 @@ If you're interested, please let me know a time that works for you, or set up a }} PaperProps={{ style: { - backgroundColor: theme.palette.surfaceColor, + backgroundColor: theme.palette.platformColor, color: "white", minWidth: "800px", minHeight: "320px", + padding: 50, }, }} > @@ -3225,7 +3250,7 @@ If you're interested, please let me know a time that works for you, or set up a style={{ margin: 4, backgroundColor: theme.palette.platformColor, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, border: "1px solid rgba(255,255,255,0.3)", color: "white", minHeight: expanded ? 250 : "inherit", @@ -4516,16 +4541,6 @@ If you're interested, please let me know a time that works for you, or set up a style={{ minWidth: 350, maxWidth: 350 }} /> - - - - { - const elementName = "copy_element_shuffle"; - var copyText = - document.getElementById(elementName); - if ( - copyText !== null && - copyText !== undefined - ) { - const clipboard = navigator.clipboard; - if (clipboard === undefined) { - toast( - "Can only copy over HTTPS (port 3443)", - ); - return; - } - - navigator.clipboard.writeText(data.apikey); - copyText.select(); - copyText.setSelectionRange( - 0, - 99999, - ); /* For mobile devices */ - - /* Copy the text inside the text field */ - document.execCommand("copy"); - - toast("Apikey copied to clipboard"); - } - }} - > - - - - ) - } - /> - )} -
    +

    Webhooks

    Webhooks used in Shuffle workflows.  @@ -5149,7 +5111,7 @@ If you're interested, please let me know a time that works for you, or set up a )} -
    +

    Pipelines

    Controls a pipeline to run things.{" "} @@ -5172,8 +5134,8 @@ If you're interested, please let me know a time that works for you, or set up a }} /> {pipelines === undefined || - pipelines === null || - pipelines.length === 0 ? ( + pipelines === null || + pipelines.length === 0 ? (
    + + - {pipelines.map((pipeline, index) => { var bgColor = "#27292d"; @@ -5210,7 +5179,7 @@ If you're interested, please let me know a time that works for you, or set up a return ( - {pipeline.workflow_id} - - } + style={{ maxWidth: 150, minWidth: 150 }} + primary={pipeline.total_runs} /> - + + @@ -5461,7 +5423,7 @@ If you're interested, please let me know a time that works for you, or set up a src={data.app.large_image} style={{ maxWidth: 50, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} /> style={{ minWidth: 75, maxWidth: 75 }} @@ -5728,7 +5690,7 @@ If you're interested, please let me know a time that works for you, or set up a -
    +
    Name @@ -5778,7 +5740,7 @@ If you're interested, please let me know a time that works for you, or set up a : - Environments failed to load. Please try again + Locations failed to load. Please try again }
    @@ -5833,7 +5795,7 @@ If you're interested, please let me know a time that works for you, or set up a -
    -

    Environments

    - - Decides what Orborus environment to run your workflow actions. If - you have scale problems, talk to our team: - support@shuffler.io.  - - Learn more - - -
    - - - { - setShowArchived(!showArchived); - }} - />{" "} - Show disabled - - - - - - - - - - - - - - {environments === undefined || environments === null - ? null - : environments.map((environment, index) => { - if (!showArchived && environment.archived) { - return null; - } - if (environment.archived === undefined) { - return null; - } + const getOrborusCommand = (environment) => { + if (environment.Type === "cloud") { + //toast("No Orborus necessary for environment cloud. Create and use a different environment to run executions on-premises.",) + return + } - var bgColor = "#27292d"; - if (index % 2 === 0) { - bgColor = "#1f2023"; - } + if ( + props.userdata.active_org === undefined || + props.userdata.active_org === null + ) { + toast( + "No active organization yet. Are you logged in?", + ); + return; + } - // Check if there's a notification for it in userdata.priorities - var showCPUAlert = false; - var foundIndex = -1; - if ( - userdata !== undefined && - userdata !== null && - userdata.priorities !== undefined && - userdata.priorities !== null && - userdata.priorities.length > 0 - ) { - foundIndex = userdata.priorities.findIndex( - (prio) => prio.name.includes("CPU") && prio.active === true, - ); + const elementName = "copy_element_shuffle"; + const auth = + environment.auth === "" + ? "cb5st3d3Z!3X3zaJ*Pc" + : environment.auth; + const newUrl = + globalUrl === "https://shuffler.io" + ? "https://shuffle-backend-stbuwivzoq-nw.a.run.app" + : globalUrl; - if ( - foundIndex >= 0 && - userdata.priorities[foundIndex].name.endsWith( - environment.Name, - ) - ) { - showCPUAlert = true; - } - } + var skipPipeline = false + if (commandController.pipelines === true) { + skipPipeline = true + } - const queueSize = - environment.queue !== undefined && environment.queue !== null - ? environment.queue < 0 - ? 0 - : environment.queue > 1000 - ? ">1000" - : environment.queue - : 0; + var addProxy = false + if (commandController.proxies === true) { + addProxy = true + } - return ( - - - - - - ) : environment.run_type === "docker" ? ( - - - - ) : environment.run_type === "k8s" ? ( - - - - ) : ( - - - - ) - } - style={{ - minWidth: 50, - maxWidth: 50, - overflow: "hidden", - }} - /> - + if (installationTab === 1) { + return (`docker run -d \\ + --restart=always \\ + --name="shuffle-orborus" \\ + --pull=always \\ + --volume "/var/run/docker.sock:/var/run/docker.sock" \\ + -e AUTH="${environment.auth}" \\ + -e ENVIRONMENT_NAME="${environment.Name}" \\ + -e ORG="${environment.org_id}" \\ + -e SHUFFLE_WORKER_IMAGE="ghcr.io/shuffle/shuffle-worker:nightly" \\ + -e SHUFFLE_SWARM_CONFIG=run \\ + -e SHUFFLE_LOGS_DISABLED=true \\ + -e BASE_URL="${newUrl}" \\${addProxy ? "\n -e HTTPS_PROXY=IP:PORT \\" : ""}${skipPipeline ? "\n -e SHUFFLE_SKIP_PIPELINES=true \\" : ""} + ghcr.io/shuffle/shuffle-orborus:latest + `) + } else if (installationTab === 2) { + return `https://shuffler.io/docs/configuration#kubernetes` + } + + const commandData = `docker rm shuffle-orborus --force; \\\ndocker run -d \\ + --restart=always \\ + --name="shuffle-orborus" \\ + --pull=always \\ + --volume "/var/run/docker.sock:/var/run/docker.sock" \\ + -e AUTH="${auth}" \\ + -e ENVIRONMENT_NAME="${environment.Name}" \\ + -e ORG="${props.userdata.active_org.id}" \\ + -e BASE_URL="${newUrl}" \\${addProxy ? "\n -e HTTPS_PROXY=IP:PORT \\" : ""}${skipPipeline ? "\n -e SHUFFLE_SKIP_PIPELINES=true \\" : ""} + ghcr.io/shuffle/shuffle-orborus:latest` + + return commandData + + + var copyText = + document.getElementById(elementName); + if ( + copyText !== null && + copyText !== undefined + ) { + const clipboard = navigator.clipboard; + if (clipboard === undefined) { + toast( + "Can only copy over HTTPS (port 3443)", + ); + return; + } + + navigator.clipboard.writeText(commandData); + copyText.select(); + copyText.setSelectionRange( + 0, + 99999, + ); /* For mobile devices */ + + /* Copy the text inside the text field */ + document.execCommand("copy"); + + toast("Orborus command copied to clipboard"); + } + } + +const environmentView = +curTab === 6 ? ( +
    +
    +

    Locations

    + + Decides where to run your workflows and actions. Uses Shuffle's Orborus runner to handle queued jobs onprem. Previously "Environments". + + If you have scale problems, talk to our team: support@shuffler.io.  + + Learn more + + +
    + + + { + setShowArchived(!showArchived); + }} + />{" "} + Show disabled + + + + + + + + + + + + + {environments === undefined || environments === null + ? null + : environments.map((environment, index) => { + if (!showArchived && environment.archived) { + return null; + } + + if (environment.archived === undefined) { + return null; + } + + var bgColor = "#27292d"; + if (index % 2 === 0) { + bgColor = "#1f2023"; + } + + // Check if there's a notification for it in userdata.priorities + var showCPUAlert = false; + var foundIndex = -1; + if ( + userdata !== undefined && + userdata !== null && + userdata.priorities !== undefined && + userdata.priorities !== null && + userdata.priorities.length > 0 + ) { + foundIndex = userdata.priorities.findIndex( + (prio) => prio.name.includes("CPU") && prio.active === true, + ); + + if ( + foundIndex >= 0 && + userdata.priorities[foundIndex].name.endsWith( + environment.Name, + ) + ) { + showCPUAlert = true; + } + } + + const queueSize = + environment.queue !== undefined && environment.queue !== null + ? environment.queue < 0 + ? 0 + : environment.queue > 1000 + ? ">1000" + : environment.queue + : 0; + + + const orborusCommandWrapper = () => { + // Check the current text + const orborusCommand = document.getElementById("orborus_command") + if (orborusCommand === undefined || orborusCommand === null) { + return getOrborusCommand(environment) + } + + return orborusCommand.textContent + } + + return ( + + { + if (environment.Type === "cloud") { + toast("Cloud environments are not configurable. To see what is possible, create a new environment.") + return + } + + setListItemExpanded(listItemExpanded === index ? -1 : index) + }} + > + + + + ) : environment.run_type === "docker" ? ( + + + + ) : environment.run_type === "k8s" ? ( + + + + ) : ( + + + + ) + } + style={{ + minWidth: 50, + maxWidth: 50, + overflow: "hidden", + }} + /> + ) : ( @@ -6410,7 +6485,7 @@ If you're interested, please let me know a time that works for you, or set up a placement="top" > @@ -6420,16 +6495,61 @@ If you're interested, please let me know a time that works for you, or set up a ) } style={{ - minWidth: 85, - maxWidth: 85, + minWidth: 60, + maxWidth: 60, + overflow: "hidden", + }} + /> + + + + : + environment?.data_lake?.enabled ? ( + + + + + + ) : ( + { + e.preventDefault() + e.stopPropagation() + + window.open("/detections/Sigma", "_blank") + }} + > + + + + + ) + } + style={{ + minWidth: 60, + maxWidth: 60, overflow: "hidden", }} /> @@ -6441,7 +6561,7 @@ If you're interested, please let me know a time that works for you, or set up a environment.running_ip.length === 0 ? (
    Not running
    ) : ( - environment.running_ip + environment.running_ip.split(":")[0] ) ) : ( "N/A" @@ -6450,100 +6570,19 @@ If you're interested, please let me know a time that works for you, or set up a style={{ minWidth: 150, maxWidth: 150, + marginRight: 10, overflow: "hidden", }} /> - - { - if (environment.Type === "cloud") { - toast( - "No Orborus necessary for environment cloud. Create and use a different environment to run executions on-premises.", - ); - return; - } - - if ( - props.userdata.active_org === undefined || - props.userdata.active_org === null - ) { - toast( - "No active organization yet. Are you logged in?", - ); - return; - } - - const elementName = "copy_element_shuffle"; - const auth = - environment.auth === "" - ? "cb5st3d3Z!3X3zaJ*Pc" - : environment.auth; - const newUrl = - globalUrl === "https://shuffler.io" - ? "https://shuffle-backend-stbuwivzoq-nw.a.run.app" - : globalUrl; - - const commandData = `docker run --restart=always --volume "/var/run/docker.sock:/var/run/docker.sock" -e ENVIRONMENT_NAME="${environment.Name}" -e 'AUTH=${auth}' -e ORG="${props.userdata.active_org.id}" -e DOCKER_API_VERSION=1.40 -e BASE_URL="${newUrl}" --name="shuffle-orborus" -d ghcr.io/shuffle/shuffle-orborus:latest`; - var copyText = - document.getElementById(elementName); - if ( - copyText !== null && - copyText !== undefined - ) { - const clipboard = navigator.clipboard; - if (clipboard === undefined) { - toast( - "Can only copy over HTTPS (port 3443)", - ); - return; - } - - navigator.clipboard.writeText(commandData); - copyText.select(); - copyText.setSelectionRange( - 0, - 99999, - ); /* For mobile devices */ - - /* Copy the text inside the text field */ - document.execCommand("copy"); - - toast("Orborus command copied to clipboard"); - } - }} - > - - - - } - /> - - {environment.default ? null : ( - - )} - - +
    +     + + {listItemExpanded === index ? : } +     + {listItemExpanded === index ? ( +
    +
    + + Your Onprem Orborus instance + + + Orborus is the Shuffle queue handler that runs your hybrid workflows and manages pipelines. It can be run in Docker/k8s container on your server or in your cluster. Follow the steps below, and configure as need be. + + + { + setInstallationTab(inputValue) + }} + aria-label="disabled tabs example" + variant="scrollable" + scrollButtons="auto" + style={{textAlign: "center", marginTop: 25, }} + > + + Verbose (default) + + /> + + Scale + + /> + + k8s + + /> + + + {installationTab === 2 ? + + Check our Kubernetes documentation for more information on how to run Shuffle on Kubernetes. The status of the node will change when connected. + + : + + 1. Ensure Docker is installed and the target server can reach '{globalUrl}' + + } + + + + {installationTab === 2 ? null : + "2. Run this command on the server you want to run workflows or store Pipeline data on"} + + + {installationTab === 2 ? null : +
    +
    + + {getOrborusCommand(environment)} + + +
    + + + Configure HTTP Proxies: { + if (commandController.proxies === undefined) { + commandController.proxies = true + } else { + commandController.proxies = !commandController.proxies + } + + setCommandController(commandController) + setUpdate(Math.random()) + }} + /> +
    + Disable Pipelines & Data Lake: { + if (commandController.pipelines === undefined) { + commandController.pipelines = true + } else { + commandController.pipelines = !commandController.pipelines + } + setCommandController(commandController) + setUpdate(Math.random()) + }} + /> +
    + } + + + {installationTab === 2 ? null : + + 3. Verify if the node is running. Try to refresh the page a little while after running the command. + + } + +
    +
    + ) : null} + {showCPUAlert === false ? null : ( - Environments + Locations /> +
    +   {modalView} {cloudSyncModal} {editUserModal} diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index d5146d66..a6ff2b63 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -69,6 +69,7 @@ import { AvatarGroup, Autocomplete, Radio, + ButtonGroup, } from "@mui/material"; import { @@ -185,6 +186,20 @@ export const triggers = [ description: "Schedule time trigger", long_description: "Create a schedule based on cron", id: "", + }, + { + name: "Pipelines", + type: "TRIGGER", + status: "uninitialized", + description: "Run a pipeline trigger", + trigger_type: "PIPELINE", + errors: null, + is_valid: true, + label: "Pipeline", + environment: "onprem", + large_image: "/images/workflows/tenzir2.png", + long_description: "Controls a pipeline to run things", + id: "", }, { name: "Shuffle Workflow", @@ -215,20 +230,6 @@ export const triggers = [ long_description: "Take user input to continue execution", id: "", }, - { - name: "Pipelines", - type: "TRIGGER", - status: "uninitialized", - description: "Run a pipeline trigger", - trigger_type: "PIPELINE", - errors: null, - is_valid: true, - label: "Pipeline", - environment: "onprem", - large_image: "/images/workflows/tenzir2.png", - long_description: "Controls a pipeline to run things", - id: "", - }, ]; // Adds specific text to items @@ -404,7 +405,6 @@ const AngularWorkflow = (defaultprops) => { const [toolsApp, setToolsApp] = React.useState({}); const [currentView, setCurrentView] = React.useState(0); const [triggerAuthentication, setTriggerAuthentication] = React.useState({}); - const [triggerFolders, setTriggerFolders] = React.useState([]); const [workflows, setWorkflows] = React.useState([]); const [parentWorkflows, setParentWorkflows] = React.useState([]); const [showEnvironment, setShowEnvironment] = React.useState(false); @@ -458,6 +458,7 @@ const AngularWorkflow = (defaultprops) => { const [showSkippedActions, setShowSkippedActions] = React.useState(false); const [lastExecution, setLastExecution] = React.useState(""); const [configureWorkflowModalOpen, setConfigureWorkflowModalOpen] = React.useState(false); + const [autoCompleting, setAutocompleting] = React.useState(false); const [authgroupModalOpen, setAuthgroupModalOpen] = React.useState(false); const [authGroups, setAuthGroups] = React.useState([]) @@ -529,6 +530,7 @@ const AngularWorkflow = (defaultprops) => { // eslint-disable-next-line no-unused-vars const [_, setUpdate] = useState(""); // Used to force rendring, don't remove + const [executionFilter, setExecutionFilter] = React.useState("ALL") const [workflowExecutions, setWorkflowExecutions] = React.useState([]); const [workflowExecutionCount, setWorkflowExecutionCount] = React.useState(0); const [defaultEnvironmentIndex, setDefaultEnvironmentIndex] = React.useState(0); @@ -543,6 +545,7 @@ const AngularWorkflow = (defaultprops) => { const [distributedFromParent, setDistributedFromParent] = React.useState("") const [suborgWorkflows, setSuborgWorkflows] = React.useState([]) + const [allTriggers, setAllTriggers] = React.useState(undefined) const [suggestionBox, setSuggestionBox] = React.useState({ "position": { @@ -563,7 +566,7 @@ const AngularWorkflow = (defaultprops) => { const releaseToConnectLabel = "Release to Connect" const integrationApps = [{ "id": "integration", - "name": "Integration Framework", + "name": "Singul", "type": "ACTION", "app_version": "1.0.0", "loop_versions": ["1.0.0"], @@ -670,6 +673,8 @@ const releaseToConnectLabel = "Release to Connect" "field_number": -1, "actionlist": [], "field_id": "", + + "example": "", }) const [loadedApps, setLoadedApps] = React.useState([]) @@ -732,8 +737,64 @@ const releaseToConnectLabel = "Release to Connect" } } + if (cy !== undefined && cy !== null) { + + // Check if any apps in the workflow has + cy.nodes().forEach((node) => { + const data = node.data() + if (data.app_id === foundapp.id) { + + if (data.name === "tmp" && data.parameters !== undefined && data.parameters !== null && data.parameters.length === 1 && data.parameters[0].name === "tmp" && foundapp.actions !== undefined && foundapp.actions !== null && foundapp.actions.length > 0) { + const startIndex = foundapp.actions.findIndex((action) => action.category_label !== undefined && action.category_label !== null && action.category_label.length > 0) + const actionIndex = startIndex < 0 ? 0 : startIndex + + node.data("name", foundapp.actions[actionIndex].name) + node.data("large_image", foundapp.large_image) + node.data("parameters", foundapp.actions[actionIndex].parameters) + node.data("finished", true) + node.data("category", foundapp.categories !== null && foundapp.categories !== undefined && foundapp.categories.length > 0 ? foundapp.categories[0] : "") + + /* + name: app.actions[actionIndex].name, + label: actionLabel, + app_name: app.name, + app_version: app.app_version, + app_id: app.id, + sharing: app.sharing, + private_id: app.private_id, + description: description, + environment: parsedEnvironments, + errors: [], + finished: false, + id_: newNodeId, + _id_: newNodeId, + id: newNodeId, + is_valid: true, + type: actionType, + parameters: parameters, + isStartNode: false, + large_image: app.large_image, + run_magic_output: false, + authentication: [], + execution_variable: undefined, + example: example, + required_body_fields: app.actions[actionIndex].required_body_fields, + authentication_id: authId, + finished: false, + template: app.template === true ? true : false, + */ + + toast("REPLACING ACTIONS") + } + } + }) + //if (action.app_id === same && app.actions.length === 1 && app.actions[0].parameters.length === 1 && app.actions[0].parameters[0].name === "tmp") { + } + + // FIXME: Add it to the existing list AND update the selected app } + }) .catch((error) => { console.log(`Failed side-loading app ${appId}: ${error}`) @@ -969,7 +1030,6 @@ const releaseToConnectLabel = "Release to Connect" if (responseJson.reason !== undefined && responseJson.reason !== undefined && responseJson.reason.length > 0) { if (!responseJson.reason.includes("404: Not Found") && responseJson.reason.length > 25) { - // Translate into markdown ![]() const imgRegex = / { if (authenticationModalOpen === true && selectedAction.app_name !== undefined) { - console.log(`Should get app docs for: ${selectedAction.app_name}`) //console.log(selectedAction) //console.log("APP: ", selectedApp) @@ -1033,6 +1092,7 @@ const releaseToConnectLabel = "Release to Connect" }; const getWorkflowExecutionCount = (workflowId) => { + fetch(`${globalUrl}/api/v1/workflows/${workflowId}/executions/count`, { method: "GET", headers: { @@ -1302,15 +1362,41 @@ const releaseToConnectLabel = "Release to Connect" }); }; - const getWorkflowExecution = (id, execution_id) => { - fetch(`${globalUrl}/api/v2/workflows/${id}/executions`, { - method: "GET", + const getWorkflowExecution = (id, execution_id, filter) => { + var url = `${globalUrl}/api/v2/workflows/${id}/executions` + var method = "GET" + if (filter === undefined || filter === null || filter.toUpperCase() === "ALL") { + + // Check for + if (executionFilter !== undefined && executionFilter !== null && executionFilter.length > 0) { + filter = executionFilter + } else { + filter = "ALL" + } + } + + var formattedBody = { + method: method, headers: { "Content-Type": "application/json", Accept: "application/json", }, credentials: "include", - }) + } + + if (filter !== "ALL") { + formattedBody.method = "POST" + + formattedBody.body = JSON.stringify({ + "status": filter, + "workflow_id": id, + }) + + url = `${globalUrl}/api/v1/workflows/search` + + } + + fetch(url, formattedBody) .then((response) => { if (response.status !== 200) { console.log("Status not 200 for WORKFLOW EXECUTION :O!"); @@ -1319,6 +1405,10 @@ const releaseToConnectLabel = "Release to Connect" return response.json(); }) .then((responseJson) => { + if (responseJson !== undefined && responseJson !== null && responseJson.runs !== undefined && responseJson.runs !== null) { + responseJson.executions = responseJson.runs + } + if (responseJson !== undefined && responseJson !== null && responseJson.executions !== undefined && responseJson.executions !== null) { // - means it's opposite @@ -1383,8 +1473,6 @@ const releaseToConnectLabel = "Release to Connect" } else { const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; var tmpView = new URLSearchParams(cursearch).get("execution_id"); - console.log("Alertnative execution id check: ", tmpView) - if (tmpView === undefined || tmpView === null || tmpView.length === 0) { const execution_id = tmpView; setExecutionModalView(1); @@ -1500,44 +1588,7 @@ const releaseToConnectLabel = "Release to Connect" if (selectedOption === "Kafka Queue") { trigger.parameters = [] - const topic = document.getElementById('topic')?.value - const bootstrapServers = document.getElementById('bootstrap_servers')?.value - const groupId = document.getElementById('group_id')?.value - const autoOffsetReset = document.getElementById('auto_offset_reset')?.value; - - if(topic) { - trigger.parameters.push({ - name: "topic", - value: topic - }) - } else { - toast("Please enter the topic name"); - return; - } - - if (bootstrapServers) { - trigger.parameters.push({ - name: "bootstrap_servers", - value: bootstrapServers - }); - } else { - toast("please enter bootstrap server details"); - return; - } - - if (groupId) { - trigger.parameters.push({ - name: "group_id", - value: groupId - }); - } - - if (autoOffsetReset) { - trigger.parameters.push({ - name: "auto_offset_reset", - value: autoOffsetReset - }); - } + const pipeline = document.getElementById('pipeline')?.value setTenzirConfigModalOpen(false); } else if (selectedOption === "Syslog listener") { @@ -2610,7 +2661,7 @@ const releaseToConnectLabel = "Release to Connect" return } - console.log("Apps loaded. JSON decoding next") + //console.log("Apps loaded. JSON decoding next") return response.json() }) @@ -3457,9 +3508,7 @@ const releaseToConnectLabel = "Release to Connect" if (responseJson.childorg_workflow_ids !== undefined && responseJson.childorg_workflow_ids !== null && responseJson.childorg_workflow_ids.length > 0) { getChildWorkflows(responseJson.id) - } - - if (responseJson.parentorg_workflow !== undefined && responseJson.parentorg_workflow !== null && responseJson.parentorg_workflow.length > 0) { + } else if (responseJson.parentorg_workflow !== undefined && responseJson.parentorg_workflow !== null && responseJson.parentorg_workflow.length > 0) { getChildWorkflows(responseJson.parentorg_workflow) } @@ -3805,13 +3854,9 @@ const releaseToConnectLabel = "Release to Connect" setSelectedAction({}); setSelectedApp({}); setSelectedComment({}) - setSelectedEdge({}); - - setSelectedEdge({}) //setSelectedActionEnvironment({}) setTriggerAuthentication({}) - setTriggerFolders([]) setLocalFirstrequest(true) setSelectedTrigger({}); @@ -3825,7 +3870,6 @@ const releaseToConnectLabel = "Release to Connect" left: 0, selected: "", }); - //console.timeEnd("UNSELECT"); }) sendStreamRequest({ @@ -4029,6 +4073,7 @@ const releaseToConnectLabel = "Release to Connect" nodedata.app_name !== "Testing" && nodedata.app_name !== "Shuffle Workflow" && nodedata.app_name !== "Integration Framework" && + nodedata.app_name !== "Singul" && nodedata.app_name !== "User Input") || nodedata.isStartNode) ) { @@ -4036,13 +4081,9 @@ const releaseToConnectLabel = "Release to Connect" var found = false; for (let nodekey in allNodes) { const currentNode = allNodes[nodekey]; - if ( - currentNode.data.attachedTo === nodedata.id && - currentNode.data.isDescriptor - ) { - found = true; - console.log("FOUND THE NODE!"); - break; + if (currentNode.data.attachedTo === nodedata.id && currentNode.data.isDescriptor) { + found = true + break } } @@ -4293,13 +4334,30 @@ const releaseToConnectLabel = "Release to Connect" } } } + + /* + // FIXME: This is the start of a highlighter for the node + // to better match it up with other elements + // 1. Get current node's position in X/Y on the screen + // 2. Draw a red line on the X and Y axis for positioning + + // Draw a red div line in the HTML + const position = event.target.position() + const redline = document.getElementById("redline") + if (redline !== null && redline !== undefined) { + redline.style.display = "block" + redline.style.position = "absolute" + redline.style.left = position.x + "px" + redline.style.top = position.y + "px" + redline.style.height = "10000px" + redline.style.width = 1 + console.log("REDLINE!") + } + */ + } - if ( - originalLocation.x === 0 && - originalLocation.y === 0 && - nodedata.position !== undefined - ) { + if (originalLocation.x === 0 && originalLocation.y === 0 && nodedata.position !== undefined) { originalLocation.x = nodedata.position.x; originalLocation.y = nodedata.position.y; } @@ -4391,10 +4449,14 @@ const releaseToConnectLabel = "Release to Connect" // This is to find sample response and parse it as string var AppContext = [] + var originalParams = [] + if (inputAction !== undefined && inputAction !== null) { + // Reload the data without copying + inputAction = JSON.parse(JSON.stringify(inputAction)) + originalParams = JSON.parse(JSON.stringify(inputAction.parameters)) const parents = getParents(inputAction) - console.log("Parents: ", parents) var actionlist = [] if (parents.length > 1) { for (let [key,keyval] in Object.entries(parents)) { @@ -4473,9 +4535,26 @@ const releaseToConnectLabel = "Release to Connect" "action_name": item.action_name, "label": item.label, "example": exampledata, - "example_response": exampledata, + //"example_response": exampledata, }) } + + var params = [] + for (var paramkey in inputAction.parameters) { + const param = inputAction.parameters[paramkey] + if (param.configuration) { + continue + } + + // Mainly for booleans + if (param.options !== undefined && param.options !== null && param.options.length > 0) { + continue + } + + params.push(param) + } + + inputAction.parameters = params } var conversationData = { @@ -4486,6 +4565,7 @@ const releaseToConnectLabel = "Release to Connect" "workflow_id": workflow.id, } + if (inputAction !== undefined) { console.log("Add app context! This should them get parameters directly") conversationData.output_format = "action_parameters" @@ -4494,16 +4574,13 @@ const releaseToConnectLabel = "Release to Connect" conversationData.app_name = inputAction.app_name conversationData.action_name = inputAction.name conversationData.parameters = inputAction.parameters - - if (!value.includes(inputAction.label)) { - conversationData.query = inputAction.label.replaceAll("_", " ") - } } // Onprem not available yet (April 2023) // Should: Make OpenAI work for them with their own key - //fetch("https://shuffler.io/api/v1/conversation", { - fetch(`${globalUrl}/api/v1/conversation`, { + //fetch(`${globalUrl}/api/v1/conversation`, { + const url = `${globalUrl}/api/v1/conversation` + fetch(url, { method: "POST", headers: { "Content-Type": "application/json", @@ -4513,12 +4590,16 @@ const releaseToConnectLabel = "Release to Connect" credentials: "include", }) .then((response) => { + setAiQueryModalOpen(false) + setAutocompleting(false) if (setSuggestionLoading !== undefined) { setSuggestionLoading(false) } if (response.status !== 200) { console.log("Status not 200 for stream results :O!"); + } else { + toast("Completion finished. Please verify your action!") } return response.json(); @@ -4543,52 +4624,71 @@ const releaseToConnectLabel = "Release to Connect" } var changed = false - - for (let paramkey in inputAction.parameters) { - const actionParam = inputAction.parameters[paramkey] - - if (actionParam.autocompleted === true) { + for (let respParamKey in responseJson.parameters) { + var respParam = responseJson.parameters[respParamKey] + if (respParam.value === undefined || respParam.value === null || respParam.value === "" ) { continue } - if (actionParam.configuration === true && actionParam.name !== "url") { - continue - } - - if (actionParam.value !== "" && actionParam.value !== actionParam.example) { - console.log("Skipping: ", actionParam) - continue - } - - for (let respParam of responseJson.parameters) { - if (respParam.name === actionParam.name) { - console.log("Found match for param: ", respParam) - - if (respParam.value === "") { - break - } - - changed = true - - inputAction.parameters[paramkey].autocompleted = true - inputAction.parameters[paramkey].value = respParam.value - break + for (let paramkey in originalParams) { + const actionParam = originalParams[paramkey] + /* + if (actionParam.value !== "" && actionParam.value !== actionParam.example) { + console.log("Skipping: ", actionParam) + continue } + */ + + if (respParam.name !== actionParam.name) { + continue + } + + const codeeditor = document.getElementById("shuffle-codeeditor") + if (codeeditor !== undefined && codeeditor !== null) { + const editorInstance = window?.ace?.edit("shuffle-codeeditor") + if (editorInstance === undefined || editorInstance === null) { + toast.error("Failed to find code editor instance") + return + } else { + editorInstance.setValue(respParam.value) + originalParams[paramkey].autocompleted = true + changed = true + } + } + + if (!changed) { + console.log("Found match for param: ", respParam) + changed = true + originalParams[paramkey].autocompleted = true + originalParams[paramkey].value = respParam.value + } + + break } } if (changed === true) { + inputAction.parameters = originalParams console.log("Setting action! Force update pls :)") setUpdate(Math.random()) + setSelectedAction(inputAction) + + // Find it in cytoscape and update the action + if (cy !== undefined && cy !== null) { + const cyAction = cy.getElementById(inputAction.id) + if (cyAction !== undefined && cyAction !== null) { + cyAction.data("parameters", inputAction.parameters) + } + } + } return } - console.log("Suggestionbox location: ", suggestionBox) - // Add action + console.log("Suggestionbox location: ", suggestionBox) if (responseJson.app_name !== undefined && responseJson.app_name !== null) { // Always added to 0, 0 // Should use suggestionBox.position.x, suggestionBox.position.y @@ -4640,6 +4740,8 @@ const releaseToConnectLabel = "Release to Connect" } }) .catch((error) => { + setAiQueryModalOpen(false) + setAutocompleting(false) if (setSuggestionLoading !== undefined) { setSuggestionLoading(false) } @@ -5278,9 +5380,6 @@ const releaseToConnectLabel = "Release to Connect" authenticationOptions[latestindex].last_modified = true } - - console.log("auth options 2: ", authenticationOptions) - curaction.authentication = authenticationOptions if ( curaction.selectedAuthentication === null || @@ -5290,7 +5389,6 @@ const releaseToConnectLabel = "Release to Connect" curaction.selectedAuthentication = {}; } } else { - console.log("auth option 3") curaction.authentication = [] curaction.authentication_id = ""; @@ -5313,37 +5411,61 @@ const releaseToConnectLabel = "Release to Connect" } } } else { - console.log("Should check APP if it has the same params as ACTION") - for (let actionKey in curapp.actions) { - const tmpaction = curapp.actions[actionKey] - if (tmpaction.name === curaction.name) { - console.log("Found action - needs change?", tmpaction) - if (tmpaction.parameters !== undefined && tmpaction.parameters !== null && tmpaction.parameters.length > 0) { - curaction.parameters = JSON.parse(JSON.stringify(tmpaction.parameters)) - } - break - } + console.log("Should check APP if it has the same params as ACTION") + for (let actionKey in curapp.actions) { + const tmpaction = curapp.actions[actionKey] + if (tmpaction.name === curaction.name) { + console.log("Found action - needs change?", tmpaction) + if (tmpaction.parameters !== undefined && tmpaction.parameters !== null && tmpaction.parameters.length > 0) { + curaction.parameters = JSON.parse(JSON.stringify(tmpaction.parameters)) + } + break + } + } + } + + // Fix authentication fields that may be missing in the UI + if (curapp.authentication.required && !curapp?.authentication?.type?.includes("oauth")) { + if (curapp.authentication.parameters !== undefined && curapp.authentication.parameters !== null) { + var actionChanged = false + for (let paramKey in curapp.authentication.parameters) { + var param = curapp.authentication.parameters[paramKey] + + if (curaction.parameters === undefined || curaction.parameters === null) { + curaction.parameters = [] + } + + var found = false + for (let actionParamKey in curaction.parameters) { + if (curaction.parameters[actionParamKey].name === param.name) { + found = true + break } } - //curaction["authentication"] = [] - //curaction["authentication_id"] = "" - // Fix parameters that are... Not ideal - //var paramnames = [] - //var newparams = [] - //for (let paramKey in curaction.parameters) { - // console.log("Name: ", curaction.parameters[paramKey].name) - // if (paramnames.includes(curaction.parameters[paramKey].name)) { - // continue - // } + if (!found) { + param.configuration = true + curaction.parameters.push(param) + + actionChanged = true + } + } - // paramnames.push(curaction.parameters[paramKey].name) - // newparams.push(curaction.parameters[paramKey]) - //} + if (actionChanged && workflow.actions !== undefined && workflow.actions !== null) { + // Find it in the workflow and set it + for (let wfActionKey in workflow.actions) { + if (workflow.actions[wfActionKey].id === curaction.id) { + workflow.actions[wfActionKey] = curaction + } + } - //curaction.parameters = newparams + setWorkflow(workflow) + + } + } + } - setSelectedApp(curapp); + setSelectedApp(curapp) setSelectedAction(curaction); cy.removeListener("drag"); @@ -5374,7 +5496,7 @@ const releaseToConnectLabel = "Release to Connect" var trigger_index = workflow.triggers.findIndex( (a) => a.id === data.id - ); + ) if (trigger_index === -1) { workflow.triggers.push(data) @@ -5393,7 +5515,6 @@ const releaseToConnectLabel = "Release to Connect" } } else if (data.app_name === "Webhook") { if (workflow.triggers[trigger_index].parameters !== undefined && workflow.triggers[trigger_index].parameters !== null && workflow.triggers[trigger_index].parameters.length > 0) { - console.log("Can set params here!") workflow.triggers[trigger_index].parameters[0] = { name: "url", value: referenceUrl + "webhook_" + workflow.triggers[trigger_index].id, @@ -5481,9 +5602,37 @@ const releaseToConnectLabel = "Release to Connect" } } + if (allTriggers !== undefined && allTriggers !== null) { + + // Just checking all three. Could just make a new list, but meh + if (allTriggers.pipelines !== undefined && allTriggers.pipelines !== null) { + for (var pipelineKey in allTriggers.pipelines) { + if (allTriggers.pipelines[pipelineKey].id === data.id) { + data.status = allTriggers.pipelines[pipelineKey].status + } + } + } + + if (allTriggers.webhooks !== undefined && allTriggers.webhooks !== null) { + for (var webhookKey in allTriggers.webhooks) { + if (allTriggers.webhooks[webhookKey].id === data.id) { + data.status = allTriggers.webhooks[webhookKey].status + } + } + } + + if (allTriggers.schedules !== undefined && allTriggers.schedules !== null) { + for (var scheduleKey in allTriggers.schedules) { + if (allTriggers.schedules[scheduleKey].id === data.id) { + data.status = allTriggers.schedules[scheduleKey].status + } + } + } + } + setSelectedTriggerIndex(trigger_index) setSelectedTrigger(data) - setSelectedActionEnvironment(data.env) + //setSelectedActionEnvironment(data.env) }, 25) } else if (data.type === "COMMENT") { setSelectedComment(data); @@ -5835,7 +5984,7 @@ const releaseToConnectLabel = "Release to Connect" const edge = event.target.data(); if (edge.source === undefined && edge.target === undefined) { - console.log("Edge added without source or target") + //console.log("Edge added without source or target") return } @@ -5966,7 +6115,7 @@ const releaseToConnectLabel = "Release to Connect" found = true break } else { - console.log("Old branch didn't exist afterall. Remove.") + //console.log("Old branch didn't exist afterall. Remove.") } } @@ -6061,28 +6210,13 @@ const releaseToConnectLabel = "Release to Connect" const node = event.target; const nodedata = event.target.data(); - //if (Object.keys(nodedata).length === 1) { - // console.log("Check if another node actually exists before adding") - //} - if (nodedata.finished === false || (nodedata.id !== undefined && nodedata.is_valid === undefined) ) { - //if (nodedata.app_id === undefined) { - //console.log("Returning because node is not valid: ", nodedata) return; } - // Check for recommendations when a new action is added - // if (isLoaded === true && firstrequest === false) { - // fetchRecommendations(workflow) - // } - - // DONT MOVE THIS LINE RIGHT HERE v setLastSaved(false) - // Dont move the line above. May break stuff. - - if (node.isNode() && cy.nodes().size() === 1) { workflow.start = node.data("id"); nodedata.isStartNode = true; @@ -6115,15 +6249,8 @@ const releaseToConnectLabel = "Release to Connect" } if (nodedata.type === "ACTION") { - // Should get recommendations to load in for all nodesma + // Should get recommendations to load in for all nodesma - /* - var curaction = workflow.actions.find((a) => a.id === nodedata.id); - if (curaction === null || curaction === undefined) { - toast("Node not found. Please remake it.") - event.target.remove(); - } - */ if (workflow.actions.length === 1 && workflow.actions[0].id === workflow.start) { const newEdgeUuid = uuidv4(); const newcybranch = { @@ -6162,11 +6289,7 @@ const releaseToConnectLabel = "Release to Connect" } - if ( - nodedata.parameters !== undefined && - nodedata.parameters !== null && - !nodedata.label.endsWith("_copy") - ) { + if (nodedata.parameters !== undefined && nodedata.parameters !== null && !nodedata.label.endsWith("_copy")) { var newparameters = []; for (let [subkey,subkeyval] in Object.entries(nodedata.parameters)) { @@ -6625,42 +6748,58 @@ const releaseToConnectLabel = "Release to Connect" return response.json(); }) .then((responseJson) => { - var found = false; - var showEnvCnt = 0; + var found = false + var showEnvCnt = 0 for (let jsonkey in responseJson) { if (responseJson[jsonkey].default && !found) { - setDefaultEnvironmentIndex(jsonkey); - found = true; + setDefaultEnvironmentIndex(jsonkey) + found = true } if (responseJson[jsonkey].archived === false) { - showEnvCnt += 1; + showEnvCnt += 1 } } if (showEnvCnt > 1) { - setShowEnvironment(true); + setShowEnvironment(true) } if (!found) { for (let jsonkey in responseJson) { if (!responseJson[jsonkey].archived) { - setDefaultEnvironmentIndex(jsonkey); + setDefaultEnvironmentIndex(jsonkey) break; } } } - // FIXME: Don't allow multiple in cloud yet. Cloud -> Onprem isn't stable. if (isCloud) { if (responseJson !== undefined && responseJson !== null && responseJson.length > 0) { - setEnvironments(responseJson); + setEnvironments(responseJson) } else { - setEnvironments([{ Name: "Cloud", Type: "cloud" }]); + setEnvironments([{ Name: "Cloud", Type: "cloud" }]) } } else { - setEnvironments(responseJson); + setEnvironments(responseJson) } + + /* + setTimeout(() => { + console.log("ACTIONS: ", workflow.actions) + if (workflow.actions !== undefined && workflow.actions !== null && workflow.actions.length > 0) { + for (var actionkey in workflow.actions) { + if (workflow.actions[actionkey].environment !== undefined && workflow.actions[actionkey].environment !== null && workflow.actions[actionkey].environment.length > 0) { + + const env = environments.findIndex((data) => data.Name === workflow.actions[actionkey].environment) + if (env !== -1) { + setSelectedActionEnvironment(environments[env]) + } + } + } + } + }, 2500) + */ }) .catch((error) => { //toast(error.toString()); @@ -7547,28 +7686,6 @@ const releaseToConnectLabel = "Release to Connect" if (nodedata.type !== "COMMENT") { parsedStyle.color = "white"; - - //if (!event.target.data("isButton") && !event.target.data("buttonId")) { - // const px = event.target.position("x") - 0; - // const py = event.target.position("y") - 50; - // const circleId = (newNodeId = uuidv4()); - - // console.log("Got px, py: ", px, py) - // - // cy.add({ - // group: "nodes", - // data: { - // weight: 30, - // id: circleId, - // isButton: true, - // attachedTo: event.target.data("id"), - // buttonType: "edgehandler", - // is_valid: true, - // }, - // position: { x: px, y: py }, - // locked: true, - // }) - //} } if (event.target !== undefined && event.target !== null) { @@ -7828,7 +7945,7 @@ const releaseToConnectLabel = "Release to Connect" } else { action.iconBackground = iconInfo.iconBackgroundColor; } - } else if (action.app_name === "Integration Framework") { + } else if (action.app_name === "Integration Framework" || action.app_name === "Singul") { const iconInfo = GetIconInfo(action) if (iconInfo !== undefined && iconInfo !== null) { action.fillGradient = iconInfo.fillGradient @@ -7865,13 +7982,17 @@ const releaseToConnectLabel = "Release to Connect" }) // What are these again? Where are they used? + const decoratorNodes = [] + + /* + // Removed for now as it wasn't really that helpful const decoratorNodes = inputworkflow.actions.map((action) => { if (!action.isStartNode) { if (action.app_name === "Testing") { return null } else if (action.app_name === "Shuffle Tools") { return null - } else if (action.app_name === "Integration Framework") { + } else if (action.app_name === "Integration Framework" || action.app_name === "Singul") { return null } } @@ -7908,6 +8029,7 @@ const releaseToConnectLabel = "Release to Connect" } return decoratorNode }) + */ const foundtriggers = inputworkflow.triggers.map((trigger) => { @@ -8315,13 +8437,39 @@ const releaseToConnectLabel = "Release to Connect" } setAllRevisions(responseJson) - setSelectedVersion(responseJson[0]) + setSelectedVersion(responseJson[0]) }) .catch((error) => { console.log("Error getting workflow revisions: ", error) }); } + const loadTriggers = () => { + const url = `${globalUrl}/api/v1/triggers` + fetch(url, + { + method: "GET", + headers: { "content-type": "application/json" }, + credentials: "include", + } + ) + .then((response) => { + if (response.status !== 200) { + throw new Error("No folders :o!"); + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success !== false) { + setAllTriggers(responseJson) + } + }) + .catch((error) => { + console.log("Get outlook folders error: ", error.toString()); + }); + } + // eslint-disable-next-line react-hooks/exhaustive-deps //useEffect(() => { if (firstrequest) { @@ -8329,6 +8477,7 @@ const releaseToConnectLabel = "Release to Connect" getWorkflow(props.match.params.key, {}); getChildWorkflows(props.match.params.key) getRevisionHistory(props.match.params.key) + loadTriggers() getApps() fetchUsecases() @@ -8374,8 +8523,6 @@ const releaseToConnectLabel = "Release to Connect" // 2nd load - configures cytoscape //} else if (!established && cy !== undefined && ((apps !== null && apps !== undefined && apps.length > 0) || workflow.public === true) && Object.getOwnPropertyNames(workflow).length > 0 && appAuthentication !== undefined && workflowRecommendations !== undefined) { } else if (!established && cy !== undefined && ((apps !== null && apps !== undefined && apps.length > 0) || workflow.public === true) && Object.getOwnPropertyNames(workflow).length > 0 && appAuthentication !== undefined) { - console.log("In POST graph setup!") - //This part has to load LAST, as it's kind of not async. //This means we need everything else to happen first. @@ -8383,8 +8530,6 @@ const releaseToConnectLabel = "Release to Connect" setEstablished(true); // Validate if the node is just a node lol - console.log("CY grid: ", cy.gridGuide) - // https://www.npmjs.com/package/cytoscape-grid-guide // if (cy.gridGuide !== undefined) { @@ -8441,7 +8586,6 @@ const releaseToConnectLabel = "Release to Connect" } // preview: true, - console.log("In POST graph setup 2") cy.fit(null, 200); cy.on("boxselect", "node", (e) => { @@ -8493,7 +8637,6 @@ const releaseToConnectLabel = "Release to Connect" document.title = "Workflow - " + workflow.name; - console.log("In POST graph setup 3") startWorkflowStream(props.match.params.key); registerKeys(); @@ -8551,7 +8694,6 @@ const releaseToConnectLabel = "Release to Connect" if (alledges !== undefined && alledges !== null && alledges.length > 0) { for (let edgekey in alledges) { const tmp = alledges[edgekey]; - console.log("TMP: ", tmp, tmp.data.source); if (tmp.data.source === trigger.id) { mappedStartnode = tmp.data.target; break; @@ -8712,7 +8854,7 @@ const releaseToConnectLabel = "Release to Connect" }; const paperAppStyle = { - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, minHeight: isMobile ? 50 : 70, maxHeight: isMobile ? 50 : 70, minWidth: isMobile ? 50 : "100%", @@ -8725,7 +8867,7 @@ const releaseToConnectLabel = "Release to Connect" }; const paperVariableStyle = { - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, minHeight: 50, maxHeight: 50, minWidth: "100%", @@ -9114,27 +9256,19 @@ const releaseToConnectLabel = "Release to Connect"
    {triggers.map((trigger, index) => { - /* - if (trigger.trigger_type === "PIPELINE") { - if (userdata.support !== true) { - return null - } - } - */ - // Hiding since March 2024 if (trigger.trigger_type === "EMAIL") { return null } const imagesize = isMobile ? 40 : trigger.large_image.includes("svg") ? 50 : 50 - var imageline = trigger.large_image.length === 0 ? + var imageline = trigger.large_image.length === 0 ? : { + if (env.archived) { + return false + } + + if (env.Name === "cloud" || env.Type === "cloud") { + return false + } + + return true + }) + + console.log("NEWENV: ", environments, newenv) + if (newenv !== undefined && newenv !== null) { + newAppData.environment = newenv.Name + } else { + newAppData.environment = "" + } + } + console.log("NEW DATA: ", newAppData) // Can all the data be in here? hmm const nodeToBeAdded = { @@ -9419,6 +9575,22 @@ const releaseToConnectLabel = "Release to Connect" const handleAppDrag = (e, app) => { const cycontainer = cy.container(); + // Handling drag of public apps + if (app.objectID !== undefined && app.objectID !== null && app.objectID.length > 0) { + // FIXME: This is still buggy for now. Not allowed + return + loadAppConfig(app.objectID, undefined) + + // Some arbitrary check + + app.id = app.objectID + app.actions = [{ + "name": "tmp", + "parameters": [{ + "name": "tmp", + }] + }] + } if (app.type === "TRIGGER") { handleTriggerDrag(e, app) @@ -9428,13 +9600,8 @@ const releaseToConnectLabel = "Release to Connect" //console.log("e: ", e) //console.log("Offset: ", cycontainer) - // Chrome lol - if ( - e.pageX > cycontainer.offsetLeft && - e.pageX < cycontainer.offsetLeft + cycontainer.offsetWidth && - e.pageY > cycontainer.offsetTop && - e.pageY < cycontainer.offsetTop + cycontainer.offsetHeight - ) { + // HTML -> Canvas overlap check + if (e.pageX > cycontainer.offsetLeft && e.pageX < cycontainer.offsetLeft + cycontainer.offsetWidth && e.pageY > cycontainer.offsetTop && e.pageY < cycontainer.offsetTop + cycontainer.offsetHeight) { if (newNodeId.length > 0) { var currentnode = cy.getElementById(newNodeId); if ( @@ -9810,13 +9977,24 @@ const releaseToConnectLabel = "Release to Connect" > {newAppname} { + if (props.index !== undefined && props.index !== null) { + // Replace the image with the default image + + const foundImage = document.getElementById(`image_${props.index}`) + if (foundImage !== undefined && foundImage !== null) { + foundImage.src = theme.palette.defaultImage + app.large_image = theme.palette.defaultImage + } + } + }} style={{ pointerEvents: "none", userDrag: "none", userSelect: "none", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, height: isMobile ? 40 : 55, width: isMobile ? 40 : 55, }} @@ -9915,7 +10093,7 @@ const releaseToConnectLabel = "Release to Connect" }}> { - toast.success(`Activating App. Please wait a moment.`) + toast.success(`Activating App. Please wait a moment, and it will show up highlighted in your apps.`) const queryID = hit.__queryID @@ -10082,6 +10260,7 @@ const releaseToConnectLabel = "Release to Connect" e.preventDefault() e.stopPropagation() + handleAppDrag(e, hit) if (!appdragged) { clickedApp(hit) } @@ -10092,8 +10271,8 @@ const releaseToConnectLabel = "Release to Connect" }} dragging={false} position={{ - x: 0, - y: 0, + x: 0, + y: 0, }} >
    { @@ -10139,7 +10318,7 @@ const releaseToConnectLabel = "Release to Connect" {/**/}
    - +
    {/*    */} :
    {extraMessage} - +
    ) })} @@ -10695,7 +10874,7 @@ const releaseToConnectLabel = "Release to Connect" maxHeight: "100vh", border: "1px solid rgb(91, 96, 100)", zIndex: 1000, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, resize: "both", overflow: "auto", @@ -10939,7 +11118,7 @@ const releaseToConnectLabel = "Release to Connect" + const submitQueryModal = () => { + const changeActionTextfield = document.getElementById("change-action-textfield") + if (changeActionTextfield === undefined || changeActionTextfield === null) { + setAiQueryModalOpen(false) + toast.error("Failed to find textfield") + return + } + + if (changeActionTextfield.value === undefined || changeActionTextfield.value === null || changeActionTextfield.value === "") { + toast("Please provide how you want formatting to happen") + return + } + + setAutocompleting(true) + if (codeEditorModalOpen === true) { + autoFormatCodemodal(changeActionTextfield.value) + } else { + aiSubmit(changeActionTextfield.value, undefined, undefined, selectedAction) + } + } + + const autoFormatCodemodal = (input) => { + if (codeEditorModalOpen !== true) { + toast.error("Code editor is not open") + return + } + + if (editorData.name === undefined || editorData.name === null || editorData.name === "") { + toast.error("Failed to find editor field name") + return + } + + const codeeditor = document.getElementById("shuffle-codeeditor") + if (codeeditor === undefined || codeeditor === null) { + toast.error("Failed to find code editor html") + return + } + + const editorInstance = window?.ace?.edit("shuffle-codeeditor") + if (editorInstance === undefined || editorInstance === null) { + toast.error("Failed to find code editor instance") + return + } + + //console.log("ACE data: ", editorInstance.getValue()) + //editorInstance.setValue("HELO") + + // Should try to automatically fix this input + console.log("Running AI input fixer: ", selectedResult) + if (aiSubmit === undefined || selectedAction === undefined) { + toast.error("Failed to find AI submit function") + return + } + + // Should remove params from selectedAction that aren't parameterName + var tmpAction = JSON.parse(JSON.stringify(selectedAction)) + var tmpParams = tmpAction.parameters.filter((param) => param.name === editorData.name) + if (tmpParams.length !== 1) { + toast.error("Failed to find correct parameter in action") + return + } + + tmpParams[0].value = editorInstance.getValue() + tmpAction.parameters = tmpParams + aiSubmit(input, undefined, undefined, tmpAction) + } + const aiQueryModal = { + setAiQueryModalOpen(false) }} > - - Condition - + + { + }} + > + + + + { + setAiQueryModalOpen(false) + }} + > + + + + Shuffle AI + + + What you write here will be fed to the Shuffle AI to generate a change for the selected action or field. Best used for when you are stuck with formatting. Uses your AI credits (resets monthly). Alpha feature. Please give feedback to support@shuffler.io {"<"}3 + + + + + + ), + onKeyPress: (e) => { + if (e.key === "Enter" && !e.shiftKey) { + submitQueryModal() + } + }, + }} + + /> @@ -12159,659 +12479,6 @@ const releaseToConnectLabel = "Release to Connect" ); }; - // 1. GET the trigger authentication data - // 2. Parse the fields that are used (outlook & gmail) - // 3. Parse the folders that are selected - // 4. Start / stop - const EmailSidebar = () => { - if (Object.getOwnPropertyNames(selectedTrigger).length === 0) { - return null; - } - - if (workflow.triggers[selectedTriggerIndex] === undefined) { - return null; - } - - if ( - workflow.triggers[selectedTriggerIndex].parameters === undefined || - workflow.triggers[selectedTriggerIndex].parameters === null || - workflow.triggers[selectedTriggerIndex].parameters.length === 0 - ) { - workflow.triggers[selectedTriggerIndex].parameters = [ - { value: "No folders selected yet", name: "outlookfolder" }, - ]; - selectedTrigger.parameters = [ - { value: "No folders selected yet", name: "outlookfolder" }, - ]; - setWorkflow(workflow); - setSelectedTrigger(selectedTrigger); - } - - const setGmailFolders = () => { - console.log("In set gmail folders"); - fetch( - globalUrl + - "/api/v1/triggers/gmail/getFolders?trigger_id=" + - selectedTrigger.id, - { - method: "GET", - headers: { "content-type": "application/json" }, - credentials: "include", - } - ) - .then((response) => { - if (response.status !== 200) { - throw new Error("No folders :o!"); - } - - return response.json(); - }) - .then((responseJson) => { - if ( - responseJson !== undefined && - responseJson !== null && - responseJson.success !== false && - responseJson.length > 0 - ) { - setTriggerFolders(responseJson); - } - - if ( - workflow.triggers[selectedTriggerIndex].parameters.length === 0 && - responseJson.length > 0 - ) { - workflow.triggers[selectedTriggerIndex].parameters = [ - { - value: responseJson[0].displayName, - name: "outlookfolder", - id: responseJson[0].id, - }, - ]; - selectedTrigger.parameters = [ - { - value: responseJson[0].displayName, - name: "outlookfolder", - id: responseJson[0].id, - }, - ]; - setWorkflow(workflow); - setSelectedTrigger(selectedTrigger); - } - }) - .catch((error) => { - console.log("Get gmail folder error: ", error.toString()); - }); - }; - - const setOutlookFolders = () => { - fetch( - globalUrl + - "/api/v1/triggers/outlook/getFolders?trigger_id=" + - selectedTrigger.id, - { - method: "GET", - headers: { "content-type": "application/json" }, - credentials: "include", - } - ) - .then((response) => { - if (response.status !== 200) { - throw new Error("No folders :o!"); - } - - return response.json(); - }) - .then((responseJson) => { - if ( - responseJson !== null && - responseJson.success !== false && - responseJson.length > 0 - ) { - console.log("Got trigger folders: ", triggerFolders) - setTriggerFolders(responseJson); - } - - if ( - workflow.triggers[selectedTriggerIndex].parameters.length === 0 && - responseJson.length > 0 - ) { - workflow.triggers[selectedTriggerIndex].parameters = [ - { - value: responseJson[0].displayName, - name: "outlookfolder", - id: responseJson[0].id, - }, - ] - selectedTrigger.parameters = [ - { - value: responseJson[0].displayName, - name: "outlookfolder", - id: responseJson[0].id, - }, - ]; - setWorkflow(workflow); - setSelectedTrigger(selectedTrigger); - } - }) - .catch((error) => { - console.log("Get outlook folders error: ", error.toString()); - }); - }; - - const getTriggerAuth = () => { - fetch(`${globalUrl}/api/v1/triggers/${selectedTrigger.id}`, { - method: "GET", - headers: { "content-type": "application/json" }, - credentials: "include", - }) - .then((response) => { - if (response.status !== 200) { - throw new Error("No trigger info :o!"); - } - - return response.json(); - }) - .then((responseJson) => { - - setTriggerAuthentication(responseJson); - }) - .catch((error) => { - //console.log(error.toString()); - console.log("Set trigger auth error: ", error.toString()); - }); - }; - - // Getting the triggers and the folders if they exist - if (localFirstrequest) { - //console.log("Trigger: ", selectedTrigger) - //console.log("Triggername: ", selectedTrigger.name) - //if (selectedTrigger.name.toLowerCase() === "gmail") { - setGmailFolders(); - setOutlookFolders(); - - getTriggerAuth(); - setLocalFirstrequest(false); - } - - const gmailButton = - selectedTrigger.name !== "Gmail" ? null : ( - - ); - - const outlookButton = - selectedTrigger.name !== "Office365" ? null : ( - - ); - - // FIXME - set everything in here to multifolder etc - var triggerInfo = "SET UP BUT NO TYPE :)"; - if (Object.getOwnPropertyNames(triggerAuthentication).length > 0) { - // Should get the folders if they don't already exist - - if ( - triggerAuthentication.type === "outlook" || - triggerAuthentication.type === "gmail" - ) { - triggerInfo = ( -
    - {selectedTrigger.status === "running" ? null : ( - -
    -
    -
    - Change auth -
    -
    - {outlookButton} - {gmailButton} - - If you have trouble using this trigger, please contact us to get access - - - )} - - {triggerFolders === undefined || triggerFolders === null ? null : ( - -
    -
    -
    - Select {triggerAuthentication.type === "gmail" ? "labels" : "a folder"} -
    -
    - {triggerFolders.length === 0 ? - - No folders found. Please authenticate first. If this persists, contact us. - - : - } - key={selectedTrigger} - > - {triggerFolders.map((folder) => { - var folderItem = ( - - ); - - if (folder.childFolderCount > 0) { - // Here to handle subfolders sometime later - folderItem = ( - - ); - } - - return folderItem; - })} - - } - - )} -
    - ); - } else if (triggerAuthentication.type === "gmail") { - console.log("AUTH: ", triggerAuthentication); - triggerInfo = "SPECIAL GMAIL"; - } - } - - // Check - const argumentView = - Object.getOwnPropertyNames(triggerAuthentication).length > 0 ? ( -
    {triggerInfo}
    - ) : ( -
    -
    -
    -
    - Login -
    -
    - {outlookButton} - {gmailButton} - - If you have trouble using this trigger, please { - if (window.drift !== undefined) { - window.drift.api.startInteraction({ interactionId: 340043 }) - } else { - console.log("Couldn't find drift in window.drift and not .drift-open-chat with querySelector: ", window.drift) - } - }}>contact us to get access - -
    - ); - - return ( -
    -

    - {selectedTrigger.app_name}: {selectedTrigger.status} -

    - - What are email triggers? - - -
    Name
    - - {/* -
    - Environment: - -
    - */} - - {argumentView} -
    -
    - -
    - - -
    -
    -
    -
    - ); - }; - const handleWorkflowSelectionUpdate = (e, isUserinput) => { if (e.target.value === undefined || e.target.value === null || e.target.value.id === undefined) { @@ -13148,7 +12815,7 @@ const releaseToConnectLabel = "Release to Connect" type: "action", id: item.id, name: item.label, - autocomplete: `${item.label.split(" ").join("_")}`, + autocomplete: `${item?.label?.split(" ")?.join("_")}`, example: exampledata, } actionlist.push(actionvalue); @@ -13380,7 +13047,7 @@ const releaseToConnectLabel = "Release to Connect" color: "white", height: 35, marginleft: 10, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, color: "rgba(255,255,255,0.4)", }} SelectDisplayProps={{ @@ -13428,7 +13095,7 @@ const releaseToConnectLabel = "Release to Connect" { setLastSaved(false) @@ -13613,7 +13280,7 @@ const releaseToConnectLabel = "Release to Connect" {data.image !== undefined && data.image !== null && data.image.length > 0 ? - {data.name} + {data.name} : null} Choose Subflow '{data.name}' @@ -13627,7 +13294,7 @@ const releaseToConnectLabel = "Release to Connect" }} value={data} onClick={() => { - getWorkflowApps(data.id); + getWorkflowApps(data.id); handleWorkflowSelectionUpdate({ target: { value: data @@ -13646,7 +13313,7 @@ const releaseToConnectLabel = "Release to Connect" { setLastSaved(false) @@ -13752,7 +13419,7 @@ const releaseToConnectLabel = "Release to Connect" { // Workaround with event lol @@ -14609,7 +14276,7 @@ const releaseToConnectLabel = "Release to Connect" variant="body1" style={{ backgroundColor: theme.palette.inputColor, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} {...params} label="Find Associated App (optional)" @@ -14628,7 +14295,20 @@ const releaseToConnectLabel = "Release to Connect" MenuProps={{ disableScrollLock: true, }} - value={selectedTrigger.environment} + value={selectedTrigger.environment === undefined || selectedTrigger.environment === null || selectedTrigger.environment === "" ? + environments?.find((env) => { + if (env.archived) { + return false + } + + if (env.name === "cloud" || env.type === "cloud") { + return false + } + + + return true + }).name + : selectedTrigger.environment} disabled={selectedTrigger.status === "running"} SelectDisplayProps={{ style: { @@ -14663,7 +14343,7 @@ const releaseToConnectLabel = "Release to Connect" > {triggerEnvironments.map((data) => { if (data.archived) { - return null; + return null } return ( @@ -14713,7 +14393,7 @@ const releaseToConnectLabel = "Release to Connect" { @@ -14845,7 +14525,7 @@ const releaseToConnectLabel = "Release to Connect" { }} @@ -14898,7 +14578,7 @@ const releaseToConnectLabel = "Release to Connect" { }} @@ -15012,85 +14692,6 @@ const releaseToConnectLabel = "Release to Connect" }); }; - const startMailSub = (trigger, triggerindex) => { - var folders = []; - - if (triggerFolders === null || triggerFolders === undefined) { - return null; - } - - const splitItem = - workflow.triggers[selectedTriggerIndex].parameters[0].value.split( - splitter - ) - - console.log("Starting mail sub: ", workflow.triggers[selectedTriggerIndex].parameters[0].value, splitItem); - for (let splitkey in splitItem) { - const item = splitItem[splitkey]; - const curfolder = triggerFolders.find((a) => a.displayName === item); - if (curfolder === undefined) { - toast("Something went wrong with folder selection: " + item); - return; - } - - folders.push(curfolder.id); - } - - const data = { - name: trigger.name, - folders: folders, - id: trigger.id, - }; - - const requesttype = triggerAuthentication.type; - toast( - "Creating " + requesttype + " subscription with name " + trigger.name - ); - - fetch( - globalUrl + - "/api/v1/workflows/" + - props.match.params.key + - "/" + - requesttype, - { - method: "POST", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - body: JSON.stringify(data), - credentials: "include", - } - ) - .then((response) => { - if (response.status !== 200) { - console.log("Status not 200 for stream results :O!"); - } - - return response.json(); - }) - .then((responseJson) => { - if (!responseJson.success) { - toast("Failed to start trigger: " + responseJson.reason); - } else { - toast( - "Successfully started folder subscription trigger. Test it by sending yoursend an email" - ); - - workflow.triggers[triggerindex].status = "running"; - trigger.status = "running"; - setWorkflow(workflow); - setSelectedTrigger(trigger); - saveWorkflow(workflow); - } - }) - .catch((error) => { - //toast(error.toString()); - console.log("Start mailsub error: ", error.toString()); - }); - }; - const newWebhook = (trigger) => { const hookname = trigger.label; if (hookname.length === 0) { @@ -15282,7 +14883,7 @@ const releaseToConnectLabel = "Release to Connect" Information - The information you want to show the user. Supports variables. + The information you want to show the user. Supports variables. Supports Markdown & HTML.
    {data.image !== undefined && data.image !== null && data.image.length > 0 ? - {data.name} + {data.name} : null} Choose Trigger '{data.name}' @@ -15502,7 +15103,7 @@ const releaseToConnectLabel = "Release to Connect" ) : null} -
    -
    +
    Required Input-Questions {workflow.input_questions !== undefined && workflow.input_questions !== null && workflow.input_questions.length > 0 ?
    @@ -15660,6 +15260,7 @@ const releaseToConnectLabel = "Release to Connect" No Input-Questions found. Click to add them!
    } +
    const defaultEnvironment = environments.find( @@ -15696,7 +15297,7 @@ const releaseToConnectLabel = "Release to Connect"
    - Parameters + What would you like to do? + {/*
    { @@ -15791,7 +15393,7 @@ const releaseToConnectLabel = "Release to Connect" }}} style={{ border: "1px solid rgba(255,255,255,0.3)", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, padding: 10, cursor: "pointer", marginTop: 5, @@ -15804,9 +15406,8 @@ const releaseToConnectLabel = "Release to Connect" { - if (selectedTrigger.status !== "running"){ setSelectedOption("Syslog listener")}} - } + value={"Syslog listener"} name="option" /> @@ -15840,7 +15441,7 @@ const releaseToConnectLabel = "Release to Connect" }}} style={{ border: "1px solid rgba(255,255,255,0.3)", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, padding: 10, cursor: "pointer", marginTop: 5, @@ -15854,7 +15455,9 @@ const releaseToConnectLabel = "Release to Connect" checked={selectedOption === "SigmaRule"} onChange={() => { if (selectedTrigger.status !== "running"){ - setSelectedOption("SigmaRule")}}} + setSelectedOption("SigmaRule") + } + }} value={"Sigma Rulesearch"} name="option" /> @@ -15863,19 +15466,22 @@ const releaseToConnectLabel = "Release to Connect" />
    + */} +
    { - if(selectedTrigger.status === "running"){ + if (selectedTrigger.status === "running"){ toast("please stop the trigger to edit the configuration"); return; } else { - setSelectedOption("Kafka Queue"); - setTenzirConfigModalOpen(true); - }}} + setSelectedOption("Kafka Queue"); + setTenzirConfigModalOpen(true); + } + }} style={{ border: "1px solid rgba(255,255,255,0.3)", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, padding: 10, cursor: "pointer", marginTop: 5, @@ -15888,76 +15494,21 @@ const releaseToConnectLabel = "Release to Connect" { - if (selectedTrigger.status !== "running"){ - setSelectedOption("Kafka Queue")}}} + if (selectedTrigger.status !== "running") { + setSelectedOption("Kafka Queue") + + } + }} + value={"Kafka Queue"} name="option" /> } - label="Follow Kafka Queue" + label="Subscribe to a Kafka Queue" />
    - - - - : null} - -
    - + + + : null} +
    ); }; @@ -16937,7 +16520,7 @@ const releaseToConnectLabel = "Release to Connect" left: leftBarSize+20, color: "white", padding: 10, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} > @@ -17155,7 +16738,7 @@ const releaseToConnectLabel = "Release to Connect" height: 235, border: "1px solid #f85a3e", cursor: "pointer", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, padding: 10, backgroundColor: hovered ? theme.palette.surfaceColor : theme.palette.platformColor, }} @@ -17221,37 +16804,6 @@ const releaseToConnectLabel = "Release to Connect" cy.edges().remove() cy.nodes().remove() } - - // Remove all cytoscape triggers first? - /* - setTimeout(() => { - setupGraph(inputworkflow) - - cy.on("select", "node", (e) => { - onNodeSelect(e, appAuthentication); - }); - cy.on("select", "edge", (e) => onEdgeSelect(e)); - - cy.on("unselect", (e) => onUnselect(e)); - - cy.on("add", "node", (e) => onNodeAdded(e)); - cy.on("add", "edge", (e) => onEdgeAdded(e)); - cy.on("remove", "node", (e) => onNodeRemoved(e)); - cy.on("remove", "edge", (e) => onEdgeRemoved(e)); - - cy.on("mouseover", "edge", (e) => onEdgeHover(e)); - cy.on("mouseout", "edge", (e) => onEdgeHoverOut(e)); - cy.on("mouseover", "node", (e) => onNodeHover(e)); - cy.on("mouseout", "node", (e) => onNodeHoverOut(e)); - - // Handles dragging - cy.on("drag", "node", (e) => onNodeDrag(e, selectedAction)); - cy.on("free", "node", (e) => onNodeDragStop(e, selectedAction)); - - cy.on("cxttap", "node", (e) => onCtxTap(e)); - }, 25) - */ - } // Uses Org-Id referencing header to create a workflow while getting it in realtime @@ -17469,7 +17021,7 @@ const releaseToConnectLabel = "Release to Connect"

    @@ -18743,21 +18293,65 @@ const releaseToConnectLabel = "Release to Connect" - + > + + + + + +
    ) : ( -
    - - There are no executions yet, or they are not loaded. - - -
    + +
    + + No executions found for the '{executionFilter}' filter. + + + +
    +     )}
    ) : ( @@ -19258,7 +18856,7 @@ const releaseToConnectLabel = "Release to Connect" { - window.open("/admin?tab=environments", "_blank") + window.open("/admin?tab=locations", "_blank") }}> {executionData.workflow.actions[0].environment} @@ -19434,7 +19032,7 @@ const releaseToConnectLabel = "Release to Connect" {environments.length > 0 && defaultEnvironmentIndex < environments.length && nonskippedResults.length === 0 && environments[defaultEnvironmentIndex].Name !== "Cloud" ? - No results yet. Is Orborus running for the "{environments[defaultEnvironmentIndex].Name}" environment?     Find out here. If the Workflow doesn't start within 30 seconds with Orborus running, contact support: support@shuffler.io + No results yet. Is Orborus running for the "{environments[defaultEnvironmentIndex].Name}" environment? Find out here. If the Workflow doesn't start within 30 seconds with Orborus running, contact support: support@shuffler.io : null} @@ -19450,7 +19048,7 @@ const releaseToConnectLabel = "Release to Connect" {environments.length > 0 && defaultEnvironmentIndex < environments.length && nonskippedResults.length === 0 && environments[defaultEnvironmentIndex].Name !== "Cloud" ? - No results yet. Is Orborus running for the "{environments[defaultEnvironmentIndex].Name}" environment? Learn more. If the Workflow doesn't start within 30 seconds with Orborus running, contact support: support@shuffler.io + No results yet. Is Orborus running for the "{environments[defaultEnvironmentIndex].Name}" environment? Learn more. If the Workflow doesn't start within 30 seconds with Orborus running, contact support: support@shuffler.io : null} @@ -19524,7 +19122,7 @@ const releaseToConnectLabel = "Release to Connect" if (triggers.length > 2) { if (data.action.app_name === "shuffle-subflow") { - const parsedImage = triggers[2].large_image; + const parsedImage = triggers[3].large_image; actionimg = ( {"ShuffleAction Logs - Logs for an action are not available without an onprem environment with the SHUFFLE_LOGS_DISABLED environment variable set to false: SHUFFLE_LOGS_DISABLED=false. Logs are enabled by default, except in scale mode. + Logs for an action are not available without an onprem environment with the SHUFFLE_LOGS_DISABLED environment variable set to false: SHUFFLE_LOGS_DISABLED=false. Logs are enabled by default, except in scale mode. ) @@ -19948,7 +19546,7 @@ const releaseToConnectLabel = "Release to Connect" cursor: "pointer", padding: 3, border: "1px solid rgba(255,255,255,0.3)", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} onClick={() => { if (!showVariable) { @@ -20141,7 +19739,7 @@ const releaseToConnectLabel = "Release to Connect" } if (isCloud && stringjson.toLowerCase().includes("timeout error")) { - return "Run this workflow in a local environment to increase the timeout. Go to https://shuffler.io/admin?tab=environments to create an environment to connect to" + return "Run this workflow in a local environment to increase the timeout. Go to https://shuffler.io/admin?tab=locationsto create an environment to connect to" } if (stringjson.toLowerCase().includes("invalid header")) { @@ -20180,8 +19778,7 @@ const releaseToConnectLabel = "Release to Connect" maxHeight: 550, overflowY: "auto", overflowX: "hidden", - // zIndex: 10012, - border: theme.palette.defaultBorder, + border: theme.palette.defaultBorder, }, }} > @@ -20362,11 +19959,11 @@ const releaseToConnectLabel = "Release to Connect"
    -
    +
    {curapp === null ? null : ( {selectedResult.action.app_name} { rightSideBarOpen && Object.getOwnPropertyNames(selectedAction).length > 0 ? -
    - -
    : null + +
    + +
    +     + : null } {/* Looks for triggers" */} {/* Only fixed the ones that require scrolling on a small screen */} @@ -21124,7 +20724,7 @@ const releaseToConnectLabel = "Release to Connect" toast( "Field " + selectedApp.authentication.parameters[paramkey].name + - " can't be empty" + " can't be empty. If you want it empty, put space." ); return; } @@ -21153,7 +20753,6 @@ const releaseToConnectLabel = "Release to Connect" var newAuthOption = JSON.parse(JSON.stringify(authenticationOption)); var newFields = []; - console.log("Fields: ", newAuthOption.fields) for (let authkey in newAuthOption.fields) { const value = newAuthOption.fields[authkey]; newFields.push({ @@ -21166,7 +20765,7 @@ const releaseToConnectLabel = "Release to Connect" setNewAppAuth(newAuthOption) if (configureWorkflowModalOpen) { - setSelectedAction({}) + //setSelectedAction({}) } setUpdate(authenticationOption.id) @@ -21199,7 +20798,7 @@ const releaseToConnectLabel = "Release to Connect" - - {data.name} +
    + + + {data?.name?.endsWith("_basic") ? data?.name?.replace("_basic", "") : data?.name} + +
    {data.schema !== undefined && data.schema !== null && @@ -21294,7 +20899,7 @@ const releaseToConnectLabel = "Release to Connect" )}
    @@ -21325,19 +20931,11 @@ const releaseToConnectLabel = "Release to Connect" - - - ); - }; - - const SuggestionBoxUi = () => { const [suggestionValue, setSuggestionValue] = useState(""); @@ -21951,7 +21471,7 @@ const releaseToConnectLabel = "Release to Connect" } return ( -
    +
    {/* */} @@ -22023,7 +21543,7 @@ const releaseToConnectLabel = "Release to Connect" }*/ const RevisionBox = (props) => { - const { revision, } = props + const { revision, showBorder, } = props if (revision === undefined || revision === null) { return null } @@ -22053,7 +21573,7 @@ const releaseToConnectLabel = "Release to Connect" return ( { if (newrevision.edited === selectedVersion.edited) { console.log("Same revision! No setting.") @@ -22146,12 +21666,14 @@ const releaseToConnectLabel = "Release to Connect" - + + +
    {/*revision.edited === originalWorkflow.edited ? @@ -22228,44 +21750,54 @@ const releaseToConnectLabel = "Release to Connect" const drawerData = originalWorkflow !== undefined && originalWorkflow !== null ?
    - - Version History - -
    -
    -
    - - Current Version - - -
    + + Version History + + + Versions are stored for every change made, up to once per minute. When restoring a version, the changes will not take effect until you save the workflow. + + +
    + {/* +
    +
    + + Current Version + + +
    - -
    + +
    + */} {allRevisions.length > 0 ?
    { allRevisions.map((revision, index) => { + /* if(revision.edited === selectedVersion.edited){ return null } + */ return ( ) }) @@ -22440,8 +21972,10 @@ const releaseToConnectLabel = "Release to Connect" } } } else { - selectedAction.parameters[count].autocompleted = false - selectedAction.parameters[count].value = data + if (selectedAction.parameters !== undefined && selectedAction.parameters !== null && selectedAction.parameters.length > count) { + selectedAction.parameters[count].autocompleted = false + selectedAction.parameters[count].value = data + } } setSelectedAction(selectedAction) @@ -22487,7 +22021,7 @@ const releaseToConnectLabel = "Release to Connect" const templatePopup = foundusecase.name === undefined || foundusecase.name === null || foundusecase.name === "" ? null : -
    +
    } + {tenzirConfigModal} {/*editWorkflowModal*/} {authgroupModal} {executionArgumentModal} @@ -22545,10 +22079,13 @@ const releaseToConnectLabel = "Release to Connect" fieldCount={editorData.field_number} actionlist={editorData.actionlist} fieldname={editorData.field_id} + editorData={editorData} changeActionParameterCodeMirror={changeActionParameterCodeMirror} - activeDialog={activeDialog} - setActiveDialog={setActiveDialog} + activeDialog={activeDialog} + setActiveDialog={setActiveDialog} + + setAiQueryModalOpen={setAiQueryModalOpen} /> : null} @@ -22565,7 +22102,7 @@ const releaseToConnectLabel = "Release to Connect" : null} {/*selectionOpen === true ? -
    +
    : null*/} +
    + {showVideo !== undefined && showVideo.length > 0 ? -
    +
    { { { { setBaseUrl(tmpstring); }} /> -
    +
    Authentication diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 1b14fe20..ee9be748 100755 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -1,4 +1,4 @@ -import React, { useEffect } from "react"; +import React, { useEffect, useContext, memo } from "react"; import { useInterval } from "react-powerhooks"; import theme from '../theme.jsx'; @@ -49,6 +49,7 @@ import { Folder as FolderIcon, LibraryBooks as LibraryBooksIcon, } from "@mui/icons-material"; +import { Context } from "../context/ContextApi.jsx"; import { ForkRight as ForkRightIcon, @@ -280,6 +281,7 @@ const searchClient = algoliasearch("JNSS5CFDZZ", "db08e40265e2941b9a7d8f644b6e52 const Apps = (props) => { const { globalUrl, isLoggedIn, isLoaded, userdata, serverside, } = props; + //const [workflows, setWorkflows] = React.useState([]); const baseRepository = "https://github.com/frikky/shuffle-apps"; //const alert = useAlert(); @@ -423,7 +425,7 @@ const Apps = (props) => { minWidth: "100%", maxWidth: 612.5, marginBottom: 5, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, color: "white", backgroundColor: surfaceColor, cursor: "pointer", @@ -875,7 +877,7 @@ const Apps = (props) => { minWidth: viewWidth, maxWidth: viewWidth, color: "white", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, backgroundColor: surfaceColor, //display: "flex", marginBottom: 10, @@ -1551,7 +1553,7 @@ const Apps = (props) => { minHeight: 150, maxHeight: 150, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, } if (!makeFancy) { @@ -1829,7 +1831,7 @@ const Apps = (props) => { }}> {
    { minWidth: viewWidth, maxWidth: viewWidth, color: "white", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, //display: "flex", marginBottom: 10, overflow: "hidden", @@ -2373,6 +2375,7 @@ const Apps = (props) => { toast("Hotloading apps from location in .env"); setIsLoading(true); fetch(globalUrl + "/api/v1/apps/run_hotload", { + method: "POST", mode: "cors", headers: { Accept: "application/json", @@ -2999,7 +3002,7 @@ const Apps = (props) => { PaperProps={{ style: { backgroundColor: theme.palette.platformColor, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, color: "white", minWidth: "800px", minHeight: "320px", @@ -3091,7 +3094,7 @@ const Apps = (props) => { PaperProps={{ style: { backgroundColor: theme.palette.platformColor, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, color: "white", minWidth: "800px", minHeight: "320px", @@ -3202,22 +3205,50 @@ const Apps = (props) => { ) : null; - const loadedCheck = - isLoaded && !firstrequest ? ( -
    - {appView} - {modalView} - {publishModal} - {generateAppView} - {appsModalLoad} - {deleteModal} -
    - ) : ( -
    - ); + const loadedCheck = isLoaded && !firstrequest ? ( + + + + ) : ( +
    + ); - // Maybe use gridview or something, idk return loadedCheck; }; export default Apps; + + +const AppsWrapper = memo(({ appView, modalView, userdata, publishModal, generateAppView, appsModalLoad, deleteModal }) => ( + <> + {appView} + {modalView} + {publishModal} + {generateAppView} + {appsModalLoad} + {deleteModal} + +)); + + +const SidebarAdjustWrapper = memo(({ userdata, children }) => { + + const {leftSideBarOpenByClick } = useContext(Context) + const marginLeft = userdata?.support + ? leftSideBarOpenByClick ? 250 : 80 + : 0; + + return ( +
    + {children} +
    + ); +}); diff --git a/frontend/src/views/Apps2.jsx b/frontend/src/views/Apps2.jsx new file mode 100644 index 00000000..079d4b2e --- /dev/null +++ b/frontend/src/views/Apps2.jsx @@ -0,0 +1,1174 @@ +import React, { useState, useEffect, useContext, useCallback, memo, useMemo, useRef } from "react"; +import theme from "../theme.jsx"; +import { isMobile } from "react-device-detect"; +import AppGrid from "../components/AppGrid.jsx"; +import { useLocation, useNavigate } from "react-router-dom"; +import { + TextField, Button, Typography, MenuItem, Select, Tabs, Tab, Zoom, + Grid, + Paper, + ButtonBase, + Tooltip, + Box, + CircularProgress, + Checkbox, +} from "@mui/material"; +import { Context } from "../context/ContextApi.jsx"; +import Add from '@mui/icons-material/Add'; +import InputAdornment from '@mui/material/InputAdornment'; +import Search from '@mui/icons-material/Search'; +import ClearIcon from '@mui/icons-material/Clear'; + +import { ClearRefinements, connectHits, connectSearchBox, connectStateResults, InstantSearch, RefinementList, connectRefinementList, Configure } from "react-instantsearch-dom"; +import { removeQuery } from "../components/ScrollToTop.jsx"; +import { toast } from "react-toastify"; +import algoliasearch from "algoliasearch/lite"; +import { debounce } from "lodash"; +import AppSelection from "../components/AppSelection.jsx"; + + +const searchClient = algoliasearch( + "JNSS5CFDZZ", + "db08e40265e2941b9a7d8f644b6e5240" +); + +// AppCard Component +const AppCard = ({ data, index, mouseHoverIndex, setMouseHoverIndex, globalUrl, deactivatedIndexes, currTab }) => { + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" || window.location.host === "localhost:3000"; + const appUrl = isCloud ? `/apps/${data.id}` : `https://shuffler.io/apps/${data.id}`; + + const paperStyle = { + backgroundColor: mouseHoverIndex === index ? "rgba(26, 26, 26, 1)" : "#1A1A1A", + color: "rgba(241, 241, 241, 1)", + cursor: "pointer", + position: "relative", + width: 365, + height: 96, + borderRadius: 8, + boxShadow: "0px 0px 10px 0px rgba(0, 0, 0, 0.1)", + marginBottom: 20, + }; + + return ( + + setMouseHoverIndex(index)} + onMouseOut={() => setMouseHoverIndex(-1)} + > + { + // if (!event.target.closest('.deactivate-button')) { + // window.location.href = appUrl; + // } + console.log("clicked"); + }} + > + {data.name} +
    +
    + {data.name.replace(/_/g, ' ').replace(/\b\w/g, char => char.toUpperCase())} +
    +
    + {data.categories ? data.categories.join(", ") : "NA"} +
    +
    +
    + {data.generated !== true && data.tags && data.tags.slice(0, 2).map((tag, tagIndex) => ( + + {tag} + {tagIndex < data.tags.length - 1 ? ", " : ""} + + ))} +
    + {/* Deactivate button */} + {currTab === 0 && !deactivatedIndexes.includes(index) && mouseHoverIndex === index && data.generated === true && ( + + )} +
    +
    +     +     +     + ); +}; + + +// Component to fetch all public app from the algolia. +const Hits = ({ + userdata, + hits, + setIsAnyAppActivated, + searchQuery, + globalUrl, + isLoading, + isLoggedIn, +}) => { + const [hoverEffect, setHoverEffect] = useState(-1); + const [allActivatedAppIds, setAllActivatedAppIds] = useState(userdata?.active_apps); + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + const [deactivatedIndexes, setDeactivatedIndexes] = React.useState([]); + + const normalizedString = (name) => { + if (typeof name === 'string') { + return name.replace(/_/g, ' '); + } else { + return name; + } + }; + + + + //Function for activation and deactivation of app + const handleActivateButton = (event, data, type) => { + + //use prevent default so it will stop redirection to the app page + event.preventDefault(); + if (!isLoggedIn) { + toast.error("Please log in to your account to activate the app.") + return; + } + if (type === "activate") { + toast.success(`The ${normalizedString(data.name)} app is activating. Please wait...`); + } + if (type === "deactivate") { + toast.success(`The ${normalizedString(data.name)} app is deactivating. Please wait...`); + } + + const baseURL = globalUrl; + const url = `${baseURL}/api/v1/apps/${data.objectID}/${type}`; + + fetch(url, { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => response.json()) + .then((responseJson) => { + if (responseJson.success === false) { + toast.error(responseJson.reason); + } else { + //toast.success(`App ${type}d Successfully!`); + if (type === 'activate') { + setAllActivatedAppIds(prev => [...prev, data.objectID]); + setIsAnyAppActivated(true); + } + if (type === 'deactivate') { + const updatedIds = allActivatedAppIds.filter(id => id !== data.objectID); + setAllActivatedAppIds(updatedIds); + } + } + }) + .catch(error => { + console.log("app error: ", error.toString()); + }); + } + + let workflowDelay = 0; + const isHeader = true; + + const [showNoAppFound, setShowNoAppFound] = useState(false); + + //show some delay to show the "App Not Found." so it doesn't not show while changing tab. + useEffect(() => { + const timer = setTimeout(() => { + setShowNoAppFound(true); + }, 1000); + return () => clearTimeout(timer); + }, []); + + + + return ( +
    + {!isLoading ? ( +
    + {hits?.length === 0 && searchQuery.length >= 0 && showNoAppFound ? ( +
    + No Apps Found +
    + ) : ( + +
    +
    + {hits?.map((data, index) => { + const appUrl = + isCloud + ? `/apps/${data.objectID}?queryID=${data.__queryID}` + : `https://shuffler.io/apps/${data.objectID}?queryID=${data.__queryID}`; + + return ( + + + + { + setHoverEffect(index); + }} + onMouseLeave={() => { + setHoverEffect(-1); + }} + > + + {data.name} +
    +
    + {(allActivatedAppIds && allActivatedAppIds.includes(data.objectID)) && } + {normalizedString(data.name)} +
    + +
    + {data.categories !== null + ? normalizedString(data.categories).join(", ") + : "NA"} +
    +
    +
    + {hoverEffect === index && isCloud ? ( +
    + {data.tags && ( + + + {data.tags.slice(0, 1).map((tag, tagIndex) => ( + + {normalizedString(tag)} + {tagIndex < 1 ? ", " : ""} + + ))} + + + )} +
    + ) : ( +
    + {data.tags && + data.tags.map((tag, tagIndex) => ( + + {normalizedString(tag)} + {tagIndex < data.tags.length - 1 ? ", " : ""} + + ))} +
    + )} +
    +
    + {hoverEffect === index && isCloud && ( +
    + {allActivatedAppIds && allActivatedAppIds.includes(data.objectID) ? ( + + ) : ( + + )} +
    + )} +
    +
    +
    +     +     +     +     +     + ); + })} +
    +
    +     + )} +
    + ) : ( +
    + )} +
    + ); +} + +// Custom Category Dropdown Component +const CategoryDropdown = ({ items, currentRefinement, refine }) => { + const handleChange = (event) => { + const value = event.target.value; + refine(value); + }; + + return ( + + ); +}; + +const CustomCategoryDropdown = connectRefinementList(CategoryDropdown); + +// Custom SearchBox Component +const SearchBox = ({ refine, searchQuery, setSearchQuery }) => { + const inputRef = useRef(null); + const [localQuery, setLocalQuery] = useState(searchQuery); + + // Debounced function to refine search + const debouncedRefine = useRef( + debounce((value) => { + setSearchQuery(value); + removeQuery("q"); + refine(value); + }, 300) // Adjust the delay as needed + ).current; + + useEffect(() => { + const urlSearchParams = new URLSearchParams(window.location.search); + const params = Object.fromEntries(urlSearchParams.entries()); + const foundQuery = params["q"]; + if (foundQuery) { + setLocalQuery(foundQuery); + debouncedRefine(foundQuery); + } + }, [debouncedRefine]); + + useEffect(() => { + if (searchQuery === "") { + return; + } + if (inputRef.current) { + inputRef.current.focus(); + } + }, [searchQuery]); + + const handleChange = (event) => { + const value = event.target.value; + setLocalQuery(value); + debouncedRefine(value); + }; + + return ( + { + if (event.key === "Enter") { + event.preventDefault(); + } + }} + style={{ width: '100%', borderRadius: '7px', fontFamily: "var(--zds-typography-base,Inter,Helvetica,arial,sans-serif)" }} + InputProps={{ + style: { + borderRadius: 8, + }, + endAdornment: ( + + {localQuery?.length === 0 ? : ( + { + setLocalQuery(''); + debouncedRefine(''); + }} + /> + )} + + ), + }} + /> + ); +}; + +const CustomSearchBox = connectSearchBox(SearchBox); +const CustomHits = connectHits(Hits); + +// Custom Label Dropdown Component +const LabelDropdown = ({ items, currentRefinement, refine }) => { + const handleChange = (event) => { + const value = event.target.value; + refine(value); + }; + + return ( + + ); +}; + +const CustomLabelDropdown = connectRefinementList(LabelDropdown); + +// Main Apps Component +const Apps2 = (props) => { + const { globalUrl, isLoaded, serverside, userdata, isLoggedIn, checkLogin } = props; + let navigate = useNavigate(); + const { leftSideBarOpenByClick } = useContext(Context); + const location = useLocation(); + const [searchQuery, setSearchQuery] = useState(""); + const [selectedCategory, setSelectedCategory] = useState([]); + const [selectedLabel, setSelectedLabel] = useState([]); + const [currTab, setCurrTab] = useState(0); + const [categories, setCategories] = useState([]); + const [labels, setLabels] = useState([]); + const [isLoading, setIsLoading] = useState(false); + const [userApps, setUserApps] = useState([]); + const [orgApps, setOrgApps] = useState([]); + const [appsToShow, setAppsToShow] = useState([]); + const [deactivatedIndexes, setDeactivatedIndexes] = React.useState([]); + const [IsAnyAppActivated, setIsAnyAppActivated] = useState(false); + const [mouseHoverIndex, setMouseHoverIndex] = useState(-1); + var counted = 0; + const [showNoAppFound, setShowNoAppFound] = useState(false); + + + const [appFramework, setAppFramework] = useState(undefined); + const [defaultSearch, setDefaultSearch] = useState(""); + // Set the current tab based on the query parameter + useEffect(() => { + const queryParams = new URLSearchParams(location.search); + const tabParam = queryParams.get('tab'); + if (tabParam !== null && tabParam !== undefined) { + if (tabParam === 'org_apps') { + setCurrTab(0); + } else if (tabParam === 'my_apps') { + setCurrTab(1); + } else { + setCurrTab(2); + } + } + }, [location.search]); + + useEffect(() => { + + const getFramework = () => { + fetch(globalUrl + "/api/v1/apps/frameworkConfiguration", { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for framework!"); + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success === false) { + setAppFramework({}) + + if (responseJson.reason !== undefined) { + //toast("Failed loading: " + responseJson.reason) + } else { + //toast("Failed to load framework for your org.") + } + } else { + setAppFramework(responseJson) + } + }) + .catch((error) => { + console.log("err in framework: ", error.toString()); + }) + } + + getFramework(); + }, []); + + // Fetch apps based on the current tab : 0 -> org_apps, 1 -> my_apps, 2 -> all_apps + useEffect(() => { + const fetchApps = async () => { + const baseUrl = globalUrl; + let url; + + const userId = userdata?.id; + if (currTab === 1 && userId) { + url = `${baseUrl}/api/v1/users/${userId}/apps`; + } else if (currTab === 0) { + url = `${baseUrl}/api/v1/apps`; + } + setIsLoading(true); + try { + const response = await fetch(url, { + method: "GET", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }); + const data = await response.json(); + if (currTab === 1) { + setAppsToShow(data); + setUserApps(data); + } else if (currTab === 0) { + setAppsToShow(data); + setOrgApps(data); + // For testing the empty state + // setAppsToShow([]); + // setOrgApps([]); + } + setIsLoading(false); + } catch (err) { + console.error("Error fetching apps:", err); + setIsLoading(false); + } + }; + + // Only fetch if we have required data + if (globalUrl && (currTab === 0 || (currTab === 1 && userdata?.id))) { + fetchApps(); + } + }, [currTab, globalUrl, userdata?.id]); // Remove location.search dependency + + useEffect(() => { + setSearchQuery(""); + }, [currTab]) + + + // Find top categories and tags based on the current tab + useEffect(() => { + if (currTab === 0) { + setCategories(findTopCategories()); + setLabels(findTopTags()); + } + }, [currTab, appsToShow]) + + + useEffect(() => { + if (serverside) { + return null; + } + }, [serverside]); + + + const findTopCategories = () => { + const categoryCountMap = {}; + const apps = currTab === 1 ? userApps : orgApps; + // Check if userAndOrgsApp is an array before iterating over it and Find top 10 Category from the apps + if (Array.isArray(apps)) { + apps.forEach((app) => { + const categories = app.categories; + + if (categories && categories.length > 0) { + categories.forEach((category) => { + categoryCountMap[category] = (categoryCountMap[category] || 0) + 1; + }); + } + }); + + const categoryArray = Object.keys(categoryCountMap).map((category) => ({ + category, + count: categoryCountMap[category], + })); + + categoryArray.sort((a, b) => b.count - a.count); + + const topCategories = categoryArray.slice(0, 7); + + return topCategories; + } + }; + + const findTopTags = () => { + const tagCountMap = {}; + const apps = currTab === 1 ? userApps : orgApps; + + if (Array.isArray(apps)) { + apps.forEach((app) => { + const tags = app.tags; + if (tags && tags.length > 0) { + tags.forEach((tag) => { + tagCountMap[tag] = (tagCountMap[tag] || 0) + 1; + }); + } + }); + } + + const tagArray = Object.keys(tagCountMap).map((tag) => ({ + tag, + count: tagCountMap[tag], + })); + + tagArray.sort((a, b) => b.count - a.count); + + const topTags = tagArray.slice(0, 8); + + return topTags; + }; + + const handleCreateApp = (e) => { + e.preventDefault(); + console.log("Create app clicked") + }; + + + useEffect(() => { + const apps = currTab === 1 ? userApps : orgApps; + // Search app based on app name, category, and tag + const filteredUserAppdata = Array.isArray(apps) ? apps.filter((app) => { + + const matchesSearchQuery = ( + searchQuery === "" || // If searchQuery is empty, match all apps + app.name.toLowerCase().includes(searchQuery.toLowerCase()) || + (app.tags && app.tags.some(tag => + tag.toLowerCase().includes(searchQuery.toLowerCase()) + )) || + (app.categories && app.categories.some((category) => + category.toLowerCase().includes(searchQuery.toLowerCase()) + )) + ); + + const matchesSelectedCategories = ( + selectedCategory.length === 0 || // If no category is selected, match all apps + (app.categories && app.categories.some(category => + selectedCategory.includes(category) + )) + ); + + const matchesSelectedTags = ( + selectedLabel.length === 0 || // If no label is selected, match all apps + (app.tags && app.tags.some(tag => + selectedLabel.includes(tag) + )) + ); + + return matchesSearchQuery && matchesSelectedCategories && matchesSelectedTags; + }) : []; + + setAppsToShow(filteredUserAppdata); + }, [searchQuery, selectedCategory, selectedLabel]); + + + const handleTabChange = (newTab) => { + setCurrTab(newTab); + if (newTab === 0) { + setAppsToShow(orgApps) + const categories = findTopCategories(); + const labels = findTopTags(); + setCategories(categories); + setLabels(labels); + } + if (newTab === 1) { + setAppsToShow(userApps) + } + const newQueryParam = newTab === 0 ? 'org_apps' : newTab === 1 ? 'my_apps' : 'all_apps'; + const queryParams = new URLSearchParams(location.search); + queryParams.set('tab', newQueryParam); + queryParams.delete('q'); + window.history.replaceState({}, '', `${location.pathname}?${queryParams.toString()}`); + console.log("Apps to show", appsToShow) + }; + + + + + const boxStyle = { + color: "white", + display: "flex", + flexDirection: "column", + width: "100%", + margin: "auto", + maxWidth: "60%", + // padding: '20px 380px', + }; + + + const handleCategoryChange = (event) => { + const value = event.target.value; + setSelectedCategory(value); + }; + + const handleLabelChange = (event) => { + const value = event.target.value; + setSelectedLabel(value); + }; + + + return ( + +
    + + Apps + +
    + handleTabChange(newTab)} + TabIndicatorProps={{ style: { height: '3px', borderRadius: 10 } }} + > + + + + +
    +
    +
    + { + (currTab === 0 || currTab === 1) && + { + setSearchQuery(event.target.value); + }} + onKeyDown={(event) => { + if (event.key === "Enter") { + event.preventDefault(); + } + }} + limit={5} + style={{ width: '100%', borderRadius: '7px', fontFamily: "var(--zds-typography-base,Inter,Helvetica,arial,sans-serif)" }} + InputProps={{ + style: { + borderRadius: 8, + }, + endAdornment: ( + + { + searchQuery.length === 0 ? : + } + + ), + }} + /> + } + { + currTab === 2 && + + } +
    +
    + {currTab === 2 ? ( + + ) : ( + + )} +
    +
    + {currTab === 2 ? ( + + ) : ( + + )} +
    +
    + +
    +
    +
    + + { + currTab === 0 && ( +
    + {isLoading ? ( +
    + +
    + ) : ( + <> + {appsToShow?.length > 0 && appsToShow !== undefined && !isLoading ? ( +
    + {appsToShow.map((data, index) => ( + + ))} +
    + ) : ( +
    + +
    + )} + + )} +
    + ) + } + { + currTab === 1 && ( +
    + {isLoading ? ( +
    + +
    + ) : ( + <> + {appsToShow?.length > 0 && appsToShow !== undefined ? ( +
    + {appsToShow.map((data, index) => ( + + ))} +
    + ) : ( +
    + No Apps Found +
    + )} + + )} +
    + ) + } + + { + currTab === 2 && + + } +
    +
    + +     + ); +}; + +export default Apps2; diff --git a/frontend/src/views/DashboardViews.jsx b/frontend/src/views/DashboardViews.jsx index 57b761a3..b87bd142 100644 --- a/frontend/src/views/DashboardViews.jsx +++ b/frontend/src/views/DashboardViews.jsx @@ -197,7 +197,7 @@ const LineChartWrapper = ({keys, height, width}) => { const name = data.metadata !== undefined && data.metadata.name !== undefined ? data.metadata.name : "No" return ( -
    +
    {name} @@ -281,7 +281,7 @@ const RadialChart = ({keys, setSelectedCategory}) => { }} content={(data, color) => { return ( -
    +
    {data.x} @@ -795,7 +795,7 @@ const Dashboard = (props) => { color: "white", height: 40, maxWidth: 150, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} > {data.available_keys.map((foundKey, index) => { diff --git a/frontend/src/views/Docs.jsx b/frontend/src/views/Docs.jsx index e05a7d90..8062408f 100755 --- a/frontend/src/views/Docs.jsx +++ b/frontend/src/views/Docs.jsx @@ -1,4 +1,4 @@ -import React, { useEffect, useLayoutEffect, useRef, useState } from "react" +import React, { useEffect, useLayoutEffect, useRef, useState, useContext, memo } from "react" import { toast } from 'react-toastify'; import Markdown from 'react-markdown' import theme from '../theme.jsx'; @@ -8,6 +8,7 @@ import { BrowserView, MobileView } from "react-device-detect"; import { useParams, useNavigate, Link } from "react-router-dom"; import { validateJson, GetIconInfo } from "../views/Workflows.jsx"; import remarkGfm from 'remark-gfm' +import { Context } from "../context/ContextApi.jsx"; import { Grid, TextField, @@ -166,7 +167,7 @@ export const Img = (props) => { return( {props.alt} @@ -213,7 +214,7 @@ export const CodeHandler = (props) => { backgroundColor: theme.palette.inputColor, overflowY: "auto", // Have it inline - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} > {validate.valid === true ? @@ -250,10 +251,8 @@ export const CodeHandler = (props) => { } const Docs = (defaultprops) => { - const { globalUrl, selectedDoc, serverside, serverMobile } = defaultprops; - + const { globalUrl, selectedDoc, serverside, serverMobile, userdata } = defaultprops; let navigate = useNavigate(); - // Quickfix for react router 5 -> 6 const params = useParams(); //var props = JSON.parse(JSON.stringify(defaultprops)) @@ -526,7 +525,7 @@ const Docs = (defaultprops) => { style={{ backgroundColor: theme.palette.inputColor, padding: 15, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, marginBottom: 30, display: "flex", }} @@ -892,7 +891,7 @@ const Docs = (defaultprops) => { target="_blank" style={{ textDecoration: "none", color: "inherit", flex: 1, margin: 10, }} > -
    { if (link === "" || link === undefined) { event.preventDefault() @@ -932,7 +931,7 @@ const Docs = (defaultprops) => { return ( -
    { setHover(true) }} @@ -968,8 +967,8 @@ const Docs = (defaultprops) => { Documentation
    - /> - link="https://discord.gg/B2CBzUm" /> + /> + link="https://discord.gg/B2CBzUm" />
    @@ -1265,15 +1264,44 @@ const Docs = (defaultprops) => {
    ); - // Padding and zIndex etc set because of footer in cloud. - const loadedCheck = ( -
    - {postDataBrowser} - {postDataMobile} -
    - ); + console.log("docs render") + + // Padding and zIndex etc set because of footer in cloud. +const loadedCheck = ( + + + +); + +return
    {loadedCheck}
    ; - return
    {loadedCheck}
    ; }; export default Docs; + +const DocsContent = memo(({postDataBrowser, postDataMobile}) => { + return( +
    + {postDataBrowser} + {postDataMobile} +
    +)}) + +const DocsWrapper = memo(({userdata, children })=>{ + + const { leftSideBarOpenByClick, windowWidth } = useContext(Context); + + return ( +
    + {children} +
    + ) +}) diff --git a/frontend/src/views/GettingStarted.jsx b/frontend/src/views/GettingStarted.jsx index 39e59f2a..fc683f4a 100644 --- a/frontend/src/views/GettingStarted.jsx +++ b/frontend/src/views/GettingStarted.jsx @@ -2467,7 +2467,7 @@ const GettingStarted = (props) => { maxWidth: 1024, zIndex: 11, border: "1px solid rgba(255,255,255,0.1)", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, textAlign: "center", overflow: "auto", }} diff --git a/frontend/src/views/RunWorkflow.jsx b/frontend/src/views/RunWorkflow.jsx index 9d741300..c8b8367f 100644 --- a/frontend/src/views/RunWorkflow.jsx +++ b/frontend/src/views/RunWorkflow.jsx @@ -43,6 +43,7 @@ import { Lock as LockIcon, LockOpen as LockOpenIcon, OpenInNew as OpenInNewIcon, + Edit as EditIcon, } from '@mui/icons-material'; const hrefStyle = { @@ -50,20 +51,13 @@ const hrefStyle = { textDecoration: "none" } -const bodyDivStyle = { - margin: "auto", - width: isMobile? "100%":"500px", - position: "relative", - - paddingBottom: 250, -} - const RunWorkflow = (defaultprops) => { const { globalUrl, userdata, isLoaded, isLoggedIn, setIsLoggedIn, setCookie, register, serverside } = defaultprops; let navigate = useNavigate(); const [_, setUpdate] = useState(""); // Used to force rendring, don't remove + const [explorerUi, setExplorerUi] = useState(false) const [message, setMessage] = useState(""); const [workflow, setWorkflow] = React.useState({}); const [executionRequest, setExecutionRequest] = React.useState({}); @@ -80,6 +74,8 @@ const RunWorkflow = (defaultprops) => { const [sharingOpen, setSharingOpen] = React.useState(false) const [realtimeMarkdown, setRealtimeMarkdown] = React.useState("") const [forms, setForms] = React.useState([]) + const [boxWidth, setBoxWidth] = React.useState(500) + const [inputQuestions, setInputQuestions] = React.useState([]) const IframeWrapper = (props) => { var propsCopy = JSON.parse(JSON.stringify(props)) @@ -94,11 +90,21 @@ const RunWorkflow = (defaultprops) => { if (propsCopy.width === undefined || propsCopy.width === null) { propsCopy.width = 400 propsCopy.height = "auto" + + propsCopy.margin = "auto" } return Img(propsCopy) } + const bodyDivStyle = { + margin: "auto", + width: isMobile? "100%" : boxWidth, + position: "relative", + + paddingBottom: 250, + } + const boxStyle = { color: "white", padding: "25px 50px 50px 50px", @@ -112,7 +118,7 @@ const RunWorkflow = (defaultprops) => { props.match = {} props.match.params = params - const defaultTitle = workflow.name !== undefined ? workflow.name : "Run Workflow" + const defaultTitle = workflow.name !== undefined ? "Shuffle - Form for " + workflow.name : "Shuffle - Form to Run Workflows" if (document != undefined && document.title != defaultTitle) { document.title = defaultTitle } @@ -136,6 +142,18 @@ const RunWorkflow = (defaultprops) => { return true } + // Check if it's an object or not + if (typeof executionArgument === "string") { + // Make it an object + try { + executionArgument = JSON.parse(executionArgument) + } catch (e) { + console.log("Error parsing execution argument: ", e) + executionArgument = {} + } + } + + console.log("EXEC: ", executionArgument) for (var key in executionArgument) { if (executionArgument[key] === undefined || executionArgument[key] === null || executionArgument[key] === "") { return false @@ -257,9 +275,12 @@ const RunWorkflow = (defaultprops) => { return (
    - + {workflowQuestion !== "" ? null : + + } - {validate.valid === false ? + {workflowQuestion !== "" ? null : + validate.valid === false ?
    { }) } - /* - useEffect(() => { - if (executionRequest === undefined || executionRequest === null || executionRequest.execution_id === undefined || executionRequest.execution_id === null || executionRequest.execution_id.length === 0) { - return - } - - if (executionRequest.start === true) { - start() - } - }, [executionRequest]) - */ - const { start, stop } = useInterval({ duration: 1500, startImmediate: true, @@ -520,8 +529,8 @@ const RunWorkflow = (defaultprops) => { if (realtimeMarkdown !== undefined && realtimeMarkdown !== null && realtimeMarkdown.length > 0) { const newmarkdown = realtimeMarkdown.replace(`{{ ${workflow_id} }}`, "", -1) setRealtimeMarkdown(newmarkdown) - } else if (inputWorkflow.input_markdown !== undefined && inputWorkflow.input_markdown !== null && inputWorkflow.input_markdown.length > 0) { - const newmarkdown = inputWorkflow.input_markdown.replace(`{{ ${workflow_id} }}`, "", -1) + } else if (inputWorkflow.form_control.input_markdown !== undefined && inputWorkflow.form_control.input_markdown !== null && inputWorkflow.form_control.input_markdown.length > 0) { + const newmarkdown = inputWorkflow.form_control.input_markdown.replace(`{{ ${workflow_id} }}`, "", -1) setRealtimeMarkdown(newmarkdown) } } @@ -531,15 +540,21 @@ const RunWorkflow = (defaultprops) => { console.log("Get workflow error: ", error.toString()) if (realtimeMarkdown !== undefined && realtimeMarkdown !== null && realtimeMarkdown.length > 0) { - const newmarkdown = inputWorkflow.input_markdown.replace(`{{ ${workflow_id} }}`, "", -1) + const newmarkdown = inputWorkflow.form_control.input_markdown.replace(`{{ ${workflow_id} }}`, "", -1) setRealtimeMarkdown(newmarkdown) - } else if (inputWorkflow.input_markdown !== undefined && inputWorkflow.input_markdown !== null && inputWorkflow.input_markdown.length > 0) { - const newmarkdown = inputWorkflow.input_markdown.replace(`{{ ${workflow_id} }}`, "", -1) + } else if (inputWorkflow.form_control.input_markdown !== undefined && inputWorkflow.form_control.input_markdown !== null && inputWorkflow.form_control.input_markdown.length > 0) { + const newmarkdown = inputWorkflow.form_control.input_markdown.replace(`{{ ${workflow_id} }}`, "", -1) setRealtimeMarkdown(newmarkdown) } }) } + const searchParams = new URLSearchParams(window.location.search) + const answer = searchParams.get("answer") + const execution_id = searchParams.get("reference_execution") + const authorization = searchParams.get("authorization") + const sourceNode = searchParams.get("source_node") + const getWorkflow = (workflow_id, selectedNode) => { setRealtimeMarkdown("") @@ -595,6 +610,56 @@ const RunWorkflow = (defaultprops) => { } } + // Override with just relevant fields + if (sourceNode !== undefined && sourceNode !== null && sourceNode.length > 0) { + for (var triggerkey in responseJson.triggers) { + const trig = responseJson.triggers[triggerkey] + if (trig.id !== sourceNode) { + continue + } + + console.log("TRIG: ", trig) + if (trig.parameters === undefined || trig.parameters === null) { + trig.parameters = [] + } + + for (var paramkey in trig.parameters) { + const param = trig.parameters[paramkey] + if (param.name !== "input_questions") { + continue + } + + // Parse as json + var keepfields = [] + try { + const parsed = JSON.parse(param.value) + + // Find this in the workflow.input_questions + for (var questionkey in responseJson.input_questions) { + var question = JSON.parse(JSON.stringify(responseJson.input_questions[questionkey])) + question.value = question.value.split(";")[0] + if (parsed.includes(question.name)) { + keepfields.push(question.value) + } + } + + //newexec = {} + } catch (e) { + console.log("Error parsing input questions: ", e) + } + + // Remapping it to exec + if (keepfields.length > 0) { + newexec = {} + for (var key in keepfields) { + newexec[keepfields[key]] = "" + } + } + } + + } + } + setExecutionArgument(newexec) } @@ -631,7 +696,8 @@ const RunWorkflow = (defaultprops) => { } } - responseJson.input_questions = relevantquestions + setInputQuestions(relevantquestions) + //responseJson.input_questions = relevantquestions } } } @@ -642,10 +708,10 @@ const RunWorkflow = (defaultprops) => { } } - if (responseJson.input_markdown !== undefined && responseJson.input_markdown !== null && responseJson.input_markdown.length > 0) { + if (responseJson.form_control.input_markdown !== undefined && responseJson.form_control.input_markdown !== null && responseJson.form_control.input_markdown.length > 0) { // Look for {{ uuid }} format, and try to run that workflow with their account // This is a hack, but a fun one. - var newmarkdown = responseJson.input_markdown.replace("", "") + var newmarkdown = responseJson.form_control.input_markdown.replace("", "") const uuidRegex = /{{\s[a-f0-9-]+\s}}/g const found = newmarkdown.match(uuidRegex) @@ -697,7 +763,21 @@ const RunWorkflow = (defaultprops) => { handleExecutionLoader() handleGetOrg(responseJson.org_id) - setWorkflow(responseJson); + + if (responseJson.form_control === undefined || responseJson.form_control === null) { + responseJson.form_control = { + "input_markdown": "", + "output_yields": [], + "form_width": 500, + } + } + + + if (responseJson.form_control.form_width !== undefined && responseJson.form_control.form_width !== null && responseJson.form_control.form_width > 300) { + setBoxWidth(responseJson.form_control.form_width) + } + + setWorkflow(responseJson) }) .catch((error) => { console.log("Get workflow error: ", error.toString()); @@ -723,7 +803,6 @@ const RunWorkflow = (defaultprops) => { if (responseJson.result.startsWith("[") && responseJson.result.endsWith("]")) { try { responseJson.result = JSON.parse(responseJson.result).length - console.log("Set length to: ", responseJson.result) } catch (e) { console.log("Error parsing length: ", e) } @@ -875,8 +954,8 @@ const RunWorkflow = (defaultprops) => { const newmarkdown = realtimeMarkdown.replace(`{{ ${responseJson.workflow.id} }}`, responseJson.result, -1) setRealtimeMarkdown(newmarkdown) - } else if (workflow.input_markdown !== undefined && workflow.input_markdown !== null && workflow.input_markdown.length > 0) { - const newmarkdown = workflow.input_markdown.replace(`{{ ${responseJson.workflow.id} }}`, responseJson.result, -1) + } else if (workflow.form_control.input_markdown !== undefined && workflow.form_control.input_markdown !== null && workflow.form_control.input_markdown.length > 0) { + const newmarkdown = workflow.form_control.input_markdown.replace(`{{ ${responseJson.workflow.id} }}`, responseJson.result, -1) setRealtimeMarkdown(newmarkdown) } @@ -893,17 +972,23 @@ const RunWorkflow = (defaultprops) => { }); }; - const searchParams = new URLSearchParams(window.location.search) - const answer = searchParams.get("answer") - const execution_id = searchParams.get("reference_execution") - const authorization = searchParams.get("authorization") - const sourceNode = searchParams.get("source_node") useEffect(() => { if (!isLoaded) { return } + if (props.match.params.key === undefined) { + setExplorerUi(true) + + if (isLoggedIn) { + loadForms(userdata.active_org.id) + handleGetOrg(userdata.active_org.id) + } + + return + } + getWorkflow(props.match.params.key, sourceNode) if (execution_id !== undefined && execution_id !== null && authorization !== undefined && authorization !== null) { console.log("Get execution: ", execution_id) @@ -915,7 +1000,6 @@ const RunWorkflow = (defaultprops) => { } }, [isLoaded]) - useEffect(() => { if (executionData === undefined || executionData === null || executionData === {}) { return @@ -947,9 +1031,14 @@ const RunWorkflow = (defaultprops) => { } if (result.status !== "WAITING") { + if (parsedresult.information !== undefined && parsedresult.information !== null && parsedresult.information.length > 0) { + setWorkflowQuestion(parsedresult.information) + } + if (parsedresult.click_info !== undefined && parsedresult.click_info !== null) { if (parsedresult.click_info.user !== undefined && parsedresult.click_info.user !== null && parsedresult.click_info.user.length > 0) { setMessage("Already answered by " + parsedresult.click_info.user) + } } else { setMessage("Answered.") @@ -965,8 +1054,12 @@ const RunWorkflow = (defaultprops) => { const buttonBackground = "linear-gradient(to right, #f86a3e, #f34079)" const buttonStyle = {borderRadius: 25, height: 50, fontSize: 18, backgroundImage: handleValidateForm(executionArgument) || executionLoading ? buttonBackground : "grey", color: "white"} - //const disabledButtons = message.length > 0 || executionData.status === "FINISHED" || executionData.status === "ABORTED" - const disabledButtons = executionLoading || executionRunning + // Check if all fields are filled in? + var disabledButtons = executionLoading || executionRunning || message.length > 0 + if (disabledButtons === false && workflow.input_questions !== undefined && workflow.input_questions !== null && workflow.input_questions.length > 0) { + // Check field values + //disabledButtons = handleValidateForm(executionArgument) + } const organization = selectedOrganization !== undefined && selectedOrganization !== null ? selectedOrganization.name : "Unknown" const contact = selectedOrganization !== undefined && selectedOrganization !== null && selectedOrganization.org !== undefined && selectedOrganization.org !== null? selectedOrganization.org : "support@shuffler.io" @@ -995,20 +1088,69 @@ const RunWorkflow = (defaultprops) => { } } + const FormList = () => { + return ( +
    + {forms.map((form, formIndex) => { + if (form.id === undefined || form.id === null) { + return null + } + + return ( +
    + { + navigate(`/forms/${form.id}`) + getWorkflow(form.id, sourceNode) + setExplorerUi(false) + }} + currentWorkflowId={workflow.id} + /> +
    + ) + })} +
    + ) + } + + const ExplorerUi = () => { + return ( +
    + + {forms !== undefined && forms !== null && forms.length > 0 ? +
    + + Available forms + + +
    + : + + No Form Found + + } +
    + ) + } + var validResults = 0 const basedata =
    - {workflow.id === undefined || workflow.id === null ? + {explorerUi === true ? + + : + workflow.id === undefined || workflow.id === null ?
    - Loding Form Details... + Loading Form Details...
    :
    - {workflow.input_markdown !== undefined && workflow.input_markdown !== null && workflow.input_markdown.length > 0 ? + {workflowQuestion !== "" || (workflow.form_control.input_markdown !== undefined && workflow.form_control.input_markdown !== null && workflow.form_control.input_markdown.length > 0) ?
    { }} rehypePlugins={[rehypeRaw]} > - {realtimeMarkdown !== undefined && realtimeMarkdown !== null && realtimeMarkdown.length > 0 ? realtimeMarkdown : workflow.input_markdown} + {workflowQuestion !== "" ? workflowQuestion : realtimeMarkdown !== undefined && realtimeMarkdown !== null && realtimeMarkdown.length > 0 ? realtimeMarkdown : workflow.form_control.input_markdown}
    : null}
    {onSubmit(e)}} style={{margin: "25px 0px 15px 0px",}}> - {workflow.input_markdown !== undefined && workflow.input_markdown !== null && workflow.input_markdown.length > 0 ? null : + {workflowQuestion !== "" || (workflow.form_control.input_markdown !== undefined && workflow.form_control.input_markdown !== null && workflow.form_control.input_markdown.length > 0) ? null :
    {workflow.name} {
    @@ -1081,7 +1223,7 @@ const RunWorkflow = (defaultprops) => { {workflow.input_questions !== undefined && workflow.input_questions !== null && workflow.input_questions.length > 0 ?
    - {workflow.input_questions.map((question, index) => { + {inputQuestions.map((question, index) => { // Multiple choice checks for semicolon-splits var multiChoiceOptions = question.value !== undefined && question.value !== null && question.value.length > 0 && question.value.includes(";") ? question.value.split(";") : [] @@ -1215,25 +1357,26 @@ const RunWorkflow = (defaultprops) => { What do you want to do? } -
    -  or  -
    @@ -1258,9 +1401,9 @@ const RunWorkflow = (defaultprops) => { } - {workflow.output_yields !== undefined && workflow.output_yields !== null && workflow.output_yields.length > 0 ? + {workflow.form_control.output_yields !== undefined && workflow.form_control.output_yields !== null && workflow.form_control.output_yields.length > 0 ?
    - {workflow.output_yields.map((yieldItem, index) => { + {workflow.form_control.output_yields.map((yieldItem, index) => { if (executionData.results === undefined || executionData.results === null || executionData.results.length === 0) { return null } @@ -1328,15 +1471,18 @@ const RunWorkflow = (defaultprops) => {
    } - - Forms are in Beta. Form submissions data includes your Organization's unique ID while logged in, or a unique identifier for your browser otherwise. Your input will be automatically sanitized. - + + {workflowQuestion !== "" ? null : + + Forms are in late Beta. Form submission data includes your Organization's unique ID while logged in, or a unique identifier for your browser otherwise. Your input will be automatically sanitized. + + }
    // const isCorrectOrg = userdata.active_org.id === undefined || userdata.active_org.id === null || workflow.org_id === null || workflow.org_id === undefined || workflow.org_id.length === 0 || userdata.active_org.id === workflow.org_id const loadedCheck = isLoaded ? -
    +
    {editWorkflowModalOpen === true ? { expanded={true} setRealtimeMarkdown={setRealtimeMarkdown} + boxWidth={boxWidth} + setBoxWidth={setBoxWidth} scrollTo={"form_fill"} /> : null} @@ -1367,7 +1515,7 @@ const RunWorkflow = (defaultprops) => { minHeight: 400, maxHeight: 400, padding: 25, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, //minWidth: isMobile ? "90%" : newWorkflow === true ? 1000 : 550, //maxWidth: isMobile ? "90%" : newWorkflow === true ? 1000 : 550, }, @@ -1407,7 +1555,7 @@ const RunWorkflow = (defaultprops) => { > Organization only - + @@ -1418,7 +1566,7 @@ const RunWorkflow = (defaultprops) => { - {isLoggedIn && userdata?.active_org?.id === workflow?.org_id ? + {isLoggedIn && userdata?.active_org?.id === workflow?.org_id || userdata?.support === true ?
    : null} @@ -1476,7 +1624,8 @@ const RunWorkflow = (defaultprops) => { // Check width const overlap = window !== undefined && window.innerWidth !== undefined && window.innerWidth < 1300 - const formSidebar = !isLoaded || overlap || !(forms !== undefined && forms !== null && forms.length > 1) ? null : + + const formSidebar = explorerUi === true || !isLoaded || overlap || !(forms !== undefined && forms !== null && forms.length > 1) ? null :
    { minHeight: 500, maxHeight: 500, position: "absolute", - left: 100, + left: 150, top: 0, border: "1px solid rgba(255,255,255,0.3)", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, padding: 25, @@ -1497,7 +1646,7 @@ const RunWorkflow = (defaultprops) => { }}> {selectedOrganization !== undefined && selectedOrganization !== null ?
    - + {selectedOrganization.name} @@ -1506,41 +1655,7 @@ const RunWorkflow = (defaultprops) => { : null} {forms !== undefined && forms !== null && forms.length > 0 ? -
    - {forms.map((form, formIndex) => { - if (form.id === undefined || form.id === null) { - return null - } - - return ( -
    - { - navigate(`/forms/${form.id}`) - getWorkflow(form.id, sourceNode) - }} - currentWorkflowId={workflow.id} - /> - - {/* - - */} -
    - ) - })} -
    + :
    No forms loaded @@ -1557,4 +1672,4 @@ const RunWorkflow = (defaultprops) => { ) } -export default RunWorkflow; +export default RunWorkflow diff --git a/frontend/src/views/Search.jsx b/frontend/src/views/Search.jsx index fa9a2268..9ebd79db 100644 --- a/frontend/src/views/Search.jsx +++ b/frontend/src/views/Search.jsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect, useMemo } from "react"; +import React, { useState, useEffect, useMemo, useContext } from "react"; import theme from "../theme.jsx"; import { isMobile } from "react-device-detect"; @@ -11,6 +11,7 @@ import { Tabs, Tab, setRef } from "@mui/material"; import { styled } from "@mui/material/styles"; import { makeStyles } from '@mui/styles'; import DiscordChat from "../components/DiscordChat.jsx"; +import { Context } from "../context/ContextApi.jsx"; import { Apps as AppsIcon, @@ -21,16 +22,14 @@ import { DescriptionOutlined as DescriptionOutlinedIcon, } from "@mui/icons-material"; -import { - Typography -} from "@mui/material" - +import {Typography} from "@mui/material"; // Should be different if logged in :| const Search = (props) => { const { globalUrl, isLoaded, serverside, userdata, hidemargins, isHeader } = props; let navigate = useNavigate(); + const { leftSideBarOpenByClick } = useContext(Context); const [curTab, setCurTab] = useState(0); const iconStyle = { marginRight: isHeader ? null : 10 }; @@ -124,14 +123,15 @@ const Search = (props) => { flex: "1", marginLeft: isHeader ? null : 10, marginRight: isHeader ? null : 10, - paddingLeft: isHeader ? null : 30, paddingRight: isHeader ? null : 30, paddingBottom: isHeader ? null : 30, paddingTop: hidemargins === true ? 0 : isHeader ? null : 30, display: "flex", flexDirection: "column", overflowX: "hidden", - minHeight: 400 + minHeight: 400, + paddingLeft: leftSideBarOpenByClick ? 250 : 0, + transition: "padding-left 0.3s ease", }; const views = { @@ -223,7 +223,7 @@ const Search = (props) => { margin: isHeader ? null : "auto", marginTop: hidemargins === true ? 0 : isHeader ? null : 25, backgroundColor: "rgba(33, 33, 33, 1)", - borderRadius: 8 + borderRadius: 8, }} value={curTab} indicatorColor="primary" diff --git a/frontend/src/views/SetAuthentication.jsx b/frontend/src/views/SetAuthentication.jsx index cbcda6dd..bb99e037 100755 --- a/frontend/src/views/SetAuthentication.jsx +++ b/frontend/src/views/SetAuthentication.jsx @@ -302,7 +302,7 @@ const SetAuthentication = (props) => { } return ( -
    +
    { display: "flex", flexDirection: "column", padding: "0px 0px 12px 0px", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }; const currentOwner = checkOwner(data, userdata); @@ -636,7 +636,7 @@ const Settings = (props) => { } return ( - + { ); const landingpageData = ( -
    +
    {imageInfo}

    Settings

    @@ -1065,7 +1065,7 @@ const Settings = (props) => { { { navigate(`/usecases?selected_object=${fixedName}`) } }}> - { + { }}> {!selectedItem ?
    @@ -799,7 +799,7 @@ const UsecaseListComponent = (props) => { style={{ backgroundColor: theme.palette.inputColor, height: 50, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} onChange={(event, newValue) => { //handleWorkflowSelectionUpdate({ target: { value: newValue} }) @@ -856,7 +856,7 @@ const UsecaseListComponent = (props) => { {data.image !== undefined && data.image !== null && data.image.length > 0 ? - {newname} + {newname} : null} Choose {newname} @@ -881,7 +881,7 @@ const UsecaseListComponent = (props) => { { target="_blank" style={{ textDecoration: "none", color: "rgba(255,255,255,0.7)", marginRight: 5, }} > -
    - +
    + {subdata.name} @@ -983,7 +983,7 @@ const UsecaseListComponent = (props) => { height: 350, width: isMobile? null:350, marginTop: isMobile ? 50:null, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, border: "1px solid rgba(255,255,255,0.3)", padding: 5, backgroundColor: theme.palette.backgroundColor, @@ -1113,7 +1113,7 @@ const RadialChart = ({keys, setSelectedCategory}) => { }} content={(data, color) => { return ( -
    +
    {data.x} diff --git a/frontend/src/views/Welcome.jsx b/frontend/src/views/Welcome.jsx index 8c57c3ff..7dd21dea 100644 --- a/frontend/src/views/Welcome.jsx +++ b/frontend/src/views/Welcome.jsx @@ -304,14 +304,14 @@ const Welcome = (props) => { minWidth: isMobile ? null : 275, backgroundColor: theme.palette.surfaceColor, color: "white", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, } const actionObject = { padding: "25px", maxHeight: 280, minHeight: 280, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, } const imageStyle = { @@ -340,7 +340,7 @@ const Welcome = (props) => { const defaultImage = "/images/experienced.png" const experienced_image = userdata !== undefined && userdata !== null && userdata.active_org !== undefined && userdata.active_org.image !== undefined && userdata.active_org.image !== null && userdata.active_org.image !== "" ? userdata.active_org.image : defaultImage return ( -
    +
    {/*
    @@ -354,7 +354,7 @@ const Welcome = (props) => { color="primary" style={{ backgroundColor: theme.palette.platformColor, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, padding: 12, border: "1px solid rgba(255,255,255,0.3)", maxWidth: 500, @@ -434,7 +434,7 @@ const Welcome = (props) => { We will use this information to personalize your automation */}
    -
    +
    { if (isCloud) { ReactGA.event({ @@ -478,7 +478,7 @@ const Welcome = (props) => { navigate("/workflows?message=Skipped intro") }}> - + Experienced diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 00eca27b..6539b0c1 100755 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -1,4 +1,4 @@ -import React, { useEffect, useContext } from "react"; +import React, { useEffect, useContext, memo } from "react"; import ReactDOM from "react-dom" import { makeStyles } from "@mui/styles"; @@ -6,7 +6,7 @@ import { Navigate } from "react-router-dom"; import SecurityFramework from '../components/SecurityFramework.jsx'; import EditWorkflow from "../components/EditWorkflow.jsx" import Priority from "../components/Priority.jsx"; - +import { Context } from "../context/ContextApi.jsx"; import { isMobile } from "react-device-detect" import { @@ -92,37 +92,40 @@ import theme from "../theme.jsx"; const svgSize = 24; const imagesize = 22; -const useStyles = makeStyles((theme) => ({ - datagrid: { - border: 0, - "& .MuiDataGrid-columnsContainer": { - backgroundColor: - theme.palette.type === "light" ? "#fafafa" : theme.palette.inputColor, - }, - "& .MuiDataGrid-iconSeparator": { - display: "none", - }, - "& .MuiDataGrid-colCell, .MuiDataGrid-cell": { - borderRight: `1px solid ${ - theme.palette.type === "light" ? "white" : "#303030" - }`, - }, - "& .MuiDataGrid-columnsContainer, .MuiDataGrid-cell": { - borderBottom: `1px solid ${ - theme.palette.type === "light" ? "#f0f0f0" : "#303030" - }`, - }, - "& .MuiDataGrid-cell": { - color: - theme.palette.type === "light" ? "white" : "rgba(255,255,255,0.65)", - }, - "& .MuiPaginationItem-root, .MuiTablePagination-actions, .MuiTablePagination-caption": - { - borderRadius: 0, - color: "white", - }, - }, -})); +const useStyles = makeStyles(() => { + + return { + datagrid: { + border: 0, + "& .MuiDataGrid-columnsContainer": { + backgroundColor: + theme?.palette?.type === "light" ? "#fafafa" : theme?.palette?.inputColor, + }, + "& .MuiDataGrid-iconSeparator": { + display: "none", + }, + "& .MuiDataGrid-colCell, .MuiDataGrid-cell": { + borderRight: `1px solid ${ + theme?.palette?.type === "light" ? "white" : "#303030" + }`, + }, + "& .MuiDataGrid-columnsContainer, .MuiDataGrid-cell": { + borderBottom: `1px solid ${ + theme?.palette?.type === "light" ? "#f0f0f0" : "#303030" + }`, + }, + "& .MuiDataGrid-cell": { + color: + theme?.palette?.type === "light" ? "white" : "rgba(255,255,255,0.65)", + }, + "& .MuiPaginationItem-root, .MuiTablePagination-actions, .MuiTablePagination-caption": + { + borderRadius: 0, + color: "white", + }, + }, + } +}) // Takes an action in Shuffle and // Returns information about the icon, the color etc to be used @@ -481,9 +484,9 @@ export const validateJson = (showResult) => { }; } } catch (e) { - showResult = showResult.split("'").join('"'); try { + showResult = showResult.split("'").join('"'); if (!showResult.includes("{") && !showResult.includes("[")) { jsonvalid = false; } @@ -590,13 +593,37 @@ export const validateJson = (showResult) => { }; }; +//Custom hook for handling styling of the dropzone +const useDropzoneStyles = () => { + const { leftSideBarOpenByClick } = useContext(Context); + + return { + maxWidth: window.innerWidth > 1366 ? 1366 : isMobile ? "100%" : 1200, + margin: "auto", + padding: 20, + paddingLeft: leftSideBarOpenByClick ? 200 : 0, + transition: "padding-left 0.3s ease", + }; +}; + +//Wrapper for the dropzone component +const DropzoneWrapper = memo(({ onDrop, WorkflowView }) => { + const dropzoneStyles = useDropzoneStyles(); + return ( + + + + ); +}); + + const Workflows = (props) => { const { globalUrl, isLoggedIn, isLoaded, userdata, checkLogin } = props; document.title = "Shuffle - Workflows"; let navigate = useNavigate(); - const classes = useStyles(theme); + const classes = useStyles(theme) const imgSize = 60; const referenceUrl = globalUrl + "/api/v1/hooks/"; @@ -702,7 +729,7 @@ const Workflows = (props) => { console.log("Using filters: ", filters) if (filters.length === 0) { setFilteredWorkflows(workflows); - handleKeysetting(allUsecases, workflows) + handleKeysetting(allUsecases, workflows) return; } @@ -778,9 +805,9 @@ const Workflows = (props) => { } } - console.log("Changing workflow filter, and finding new usecase mappings!") + console.log("Changing workflow filter, and finding new usecase mappings!") if (newWorkflows.length !== workflows.length) { - handleKeysetting(allUsecases, newWorkflows) + handleKeysetting(allUsecases, newWorkflows) setFilteredWorkflows(newWorkflows); } @@ -1194,8 +1221,8 @@ const Workflows = (props) => { return response.json(); }) .then((responseJson) => { - if (responseJson !== undefined) { + if (responseJson !== undefined) { var newarray = [] for (var wfkey in responseJson) { const wf = responseJson[wfkey] @@ -1228,12 +1255,10 @@ const Workflows = (props) => { } } - if (newarray.length > 0) { - try { - localStorage.setItem("workflows", JSON.stringify(newarray)) - } catch (e) { - console.log("Failed to set workflows in localstorage: ", e) - } + try { + localStorage.setItem("workflows", JSON.stringify(newarray)) + } catch (e) { + console.log("Failed to set workflows in localstorage: ", e) } // Ensures the zooming happens only once per load @@ -1285,7 +1310,41 @@ const Workflows = (props) => { .catch((error) => { toast(error.toString()); }); - }; + } + + const findMatches = (category, workflows) => { + category.matches = [] + for (var subcategorykey in category.list) { + var subcategory = category.list[subcategorykey] + subcategory.matches = [] + + for (var workflowkey in workflows) { + const workflow = workflows[workflowkey] + + if (workflow.usecase_ids !== undefined && workflow.usecase_ids !== null) { + for (var usecasekey in workflow.usecase_ids) { + + if (workflow.usecase_ids[usecasekey].toLowerCase() === subcategory.name.toLowerCase()) { + //console.log("Got match: ", workflow.usecase_ids[usecasekey]) + + category.matches.push({ + "workflow": workflow.id, + "category": subcategory.name, + }) + subcategory.matches.push(workflow.id) + break + } + } + } + + if (subcategory.matches.length > 0) { + break + } + } + } + + return category + } const handleKeysetting = (categorydata, workflows) => { if (workflows !== undefined && workflows !== null) { @@ -1297,37 +1356,7 @@ const Workflows = (props) => { continue } - category.matches = [] - - for (var subcategorykey in category.list) { - var subcategory = category.list[subcategorykey] - subcategory.matches = [] - - for (var workflowkey in workflows) { - const workflow = workflows[workflowkey] - - if (workflow.usecase_ids !== undefined && workflow.usecase_ids !== null) { - for (var usecasekey in workflow.usecase_ids) { - - if (workflow.usecase_ids[usecasekey].toLowerCase() === subcategory.name.toLowerCase()) { - //console.log("Got match: ", workflow.usecase_ids[usecasekey]) - - category.matches.push({ - "workflow": workflow.id, - "category": subcategory.name, - }) - subcategory.matches.push(workflow.id) - break - } - } - } - - if (subcategory.matches.length > 0) { - break - } - } - } - + category = findMatches(category, workflows) newcategories.push(category) } @@ -1440,7 +1469,7 @@ const Workflows = (props) => { display: "flex", boxSizing: "border-box", position: "relative", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, backgroundColor: theme.palette.surfaceColor, }; @@ -2229,7 +2258,7 @@ const Workflows = (props) => { } return ( -
    +
    {selectedCategory !== "" ? @@ -2277,7 +2306,7 @@ const Workflows = (props) => { title={
    {data.image !== undefined && data.image !== null && data.image.length > 0 ? - {data.name} + {data.name} : null} Edit '{data.name}' @@ -2493,7 +2522,7 @@ const Workflows = (props) => { {workflowMenuButtons}
    ) : null} - {(data.sharing !== undefined && data.sharing !== null && data.sharing === "form") || (data.input_markdown !== undefined && data.input_markdown !== null && data.input_markdown !== "") ? + {(data.sharing !== undefined && data.sharing !== null && data.sharing === "form") || (data?.form_control?.input_markdown !== undefined && data?.form_control?.input_markdown !== null && data?.form_control?.input_markdown !== "") ?
    { autoHeight density="standard" onSelectionModelChange={(newSelection) => { - //setSelectedWorkflows(newSelection.selectionModel); - console.log(newSelection) - setSelectedWorkflowIndexes(newSelection) }} selectionModel={selectedWorkflowIndexes} @@ -3210,7 +3236,6 @@ const Workflows = (props) => { None {usecases.map((usecase, index) => { - //console.log(usecase) return ( { // ); // } - const WorkflowView = () => { + const WorkflowView = memo(() => { if (workflows.length === 0) { } @@ -3660,9 +3685,14 @@ const Workflows = (props) => { {!isMobile && !hasWorkflows && usecases !== null && usecases !== undefined && usecases.length > 0 ?
    {usecases.map((usecase, index) => { - //console.log(usecase) + if (usecase.name === "5. Verify") { + return null + } + const percentDone = usecase.matches.length > 0 ? parseInt(usecase.matches.length/usecase.list.length*100) : 0 - //console.log("Usecase Matches: ", usecase.matches, ", Percent: ", percentDone) + if (percentDone === 0) { + usecase = findMatches(usecase, workflows) + } return ( { flex: 1, backgroundImage: `linear-gradient(to right, ${usecase.color}, ${usecase.color} ${percentDone}%, transparent ${percentDone}%, transparent 100%)`, backgroundColor: filters.includes(usecase.name.toLowerCase()) ? null : theme.palette.surfaceColor, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, marginRight: index === usecases.length-1 ? 0 : 10, cursor: "pointer", border: `2px solid ${usecase.color}`, @@ -3692,7 +3722,7 @@ const Workflows = (props) => { {usecase.name} - + {usecase.matches.length}/{usecase.list.length} @@ -3715,7 +3745,7 @@ const Workflows = (props) => { minWidth: isMobile ? "100%" : 1024, zIndex: 11, border: "1px solid rgba(255,255,255,0.1)", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, textAlign: "center", overflow: "auto", }} @@ -3734,7 +3764,6 @@ const Workflows = (props) => { } if (data.app_name.toLowerCase() === "integration framework") { - console.log("Skipping: ", data.app_name) return null } @@ -3813,7 +3842,7 @@ const Workflows = (props) => { ) : null} {userdata.priorities !== undefined && userdata.priorities !== null && userdata.priorities.length > 0 && userdata.priorities[0].name.includes("CPU") && userdata.priorities[0].active === true ? -
    @@ -3905,7 +3934,7 @@ const Workflows = (props) => {
    ); - }; + }); const importWorkflowsFromUrl = (url) => { console.log("IMPORT WORKFLOWS FROM ", downloadUrl); @@ -4173,7 +4202,7 @@ const Workflows = (props) => { Setup progress: {isNaN(percentDone) ? 0 : percentDone}% - + Follow these steps to get you up and running! @@ -4251,17 +4280,7 @@ const Workflows = (props) => { */} - 1366 ? 1366 : isMobile ? "100%" : 1200, - margin: "auto", - padding: 20, - paddingLeft: userdata?.support ? 80 : 0 - }} - onDrop={uploadFile} - > - - + {/*modalView*/} {deleteModal} {exportVerifyModal} @@ -4269,7 +4288,7 @@ const Workflows = (props) => { {workflowDownloadModalOpen} {/*!drawerOpen ? -
    +
    { setDrawerOpen(true) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index f6307c98..893fe037 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -782,7 +782,6 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] // image on every Orborus/new worker restart. // Running as coroutine for eventual completeness - //go shuffle.DownloadDockerImageBackend(&http.Client{}, image) // FIXME: With goroutines it got too much trouble of deploying with an older version // Allowing slow startups, as long as it's eventually fast, and uses the same registry as on host. shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image) diff --git a/startup.sh b/startup.sh deleted file mode 100644 index 4fac7596..00000000 --- a/startup.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash - -# Update and install dependencies -apt-get update -apt-get install -y docker.io docker-compose curl git - -# Start and enable Docker -systemctl start docker -systemctl enable docker - -# Clone Shuffle repository -git clone --branch 2.0.0 https://github.com/Shuffle/Shuffle.git -cd Shuffle - -# Setup directories and permissions -mkdir shuffle-database && chmod -R 777 shuffle-database - -# Start services -docker-compose up -d - -# Wait for initial startup -echo "Waiting 30 seconds for initial startup..." -sleep 30 - -# Check for restarting containers -echo "Checking for restarting containers..." -ATTEMPTS=30 -for i in $(seq 1 $ATTEMPTS); do - RESTARTING_CONTAINERS=$(docker ps --filter "status=restarting" --format "{{.Names}}") - if [ -n "$RESTARTING_CONTAINERS" ]; then - echo "The following containers are restarting:" - echo "$RESTARTING_CONTAINERS" - exit 1 - fi - echo "No containers are restarting. Attempt $i/$ATTEMPTS." - sleep 1 -done -echo "No containers were found in a restarting state after $ATTEMPTS checks." - -# Check frontend response -echo "Checking frontend response..." -RESPONSE=$(curl -s http://localhost:3001) -if echo "$RESPONSE" | grep -q "Shuffle"; then - echo "The word 'Shuffle' was found in the response." -else - echo "The word 'Shuffle' was not found in the response." - exit 1 -fi - -# Register user -echo "Attempting to register user..." -MAX_RETRIES=30 -RETRY_INTERVAL=10 -CONTAINER_NAME="shuffle-backend" - -for (( i=1; i<=$MAX_RETRIES; i++ )) -do - STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" 'http://localhost:3001/api/v1/register' \ - -H 'Accept: */*' \ - -H 'Accept-Language: en-US,en;q=0.9' \ - -H 'Connection: keep-alive' \ - -H 'Content-Type: application/json' \ - --data-raw '{"username":"demo@demo.io","password":"supercoolpassword"}') - - if [ "$STATUS_CODE" -eq 200 ]; then - echo "User registration was successful with status code 200." - exit 0 - elif [ "$STATUS_CODE" -ne 502 ]; then - echo "User registration failed with status code $STATUS_CODE." - exit 1 - fi - - echo "Received status code $STATUS_CODE. Retrying in $RETRY_INTERVAL seconds... ($i/$MAX_RETRIES)" - echo "Fetching last 30 lines of logs from container $CONTAINER_NAME..." - docker logs --tail 30 "$CONTAINER_NAME" - echo "Fetching last 30 lines of logs from container shuffle-opensearch..." - docker logs --tail 30 shuffle-opensearch - - sleep $RETRY_INTERVAL -done - -echo "User registration failed after $MAX_RETRIES attempts." -exit 1 \ No newline at end of file