diff --git a/.env b/.env index 298128cc..b7b65b8f 100755 --- a/.env +++ b/.env @@ -40,6 +40,7 @@ BACKEND_HOSTNAME=shuffle-backend BACKEND_PORT=5001 FRONTEND_PORT=3001 FRONTEND_PORT_HTTPS=3443 +AUTH_FOR_ORBORUS = # CHANGE THIS IF YOU WANT GOOD LOCAL EXECUTIONS: OUTER_HOSTNAME=shuffle-backend @@ -51,8 +52,8 @@ HTTP_PROXY= HTTPS_PROXY= SHUFFLE_PASS_WORKER_PROXY=TRUE SHUFFLE_PASS_APP_PROXY=TRUE -SHUFFLE_INTERNAL_HTTP_PROXY=NOPROXY -SHUFFLE_INTERNAL_HTTPS_PROXY=NOPROXY +SHUFFLE_INTERNAL_HTTP_PROXY=noproxy +SHUFFLE_INTERNAL_HTTPS_PROXY=noproxy # Timezone-handler in Orborus, Worker and Apps TZ=Europe/Amsterdam # Used to FIND the containername. cgroup v2: issue 501 @@ -68,6 +69,10 @@ IS_KUBERNETES=false SHUFFLE_BASE_IMAGE_REPOSITORY=frikky #SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-1.4.0" +# For environments using their own docker registry +# where they don't want to update http, subflow and shuffle tools again +SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY=true + # The eth0 interface inside a container corresponds # to the virtual Ethernet interface that connects # the container to the docker0 @@ -97,14 +102,15 @@ SHUFFLE_MAX_EXECUTION_DEPTH= DATASTORE_EMULATOR_HOST=shuffle-database:8000 #SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200 SHUFFLE_OPENSEARCH_URL=https://shuffle-opensearch:9200 -SHUFFLE_OPENSEARCH_USERNAME="admin" -SHUFFLE_OPENSEARCH_PASSWORD="StrongShufflePassword321!" SHUFFLE_OPENSEARCH_CERTIFICATE_FILE= SHUFFLE_OPENSEARCH_APIKEY= SHUFFLE_OPENSEARCH_CLOUDID= SHUFFLE_OPENSEARCH_PROXY= SHUFFLE_OPENSEARCH_INDEX_PREFIX= SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY=true +SHUFFLE_OPENSEARCH_USERNAME="admin" +SHUFFLE_OPENSEARCH_PASSWORD="StrongShufflePassword321!" # In use for the first time setup of OpenSearch + backend of Shuffle +OPENSEARCH_INITIAL_ADMIN_PASSWORD="StrongShufflePassword321!" # In use for the first time setup of OpenSearch #Tenzir related SHUFFLE_TENZIR_URL= diff --git a/.github/workflows/dockerbuild.yaml b/.github/workflows/dockerbuild.yaml index f307ce08..a50fe7be 100644 --- a/.github/workflows/dockerbuild.yaml +++ b/.github/workflows/dockerbuild.yaml @@ -19,23 +19,19 @@ jobs: include: - app: frontend path: frontend - version: 1.4.2 + version: 2.0.0 experimental: true - app: backend path: backend - version: 1.4.2 - experimental: true - - app: app_sdk - path: backend/app_sdk - version: 1.4.2 + version: nightly experimental: true - app: orborus path: functions/onprem/orborus - version: 1.4.2 + version: 2.0.0 experimental: true - app: worker path: functions/onprem/worker - version: 1.4.2 + version: 2.0.0 experimental: true steps: - name: Checkout diff --git a/.github/workflows/nightly-release.yaml b/.github/workflows/nightly-release.yaml new file mode 100644 index 00000000..ef888ced --- /dev/null +++ b/.github/workflows/nightly-release.yaml @@ -0,0 +1,85 @@ +name: Nightly Release +on: + release: + types: [published] + branches: + - 2.0.0 + +jobs: + main: + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + include: + - app: frontend + path: frontend + experimental: true + - app: backend + path: backend + experimental: true + - app: app_sdk + path: backend/app_sdk + experimental: true + - app: orborus + path: functions/onprem/orborus + experimental: true + - app: worker + path: functions/onprem/worker + experimental: true + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set version + id: set_version + run: | + if [[ ${{ github.event_name }} == 'release' ]]; then + echo "VERSION=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT + else + echo "VERSION=nightly-untagged-latest" >> $GITHUB_OUTPUT + fi + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: "amd64,arm64,arm" + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to Ghcr + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Ghcr Build and push + id: docker_build + uses: docker/build-push-action@v4 + env: + BUILDX_NO_DEFAULT_LOAD: true + with: + logout: false + context: ${{ matrix.path }}/ + file: ${{ matrix.path }}/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + tags: | + ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }} + ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }} + frikky/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }} + frikky/shuffle:${{ matrix.app }} + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml deleted file mode 100644 index b3d1927e..00000000 --- a/.github/workflows/release-please.yml +++ /dev/null @@ -1,13 +0,0 @@ -on: - push: - branches: - - launch -name: release-please -jobs: - release-please: - runs-on: ubuntu-latest - steps: - - uses: google-github-actions/release-please-action@v3 - with: - release-type: node - package-name: release-please-action diff --git a/.github/workflows/upload_sdk.yml b/.github/workflows/upload_sdk.yml deleted file mode 100644 index 9faa33e4..00000000 --- a/.github/workflows/upload_sdk.yml +++ /dev/null @@ -1,51 +0,0 @@ -# This is a basic workflow to help you get started with Actions - -name: App SDK upload - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the main branch - push: - branches: [ master, launch ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - # This workflow contains a single job called "build" - build: - # The type of runner that the job will run on - runs-on: ubuntu-latest - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v3 - - - id: 'auth' - name: 'Authenticate to Google Cloud' - uses: 'google-github-actions/auth@v0' - with: - credentials_json: '${{ secrets.SANDBOX_CREDENTIALS }}' - - - id: 'upload_sdk' - name: Cloud Storage Uploader - uses: google-github-actions/upload-cloud-storage@v0.9.0 - with: - path: 'backend/app_sdk/app_base.py' - destination: 'shuffle-sandbox-337810.appspot.com/generated_apps/baseline' - - - id: 'upload_requirement' - name: Cloud Storage Uploader - uses: google-github-actions/upload-cloud-storage@v0.9.0 - with: - path: 'backend/app_sdk/requirements.txt' - destination: 'shuffle-sandbox-337810.appspot.com/generated_apps/baseline' - - - id: 'upload_Dockerfile' - name: Cloud Storage Uploader - uses: google-github-actions/upload-cloud-storage@v0.9.0 - with: - path: 'backend/app_sdk/Dockerfile' - destination: 'shuffle-sandbox-337810.appspot.com/generated_apps/baseline' diff --git a/README.md b/README.md index 668ebed7..b90944bf 100755 --- a/README.md +++ b/README.md @@ -4,15 +4,22 @@ Shuffle Automation +[![CodeQL](https://github.com/Shuffle/Shuffle/actions/workflows/codeql-analysis.yml/badge.svg?branch=launch)](https://github.com/Shuffle/Shuffle/actions/workflows/codeql-analysis.yml) +[![Autobuild](https://github.com/Shuffle/Shuffle/actions/workflows/dockerbuild.yaml/badge.svg?branch=launch)](https://github.com/Shuffle/Shuffle/actions/workflows/dockerbuild.yaml) +[![Deploy to AWS](https://d1.awsstatic.com/cloudformation-deploy-to-aws-button.png)](https://console.aws.amazon.com/cloudformation/home?#/stacks/new?stackName=Shuffle-Instance&templateURL=https://shuffle-public-amis.s3.eu-north-1.amazonaws.com/template.yaml) +

[Shuffle](https://shuffler.io) is an open source automation platform, built for and by the security professionals. Security operations is complex, but it doesn't have to be. Built to work well with MSSP's and other service providers in mind. +[ Get training ](https://shuffler.io/training) [_Key Features_](https://shuffler.io/docs/features) — [_Community & Support_](https://discord.gg/B2CBzUm) — [ Get training ](https://shuffler.io/training) - [_Documentation_](https://shuffler.io/docs) — -[_Getting Started_](https://shuffler.io/docs/getting_started) +[_Getting Started_](https://shuffler.io/docs/getting_started) — +[_Development_](https://github.com/shuffle/Shuffle/blob/master/.github/CONTRIBUTING.md) +[ Set up a demo call ](https://shuffler.io/contact) Follow us on Twitter at [@shuffleio](https://twitter.com/shuffleio). diff --git a/backend/Dockerfile b/backend/Dockerfile index e34f9c11..c591da21 100755 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -11,7 +11,7 @@ ADD ./go-app/docker.go /app ADD ./go-app/go.mod /app # Required files for code generation -ADD ./app_sdk/app_base.py /app_sdk +RUN wget -O /app_sdk/app_base.py https://raw.githubusercontent.com/Shuffle/app_sdk/refs/heads/main/shuffle_sdk/shuffle_sdk.py ADD ./app_gen /app_gen RUN go get -v diff --git a/backend/app_sdk/Dockerfile b/backend/app_sdk/Dockerfile deleted file mode 100755 index d3709032..00000000 --- a/backend/app_sdk/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -#FROM python:3.9.1-alpine as base -FROM python:3.10.0-alpine as base -#FROM python:3.11.3-alpine as base - -FROM base as builder -RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev tzdata coreutils - -RUN mkdir /install -WORKDIR /install - -FROM base - -#--no-cache -RUN apk update && apk add --update tzdata libmagic alpine-sdk libffi libffi-dev musl-dev openssl-dev coreutils - -COPY --from=builder /install /usr/local -COPY requirements.txt /requirements.txt -RUN pip3 install -r /requirements.txt - -COPY __init__.py /app/walkoff_app_sdk/__init__.py -COPY app_base.py /app/walkoff_app_sdk/app_base.py diff --git a/backend/app_sdk/Dockerfile_alpine_grpc b/backend/app_sdk/Dockerfile_alpine_grpc deleted file mode 100644 index dedc95b1..00000000 --- a/backend/app_sdk/Dockerfile_alpine_grpc +++ /dev/null @@ -1,42 +0,0 @@ -FROM python:3.10.0-alpine as base - -FROM base as builder -RUN apk --no-cache add --update \ - alpine-sdk \ - build-base \ - g++ \ - gcc \ - libffi \ - libffi-dev \ - libstdc++ \ - linux-headers \ - musl-dev \ - openssl-dev \ - tzdata \ - coreutils - -RUN pip install --upgrade pip && \ - pip install --prefix="/install" --no-cache-dir grpcio grpcio-tools && \ - apk del --purge \ - g++ \ - gcc \ - musl-dev \ - libffi-dev \ - libstdc++ \ - build-base \ - linux-headers - -RUN mkdir -p /install -WORKDIR /install - -FROM base - -#--no-cache -RUN apk update && apk add --update tzdata libmagic alpine-sdk libffi libffi-dev musl-dev openssl-dev coreutils - -COPY --from=builder /install /usr/local -COPY requirements.txt /requirements.txt -RUN pip3 install -r /requirements.txt - -COPY __init__.py /app/walkoff_app_sdk/__init__.py -COPY app_base.py /app/walkoff_app_sdk/app_base.py diff --git a/backend/app_sdk/Dockerfile_blackarch b/backend/app_sdk/Dockerfile_blackarch deleted file mode 100755 index e7469166..00000000 --- a/backend/app_sdk/Dockerfile_blackarch +++ /dev/null @@ -1,19 +0,0 @@ -FROM blackarchlinux/blackarch as base - -FROM base as builder - -RUN /bin/pacman -Syu --noconfirm - -RUN /bin/pacman -Sy --noconfirm base-devel libffi musl openssl python python-pip -y - -RUN mkdir /install -WORKDIR /install - -COPY requirements.txt /requirements.txt -RUN pip install --prefix="/install" -r /requirements.txt - -FROM base - -COPY --from=builder /install /usr/local -COPY __init__.py /app/walkoff_app_sdk/__init__.py -COPY app_base.py /app/walkoff_app_sdk/app_base.py diff --git a/backend/app_sdk/Dockerfile_kali b/backend/app_sdk/Dockerfile_kali deleted file mode 100755 index af7fd24f..00000000 --- a/backend/app_sdk/Dockerfile_kali +++ /dev/null @@ -1,19 +0,0 @@ -FROM kalilinux/kali-rolling as base - -FROM base as builder - -RUN apt-get update -RUN apt-get dist-upgrade -y -RUN apt install build-essential libffi-dev musl-dev openssl python3 python3-pip -y - -RUN mkdir /install -WORKDIR /install - -COPY requirements.txt /requirements.txt -RUN pip install --prefix="/install" -r /requirements.txt - -FROM base - -COPY --from=builder /install /usr/local -COPY __init__.py /app/walkoff_app_sdk/__init__.py -COPY app_base.py /app/walkoff_app_sdk/app_base.py diff --git a/backend/app_sdk/Dockerfile_ubuntu b/backend/app_sdk/Dockerfile_ubuntu deleted file mode 100644 index 3f34d4bd..00000000 --- a/backend/app_sdk/Dockerfile_ubuntu +++ /dev/null @@ -1,22 +0,0 @@ -FROM ubuntu as base - -FROM base as builder - -RUN apt-get update -RUN apt-get dist-upgrade -y -RUN apt install build-essential libffi-dev musl-dev openssl python3 python3-pip -y - -RUN mkdir /install -WORKDIR /install - -COPY requirements.txt /requirements.txt -RUN pip install --prefix="/install" -r /requirements.txt - -FROM base -RUN apt-get update -RUN apt-get dist-upgrade -y -RUN apt install build-essential libffi-dev musl-dev openssl python3 python3-pip -y - -COPY --from=builder /install /usr/local -COPY __init__.py /app/walkoff_app_sdk/__init__.py -COPY app_base.py /app/walkoff_app_sdk/app_base.py diff --git a/backend/app_sdk/LICENSE b/backend/app_sdk/LICENSE deleted file mode 100755 index ce11f6f3..00000000 --- a/backend/app_sdk/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2020 Frikkylikeme - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/backend/app_sdk/README.md b/backend/app_sdk/README.md old mode 100755 new mode 100644 index 478fb394..1e4dc72e --- a/backend/app_sdk/README.md +++ b/backend/app_sdk/README.md @@ -1,22 +1,2 @@ -# app_sdk.py -This is the SDK used for apps to behave like they should. - -## If you want to update apps.. PS: downloads from docker hub do overrides.. :) -1. Write your code & check if runtime works -2. Build app_base image -3. docker rm $(docker ps -aq) # Remove all stopped containers -4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...) -5. Rebuild the Docker image (click load in GUI?) - -## Cloud updates -1. Go to shuffle cloud on GCP -2. Go to Cloud Storage -3. Find shuffler.appspot.com -4. Navigate to generated_apps/baseline -5. Update SDK there. This will make all new apps run with the new SDK - -## Cloud app force-updates -1. Run the "stitcher.go" program in the public shuffle-shared repository. - -# LICENSE -Everything in here is MIT, not AGPLv3 as indicated by the license. +## CHANGES +In November 2024, we moved this to its own repistory: https://github.com/shuffle/app_sdk diff --git a/backend/app_sdk/__init__.py b/backend/app_sdk/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py deleted file mode 100755 index 6ef571aa..00000000 --- a/backend/app_sdk/app_base.py +++ /dev/null @@ -1,4143 +0,0 @@ -import os -import ast -import sys -import re -import copy -import time -import base64 -import json -import random -import liquid -import logging -import urllib3 -import hashlib -import zipfile -import asyncio -import requests -import http.client -import urllib.parse -import jinja2 -import datetime -import dateutil - -import threading -import concurrent.futures - -from io import StringIO as StringBuffer, BytesIO -from liquid import Liquid, defaults -from requests.packages.urllib3.exceptions import InsecureRequestWarning - -# Suppress the warning -requests.packages.urllib3.disable_warnings(InsecureRequestWarning) - - -runtime = os.getenv("SHUFFLE_SWARM_CONFIG", "") - -### -### -### -#### Filters for liquidpy -### -### -### - -defaults.MODE = 'wild' -defaults.FROM_FILE = False -from liquid.filters.manager import FilterManager -from liquid.filters.standard import standard_filter_manager - -shuffle_filters = FilterManager() -for key, value in standard_filter_manager.filters.items(): - shuffle_filters.filters[key] = value - -#@shuffle_filters.register -#def plus(a, b): -# try: -# a = int(a) -# except: -# a = 0 -# -# try: -# b = int(b) -# except: -# b = 0 -# -# return standard_filter_manager.filters["plus"](a, b) -# -#@shuffle_filters.register -#def minus(a, b): -# a = int(a) -# b = int(b) -# return standard_filter_manager.filters["minus"](a, b) -# -#@shuffle_filters.register -#def multiply(a, b): -# a = int(a) -# b = int(b) -# return standard_filter_manager.filters["multiply"](a, b) -# -#@shuffle_filters.register -#def divide(a, b): -# a = int(a) -# b = int(b) -# return standard_filter_manager.filters["divide"](a, b) - -@shuffle_filters.register -def md5(a): - a = str(a) - return hashlib.md5(a.encode('utf-8')).hexdigest() - -@shuffle_filters.register -def sha256(a): - a = str(a) - return hashlib.sha256(str(a).encode("utf-8")).hexdigest() - -@shuffle_filters.register -def md5_base64(a): - a = str(a) - foundhash = hashlib.md5(a.encode('utf-8')).hexdigest() - return base64.b64encode(foundhash.encode('utf-8')) - -@shuffle_filters.register -def base64_encode(a): - a = str(a) - - try: - return base64.b64encode(a.encode('utf-8')).decode() - except: - return base64.b64encode(a).decode() - -@shuffle_filters.register -def base64_decode(a): - a = str(a) - - if "-" in a: - a = a.replace("-", "+", -1) - - if "_" in a: - a = a.replace("_", "/", -1) - - # Fix padding - if len(a) % 4 != 0: - a += "=" * (4 - len(a) % 4) - - try: - return base64.b64decode(a).decode("unicode_escape") - except: - try: - return base64.b64decode(a).decode() - except: - return base64.b64decode(a) - -@shuffle_filters.register -def json_parse(a): - return json.loads(str(a)) - -@shuffle_filters.register -def as_object(a): - return json.loads(str(a)) - -@shuffle_filters.register -def ast_eval(a): - return ast.literal_eval(str(a)) - -@shuffle_filters.register -def escape_string(a): - a = str(a) - return a.replace("\\\'", "\'", -1).replace("\\\"", "\"", -1).replace("'", "\\\'", -1).replace("\"", "\\\"", -1) - -@shuffle_filters.register -def json_escape(a): - a = str(a) - return a.replace("\\\'", "\'", -1).replace("\\\"", "\"", -1).replace("'", "\\\\\'", -1).replace("\"", "\\\\\"", -1) - -@shuffle_filters.register -def escape_json(a): - a = str(a) - return a.replace("\\\'", "\'", -1).replace("\\\"", "\"", -1).replace("'", "\\\\\'", -1).replace("\"", "\\\\\"", -1) - -# By default using json escape to add all backslashes -@shuffle_filters.register -def escape(a): - a = str(a) - return json_escape(a) - - -@shuffle_filters.register -def neat_json(a): - try: - a = json.loads(a) - except: - pass - - return json.dumps(a, indent=4, sort_keys=True) - -@shuffle_filters.register -def flatten(a): - a = list(a) - - flat_list = [a for xs in a for a in xs] - return flat_list - -@shuffle_filters.register -def last(a): - try: - a = json.loads(a) - except: - pass - - if len(a) == 0: - return "" - - return a[-1] - -@shuffle_filters.register -def first(a): - try: - a = json.loads(a) - except: - pass - - if len(a) == 0: - return "" - - return a[0] - - -@shuffle_filters.register -def csv_parse(a): - a = str(a) - splitdata = a.split("\n") - columns = [] - if len(splitdata) > 1: - columns = splitdata[0].split(",") - else: - return a.split("\n") - - allitems = [] - cnt = -1 - for item in splitdata[1:]: - cnt += 1 - commasplit = item.split(",") - - fullitem = {} - fullitem["unparsed"] = item - fullitem["index"] = cnt - fullitem["parsed"] = {} - if len(columns) != len(commasplit): - - if len(commasplit) > len(columns): - diff = len(commasplit)-len(columns) - - try: - commasplit = commasplit[0:len(commasplit)-diff] - except: - pass - else: - for item in range(0, len(columns)-len(commasplit)): - commasplit.append("") - - for key in range(len(columns)): - try: - fullitem["parsed"][columns[key]] = commasplit[key] - except: - continue - - allitems.append(fullitem) - - try: - return json.dumps(allitems) - except: - return allitems - -@shuffle_filters.register -def parse_csv(a): - return csv_parse(a) - -@shuffle_filters.register -def format_csv(a): - return csv_parse(a) - -@shuffle_filters.register -def csv_format(a): - return csv_parse(a)@standard_filter_manager.register - -@shuffle_filters.register -def split(base, sep): - if not sep: - try: - return json.dumps(list(base)) - except: - return list(base) - - try: - return json.dumps(base.split(sep)) - except: - return base.split(sep) - - -### -### -### -### -### -### -### - - -class AppBase: - __version__ = None - app_name = None - - def __init__(self, redis=None, logger=None, console_logger=None):#, docker_client=None): - self.logger = logger if logger is not None else logging.getLogger("AppBaseLogger") - - if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true": - self.log_capture_string = StringBuffer() - ch = logging.StreamHandler(self.log_capture_string) - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - ch.setFormatter(formatter) - logger.addHandler(ch) - - self.redis=redis - self.console_logger = logger if logger is not None else logging.getLogger("AppBaseLogger") - - # apikey is for the user / org - # authorization is for the specific workflow - - self.url = os.getenv("CALLBACK_URL", "https://shuffler.io") - self.base_url = os.getenv("BASE_URL", "https://shuffler.io") - self.action = os.getenv("ACTION", "") - self.original_action = os.getenv("ACTION", "") - self.authorization = os.getenv("AUTHORIZATION", "") - self.current_execution_id = os.getenv("EXECUTIONID", "") - self.full_execution = os.getenv("FULL_EXECUTION", "") - self.result_wrapper_count = 0 - - # Make start time with milliseconds - self.start_time = int(time.time_ns()) - - self.action_result = { - "action": self.action, - "authorization": self.authorization, - "execution_id": self.current_execution_id, - "result": f"", - "started_at": self.start_time, - "status": "", - "completed_at": int(time.time_ns()), - } - - self.proxy_config = { - "http": os.getenv("HTTP_PROXY", ""), - "https": os.getenv("HTTPS_PROXY", ""), - "no_proxy": os.getenv("NO_PROXY", ""), - } - - if len(os.getenv("SHUFFLE_INTERNAL_HTTP_PROXY", "")) > 0: - self.proxy_config["http"] = os.getenv("SHUFFLE_INTERNAL_HTTP_PROXY", "") - - if len(os.getenv("SHUFFLE_INTERNAL_HTTPS_PROXY", "")) > 0: - self.proxy_config["https"] = os.getenv("SHUFFLE_INTERNAL_HTTP_PROXY", "") - - if len(os.getenv("SHUFFLE_INTERNAL_NO_PROXY", "")) > 0: - self.proxy_config["no_proxy"] = os.getenv("SHUFFLE_INTERNAL_NO_PROXY", "") - - try: - if self.proxy_config["http"].lower() == "noproxy": - self.proxy_config["http"] = "" - if self.proxy_config["https"].lower() == "noproxy": - self.proxy_config["https"] = "" - except Exception as e: - self.logger.info(f"[WARNING] Failed setting proxy config: {e}. NOT important if running apps with webserver. This is NOT critical.") - - - if isinstance(self.action, str): - try: - self.action = json.loads(self.action) - self.original_action = json.loads(self.action) - except Exception as e: - pass - - if len(self.base_url) == 0: - self.base_url = self.url - - - self.local_storage = [] - - # Checks output for whether it should be automatically parsed or not - def run_magic_parser(self, input_data): - if not isinstance(input_data, str): - return input_data - - # Don't touch existing JSON/lists - if (input_data.startswith("[") and input_data.endswith("]")) or (input_data.startswith("{") and input_data.endswith("}")): - return input_data - - if len(input_data) < 3: - return input_data - - # Don't touch large data. - if len(input_data) > 100000: - return input_data - - if not "\n" in input_data and not "," in input_data: - return input_data - - new_input = input_data - try: - #new_input.strip() - new_input = input_data.split() - new_return = [] - - index = 0 - for item in new_input: - splititem = "," - if ", " in item: - splititem = ", " - elif "," in item: - splititem = "," - else: - new_return.append(item) - - index += 1 - continue - - for subitem in item.split(splititem): - new_return.insert(index, subitem) - - index += 1 - - # Prevent large data or infinite loops - if index > 10000: - #self.logger.info(f"[DEBUG] Infinite loop. Returning default data.") - return input_data - - fixed_return = [] - for item in new_return: - if not item: - continue - - if not isinstance(item, str): - fixed_return.append(item) - continue - - if item.endswith(","): - item = item[0:-1] - - fixed_return.append(item) - - new_input = fixed_return - except Exception as e: - # Not used anymore - #self.logger.info(f"[ERROR] Failed to run magic parser (2): {e}") - return input_data - - try: - new_input = input_data.split() - except Exception as e: - self.logger.info(f"[ERROR] Failed to run parser during split (1): {e}") - return input_data - - # Won't ever touch this one? - if isinstance(new_input, list) or isinstance(new_input, object): - try: - return json.dumps(new_input) - except Exception as e: - self.logger.info(f"[ERROR] Failed to run magic parser (3): {e}") - - return new_input - - def prepare_response(self, request): - try: - parsedheaders = {} - for key, value in request.headers.items(): - parsedheaders[key] = value - - cookies = {} - if request.cookies: - for key, value in request.cookies.items(): - cookies[key] = value - - - jsondata = request.text - try: - jsondata = json.loads(jsondata) - except: - pass - - return json.dumps({ - "success": True, - "status": request.status_code, - "url": request.url, - "body": jsondata, - "headers": parsedheaders, - "cookies":cookies, - }) - except Exception as e: - return request.text - - # Fixes pattern issues in json/liquid based on input and supplied patterns - def patternfix_string(self, liquiddata, patterns, regex_patterns, inputtype="liquid"): - if not inputtype or inputtype == "liquid": - if "{{" not in liquiddata or "}}" not in liquiddata: - return liquiddata - elif inputtype == "json": - liquiddata = liquiddata.strip() - - # Validating if it looks like json or not - if liquiddata[0] == "{" and liquiddata[len(liquiddata)-1] == "}": - pass - else: - if liquiddata[0] == "[" and liquiddata[len(liquiddata)-1] == "]": - pass - else: - return liquiddata - - # If it's already json, don't touch it - try: - json.loads(liquiddata) - return liquiddata - except Exception as e: - pass - else: - print("No replace handler for %s" % inputtype) - return liquiddata - - skipkeys = [" "] - newoutput = liquiddata[:] - for pattern in patterns: - keylocations = [] - parsedvalue = "" - record = False - index = -1 - for key in liquiddata: - - # Return instant if possible - if inputtype == "json": - try: - json.loads(newoutput) - return newoutput - except: - pass - - index += 1 - if not key: - if record: - keylocations.append(index) - parsedvalue += key - - continue - - if key in skipkeys: - if record: - keylocations.append(index) - parsedvalue += key - - continue - - if key == pattern[0] and not record: - record = True - - if key not in pattern: - keylocations = [] - parsedvalue = "" - record = False - - if record: - keylocations.append(index) - parsedvalue += key - - if len(parsedvalue) == 0: - continue - - evaluated_value = parsedvalue[:] - for skipkey in skipkeys: - evaluated_value = "".join(evaluated_value.split(skipkey)) - - if evaluated_value == pattern: - #print("Found matching: %s (%s)" % (parsedvalue, keylocations)) - #print("Should replace with: %s" % patterns[pattern]) - - newoutput = newoutput.replace(parsedvalue, patterns[pattern], -1) - - # Return instant if possible - if inputtype == "json": - try: - json.loads(newoutput) - return newoutput - except: - pass - - - for pattern in regex_patterns: - newlines = [] - for line in newoutput.split("\n"): - replaced_line = re.sub(pattern, regex_patterns[pattern], line) - newlines.append(replaced_line) - - newoutput = "\n".join(newlines) - - # Return instant if possible - if inputtype == "json": - try: - json.loads(newoutput) - return newoutput - except: - pass - - # Dont return json properly unless actually json - if inputtype == "json": - try: - json.loads(newoutput) - return newoutput - except: - # Returns original if json fixing didn't work - return liquiddata - - return newoutput - - # FIXME: Add more info like logs in here. - # Docker logs: https://forums.docker.com/t/docker-logs-inside-the-docker-container/68190/2 - def send_result(self, action_result, headers, stream_path): - if action_result["status"] == "EXECUTING": - action_result["status"] = "FAILURE" - - try: - if self.action["run_magic_output"] == True: - action_result["result"] = self.run_magic_parser(action_result["result"]) - except KeyError as e: - pass - except Exception as e: - pass - - # Try it with some magic - - action_result["completed_at"] = int(time.time_ns()) - #if isinstance(action_result, - - # FIXME: Add cleanup of parameters to not send to frontend here - params = {} - - # I wonder if this actually works - url = "%s%s" % (self.base_url, stream_path) - - try: - log_contents = "disabled: add env SHUFFLE_LOGS_DISABLED=true to Orborus to re-enable logs for apps. Can not be enabled natively in Cloud except in Hybrid mode." - if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true": - log_contents = self.log_capture_string.getvalue() - - if len(action_result["action"]["parameters"]) == 0: - action_result["action"]["parameters"] = [] - - param_found = False - for param in action_result["action"]["parameters"]: - if param["name"] == "shuffle_action_logs": - param_found = True - break - - if not param_found: - action_result["action"]["parameters"].append({ - "name": "shuffle_action_logs", - "value": log_contents, - }) - - except Exception as e: - pass - - try: - finished = False - ret = {} - for i in range (0, 10): - # Random sleeptime between 0 and 1 second, with 0.1 increments - sleeptime = float(random.randint(0, 10) / 10) - - try: - ret = requests.post(url, headers=headers, json=action_result, timeout=10, verify=False, proxies=self.proxy_config) - - #self.logger.info(f"""[DEBUG] Successful result request: Status= {ret.status_code} (break on 200/201) & Action status: {action_result["status"]}. Response= {ret.text}""") - if ret.status_code == 200 or ret.status_code == 201: - finished = True - break - else: - # FIXME: Add a checker for 403, and Proxy logs failing - self.logger.info(f"[ERROR] Bad resp ({ret.status_code}) in send_result for url '{url}'") - time.sleep(sleeptime) - - - # Proxyerrror - except requests.exceptions.ProxyError as e: - self.proxy_config = {} - continue - - except requests.exceptions.RequestException as e: - time.sleep(sleeptime) - - # Check if we have a read timeout. If we do, exit as we most likely sent the result without getting a good result - if "Read timed out" in str(e): - self.logger.warning(f"[WARNING] Read timed out: {e}") - finished = True - break - - if "Max retries exceeded with url" in str(e): - self.logger.warning(f"[WARNING] Max retries exceeded with url: {e}") - finished = True - break - - #time.sleep(5) - continue - except TimeoutError as e: - time.sleep(sleeptime) - - #time.sleep(5) - continue - except requests.exceptions.ConnectionError as e: - time.sleep(sleeptime) - - #time.sleep(5) - continue - except http.client.RemoteDisconnected as e: - time.sleep(sleeptime) - - #time.sleep(5) - continue - except urllib3.exceptions.ProtocolError as e: - time.sleep(0.1) - - #time.sleep(5) - continue - - #time.sleep(5) - - if not finished: - # Not sure why this would work tho :) - action_result["status"] = "FAILURE" - action_result["result"] = json.dumps({"success": False, "reason": "POST error: Failed connecting to %s over 10 retries to the backend" % url}) - self.send_result(action_result, {"Content-Type": "application/json", "Authorization": "Bearer %s" % self.authorization}, "/api/v1/streams") - return - - except requests.exceptions.ConnectionError as e: - #self.logger.info(f"[DEBUG] Unexpected ConnectionError happened: {e}") - pass - except TypeError as e: - action_result["status"] = "FAILURE" - action_result["result"] = json.dumps({"success": False, "reason": "Typeerror when sending to backend URL %s" % url}) - - ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False, proxies=self.proxy_config) - #self.logger.info(f"[DEBUG] Result: {ret.status_code}") - #if ret.status_code != 200: - # pr - - #self.logger.info(f"[DEBUG] TypeError request: Status= {ret.status_code} & Response= {ret.text}") - except http.client.RemoteDisconnected as e: - self.logger.info(f"[DEBUG] Expected Remotedisconnect happened: {e}") - except urllib3.exceptions.ProtocolError as e: - self.logger.info(f"[DEBUG] Expected ProtocolError happened: {e}") - - - # FIXME: Re-enable data flushing otherwise we'll overload it all - # Or nah? - if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true": - try: - self.log_capture_string.flush() - #self.log_capture_string.close() - #pass - except Exception as e: - pass - - #async def cartesian_product(self, L): - def cartesian_product(self, L): - if L: - #return {(a, ) + b for a in L[0] for b in await self.cartesian_product(L[1:])} - return {(a, ) + b for a in L[0] for b in self.cartesian_product(L[1:])} - else: - return {()} - - # Handles unique fields by negoiating with the backend - def validate_unique_fields(self, params): - #self.logger.info("IN THE UNIQUE FIELDS PLACE!") - - newlist = [params] - if isinstance(params, list): - #self.logger.info("ITS A LIST!") - newlist = params - - # FIXME: Also handle MULTI PARAM - values = [] - param_names = [] - all_values = {} - index = 0 - for outerparam in newlist: - - #self.logger.info(f"INNERTYPE: {type(outerparam)}") - #self.logger.info(f"HANDLING PARAM {key}") - param_value = "" - for key, value in outerparam.items(): - #self.logger.info("KEY: %s" % key) - #value = params[key] - for param in self.action["parameters"]: - try: - if param["name"] == key and param["unique_toggled"]: - self.logger.info(f"[DEBUG] FOUND: {key} with param {param}!") - if isinstance(value, dict) or isinstance(value, list): - try: - value = json.dumps(value) - except json.decoder.JSONDecodeError as e: - self.logger.info(f"[WARNING] Error in json decode for param {value}: {e}") - continue - elif isinstance(value, int) or isinstance(value, float): - value = str(value) - elif value == False: - value = "False" - elif value == True: - value = "True" - - self.logger.info(f"[DEBUG] VALUE APPEND: {value}") - param_value += value - if param["name"] not in param_names: - param_names.append(param["name"]) - - except (KeyError, NameError) as e: - self.logger.info(f"""Key/NameError in param handler for {param["name"]}: {e}""") - - #self.logger.info(f"[DEBUG] OUTER VALUE: {param_value}") - if len(param_value) > 0: - md5 = hashlib.md5(param_value.encode('utf-8')).hexdigest() - values.append(md5) - all_values[md5] = { - "index": index, - } - - index += 1 - - # When in here, it means it should be unique - # Should this be done by the backend? E.g. ask it if the value is valid? - # 1. Check if it's unique towards key:value store in org for action - # 2. Check if COMBINATION is unique towards key:value store of action for org - # 3. Have a workflow configuration for unique ID's in unison or per field? E.g. if toggled, then send a hash of all fields together alphabetically, but if not, send one field at a time - - # org_id = full_execution["workflow"]["execution_org"]["id"] - - # USE ARRAY? - - new_params = [] - if len(values) > 0: - org_id = self.full_execution["workflow"]["execution_org"]["id"] - data = { - "append": True, - "workflow_check": False, - "authorization": self.authorization, - "execution_ref": self.current_execution_id, - "org_id": org_id, - "values": [{ - "app": self.action["app_name"], - "action": self.action["name"], - "parameternames": param_names, - "parametervalues": values, - }] - } - - #self.logger.info(f"DATA: {data}") - # 1594869a676630b397bc34f7dc0951a3 - - url = f"{self.url}/api/v1/orgs/{org_id}/validate_app_values" - ret = requests.post(url, json=data, verify=False, proxies=self.proxy_config) - if ret.status_code == 200: - json_value = ret.json() - if len(json_value["found"]) > 0: - modifier = 0 - for item in json_value["found"]: - self.logger.info(f"Should remove {item}") - - try: - self.logger.info(f"FOUND: {all_values[item]}") - self.logger.info(f"SHOULD REMOVE INDEX: {all_values[item]['index']}") - - try: - newlist.pop(all_values[item]["index"]-modifier) - modifier += 1 - except IndexError as e: - self.logger.info(f"Error popping value from array: {e}") - except (NameError, KeyError) as e: - self.logger.info(f"Failed removal: {e}") - - - #return False - else: - self.logger.info("None of the items were found!") - return newlist - else: - self.logger.info(f"[WARNING] Failed checking values with status code {ret.status_code}!") - - #return True - return newlist - - # Returns a list of all the executions to be done in the inner loop - # FIXME: Doesn't take into account whether you actually WANT to loop or not - # Check if the last part of the value is #? - #async def get_param_multipliers(self, baseparams): - def get_param_multipliers(self, baseparams): - # Example: - # {'call': ['hello', 'hello4'], 'call2': ['hello2', 'hello3'], 'call3': '1'} - # - # Should become this because of pairs (all same-length arrays, PROBABLY indicates same source node's values. - # [ - # {'call': 'hello', 'call2': 'hello2', 'call3': '1'}, - # {'call': 'hello4', 'call2': 'hello3', 'call3': '1'} - # ] - # - # ---------------------------------------------------------------------- - # Example2: - # {'call': ['hello'], 'call2': ['hello2', 'hello3'], 'call3': '1'} - # - # Should become this because NOT pairs/triplets: - # [ - # {'call': 'hello', 'call2': 'hello2', 'call3': '1'}, - # {'call': 'hello', 'call2': 'hello3', 'call3': '1'} - # ] - # - # ---------------------------------------------------------------------- - # Example3: - # {'call': ['hello', 'hello2'], 'call2': ['hello3', 'hello4', 'hello5'], 'call3': '1'} - # - # Should become this because arrays are not same length, aka no pairs/triplets. This is the multiplier effect. 2x3 arrays = 6 iterations - # [ - # {'call': 'hello', 'call2': 'hello3', 'call3': '1'}, - # {'call': 'hello', 'call2': 'hello4', 'call3': '1'}, - # {'call': 'hello', 'call2': 'hello5', 'call3': '1'}, - # {'call': 'hello2', 'call2': 'hello3', 'call3': '1'}, - # {'call': 'hello2', 'call2': 'hello4', 'call3': '1'}, - # {'call': 'hello2', 'call2': 'hello5', 'call3': '1'} - # ] - # To achieve this, we'll do this: - # 1. For the first array, take the total amount(y) (2x3=6) and divide it by the current array (x): 2. x/y = 3. This means do 3 of each value - # 2. For the second array, take the total amount(y) (2x3=6) and divide it by the current array (x): 3. x/y = 2. - # 3. What does the 3rd array do? Same, but ehhh? - # - # Example4: - # What if there are multiple loops inside a single item? - # - # - - paramlist = [] - listitems = [] - listlengths = [] - all_lists = [] - all_list_keys = [] - - #check_value = "$Filter_list_testing.wrapper.#.tmp" - #self.action = action - - loopnames = [] - self.logger.info(f"Baseparams to check: {baseparams}") - for key, value in baseparams.items(): - check_value = "" - for param in self.original_action["parameters"]: - if param["name"] == key: - check_value = param["value"] - # self.result_wrapper_count = 0 - - octothorpe_count = param["value"].count(".#") - if octothorpe_count > self.result_wrapper_count: - self.result_wrapper_count = octothorpe_count - self.logger.info("[INFO] NEW OCTOTHORPE WRAPPER: %d" % octothorpe_count) - - - # This whole thing is hard. - # item = [{"data": "1.2.3.4", "dataType": "ip"}] - # $item = DONT loop items. - # $item.# = Loop items - # $item.#.data = Loop items - # With a single item, this is fine. - - # item = [{"list": [{"data": "1.2.3.4", "dataType": "ip"}]}] - # $item = DONT loop items - # $item.# = Loop items - # $item.#.list = DONT loop items - # $item.#.list.# = Loop items - # $item.#.list.#.data = Loop items - # If the item itself is a list.. hmm - - # FIXME: Check the above, and fix so that nested looped items can be - # Skipped if wanted - - #self.logger.info("\nCHECK: %s" % check_value) - #try: - # values = parameter["value_replace"] - # if values != None: - # self.logger.info(values) - # for val in values: - # self.logger.info(val) - #except: - # pass - - should_merge = False - if "#" in check_value: - should_merge = True - - # Specific for OpenAPI body replacement - #self.logger.info("\n\n\nDOING STUFF BELOW HERE") - if not should_merge: - for parameter in self.original_action["parameters"]: - if parameter["name"] == key: - #self.logger.info("CHECKING BODY FOR VALUE REPLACE DATA!") - try: - values = parameter["value_replace"] - if values != None: - self.logger.info(values) - for val in values: - if "#" in val["value"]: - should_merge = True - break - except: - pass - - #self.logger.info(f"VALUE LENGTH: {len(value)}") - if isinstance(value, list): - #subvalue = [] - # Override for single vs multi items - #if len(value) > 0: - # if isinstance(value[0], list) and len(value[0]) == 1: - # subvalue = value[0] - - # subvalue = value[0] - - - if len(value) <= 1: - # FIXME: This broke some shit for a single item fml - # Necessary as override again :( - if len(value) == 1: - baseparams[key] = value[0] - - #if "#" in check_value: - # should_merge = True - else: - #if len(value) > 1: - if not should_merge: - self.logger.info("[DEBUG] Adding WITHOUT looping list") - else: - if len(value) not in listlengths: - listlengths.append(len(value)) - #listlength - - listitems.append( - { - key: len(value) - } - ) - - all_list_keys.append(key) - all_lists.append(baseparams[key]) - else: - #self.logger.info(f"{value} is not a list") - pass - - self.logger.info("[DEBUG] Listlengths: %s - listitems: %d" % (listlengths, len(listitems))) - #if len(listitems) == 0: - if len(listlengths) == 0: - self.logger.info("[DEBUG] NO multiplier. Running a single iteration.") - paramlist.append(baseparams) - - #elif len(listitems) == 1: - elif len(listlengths) == 1: - self.logger.info("All subitems are the same length") - - for item in listitems: - # This loops should always be length 1 - for key, value in item.items(): - if not isinstance(value, int): - continue - - if len(paramlist) == value: - for subloop in range(value): - baseitem = copy.deepcopy(baseparams) - paramlist[subloop][key] = baseparams[key][subloop] - else: - for subloop in range(value): - baseitem = copy.deepcopy(baseparams) - baseitem[key] = baseparams[key][subloop] - paramlist.append(baseitem) - - else: - newlength = 1 - for item in listitems: - for key, value in item.items(): - newlength = newlength * value - - self.logger.info("[DEBUG] Newlength of array: %d. Lists: %s" % (newlength, all_lists)) - # Get the cartesian product of the arrays - #cartesian = await self.cartesian_product(all_lists) - try: - cartesian = self.cartesian_product(all_lists) - newlist = [] - for item in cartesian: - newlist.append(list(item)) - except Exception as e: - self.logger.info(f"[ERROR] Error in cartesian product: {e}") - newlist = [] - - newobject = {} - for subitem in range(len(newlist)): - baseitem = copy.deepcopy(baseparams) - for key in range(len(newlist[subitem])): - baseitem[all_list_keys[key]] = newlist[subitem][key] - - paramlist.append(baseitem) - - self.logger.info("CARTESIAN PARAMLIST: %s" % paramlist) - - #newlist[subitem[0]] - #if len(newlist) > 0: - # itemlength = len(newlist[0]) - - # How do we get it back, ordered? - #for item in cartesian: - #self.logger.info("Listlengths: %s" % listlengths) - #paramlist = [baseparams] - - #self.logger.info("[INFO] Return paramlist (1): %s" % paramlist) - return paramlist - - - # Runs recursed versions with inner loops and such - #async def run_recursed_items(self, func, baseparams, loop_wrapper): - def run_recursed_items(self, func, baseparams, loop_wrapper): - self.logger.info(f"PRE RECURSED ITEMS: {baseparams}") - has_loop = False - - newparams = {} - for key, value in baseparams.items(): - if isinstance(value, list) and len(value) > 0: - self.logger.info(f"[DEBUG] In list check for {key}") - - for value_index in range(len(value)): - try: - # Added skip for body (OpenAPI) which uses data= in requests - # Can be screwed up if they name theirs body too - if key != "body": - value[value_index] = json.loads(value[value_index]) - except json.decoder.JSONDecodeError as e: - pass - except TypeError as e: - pass - - try: - #if isinstance(value, list) and len(value) == 1 and isinstance(value[0], list): - # try: - # loop_wrapper[key] += 1 - # except Exception as e: - # self.logger.info(f"[WARNING] Exception in loop wrapper: {e}") - # loop_wrapper[key] = 1 - - # newparams[key] = value[0] - # has_loop = True - #else: - #self.logger.info(f"Key {key} is NOT a list within a list. Value: {value}") - newparams[key] = value - except Exception as e: - self.logger.info(f"[WARNING] Error in baseparams list: {e}") - newparams[key] = value - - results = [] - if has_loop: - #self.logger.info(f"[DEBUG] Should run inner loop: {newparams}") - self.logger.info(f"[DEBUG] Should run inner loop") - #ret = await self.run_recursed_items(func, newparams, loop_wrapper) - ret = self.run_recursed_items(func, newparams, loop_wrapper) - else: - self.logger.info(f"[DEBUG] Should run multiplier check with params (inner): {newparams}") - #self.logger.info(f"[DEBUG] Should run multiplier check with params (inner)") - - # 1. Find the loops that are required and create new multipliers - # If here: check for multipliers within this scope. - ret = [] - param_multiplier = self.get_param_multipliers(newparams) - - #self.logger.info("PARAM MULTIPLIER: %s" % param_multiplier) - - # FIXME: This does a deduplication of the data - new_params = self.validate_unique_fields(param_multiplier) - if len(new_params) == 0: - self.logger.info("[WARNING] SHOULD STOP MULTI-EXECUTION BECAUSE FIELDS AREN'T UNIQUE") - self.action_result = { - "action": self.action, - "authorization": self.authorization, - "execution_id": self.current_execution_id, - "result": f"All {len(param_multiplier)} values were non-unique", - "started_at": self.start_time, - "status": "SKIPPED", - "completed_at": int(time.time_ns()), - } - - self.send_result(self.action_result, {"Content-Type": "application/json", "Authorization": "Bearer %s" % self.authorization}, "/api/v1/streams") - if runtime != "run": - exit() - else: - return - else: - #subparams = new_params - param_multiplier = new_params - - #self.logger.info(f"NEW PARAM MULTIPLIER: {param_multiplier}") - - #if isinstance(new_params, list) and len(new_params) == 1: - # params = new_params[0] - #else: - # self.logger.info("[WARNING] SHOULD STOP EXECUTION BECAUSE FIELDS AREN'T UNIQUE") - # action_result["status"] = "SKIPPED" - # action_result["result"] = f"A non-unique value was found" - # action_result["completed_at"] = int(time.time()) - # self.send_result(action_result, headers, stream_path) - # return - - for subparams in param_multiplier: - #self.logger.info(f"SUBPARAMS IN MULTI: {subparams}") - tmp = "" - try: - - while True: - try: - tmp = func(**subparams) - break - except TypeError as e: - self.logger.info("BASE TYPEERROR: %s" % e) - errorstring = "%s" % e - if "got an unexpected keyword argument" in errorstring: - fieldsplit = errorstring.split("'") - if len(fieldsplit) > 1: - field = fieldsplit[1] - - try: - del subparams[field] - self.logger.info("Removed invalid field %s (1)" % field) - except KeyError: - break - else: - raise Exception(json.dumps({ - "success": False, - "exception": f"TypeError: {e}", - "reason": "You may be running an old version of this action. Please delete and remake the node.", - })) - break - - - except: - e = "" - try: - e = sys.exc_info()[1] - except: - self.logger.info("Exec check fail: %s" % e) - pass - - tmp = json.dumps({ - "success": False, - "reason": f"An error occured during the App Function Run (not Shuffle)", - "details": f"{e}", - }) - - - # An attempt at decomposing coroutine results - # Backwards compatibility - try: - if asyncio.iscoroutine(tmp): - self.logger.info("[DEBUG] In coroutine (2)") - async def parse_value(tmp): - value = await asyncio.gather( - tmp - ) - - return value[0] - - - tmp = asyncio.run(parse_value(tmp)) - else: - #self.logger.info("[DEBUG] Not in coroutine (2)") - pass - except Exception as e: - self.logger.warning("[ERROR] Failed to parse coroutine value for old app: {e}") - - new_value = tmp - if tmp == None: - new_value = "" - elif isinstance(tmp, dict): - new_value = json.dumps(tmp) - elif isinstance(tmp, list): - new_value = json.dumps(tmp) - #else: - #tmp = tmp.replace("\"", "\\\"", -1) - - try: - new_value = json.loads(new_value) - except json.decoder.JSONDecodeError as e: - pass - except TypeError as e: - pass - except: - pass - #self.logger.info("Json: %s" % e) - #ret.append(tmp) - - #if self.result_wrapper_count > 0: - # ret.append("["*(self.result_wrapper_count-1)+new_value+"]"*(self.result_wrapper_count-1)) - #else: - ret.append(new_value) - - self.logger.info("[INFO] Function return length: %d" % len(ret)) - if len(ret) == 1: - #ret = ret[0] - self.logger.info("[DEBUG] DONT make list of 1 into 0!!") - - #self.logger.info("Return from execution: %s" % ret) - if ret == None: - results.append("") - json_object = False - elif isinstance(ret, dict): - results.append(ret) - json_object = True - elif isinstance(ret, list): - results = ret - json_object = True - else: - ret = ret.replace("\"", "\\\"", -1) - - try: - results.append(json.loads(ret)) - json_object = True - except json.decoder.JSONDecodeError as e: - #self.logger.info("Json: %s" % e) - results.append(ret) - except TypeError as e: - results.append(ret) - except: - results.append(ret) - - #if len(results) == 1: - # #results = results[0] - # #self.logger.info("DONT MAKE LIST FROM 1 TO 0!!") - # pass - - #self.logger.info("\nLOOP: %s\nRESULTS: %s" % (loop_wrapper, results)) - return results - - # Downloads all files from a namespace - # Currently only working on local version of Shuffle (2023) - def get_file_category_ids(self, category): - org_id = self.full_execution["workflow"]["execution_org"]["id"] - - get_path = "/api/v1/files/namespaces/%s?execution_id=%s&ids=true" % (category, self.full_execution["execution_id"]) - headers = { - "Authorization": "Bearer %s" % self.authorization, - "User-Agent": "Shuffle 1.1.0", - } - - ret = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False, proxies=self.proxy_config) - return ret.json() - #if ret1.status_code != 200: - # return { - # "success": False, - # "reason": "Status code is %d from backend for category %s" % category, - # "list": [], - # } - - #return { - # "success": True, - # "ids": ret1.json(), - #} - - - # Downloads all files from a namespace - # Currently only working on local version of Shuffle - def get_file_namespace(self, namespace): - org_id = self.full_execution["workflow"]["execution_org"]["id"] - - get_path = "/api/v1/files/namespaces/%s?execution_id=%s" % (namespace, self.full_execution["execution_id"]) - headers = { - "Authorization": "Bearer %s" % self.authorization, - "User-Agent": "Shuffle 1.1.0", - } - - ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False, proxies=self.proxy_config) - if ret1.status_code != 200: - return None - - filebytes = BytesIO(ret1.content) - myzipfile = zipfile.ZipFile(filebytes) - - # Unzip and build here! - #for member in files.namelist(): - # filename = os.path.basename(member) - # if not filename: - # continue - - # self.logger.info("File: %s" % member) - # source = files.open(member) - # with open("%s/%s" % (basedir, source.name), "wb+") as tmp: - # filedata = source.read() - # self.logger.info("Filedata (%s): %s" % (source.name, filedata)) - # tmp.write(filedata) - - return myzipfile - - def get_file_namespace_ids(self, namespace): - return self.get_file_category_ids(self, namespace) - - def get_file_category(self, category): - return self.get_file_namespace(self, category) - - # Things to consider for files: - # - How can you download / stream a file? - # - Can you decide if you want a stream or the files directly? - def get_file(self, value): - full_execution = self.full_execution - org_id = full_execution["workflow"]["execution_org"]["id"] - - if isinstance(value, list): - self.logger.info("IS LIST!") - #if len(value) == 1: - # value = value[0] - else: - value = [value] - - returns = [] - for item in value: - self.logger.info("FILE VALUE: %s" % item) - # Check if item is a dict, and if it is, check if it has the key "id" - if isinstance(item, dict): - if "file_id" in item: - item = item["file_id"] - elif "id" in item: - item = item["id"] - - if len(item) != 36 and not item.startswith("file_"): - self.logger.info("Bad length for file value: '%s'" % item) - continue - #return { - # "filename": "", - # "data": "", - # "success": False, - #} - - get_path = "/api/v1/files/%s?execution_id=%s" % (item, full_execution["execution_id"]) - headers = { - "Content-Type": "application/json", - "Authorization": "Bearer %s" % self.authorization, - "User-Agent": "Shuffle 1.1.0", - } - - ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False, proxies=self.proxy_config) - if ret1.status_code != 200: - returns.append({ - "filename": "", - "data": "", - "success": False, - }) - continue - - content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"]) - ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers, verify=False, proxies=self.proxy_config) - if ret2.status_code == 200: - tmpdata = ret1.json() - returndata = { - "success": True, - "filename": tmpdata["filename"], - "data": ret2.content, - } - returns.append(returndata) - - if len(returns) == 0: - return { - "success": False, - "filename": "", - "data": b"", - } - elif len(returns) == 1: - return returns[0] - else: - return returns - - def delete_cache(self, key): - org_id = self.full_execution["workflow"]["execution_org"]["id"] - url = "%s/api/v1/orgs/%s/delete_cache" % (self.url, org_id) - - data = { - "workflow_id": self.full_execution["workflow"]["id"], - "execution_id": self.current_execution_id, - "authorization": self.authorization, - "org_id": org_id, - "key": key, - } - - try: - newstorage = [] - for item in self.local_storage: - if item["execution_id"] == self.current_execution_id and item["key"] == key: - continue - - newstorage.append(item) - - self.local_storage = newstorage - - except Exception as e: - print("[ERROR] Failed DELETING current execution id local storage: %s" % e) - - response = requests.post(url, json=data, verify=False, proxies=self.proxy_config) - try: - allvalues = response.json() - return json.dumps(allvalues) - except Exception as e: - self.logger.info("[ERROR} Failed to parse response from delete_cache: %s" % e) - #return response.json() - return json.dumps({"success": False, "reason": f"Failed to delete cache for key '{key}'"}) - - def set_cache(self, key, value): - org_id = self.full_execution["workflow"]["execution_org"]["id"] - url = "%s/api/v1/orgs/%s/set_cache" % (self.url, org_id) - data = { - "workflow_id": self.full_execution["workflow"]["id"], - "execution_id": self.current_execution_id, - "authorization": self.authorization, - "org_id": org_id, - "key": key, - "value": str(value), - } - - try: - newstorage = [] - for item in self.local_storage: - if item["execution_id"] == self.current_execution_id and item["key"] == key: - continue - - newstorage.append(item) - - self.local_storage = newstorage - - except Exception as e: - print("[ERROR] Failed SETTING current execution id local storage: %s" % e) - - response = requests.post(url, json=data, verify=False, proxies=self.proxy_config) - try: - allvalues = response.json() - allvalues["key"] = key - allvalues["value"] = str(value) - return allvalues - except Exception as e: - self.logger.info("[ERROR} Failed to parse response from set cache: %s" % e) - #return response.json() - return {"success": False} - - def get_cache(self, key): - org_id = self.full_execution["workflow"]["execution_org"]["id"] - url = "%s/api/v1/orgs/%s/get_cache" % (self.url, org_id) - data = { - "workflow_id": self.full_execution["workflow"]["id"], - "execution_id": self.current_execution_id, - "authorization": self.authorization, - "org_id": org_id, - "key": key, - } - - # Makes it so that loops for the same action doesn't re-ask the db unless necessary - try: - for item in self.local_storage: - if item["execution_id"] == self.current_execution_id and item["key"] == key: - # Max keeping the local cache properly for 5 seconds due to workflow continuations - elapsed_time = time.time() - item["time_set"] - if elapsed_time > 5: - break - - return item["data"] - except Exception as e: - print("[ERROR] Failed getting current execution id local storage: %s" % e) - - value = requests.post(url, json=data, verify=False, proxies=self.proxy_config) - try: - allvalues = value.json() - allvalues["key"] = key - - try: - parsedvalue = json.loads(allvalues["value"]) - allvalues["value"] = parsedvalue - except: - self.logger.info("Parsing of value as JSON failed. Continue anyway!") - - try: - newdata = json.loads(json.dumps(data)) - newdata["time_set"] = time.time() - newdata["data"] = allvalues - self.local_storage.append(newdata) - except Exception as e: - print("[ERROR] Failed in local storage append: %s" % e) - - return allvalues - except: - self.logger.info("Value couldn't be parsed, or json dump of value failed") - #return value.json() - return {"success": False} - - # Wrapper for set_files - def set_file(self, infiles): - return self.set_files(infiles) - - # Sets files in the backend - def set_files(self, infiles): - full_execution = self.full_execution - workflow_id = full_execution["workflow"]["id"] - org_id = full_execution["workflow"]["execution_org"]["id"] - headers = { - "Content-Type": "application/json", - "Authorization": "Bearer %s" % self.authorization, - "User-Agent": "Shuffle 1.1.0", - } - - if not isinstance(infiles, list): - infiles = [infiles] - - create_path = "/api/v1/files/create?execution_id=%s" % full_execution["execution_id"] - file_ids = [] - for curfile in infiles: - filename = "unspecified" - data = { - "filename": filename, - "workflow_id": workflow_id, - "org_id": org_id, - } - - try: - data["filename"] = curfile["filename"] - filename = curfile["filename"] - except KeyError as e: - self.logger.info(f"KeyError in file setup: {e}") - pass - - ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data, verify=False, proxies=self.proxy_config) - #self.logger.info(f"Ret CREATE: {ret.text}") - cur_id = "" - if ret.status_code == 200: - ret_json = ret.json() - if not ret_json["success"]: - self.logger.info("Not success in file upload creation.") - continue - - self.logger.info("Should handle ID %s" % ret_json["id"]) - file_ids.append(ret_json["id"]) - cur_id = ret_json["id"] - else: - self.logger.info("Bad status code: %d" % ret.status_code) - continue - - if len(cur_id) == 0: - self.logger.info("No file ID specified from backend") - continue - - new_headers = { - "Authorization": f"Bearer {self.authorization}", - "User-Agent": "Shuffle 1.1.0", - } - - upload_path = "/api/v1/files/%s/upload?execution_id=%s" % (cur_id, full_execution["execution_id"]) - - files={"shuffle_file": (filename, curfile["data"])} - #open(filename,'rb')} - - ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers, verify=False, proxies=self.proxy_config) - - return file_ids - - #async def execute_action(self, action): - def execute_action(self, action): - # !!! Let this line stay - its used for some horrible codegeneration / stitching !!! # - #STARTCOPY - stream_path = "/api/v1/streams" - self.action_result = { - "action": action, - "authorization": self.authorization, - "execution_id": self.current_execution_id, - "result": "", - "started_at": int(time.time_ns()), - "status": "EXECUTING" - } - - # Simple validation of parameters in general - replace_params = False - try: - tmp_parameters = action["parameters"] - for param in tmp_parameters: - if param["value"] == "SHUFFLE_AUTO_REMOVED": - replace_params = True - except KeyError: - action["parameters"] = [] - except TypeError: - pass - - self.action = copy.deepcopy(action) - - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {self.authorization}", - "User-Agent": "Shuffle 1.1.0", - } - - if len(self.action) == 0: - self.logger.info("[WARNING] ACTION env not defined") - self.action_result["result"] = "Error in setup ENV: ACTION not defined" - self.send_result(self.action_result, headers, stream_path) - return - - if len(self.authorization) == 0: - self.logger.info("[WARING] AUTHORIZATION env not defined") - self.action_result["result"] = "Error in setup ENV: AUTHORIZATION not defined" - self.send_result(self.action_result, headers, stream_path) - return - - if len(self.current_execution_id) == 0: - self.logger.info("[WARNING] EXECUTIONID env not defined") - self.action_result["result"] = "Error in setup ENV: EXECUTIONID not defined" - self.send_result(self.action_result, headers, stream_path) - return - - - # Add async logger - # self.console_logger.handlers[0].stream.set_execution_id() - - # FIXME: Shouldn't skip this, but it's good for minimzing API calls - #try: - # ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False) - # self.logger.info("Workflow: %d" % ret.status_code) - # if ret.status_code != 200: - # self.logger.info(ret.text) - #except requests.exceptions.ConnectionError as e: - # self.logger.info("Connectionerror: %s" % e) - - # action_result["result"] = "Bad setup during startup: %s" % e - # self.send_result(action_result, headers, stream_path) - # return - - # Verify whether there are any parameters with ACTION_RESULT required - # If found, we get the full results list from backend - fullexecution = {} - if isinstance(self.full_execution, str) and len(self.full_execution) == 0: - #self.logger.info("[DEBUG] NO EXECUTION - LOADING!") - try: - failed = False - rettext = "" - for i in range(0, 5): - tmpdata = { - "authorization": self.authorization, - "execution_id": self.current_execution_id - } - - resultsurl = "%s/api/v1/streams/results" % (self.base_url) - ret = requests.post( - resultsurl, - headers=headers, - json=tmpdata, - verify=False, - proxies=self.proxy_config, - ) - - if ret.status_code == 200: - fullexecution = ret.json() - failed = False - break - - #elif ret.status_code == 500 or ret.status_code == 400: - elif ret.status_code >= 400: - self.logger.info("[ERROR] (fails: %d) Error in app with status code %d for results (1). RETRYING because results can't be handled" % (i+1, ret.status_code)) - - rettext = ret.text - failed = True - time.sleep(8) - continue - - else: - self.logger.info("[ERROR] (fails: %d) Error in app with status code %d for results (2). Crashing because results can't be handled. Details: %s" % (i+1, ret.status_code, ret.text)) - - rettext = ret.text - failed = True - time.sleep(8) - break - - if failed: - self.action_result["result"] = json.dumps({ - "success": False, - "reason": f"Bad result from backend during startup of app: {ret.status_code}", - "extended_reason": f"{rettext}" - }) - - self.send_result(self.action_result, headers, stream_path) - return - - except requests.exceptions.ConnectionError as e: - self.logger.info("[ERROR] FullExec Connectionerror: %s" % e) - self.action_result["result"] = json.dumps({ - "success": False, - "reason": f"Connection error during startup (connection error): {e}" - }) - - self.send_result(self.action_result, headers, stream_path) - return - except Exception as e: - self.logger.info("[ERROR] FullExec Exception outer: %s" % e) - self.action_result["result"] = json.dumps({ - "success": False, - "reason": f"Exception during startup of app (general error): {e}" - }) - - self.send_result(self.action_result, headers, stream_path) - return - - else: - self.logger.info(f"[DEBUG] Setting execution to default value with type {type(self.full_execution)}") - try: - fullexecution = json.loads(self.full_execution) - except json.decoder.JSONDecodeError as e: - self.logger.info("[ERROR] Json decode execution error: %s" % e) - self.action_result["result"] = "Json error during startup: %s" % e - self.send_result(self.action_result, headers, stream_path) - return - - self.logger.info("") - - - self.full_execution = fullexecution - - found_id = "" - try: - if "execution_id" in self.full_execution and len(self.full_execution["execution_id"]) > 0: - found_id = self.full_execution["execution_id"] - elif len(self.current_execution_id) > 0: - found_id = self.current_execution_id - except Exception as e: - print("[ERROR] Failed in get full exec") - - try: - contains_body = False - parameter_count = 0 - - if "parameters" in self.action: - parameter_count = len(self.action["parameters"]) - for param in self.action["parameters"]: - if param["name"] == "body": - contains_body = True - - print("[DEBUG][%s] Action name: %s, Params: %d, Has Body: %s" % (self.current_execution_id, self.action["name"], parameter_count, str(contains_body))) - except Exception as e: - print("[ERROR] Failed in init print handler: %s" % e) - - try: - if replace_params == True: - for inner_action in self.full_execution["workflow"]["actions"]: - self.logger.info("[DEBUG] ID: %s vs %s" % (inner_action["id"], self.action["id"])) - - # In case of some kind of magic, we're just doing params - if inner_action["id"] != self.action["id"]: - continue - self.logger.info("FOUND!") - - if isinstance(self.action, str): - self.logger.info("Params is in string object for self.action?") - else: - self.action["parameters"] = inner_action["parameters"] - self.action_result["action"]["parameters"] = inner_action["parameters"] - - if isinstance(self.original_action, str): - self.logger.info("Params for original actions is in string object?") - else: - self.original_action["parameters"] = inner_action["parameters"] - - break - - except Exception as e: - self.logger.info(f"[WARNING] Failed in replace params action parsing: {e}") - - # Gets the value at the parenthesis level you want - def parse_nested_param(string, level): - """ - Generate strings contained in nested (), indexing i = level - """ - if len(re.findall("\(", string)) == len(re.findall("\)", string)): - LeftRightIndex = [x for x in zip( - [Left.start()+1 for Left in re.finditer('\(', string)], - reversed([Right.start() for Right in re.finditer('\)', string)]))] - - elif len(re.findall("\(", string)) > len(re.findall("\)", string)): - return parse_nested_param(string + ')', level) - elif len(re.findall("\(", string)) < len(re.findall("\)", string)): - return parse_nested_param('(' + string, level) - else: - return 'Failed to parse params' - - try: - return [string[LeftRightIndex[level][0]:LeftRightIndex[level][1]]] - except IndexError: - return [string[LeftRightIndex[level+1][0]:LeftRightIndex[level+1][1]]] - - # Finds the deepest level parenthesis in a string - def maxDepth(S): - current_max = 0 - max = 0 - n = len(S) - - # Traverse the input string - for i in range(n): - if S[i] == '(': - current_max += 1 - - if current_max > max: - max = current_max - elif S[i] == ')': - if current_max > 0: - current_max -= 1 - else: - return -1 - - # finally check for unbalanced string - if current_max != 0: - return -1 - - return max-1 - - # Specific type parsing - def parse_type(data, thistype): - if data == None: - return "Empty" - - if "int" in thistype or "number" in thistype: - try: - return int(data) - except ValueError: - return data - - if "lower" in thistype: - return data.lower() - if "upper" in thistype: - return data.upper() - if "trim" in thistype: - return data.strip() - if "strip" in thistype: - return data.strip() - if "split" in thistype: - return data.split() - if "replace" in thistype: - splitvalues = data.split(",") - - if len(splitvalues) > 2: - for i in range(len(splitvalues)): - if i != 0: - if splitvalues[i] == " ": - splitvalues[i] = " " - continue - - splitvalues[i] = splitvalues[i].strip() - - if splitvalues[i] == "\"\"": - splitvalues[i] = "" - if splitvalues[i] == "\" \"": - splitvalues[i] = " " - if len(splitvalues[i]) > 2: - if splitvalues[i][0] == "\"" and splitvalues[i][len(splitvalues[i])-1] == "\"": - splitvalues[i] = splitvalues[i][1:-1] - if splitvalues[i][0] == "'" and splitvalues[i][len(splitvalues[i])-1] == "'": - splitvalues[i] = splitvalues[i][1:-1] - - - replacementvalue = splitvalues[0] - return replacementvalue.replace(splitvalues[1], splitvalues[2], -1) - else: - return f"replace({data})" - if "join" in thistype: - try: - splitvalues = data.split(",") - if "," not in data: - return f"join({data})" - - if len(splitvalues) >= 2: - - # 1. Take the list and parse it from string - # 2. Take all the items and join them - # 3. Parse them back as string and return - values = ",".join(splitvalues[0:-1]) - tmp = json.loads(values) - try: - newvalues = splitvalues[-1].join(str(item).strip() for item in tmp) - except TypeError: - newvalues = splitvalues[-1].join(json.dumps(item).strip() for item in tmp) - - return newvalues - else: - return f"join({data})" - - except (KeyError, IndexError) as e: - pass - except json.decoder.JSONDecodeError as e: - pass - - if "len" in thistype or "length" in thistype or "lenght" in thistype: - #self.logger.info(f"Trying to length-parse: {data}") - try: - tmp_len = json.loads(data, parse_float=str, parse_int=str, parse_constant=str) - except (NameError, KeyError, TypeError, json.decoder.JSONDecodeError) as e: - try: - #self.logger.info(f"[WARNING] INITIAL Parsing bug for length in app sdk: {e}") - # data = data.replace("\'", "\"") - data = data.replace("True", "true", -1) - data = data.replace("False", "false", -1) - data = data.replace("None", "null", -1) - data = data.replace("\"", "\\\"", -1) - data = data.replace("'", "\"", -1) - - tmp_len = json.loads(data, parse_float=str, parse_int=str, parse_constant=str) - except (NameError, KeyError, TypeError, json.decoder.JSONDecodeError) as e: - tmp_len = str(data) - - return str(len(tmp_len)) - - if "parse" in thistype: - splitvalues = [] - default_error = """Error. Expected syntax: parse(["hello","test1"],0:1)""" - if "," in data: - splitvalues = data.split(",") - - for item in range(len(splitvalues)): - splitvalues[item] = splitvalues[item].strip() - else: - return default_error - - lastsplit = [] - if ":" in splitvalues[-1]: - lastsplit = splitvalues[-1].split(":") - else: - try: - lastsplit = [int(splitvalues[-1])] - except ValueError: - return default_error - - try: - parsedlist = ",".join(splitvalues[0:-1]) - if len(lastsplit) > 1: - tmp = json.loads(parsedlist)[int(lastsplit[0]):int(lastsplit[1])] - else: - tmp = json.loads(parsedlist)[lastsplit[0]] - - return tmp - except IndexError as e: - return default_error - - # Parses the INNER value and recurses until everything is done - # Looks for a way to use e.g. int() or number() as a value - def parse_wrapper(data): - try: - if "(" not in data or ")" not in data: - return data, False - except TypeError: - return data, False - - # Because liquid can handle ALL of this now. - # Implemented for >0.9.25 - #self.logger.info("[DEBUG] Skipping parser because use of its been deprecated >0.9.25 due to Liquid implementation") - return data, False - - wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length", "lenght", "join", "replace"] - - if not any(wrapper in data for wrapper in wrappers): - return data, False - - # Do stuff here. - inner_value = parse_nested_param(data, maxDepth(data) - 0) - outer_value = parse_nested_param(data, maxDepth(data) - 1) - - wrapper_group = "|".join(wrappers) - parse_string = data - max_depth = maxDepth(parse_string) - - if outer_value != inner_value: - for casting_items in reversed(range(max_depth + 1)): - c_parentheses = parse_nested_param(parse_string, casting_items)[0] - match_string = re.escape(c_parentheses) - custom_casting = re.findall(fr"({wrapper_group})\({match_string}", parse_string) - - # no matching ; go next group - if len(custom_casting) == 0: - continue - - inner_result = parse_type(c_parentheses, custom_casting[0]) - - # if result is a string then parse else return - if isinstance(inner_result, str): - parse_string = parse_string.replace(f"{custom_casting[0]}({c_parentheses})", inner_result, 1) - elif isinstance(inner_result, list): - parse_string = parse_string.replace(f"{custom_casting[0]}({c_parentheses})", json.dumps(inner_result), 1) - else: - parse_string = inner_result - break - else: - c_parentheses = parse_nested_param(parse_string, 0)[0] - match_string = re.escape(c_parentheses) - custom_casting = re.findall(fr"({wrapper_group})\({match_string}", parse_string) - # check if a wrapper was found - if len(custom_casting) != 0: - inner_result = parse_type(c_parentheses, custom_casting[0]) - if isinstance(inner_result, str): - parse_string = parse_string.replace(f"{custom_casting[0]}({c_parentheses})", inner_result) - elif isinstance(inner_result, list): - parse_string = parse_string.replace(f"{custom_casting[0]}({c_parentheses})", - json.dumps(inner_result)) - else: - parse_string = inner_result - - return parse_string, True - - # Looks for parantheses to grab special cases within a string, e.g: - # int(1) lower(HELLO) or length(what's the length) - # FIXME: - # There is an issue in here where it returns data wrong. Example: - # Authorization=Bearer authkey - # = - # Authorization=Bearer authkey - # ^ Double space. - def parse_wrapper_start(data, self): - try: - data = parse_liquid(data, self) - except: - pass - - if "(" not in data or ")" not in data: - return data - - if isinstance(data, str) and len(data) > 4: - if (data[0] == "{" or data[0] == "[") and (data[len(data)-1] == "]" or data[len(data)-1] == "}"): - self.logger.info("[DEBUG] Skipping parser because use of {[ and ]}") - return data - - newdata = [] - newstring = "" - record = True - paranCnt = 0 - charcnt = 0 - for char in data: - if char == "(": - charskip = False - if charcnt > 0: - if data[charcnt-1] == " ": - charskip = True - - if not charskip: - paranCnt += 1 - - if not record: - record = True - - if record: - newstring += char - - if paranCnt == 0 and char == " ": - newdata.append(newstring) - newstring = "" - record = True - - if char == ")": - paranCnt -= 1 - - if paranCnt == 0: - record = False - - charcnt += 1 - - if len(newstring) > 0: - newdata.append(newstring) - - parsedlist = [] - non_string = False - parsed = False - for item in newdata: - ret = parse_wrapper(item) - if not isinstance(ret[0], str): - non_string = True - - parsedlist.append(ret[0]) - if ret[1]: - parsed = True - - if not parsed: - return data - - if len(parsedlist) > 0 and not non_string: - #self.logger.info("Returning parsed list: ", parsedlist) - return " ".join(parsedlist) - elif len(parsedlist) == 1 and non_string: - return parsedlist[0] - else: - #self.logger.info("Casting back to string because multi: ", parsedlist) - newlist = [] - for item in parsedlist: - try: - newlist.append(str(item)) - except ValueError: - newlist.append("parsing_error") - - # Does this create the issue? - return " ".join(newlist) - - # Parses JSON loops and such down to the item you're looking for - # Check recurse_test.py for examples and tests of this function - # $nodename.#.id - # $nodename.data.#min-max.info.id - # $nodename.data.#1-max.info.id - # $nodename.data.#min-1.info.id - def recurse_json(basejson, parsersplit): - match = "#([0-9a-z]+):?-?([0-9a-z]+)?#?" - try: - outercnt = 0 - - # Loops over split values - splitcnt = -1 - for value in parsersplit: - splitcnt += 1 - #if " " in value: - # value = value.replace(" ", "_", -1) - - actualitem = re.findall(match, value, re.MULTILINE) - # Goes here if loop - if value == "#": - newvalue = [] - - if basejson == None: - return "", False - - for innervalue in basejson: - # 1. Check the next item (message) - # 2. Call this function again - - try: - ret, is_loop = recurse_json(innervalue, parsersplit[outercnt+1:]) - except IndexError: - # Only in here if it's the last loop without anything in it? - ret, is_loop = recurse_json(innervalue, parsersplit[outercnt:]) - - newvalue.append(ret) - - # Magical way of returning which makes app sdk identify - # it as multi execution - return newvalue, True - - # Checks specific regex like #1-2 for index 1-2 in a loop - elif len(actualitem) > 0: - - is_loop = True - newvalue = [] - firstitem = actualitem[0][0] - seconditem = actualitem[0][1] - if isinstance(firstitem, int): - firstitem = str(firstitem) - if isinstance(seconditem, int): - seconditem = str(seconditem) - - # Means it's a single item -> continue - if seconditem == "": - if str(firstitem).lower() == "max" or str(firstitem).lower() == "last" or str(firstitem).lower() == "end": - firstitem = len(basejson)-1 - elif str(firstitem).lower() == "min" or str(firstitem).lower() == "first": - firstitem = 0 - else: - firstitem = int(firstitem) - - tmpitem = basejson[int(firstitem)] - try: - newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:]) - except IndexError: - newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:]) - else: - if isinstance(firstitem, str): - if firstitem.lower() == "max" or firstitem.lower() == "last" or firstitem.lower() == "end": - firstitem = len(basejson)-1 - elif firstitem.lower() == "min" or firstitem.lower() == "first": - firstitem = 0 - else: - firstitem = int(firstitem) - else: - firstitem = int(firstitem) - - if isinstance(seconditem, str): - if str(seconditem).lower() == "max" or str(seconditem).lower() == "last" or str(firstitem).lower() == "end": - seconditem = len(basejson)-1 - elif str(seconditem).lower() == "min" or str(seconditem).lower() == "first": - seconditem = 0 - else: - seconditem = int(seconditem) - else: - seconditem = int(seconditem) - - newvalue = [] - if int(seconditem) > len(basejson): - seconditem = len(basejson) - - for i in range(int(firstitem), int(seconditem)+1): - # 1. Check the next item (message) - # 2. Call this function again - - try: - ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt+1:]) - except IndexError: - #ret = innervalue - ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt:]) - - newvalue.append(ret) - - return newvalue, is_loop - - else: - if len(value) == 0: - return basejson, False - - try: - if isinstance(basejson, list): - return basejson, False - elif isinstance(basejson, bool): - return basejson, False - elif isinstance(basejson, int): - return basejson, False - elif isinstance(basejson[value], str): - try: - if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")): - basejson = json.loads(basejson[value]) - else: - # Should we sanitize here? - # Check if we are on the last item? - if outercnt == len(parsersplit)-1: - return str(basejson[value]), False - else: - pass - - except json.decoder.JSONDecodeError as e: - return str(basejson[value]), False - else: - basejson = basejson[value] - except KeyError as e: - if "_" in value: - value = value.replace("_", " ", -1) - elif " " in value: - value = value.replace(" ", "_", -1) - - try: - if isinstance(basejson, list): - return basejson, False - elif isinstance(basejson, bool): - return basejson, False - elif isinstance(basejson, int): - return basejson, False - elif isinstance(basejson[value], str): - try: - if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")): - basejson = json.loads(basejson[value]) - else: - - if outercnt == len(parsersplit)-1: - return str(basejson[value]), False - else: - pass - - except json.decoder.JSONDecodeError as e: - return str(basejson[value]), False - else: - basejson = basejson[value] - except KeyError as e: - # Check if previous key was handled or not - previouskey = parsersplit[outercnt-1] - - tmpval = previouskey + "." + value - if tmpval in basejson: - return basejson[tmpval], False - - try: - currentsplitcnt = splitcnt - - recursed_value = value - handled = False - - #tmpbase = basejson - previouskey = value - while True: - newvalue = parsersplit[currentsplitcnt+1] - if newvalue == "#" or newvalue == "": - break - - recursed_value += "." + newvalue - - found = False - for key, value in basejson.items(): - if recursed_value.lower() in key.lower(): - found = True - - if found == False: - # Check if we are on the last key or not - return "", False - - if recursed_value in basejson: - basejson = basejson[recursed_value] - - # Whether to dig deeper or not - if isinstance(basejson, bool) or isinstance(basejson, int) or isinstance(basejson, str): - handled = False - else: - handled = True - - break - - currentsplitcnt += 1 - - if handled: - continue - - break - except IndexError as e: - return "", False - - outercnt += 1 - - except KeyError as e: - return "", False - except Exception as e: - return "", False - - return basejson, False - - # Takes a workflow execution as argument - # Returns a string if the result is single, or a list if it's a list - def get_json_value(execution_data, input_data): - parsersplit = input_data.split(".") - actionname_lower = parsersplit[0][1:].lower() - - #Actionname: Start_node - - # 1. Find the action - baseresult = "" - - appendresult = "" - if (actionname_lower.startswith("exec ") or actionname_lower.startswith("webhook ") or actionname_lower.startswith("schedule ") or actionname_lower.startswith("userinput ") or actionname_lower.startswith("email_trigger ") or actionname_lower.startswith("trigger ")) and len(parsersplit) == 1: - record = False - for char in actionname_lower: - if char == " ": - record = True - - if record: - appendresult += char - - actionname_lower = "exec" - elif actionname_lower.startswith("shuffle_cache ") or actionname_lower.startswith("shuffle_db "): - actionname_lower = "shuffle_cache" - - actionname_lower = actionname_lower.replace(" ", "_", -1) - - try: - if actionname_lower == "exec" or actionname_lower == "webhook" or actionname_lower == "schedule" or actionname_lower == "userinput" or actionname_lower == "email_trigger" or actionname_lower == "trigger": - baseresult = execution_data["execution_argument"] - elif actionname_lower == "shuffle_cache": - if len(parsersplit) > 1: - actual_key = parsersplit[1] - cachedata = self.get_cache(actual_key) - parsersplit.pop(1) - try: - baseresult = json.dumps(cachedata) - except json.decoder.JSONDecodeError as e: - pass - - - else: - if execution_data["results"] != None: - for result in execution_data["results"]: - resultlabel = result["action"]["label"].replace(" ", "_", -1).lower() - if resultlabel.lower() == actionname_lower: - baseresult = result["result"] - break - else: - baseresult = "$" + parsersplit[0][1:] - - if len(baseresult) == 0: - try: - for variable in execution_data["workflow"]["workflow_variables"]: - variablename = variable["name"].replace(" ", "_", -1).lower() - - if variablename.lower() == actionname_lower: - baseresult = variable["value"] - break - - except KeyError as e: - pass - except TypeError as e: - pass - - if len(baseresult) == 0: - try: - for variable in execution_data["execution_variables"]: - variablename = variable["name"].replace(" ", "_", -1).lower() - if variablename.lower() == actionname_lower: - baseresult = variable["value"] - break - except KeyError as e: - pass - except TypeError as e: - pass - - except KeyError as error: - pass - - # 2. Find the JSON data - # Returns if there isn't any JSON in the base ($nodename) - if len(baseresult) == 0: - return ""+appendresult, False - - # Returns if the result is JUST something like $nodename, not $nodename.value - if len(parsersplit) == 1: - returndata = str(baseresult)+str(appendresult) - return returndata, False - - baseresult = baseresult.replace(" True,", " true,") - baseresult = baseresult.replace(" False", " false,") - - # Tries to actually read it as JSON with some stupid formatting - basejson = {} - try: - basejson = json.loads(baseresult) - except json.decoder.JSONDecodeError as e: - try: - baseresult = baseresult.replace("\'", "\"") - basejson = json.loads(baseresult) - except json.decoder.JSONDecodeError as e: - return str(baseresult)+str(appendresult), False - - # Finds the ACTUAL value which is in the $nodename.value.test - focusing on value.test - data, is_loop = recurse_json(basejson, parsersplit[1:]) - parseditem = data - - if isinstance(parseditem, dict) or isinstance(parseditem, list): - try: - parseditem = json.dumps(parseditem) - except json.decoder.JSONDecodeError as e: - pass - - if is_loop: - if parsersplit[-1] == "#": - parseditem = "${SHUFFLE_NO_SPLITTER%s}$" % json.dumps(data) - else: - # Return value: ${id[12345, 45678]}$ - parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data)) - - - returndata = str(parseditem)+str(appendresult) - - # New in 0.8.97: Don't return items without lists - #return returndata, is_loop - - # 0.9.70: - # The {} and [] checks are required because e.g. 7e7 is valid JSON for some reason... - # This breaks EVERYTHING - try: - if (returndata.endswith("}") and returndata.endswith("}")) or (returndata.startswith("[") and returndata.endswith("]")): - return json.dumps(json.loads(returndata)), is_loop - else: - return returndata, is_loop - except json.decoder.JSONDecodeError as e: - return returndata, is_loop - - - - # Sending self as it's not a normal function - def parse_liquid(template, self): - - errors = False - error_msg = "" - try: - if len(template) > 10000000: - self.logger.info("[DEBUG] Skipping liquid - size too big (%d)" % len(template)) - return template - - if "${" in template and "}$" in template: - #self.logger.info("[DEBUG] Shuffle loop shouldn't run in liquid. Data length: %d" % len(template)) - return template - - - # New pattern fixer to help with bad liquid formats - try: - newoutput = self.patternfix_string(template, - { - "{{|": '{{ "" |', - }, - { - r'\{\{\s*\$[^|}]+\s*\|': '{{ "" |', - } - , - inputtype="liquid" - ) - - template = newoutput - except Exception as e: - print("[ERROR] Failed liquid parsing fix: %s" % e) - - all_globals = globals() - all_globals["self"] = self - run = Liquid(template, mode="wild", from_file=False, filters=shuffle_filters.filters, globals=all_globals) - - # Add locals that are missing to globals - ret = run.render() - return ret - except jinja2.exceptions.TemplateNotFound as e: - self.logger.info(f"[ERROR] Liquid Template error: {e}") - error = True - error_msg = e - - self.action["parameters"].append({ - "name": "liquid_template_error", - "value": f"There was a Liquid input error (1). Details: {e}", - }) - - self.action_result["action"] = self.action - except SyntaxError as e: - self.logger.info(f"[ERROR] Liquid Syntax error: {e}") - error = True - error_msg = e - - self.action["parameters"].append({ - "name": "liquid_python_syntax_error", - "value": f"There was a syntax error in your Liquid input (2). Details: {e}", - }) - - self.action_result["action"] = self.action - except IndentationError as e: - self.logger.info(f"[ERROR] Liquid IndentationError: {e}") - error = True - error_msg = e - - self.action["parameters"].append({ - "name": "liquid_indentiation_error", - "value": f"There was an indentation error in your Liquid input (2). Details: {e}", - }) - - self.action_result["action"] = self.action - except jinja2.exceptions.TemplateSyntaxError as e: - self.logger.info(f"[ERROR] Liquid Syntax error: {e}") - error = True - error_msg = e - - self.action["parameters"].append({ - "name": "liquid_syntax_error", - "value": f"There was a syntax error in your Liquid input (2). Details: {e}", - }) - - self.action_result["action"] = self.action - except json.decoder.JSONDecodeError as e: - self.logger.info(f"[ERROR] Liquid JSON Syntax error: {e}") - - replace = False - skip_next = False - newlines = [] - thisline = [] - for line in template.split("\n"): - if "\"\"\"" in line or "\'\'\'" in line: - if replace: - skip_next = True - else: - replace = not replace - - if replace == True: - thisline.append(line) - if skip_next == True: - if len(thisline) > 0: - newlines.append(" ".join(thisline)) - thisline = [] - - replace = False - else: - newlines.append(line) - - new_template = "\n".join(newlines) - if new_template != template: - #check_template(new_template) - return parse_liquid(new_template, self) - else: - error = True - error_msg = e - - self.action["parameters"].append({ - "name": "liquid_json_error", - "value": f"There was a syntax error in your input JSON(2). This is typically an issue with escaping newlines. Details: {e}", - }) - - self.action_result["action"] = self.action - except TypeError as e: - try: - if "string as left operand" in f"{e}": - split_left = template.split("|") - if len(split_left) < 2: - return template - - splititem = split_left[0] - additem = "{{" - if "{{" in splititem: - splititem = splititem.replace("{{", "", -1) - - if "{%" in splititem: - splititem = splititem.replace("{%", "", -1) - additem = "{%" - - splititem = "%s \"%s\"" % (additem, splititem.strip()) - parsed_template = template.replace(split_left[0], splititem) - run = Liquid(parsed_template, mode="wild", from_file=False) - return run.render(**globals()) - - except Exception as e: - self.action["parameters"].append({ - "name": "liquid_general_error", - "value": f"There was general error Liquid input (2). Details: {e}", - }) - - self.action_result["action"] = self.action - #return template - - self.logger.info(f"[ERROR] Liquid TypeError error: {e}") - error = True - error_msg = e - - except Exception as e: - self.logger.info(f"[ERROR] General exception for liquid: {e}") - error = True - error_msg = e - - self.action["parameters"].append({ - "name": "liquid_general_exception", - "value": f"There was general exception Liquid input (2). Details: {e}", - }) - - self.action_result["action"] = self.action - - if "fmt" in error_msg and "liquid_date" in error_msg: - return template - - self.logger.info("Done in liquid") - if error == True: - self.action_result["status"] = "FAILURE" - data = { - "success": False, - "reason": f"Failed to parse LiquidPy: {error_msg}", - "input": template, - } - - try: - self.action_result["result"] = json.dumps(data) - except Exception as e: - self.action_result["result"] = f"Failed to parse LiquidPy: {error_msg}" - - self.action_result["completed_at"] = int(time.time_ns()) - self.send_result(self.action_result, headers, stream_path) - - self.logger.info(f"[ERROR] Sent FAILURE response to backend due to : {e}") - - if runtime == "run": - return template - else: - os.exit() - - return template - - # Suboptimal cleanup script for BOdy parsing of OpenAPI - # Should have a regex which looks for the value, then goes out and cleans up the key - def recurse_cleanup_script(data): - deletekeys = [] - newvalue = data - try: - if not isinstance(data, dict): - newvalue = json.loads(data) - else: - newvalue = data - - for key, value in newvalue.items(): - if isinstance(value, str) and len(value) == 0: - deletekeys.append(key) - continue - - if isinstance(value, list): - try: - value = json.dumps(value) - except: - pass - - if value == "${%s}" % key: - deletekeys.append(key) - continue - elif "${" in value and "}" in value: - deletekeys.append(key) - continue - - if isinstance(value, dict): - newvalue[key] = recurse_cleanup_script(value) - - except json.decoder.JSONDecodeError as e: - # Since here the data isn't at all JSON compatible..? - # Seems to happen with newlines in variables being parsed in as strings? - pass - except Exception as e: - pass - - try: - for deletekey in deletekeys: - try: - del newvalue[deletekey] - except: - pass - except Exception as e: - return data - - try: - for key, value in newvalue.items(): - if isinstance(value, bool): - continue - elif isinstance(value, dict) and not bool(value): - continue - - try: - value = json.loads(value) - newvalue[key] = value - except json.decoder.JSONDecodeError as e: - continue - except Exception as e: - continue - - try: - data = json.dumps(newvalue) - except json.decoder.JSONDecodeError as e: - data = newvalue - - except json.decoder.JSONDecodeError as e: - pass - except Exception as e: - pass - - return data - - # Makes JSON string values into valid strings in JSON - # Mainly by removing newlines and such - def fix_json_string_value(value): - try: - value = value.replace("\r\n", "\\r\\n") - value = value.replace("\n", "\\n") - value = value.replace("\r", "\\r") - - # Fix quotes in the string - value = value.replace("\\\"", "\"") - value = value.replace("\"", "\\\"") - - value = value.replace("\\\'", "\'") - value = value.replace("\'", "\\\'") - except Exception as e: - pass - - return value - - - - # Parses parameters sent to it and returns whether it did it successfully with the values found - def parse_params(action, fullexecution, parameter, self): - # Skip if it starts with $? - jsonparsevalue = "$." - is_loop = False - - # Matches with space in the first part, but not in subsequent parts. - # JSON / yaml etc shouldn't have spaces in their fields anyway. - #match = ".*?([$]{1}([a-zA-Z0-9 _-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})[$/, ]?" - #match = ".*?([$]{1}([a-zA-Z0-9 _-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})" - - #match = ".*?([$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})" # Removed space - no longer ok. Force underscore. - #match = "([$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})" # Removed .*? to make it work with large amounts of data - match = "([$]{1}([a-zA-Z0-9_@-]+\.?){1}([a-zA-Z0-9#_@-]+\.?){0,})" # Added @ to the regex - - # Extra replacements for certain scenarios - escaped_dollar = "\\$" - escape_replacement = "\\%\\%\\%\\%\\%" - end_variable = "^_^" - - #self.logger.info("Input value: %s" % parameter["value"]) - try: - parameter["value"] = parameter["value"].replace(escaped_dollar, escape_replacement, -1) - except: - self.logger.info("Error in initial replacement of escaped dollar!") - - paramname = "" - try: - paramname = parameter["name"] - except: - pass - - # Basic fix in case variant isn't set - # Variant is ALWAYS STATIC_VALUE from mid 2021~ - try: - parameter["variant"] = parameter["variant"] - except: - parameter["variant"] = "STATIC_VALUE" - - # Regex to find all the things - # Should just go in here if data is ... not so big - #if parameter["variant"] == "STATIC_VALUE" and len(parameter["value"]) < 1000000: - #if parameter["variant"] == "STATIC_VALUE" and len(parameter["value"]) < 5000000: - if parameter["variant"] == "STATIC_VALUE": - data = parameter["value"] - actualitem = re.findall(match, data, re.MULTILINE) - #self.logger.debug(f"\n\nHandle static data with JSON: {data}\n\n") - #self.logger.info("STATIC PARSED: %s" % actualitem) - #self.logger.info("[INFO] Done with regex matching") - if len(actualitem) > 0: - for replace in actualitem: - try: - to_be_replaced = replace[0] - except IndexError: - continue - - # Handles for loops etc. - # FIXME: Should it dump to string here? Doesn't that defeat the purpose? - # Trying without string dumping. - #self.logger.info("TO BE REPLACED: %s" % to_be_replaced) - value, is_loop = get_json_value(fullexecution, to_be_replaced) - - #self.logger.info(f"\n\nType of value: {type(value)}") - if isinstance(value, str): - # Could we take it here? - #self.logger.info(f"[DEBUG] Got value %s for parameter {paramname}" % value) - # Should check if there is are quotes infront of and after the to_be_replaced - # If there are, then we need to sanitize the value - # 1. Look for the to_be_replaced in the data - # 2. Check if there is a quote infront of it and also if there are {} in the data to validate JSON - # 3. If there are, sanitize! - #if data.find(f'"{to_be_replaced}"') != -1 and data.find("{") != -1 and data.find("}") != -1: - # returnvalue = fix_json_string_value(value) - # value = returnvalue - - parameter["value"] = parameter["value"].replace(to_be_replaced, value, 1) - elif isinstance(value, dict) or isinstance(value, list): - # Changed from JSON dump to str() 28.05.2021 - # This makes it so the parameters gets lists and dicts straight up - parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value), 1) - - #try: - # parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) - #except: - # parameter["value"] = parameter["value"].replace(to_be_replaced, str(value)) - # self.logger.info("Failed parsing value as string?") - else: - self.logger.error("[ERROR] Unknown type %s" % type(value)) - try: - parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value), 1) - except json.decoder.JSONDecodeError as e: - parameter["value"] = parameter["value"].replace(to_be_replaced, value, 1) - - else: - #self.logger.info(f"[ERROR] Not running static variant regex parsing (slow) on value with length {len(parameter['value'])}. Max is 5Mb~.") - pass - - if parameter["variant"] == "WORKFLOW_VARIABLE": - self.logger.info("[DEBUG] Handling workflow variable") - found = False - try: - for item in fullexecution["workflow"]["workflow_variables"]: - if parameter["action_field"] == item["name"]: - found = True - parameter["value"] = item["value"] - break - except KeyError as e: - self.logger.info("KeyError WF variable 1: %s" % e) - pass - except TypeError as e: - self.logger.info("TypeError WF variables 1: %s" % e) - pass - - if not found: - try: - for item in fullexecution["execution_variables"]: - if parameter["action_field"] == item["name"]: - parameter["value"] = item["value"] - break - except KeyError as e: - self.logger.info("KeyError WF variable 2: %s" % e) - pass - except TypeError as e: - self.logger.info("TypeError WF variables 2: %s" % e) - pass - - elif parameter["variant"] == "ACTION_RESULT": - # FIXME - calculate value based on action_field and $if prominent - # FIND THE RIGHT LABEL - # GET THE LABEL'S RESULT - - tmpvalue = "" - self.logger.info("ACTION FIELD: %s" % parameter["action_field"]) - - fullname = "$" - if parameter["action_field"] == "Execution Argument": - tmpvalue = fullexecution["execution_argument"] - fullname += "exec" - else: - fullname += parameter["action_field"] - - self.logger.info("PRE Fullname: %s" % fullname) - - if parameter["value"].startswith(jsonparsevalue): - fullname += parameter["value"][1:] - #else: - # fullname = "$%s" % parameter["action_field"] - - self.logger.info("Fullname: %s" % fullname) - actualitem = re.findall(match, fullname, re.MULTILINE) - self.logger.info("ACTION PARSED: %s" % actualitem) - if len(actualitem) > 0: - for replace in actualitem: - try: - to_be_replaced = replace[0] - except IndexError: - self.logger.info("Nothing to replace?: " % e) - continue - - # This will never be a loop aka multi argument - parameter["value"] = to_be_replaced - - value, is_loop = get_json_value(fullexecution, to_be_replaced) - self.logger.info("Loop: %s" % is_loop) - if isinstance(value, str): - parameter["value"] = parameter["value"].replace(to_be_replaced, value) - elif isinstance(value, dict): - parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) - else: - self.logger.info("Unknown type %s" % type(value)) - try: - parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) - except json.decoder.JSONDecodeError as e: - parameter["value"] = parameter["value"].replace(to_be_replaced, value) - - #self.logger.info("PRE Replaced data: %s" % parameter["value"]) - - try: - parameter["value"] = parameter["value"].replace(end_variable, "", -1) - parameter["value"] = parameter["value"].replace(escape_replacement, "$", -1) - except: - self.logger.info(f"[ERROR] Problem in datareplacement: {e}") - - # Just here in case it breaks - # Implemented 02.08.2021 - #self.logger.info("Pre liquid: %s" % parameter["value"]) - try: - parameter["value"] = parse_liquid(parameter["value"], self) - except: - pass - - return "", parameter["value"], is_loop - - def run_validation(sourcevalue, check, destinationvalue): - #self.logger.info("[DEBUG] Checking %s '%s' %s" % (sourcevalue, check, destinationvalue)) - - if check == "=" or check.lower() == "equals": - if str(sourcevalue).lower() == str(destinationvalue).lower(): - return True - elif check == "!=" or check.lower() == "does not equal": - if str(sourcevalue).lower() != str(destinationvalue).lower(): - return True - elif check.lower() == "startswith": - if str(sourcevalue).lower().startswith(str(destinationvalue).lower()): - return True - elif check.lower() == "endswith": - if str(sourcevalue).lower().endswith(str(destinationvalue).lower()): - return True - elif check.lower() == "contains": - if destinationvalue.lower() in sourcevalue.lower(): - return True - - elif check.lower() == "is empty" or check.lower() == "is_empty": - try: - if len(json.loads(sourcevalue)) == 0: - return True - except Exception as e: - self.logger.info(f"[WARNING] Failed to check if empty as list: {e}") - - if len(str(sourcevalue)) == 0: - return True - - elif check.lower() == "contains_any_of": - newvalue = [destinationvalue.lower()] - if "," in destinationvalue: - newvalue = destinationvalue.split(",") - elif ", " in destinationvalue: - newvalue = destinationvalue.split(", ") - - for item in newvalue: - if not item: - continue - - if item.strip() in sourcevalue: - return True - - elif check.lower() == "larger than" or check.lower() == "bigger than": - try: - if str(sourcevalue).isdigit() and str(destinationvalue).isdigit(): - if int(sourcevalue) > int(destinationvalue): - return True - - except AttributeError as e: - self.logger.info("[WARNING] Condition larger than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) - - try: - destinationvalue = len(json.loads(destinationvalue)) - except Exception as e: - self.logger.info(f"[WARNING] Failed to convert destination to list: {e}") - try: - # Check if it's a list in autocast and if so, check the length - if len(json.loads(sourcevalue)) > int(destinationvalue): - return True - except Exception as e: - self.logger.info(f"[WARNING] Failed to check if larger than as list: {e}") - - - elif check.lower() == "smaller than" or check.lower() == "less than": - self.logger.info("In smaller than check: %s %s" % (sourcevalue, destinationvalue)) - - try: - if str(sourcevalue).isdigit() and str(destinationvalue).isdigit(): - if int(sourcevalue) < int(destinationvalue): - return True - - except AttributeError as e: - pass - - try: - destinationvalue = len(json.loads(destinationvalue)) - except Exception as e: - self.logger.info(f"[WARNING] Failed to convert destination to list: {e}") - - try: - # Check if it's a list in autocast and if so, check the length - if len(json.loads(sourcevalue)) < int(destinationvalue): - return True - except Exception as e: - self.logger.info(f"[WARNING] Failed to check if smaller than as list: {e}") - - elif check.lower() == "re" or check.lower() == "matches regex": - try: - found = re.search(str(destinationvalue), str(sourcevalue)) - except re.error as e: - return False - except Exception as e: - return False - - if found == None: - return False - - return True - else: - self.logger.error("[DEBUG] Condition: can't handle %s yet. Setting to true" % check) - - return False - - def check_branch_conditions(action, fullexecution, self): - # relevantbranches = workflow.branches where destination = action - try: - if fullexecution["workflow"]["branches"] == None or len(fullexecution["workflow"]["branches"]) == 0: - return True, "" - except KeyError: - return True, "" - - # Startnode should always run - no need to check incoming - # Removed November 2023 due to people wanting startnode to also check - # This is to make it possible ot - try: - if action["id"] == fullexecution["start"]: - return True, "" - - except Exception as error: - self.logger.info(f"[WARNING] Failed checking startnode: {error}") - #return True, "" - #return True, "" - - available_checks = [ - "=", - "equals", - "!=", - "does not equal", - ">", - "larger than", - "<", - "less than", - ">=", - "<=", - "startswith", - "endswith", - "contains", - "contains_any_of", - "re", - "matches regex", - "is empty", - "is_empty", - ] - - relevantbranches = [] - correct_branches = 0 - matching_branches = 0 - for branch in fullexecution["workflow"]["branches"]: - if branch["destination_id"] != action["id"]: - continue - - matching_branches += 1 - - # Find if previous is skipped or failed. Skipped != correct branch - try: - should_skip = False - for res in fullexecution["results"]: - if res["action"]["id"] == branch["source_id"]: - if res["status"] == "FAILURE" or res["status"] == "SKIPPED": - should_skip = True - - break - - if should_skip: - continue - except Exception as e: - self.logger.info("[WARNING] Failed handling check of if parent is skipped") - - - # Remove anything without a condition - try: - if (branch["conditions"]) == 0 or branch["conditions"] == None: - correct_branches += 1 - continue - except KeyError: - correct_branches += 1 - continue - - successful_conditions = [] - failed_conditions = [] - successful_conditions = 0 - total_conditions = len(branch["conditions"]) - for condition in branch["conditions"]: - # Parse all values first here - sourcevalue = condition["source"]["value"] - check, sourcevalue, is_loop = parse_params(action, fullexecution, condition["source"], self) - if check: - continue - - sourcevalue = parse_wrapper_start(sourcevalue, self) - destinationvalue = condition["destination"]["value"] - - check, destinationvalue, is_loop = parse_params(action, fullexecution, condition["destination"], self) - if check: - continue - - destinationvalue = parse_wrapper_start(destinationvalue, self) - - if not condition["condition"]["value"] in available_checks: - self.logger.error("[ERROR] Skipping '%s' -> %s -> '%s' because %s is invalid." % (sourcevalue, condition["condition"]["value"], destinationvalue, condition["condition"]["value"])) - continue - - # Configuration = negated because of WorkflowAppActionParam.. - validation = run_validation(sourcevalue, condition["condition"]["value"], destinationvalue) - try: - if condition["condition"]["configuration"]: - validation = not validation - except KeyError: - pass - - if validation == True: - successful_conditions += 1 - - if total_conditions == successful_conditions: - correct_branches += 1 - - if matching_branches == 0: - return True, "" - - if matching_branches > 0 and correct_branches > 0: - return True, "" - - #self.logger.info("[DEBUG] Correct branches vs matching branches: %d vs %d" % (correct_branches, matching_branches)) - return False, {"success": False, "reason": "Minimum of one branch's conditions must be correct to continue. Total: %d of %d" % (correct_branches, matching_branches)} - - - # - # - # - # - # CONT - # CONT - # CONT - # CONT - # CONT - # CONT - # CONT - # CONT - # CONT - # CONT - # CONT - # CONT - # CONT - # - # - # - # - - # THE START IS ACTUALLY RIGHT HERE :O - # Checks whether conditions are met, otherwise set - branchcheck, tmpresult = check_branch_conditions(action, fullexecution, self) - if isinstance(tmpresult, object) or isinstance(tmpresult, list) or isinstance(tmpresult, dict): - #self.logger.info("[DEBUG] Fixing branch return as object -> string") - try: - #tmpresult = tmpresult.replace("'", "\"") - tmpresult = json.dumps(tmpresult) - except json.decoder.JSONDecodeError as e: - pass - - - # IF branches fail: Exit! - if not branchcheck: - self.action_result["result"] = tmpresult - self.action_result["status"] = "SKIPPED" - self.action_result["completed_at"] = int(time.time_ns()) - - self.send_result(self.action_result, headers, stream_path) - return - - # Replace name cus there might be issues - # Not doing lower() as there might be user-made functions - actionname = action["name"] - if " " in actionname: - actionname.replace(" ", "_", -1) - - #if action.generated: - # actionname = actionname.lower() - - # Runs the actual functions - try: - func = getattr(self, actionname, None) - if func == None: - self.logger.debug(f"[DEBUG] Failed executing {actionname} because func is None (no function specified).") - self.action_result["status"] = "FAILURE" - self.action_result["result"] = json.dumps({ - "success": False, - "reason": f"Function {actionname} doesn't exist, or the App is out of date.", - "details": "If this persists, please delete the Docker image locally, then restart your Orborus instance before trying again. This will force-download the latest version. Contact support@shuffler.io with this data if the issue persists.", - }) - elif callable(func): - try: - if len(action["parameters"]) < 1: - #result = await func() - result = func() - else: - # Potentially parse JSON here - # FIXME - add potential authentication as first parameter(s) here - # params[parameter["name"]] = parameter["value"] - #self.logger.info(fullexecution["authentication"] - # What variables are necessary here tho hmm - - params = {} - - # Fixes OpenAPI body parameters for later. - newparams = [] - counter = -1 - bodyindex = -1 - for parameter in action["parameters"]: - counter += 1 - - # Hack for key:value in options using || - try: - if parameter["options"] != None and len(parameter["options"]) > 0: - #self.logger.info(f'OPTIONS: {parameter["options"]}') - #self.logger.info(f'OPTIONS VAL: {parameter}') - if "||" in parameter["value"]: - splitvalue = parameter["value"].split("||") - if len(splitvalue) > 1: - #self.logger.info(f'[INFO] Parsed split || options of actions["parameters"]["name"]') - action["parameters"][counter]["value"] = splitvalue[1] - - except (IndexError, KeyError, TypeError) as e: - self.logger.info("[WARNING] Options err: {e}") - - # This part is purely for OpenAPI accessibility. - # It replaces the data back into the main item - # Earlier, we handled each of the items and did later string replacement, - # but this has changed to do lists within items and such - if parameter["name"] == "body": - bodyindex = counter - - try: - values = parameter["value_replace"] - if values != None: - added = 0 - for val in values: - replace_value = val["value"] - replace_key = val["key"] - - if (val["value"].startswith("{") and val["value"].endswith("}")) or (val["value"].startswith("[") and val["value"].endswith("]")): - self.logger.info(f"""Trying to parse as JSON: {val["value"]}""") - try: - newval = val["value"] - - # If it gets here, remove the "" infront and behind the key as well - # since this is preventing the JSON from being loaded - tmpvalue = json.loads(newval) - replace_key = f"\"{replace_key}\"" - except json.decoder.JSONDecodeError as e: - self.logger.info("[WARNING] Failed JSON replacement for OpenAPI %s", val["key"]) - - elif val["value"].lower() == "true" or val["value"].lower() == "false": - replace_key = f"\"{replace_key}\"" - else: - if "\"" in replace_value and not "\\\"" in replace_value: - replace_value = replace_value.replace("\"", "\\\"", -1) - - action["parameters"][counter]["value"] = action["parameters"][counter]["value"].replace(replace_key, replace_value, 1) - - self.logger.info(f'[INFO] Added param {val["key"]} for body (using OpenAPI)') - added += 1 - - #action["parameters"]["body"] - - self.logger.info("ADDED %d parameters for body" % added) - except KeyError as e: - self.logger.info("KeyError body OpenAPI: %s" % e) - pass - - - action["parameters"][counter]["value"] = recurse_cleanup_script(action["parameters"][counter]["value"]) - - #self.logger.info(action["parameters"]) - - # This seems redundant now - for parameter in newparams: - action["parameters"].append(parameter) - - self.action = action - - # Setting due to them being overwritten, but still later useful - try: - self.original_action = json.loads(json.dumps(action)) - except Exception as e: - pass - - # calltimes is used to handle forloops in the app itself. - # 2 kinds of loop - one in gui with one app each, and one like this, - # which is super fast, but has a bad overview (potentially good tho) - calltimes = 1 - result = "" - - all_executions = [] - - # Multi_parameter has the data for each. variable - minlength = 0 - multi_parameters = json.loads(json.dumps(params)) - multiexecution = False - multi_execution_lists = [] - remove_params = [] - for parameter in action["parameters"]: - check, value, is_loop = parse_params(action, fullexecution, parameter, self) - if check: - raise Exception(json.dumps({ - "success": False, - "exception": f"Value Error: {check}", - "reason": "Parameter {parameter} has an issue", - })) - - #if parameter["name"] == "body": - # #self.logger.info(f"[INFO] Should debug field with liquid and other checks as it's BODY: {value}") - - # Custom format for ${name[0,1,2,...]}$ - #submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" - #self.logger.info(f"Returnedvalue: {value}") - # OLD: Used until 13.03.2021: submatch = "([${]{2}#?([0-9a-zA-Z_-]+)#?(\[.*\])[}$]{2})" - # \${[0-9a-zA-Z_-]+#?(\[.*?]}\$) - submatch = "([${]{2}#?([0-9a-zA-Z_-]+)#?(\[.*?]}\$))" - actualitem = re.findall(submatch, value, re.MULTILINE) - try: - if action["skip_multicheck"]: - self.logger.info("Skipping multicheck") - actualitem = [] - except KeyError: - pass - - actionname = action["name"] - - # Loops in general goes in here to be parsed out as one->multi - if len(actualitem) > 0: - self.logger.info(f"[INFO] Found {len(actualitem)} items in {parameter['name']}. MULTI EXEC.") - multiexecution = True - - handled = False - - # Has a loop without a variable used inside - - # This is here to handle for loops within variables.. kindof - # 1. Find the length of the longest array - # 2. Build an array with the base values based on parameter["value"] - # 3. Get the n'th value of the generated list from values - # 4. Execute all n answers - replacements = {} - curminlength = 0 - for replace in actualitem: - try: - to_be_replaced = replace[0] - actualitem = replace[2] - if actualitem.endswith("}$"): - actualitem = actualitem[:-2] - - except IndexError: - self.logger.info("[WARNING] Indexerror") - continue - - try: - itemlist = json.loads(actualitem) - if len(itemlist) > minlength: - minlength = len(itemlist) - - if len(itemlist) > curminlength: - curminlength = len(itemlist) - - except json.decoder.JSONDecodeError as e: - self.logger.info("JSON Error (replace): %s in %s" % (e, actualitem)) - - replacements[to_be_replaced] = actualitem - - - # Parses the data as string with length, split etc. before moving on. - #self.logger.info("In second part of else: %s" % (len(itemlist))) - # This is a result array for JUST this value.. - # What if there are more? - resultarray = [] - for i in range(0, curminlength): - tmpitem = json.loads(json.dumps(parameter["value"])) - for key, value in replacements.items(): - replacement = value - try: - replacement = json.dumps(json.loads(value)[i]) - except IndexError as e: - self.logger.info(f"[ERROR] Failed handling value parsing with index: {e}") - pass - - if replacement.startswith("\"") and replacement.endswith("\""): - replacement = replacement[1:len(replacement)-1] - - #except json.decoder.JSONDecodeError as e: - - #self.logger.info("REPLACING %s with %s" % (key, replacement)) - #replacement = parse_wrapper_start(replacement) - tmpitem = tmpitem.replace(key, replacement, -1) - try: - tmpitem = parse_liquid(tmpitem, self) - except Exception as e: - self.logger.info(f"[WARNING] Failed liquid parsing in loop (2): {e}") - - - # This code handles files. - isfile = False - try: - if parameter["schema"]["type"] == "file" and len(value) > 0: - self.logger.info("(2) SHOULD HANDLE FILE IN MULTI. Get based on value %s" % parameter["value"]) - - for tmp_file_split in json.loads(parameter["value"]): - file_value = self.get_file(tmp_file_split) - resultarray.append(file_value) - - - isfile = True - except KeyError as e: - self.logger.info("(2) SCHEMA ERROR IN FILE HANDLING: %s" % e) - except json.decoder.JSONDecodeError as e: - self.logger.info("(2) JSON ERROR IN FILE HANDLING: %s" % e) - - if not isfile: - #tmpitem = tmpitem.replace("\\\\", "\\", -1) - resultarray.append(tmpitem) - - # With this parameter ready, add it to... a greater list of parameters. Rofl - if len(resultarray) == 0: - self.logger.info("[WARNING] Returning empty array because the array length to be looped is 0 (0)") - self.action_result["status"] = "SUCCESS" - self.action_result["result"] = "[]" - self.send_result(self.action_result, headers, stream_path) - return - - #self.logger.info("RESULTARRAY: %s" % resultarray) - if resultarray not in multi_execution_lists: - multi_execution_lists.append(resultarray) - - multi_parameters[parameter["name"]] = resultarray - else: - # Parses things like int(value) - #self.logger.info("[DEBUG] Normal parsing (not looping)")#with data %s" % value) - # This part has fucked over so many random JSON usages because of weird paranthesis parsing - - value = parse_wrapper_start(value, self) - - try: - if str(value).startswith("b'") and str(value).endswith("'"): - value = value[2:-1] - except Exception as e: - pass - - params[parameter["name"]] = value - multi_parameters[parameter["name"]] = value - - # This code handles files. - try: - if parameter["schema"]["type"] == "file" and len(value) > 0: - self.logger.info("\n SHOULD HANDLE FILE. Get based on value %s. <--- is this a valid ID?" % parameter["value"]) - file_value = self.get_file(value) - self.logger.info("FILE VALUE: %s \n" % file_value) - - params[parameter["name"]] = file_value - multi_parameters[parameter["name"]] = file_value - except KeyError as e: - self.logger.info("SCHEMA ERROR IN FILE HANDLING: %s" % e) - - - # Fix lists here - # FIXME: This doesn't really do anything anymore - #self.logger.info("[DEBUG] CHECKING multi execution list: %d!" % len(multi_execution_lists)) - if len(multi_execution_lists) > 0: - filteredlist = [] - for listitem in multi_execution_lists: - if listitem in filteredlist: - continue - - # FIXME: Subsub required?. Recursion! - # Basically multiply what we have with the outer loop? - # - #if isinstance(listitem, list): - # for subitem in listitem: - # filteredlist.append(subitem) - #else: - # filteredlist.append(listitem) - - #self.logger.info("New list length: %d" % len(filteredlist)) - if len(filteredlist) > 1: - self.logger.info(f"Calculating new multi-loop length with {len(filteredlist)} lists") - tmplength = 1 - for innerlist in filteredlist: - tmplength = len(innerlist)*tmplength - self.logger.info("List length: %d. %d*%d" % (tmplength, len(innerlist), tmplength)) - - minlength = tmplength - - self.logger.info("New multi execution length: %d\n" % tmplength) - - # Cleaning up extra list params - for subparam in remove_params: - #self.logger.info(f"DELETING {subparam}") - try: - del params[subparam] - except: - pass - #self.logger.info(f"Error with subparam deletion of {subparam} in {params}") - try: - del multi_parameters[subparam] - except: - #self.logger.info(f"Error with subparam deletion of {subparam} in {multi_parameters} (2)") - pass - - #self.logger.info() - #self.logger.info(f"Param: {params}") - #self.logger.info(f"Multiparams: {multi_parameters}") - #self.logger.info() - - if not multiexecution: - self.logger.info("NOT MULTI EXEC") - # Runs a single iteration here - new_params = self.validate_unique_fields(params) - if isinstance(new_params, list) and len(new_params) == 1: - params = new_params[0] - #params = new_params - else: - #self.logger.info("[WARNING] SHOULD STOP EXECUTION BECAUSE FIELDS AREN'T UNIQUE") - self.action_result["status"] = "SKIPPED" - self.action_result["result"] = f"A non-unique value was found" - self.action_result["completed_at"] = int(time.time_ns()) - self.send_result(self.action_result, headers, stream_path) - return - - #self.logger.info("[INFO] Running normal execution (not loop)\n\n") - - # Added literal evaluation of anything resembling a string - # The goal is to parse objects that e.g. use single quotes and the like - # FIXME: add this to Multi exec as well. - try: - for key, value in params.items(): - if "-" in key: - try: - newkey = key.replace("-", "_", -1).lower() - params[newkey] = params[key] - except Exception as e: - self.logger.info("[DEBUG] Failed updating key with dash in it: %s" % e) - - try: - if isinstance(value, str) and ((value.startswith("{") and value.endswith("}")) or (value.startswith("[") and value.endswith("]"))): - params[key] = json.loads(value) - except Exception as e: - try: - if isinstance(value, str) and ((value.startswith("{") and value.endswith("}")) or (value.startswith("[") and value.endswith("]"))): - params[key] = ast.literal_eval(value) - except Exception as e: - self.logger.info(f"[DEBUG] Failed parsing value with ast and json.loads - noncritical. Trying next: {e}") - continue - except Exception as e: - self.logger.info("[DEBUG] Failed looping objects. Non critical: {e}") - - # Uncomment below to get the param input - # self.logger.info(f"[DEBUG] PARAMS: {params}") - - #newres = "" - iteration_count = 0 - found_error = "" - while True: - iteration_count += 1 - if iteration_count >= 10: - newres = { - "success": False, - "reason": "Iteration count more than 10. This happens if the input to the action is wrong. Try remaking the action, and contact support@shuffler.io if this persists.", - "details": f"{found_error}", - } - break - - try: - #try: - # Individual functions shouldn't take longer than this - # This is an attempt to make timeouts occur less, incentivizing users to make use efficient API's - # PS: Not implemented for lists - only single actions as of May 2023 - timeout = 30 - - # Check if current app is Shuffle Tools, then set to 55 due to certain actions being slow (ioc parser..) - # In general, this should be disabled for onprem - if self.action["app_name"].lower() == "shuffle tools": - timeout = 55 - - timeout_env = os.getenv("SHUFFLE_APP_SDK_TIMEOUT", timeout) - try: - timeout = int(timeout_env) - #self.logger.info(f"[DEBUG] Timeout set to {timeout} seconds") - except Exception as e: - self.logger.info(f"[ERROR] Failed parsing timeout to int: {e}") - - #timeout = 30 - self.logger.info("[DEBUG][%s] Running function '%s' with timeout %d" % (self.current_execution_id, action["name"], timeout)) - - try: - executor = concurrent.futures.ThreadPoolExecutor() - future = executor.submit(func, **params) - newres = future.result(timeout) - - if not future.done(): - # The future is still running, so we need to cancel it - future.cancel() - newres = json.dumps({ - "success": False, - "exception": str(e), - "reason": "Timeout error within %d seconds (1). This happens if we can't reach or use the API you're trying to use within the time limit. Configure SHUFFLE_APP_SDK_TIMEOUT=100 in Orborus to increase it to 100 seconds. Not changeable for cloud." % timeout, - }) - - else: - # The future is done, so we can just get the result from newres :) - #newres = future.result() - pass - - except concurrent.futures.TimeoutError as e: - newres = json.dumps({ - "success": False, - "reason": "Timeout error (2) within %d seconds (2). This happens if we can't reach or use the API you're trying to use within the time limit. Configure SHUFFLE_APP_SDK_TIMEOUT=100 in Orborus to increase it to 100 seconds. Not changeable for cloud." % timeout, - }) - - break - except TypeError as e: - newres = "" - self.logger.info(f"[ERROR] Got function exec type error: {e}") - try: - e = json.loads(f"{e}") - except: - e = f"{e}" - - found_error = e - errorstring = f"{e}" - - if "the JSON object must be" in errorstring: - self.logger.info("[ERROR] Something is wrong with the input for this function. Are lists and JSON data handled parsed properly (0)? the JSON object must be in...") - - newres = json.dumps({ - "success": False, - "exception": f"{type(e).__name__} - {e}", - "reason": "An exception occurred while running this function (1). See exception for more details and contact support if this persists (support@shuffler.io)", - }) - break - elif "got an unexpected keyword argument" in errorstring: - fieldsplit = errorstring.split("'") - if len(fieldsplit) > 1: - field = fieldsplit[1] - - try: - del params[field] - self.logger.info("[WARNING] Removed invalid field %s (2)" % field) - except KeyError: - break - else: - newres = json.dumps({ - "success": False, - "exception": f"TypeError: {e}", - "reason": "You may be running an old version of this action. Try remaking the node, then contact us at support@shuffler.io if it doesn't work with all these details.", - }) - break - except Exception as e: - self.logger.info(f"[ERROR] Something is wrong with the input for this function. Are lists and JSON data handled parsed properly (1)? err: {e}") - - #try: - # e = json.loads(f"{e}") - #except: - # e = f"{e}" - - newres = json.dumps({ - "success": False, - "exception": f"{type(e).__name__} - {e}", - "reason": "An exception occurred while running this function (2). See exception for more details and contact support if this persists (support@shuffler.io)", - - }) - break - - # Forcing async wait in case of old apps that use async (backwards compatibility) - try: - if asyncio.iscoroutine(newres): - self.logger.info("[DEBUG] In coroutine (1)") - async def parse_value(newres): - value = await asyncio.gather( - newres - ) - - return value[0] - - newres = asyncio.run(parse_value(newres)) - else: - #self.logger.info("[DEBUG] Not in coroutine (1)") - pass - except Exception as e: - self.logger.warning("[ERROR] Failed to parse coroutine value for old app: {e}") - - #self.logger.info("\n\n\n[INFO] Returned from execution with type(s) %s" % type(newres)) - #self.logger.info("\n[INFO] Returned from execution with %s of types %s" % (newres, type(newres)))#, newres) - if isinstance(newres, tuple): - #self.logger.info(f"[INFO] Handling return as tuple: {newres}") - # Handles files. - filedata = "" - file_ids = [] - if isinstance(newres[1], list): - self.logger.info("[INFO] HANDLING LIST FROM RET") - file_ids = self.set_files(newres[1]) - elif isinstance(newres[1], object): - self.logger.info("[INFO] Handling JSON from ret") - file_ids = self.set_files([newres[1]]) - elif isinstance(newres[1], str): - self.logger.info("[INFO] Handling STRING from ret") - file_ids = self.set_files([newres[1]]) - else: - self.logger.info("[INFO] NO FILES TO HANDLE") - - tmp_result = { - "success": True, - "result": newres[0], - "file_ids": file_ids - } - - result = json.dumps(tmp_result) - elif isinstance(newres, str): - #self.logger.info("[INFO] Handling return as string of length %d" % len(newres)) - result += newres - elif isinstance(newres, dict) or isinstance(newres, list): - try: - result += json.dumps(newres, indent=4) - except json.JSONDecodeError as e: - self.logger.info("[WARNING] Failed decoding result: %s" % e) - try: - result += str(newres) - except ValueError: - result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) - self.logger.info("[ERROR] Can't handle type %s value from function" % (type(newres))) - except Exception as e: - self.logger.info("[ERROR] Failed to json dump. Returning as string.") - result += str(newres) - else: - try: - result += str(newres) - except ValueError: - result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) - self.logger.info("Can't handle type %s value from function" % (type(newres))) - - else: - #self.logger.info("[INFO] APP_SDK DONE: Starting MULTI execution (length: %d) with values %s" % (minlength, multi_parameters)) - # 1. Use number of executions based on the arrays being similar - # 2. Find the right value from the parsed multi_params - - #self.logger.info("[INFO] Running WITH loop. MULTI: %s", multi_parameters) - self.logger.info("[INFO] Running WITH loop") - json_object = False - #results = await self.run_recursed_items(func, multi_parameters, {}) - results = self.run_recursed_items(func, multi_parameters, {}) - if isinstance(results, dict) or isinstance(results, list): - json_object = True - - # Dump the result as a string of a list - #self.logger.info("RESULTS: %s" % results) - if isinstance(results, list) or isinstance(results, dict): - - # This part is weird lol - if json_object: - try: - result = json.dumps(results) - except json.JSONDecodeError as e: - self.logger.info(f"Failed to decode: {e}") - result = results - else: - result = "[" - for item in results: - try: - json.loads(item) - result += item - except json.decoder.JSONDecodeError as e: - # Common nested issue which puts " around everything - self.logger.info("Decodingerror: %s" % e) - try: - tmpitem = item.replace("\\\"", "\"", -1) - json.loads(tmpitem) - result += tmpitem - - except: - result += "\"%s\"" % item - - result += ", " - - result = result[:-2] - result += "]" - else: - self.logger.info("Normal result - no list?") - result = results - - self.action_result["status"] = "SUCCESS" - self.action_result["result"] = str(result) - if self.action_result["result"] == "": - self.action_result["result"] = result - - #self.logger.debug(f"[DEBUG] Executed {action['label']}-{action['id']}")#with result: {result}") - #self.logger.debug(f"Data: %s" % action_result) - except TypeError as e: - self.logger.info("[ERROR] TypeError issue: %s" % e) - self.action_result["status"] = "FAILURE" - self.action_result["result"] = json.dumps({ - "success": False, - "reason": f"Typeerror. Most likely due to a list that should've been a string. See details for more info.", - "details": f"{e}", - }) - #self.action_result["result"] = "TypeError: %s" % str(e) - else: - self.logger.info("[DEBUG] Function %s doesn't exist?" % action["name"]) - self.logger.error(f"[ERROR] App {self.__class__.__name__}.{action['name']} is not callable") - self.action_result["status"] = "FAILURE" - #self.action_result["result"] = "Function %s is not callable." % actionname - - self.action_result["result"] = json.dumps({ - "success": False, - "reason": f"Function %s doesn't exist." % actionname, - }) - - # https://ptb.discord.com/channels/747075026288902237/882017498550112286/882043773138382890 - except (requests.exceptions.RequestException, TimeoutError) as e: - self.logger.info(f"[ERROR] Failed to execute request (requests): {e}") - self.logger.exception(f"[ERROR] Failed to execute {e}-{action['id']}") - self.action_result["status"] = "SUCCESS" - try: - e = json.loads(f"{e}") - except: - e = f"{e}" - - try: - self.action_result["result"] = json.dumps({ - "success": False, - "reason": f"Request error - failing silently. Details in detail section", - "details": f"{e}", - }) - except json.decoder.JSONDecodeError as e: - self.action_result["result"] = f"Request error: {e}" - - except Exception as e: - self.logger.info(f"[ERROR] Failed to execute: {e}") - self.logger.exception(f"[ERROR] Failed to execute {e}-{action['id']}") - self.action_result["status"] = "FAILURE" - try: - e = json.loads(f"{e}") - except: - e = f"{e}" - - self.action_result["result"] = json.dumps({ - "success": False, - "reason": f"General exception in the app. See shuffle action logs for more details.", - "details": f"{e}", - }) - - # Send the result :) - self.action_result["completed_at"] = int(time.time_ns()) - self.send_result(self.action_result, headers, stream_path) - - #try: - # try: - # self.log_capture_string.flush() - # except Exception as e: - # print(f"[WARNING] Failed to flush logs (2): {e}") - # pass - - # self.log_capture_string.close() - #except: - # print(f"[WARNING] Failed to close logs (2): {e}") - - return - - @classmethod - def run(cls, action=""): - logging.basicConfig(format="{asctime} - {name} - {levelname}:{message}", style='{') - logger = logging.getLogger(f"{cls.__name__}") - logger.setLevel(logging.DEBUG) - - #logger.info("[DEBUG] Normal execution.") - - ############################################## - - exposed_port = os.getenv("SHUFFLE_APP_EXPOSED_PORT", "") - #logger.info(f"[DEBUG] \"{runtime}\" - run indicates microservices. Port: \"{exposed_port}\"") - if runtime == "run" and exposed_port != "": - # Base port is 33334. Exposed port may differ based on discovery from Worker - port = int(exposed_port) - #logger.info(f"[DEBUG] Starting webserver on port {port} (same as exposed port)") - from flask import Flask, request - from waitress import serve - - flask_app = Flask(__name__) - #flask_app.config['PERMANENT_SESSION_LIFETIME'] = datetime.timedelta(minutes=5) - - #async def execute(): - @flask_app.route("/api/v1/health", methods=["GET", "POST"]) - def check_health(): - return "OK" - - @flask_app.route("/api/v1/run", methods=["POST"]) - def execute(): - if request.method == "POST": - requestdata = {} - try: - requestdata = json.loads(request.data) - except Exception as e: - return { - "success": False, - "reason": f"Invalid Action data {e}", - } - - # Remaking class for each request - - app = cls(redis=None, logger=logger, console_logger=logger) - extra_info = "" - try: - #asyncio.run(AppBase.run(action=requestdata), debug=True) - #value = json.dumps(value) - try: - app.full_execution = json.dumps(requestdata["workflow_execution"]) - except Exception as e: - extra_info += f"\n{e}" - - try: - app.action = requestdata["action"] - except Exception as e: - extra_info += f"\n{e}" - - try: - app.authorization = requestdata["authorization"] - app.current_execution_id = requestdata["execution_id"] - except Exception as e: - extra_info += f"\n{e}" - - # BASE URL (backend) - try: - app.url = requestdata["url"] - except Exception as e: - extra_info += f"\n{e}" - - # URL (worker) - try: - app.base_url = requestdata["base_url"] - except Exception as e: - extra_info += f"\n{e}" - - #await - app.execute_action(app.action) - except Exception as e: - return { - "success": False, - "reason": f"Problem in execution {e}", - "execution_issues": extra_info, - } - - return { - "success": True, - "reason": "App successfully finished", - "execution_issues": extra_info, - } - else: - return { - "success": False, - "reason": f"HTTP method {request.method} not allowed", - } - - logger.info(f"[DEBUG] Serving on port {port}") - - #flask_app.run( - # host="0.0.0.0", - # port=port, - # threaded=True, - # processes=1, - # debug=False, - #) - - serve( - flask_app, - host="0.0.0.0", - port=port, - threads=8, - channel_timeout=30, - expose_tracebacks=True, - asyncore_use_poll=True, - ) - ####################### - else: - # Has to start like this due to imports in other apps - # Move it outside everything? - app = cls(redis=None, logger=logger, console_logger=logger) - - if isinstance(action, str): - #logger.info("[DEBUG] Normal execution (env var). Action is a string.") - pass - elif isinstance(action, object): - #logger.info("[DEBUG] OBJECT execution (cloud). Action is NOT a string.") - app.action = action - - try: - app.authorization = action["authorization"] - app.current_execution_id = action["execution_id"] - except: - pass - - # BASE URL (worker) - try: - app.url = action["url"] - except: - pass - - # Callback URL (backend) - try: - app.base_url = action["base_url"] - except: - pass - else: - #self.logger.info("ACTION TYPE (unhandled): %s" % type(action)) - pass - - app.execute_action(app.action) - -if __name__ == "__main__": - AppBase.run() diff --git a/backend/app_sdk/autocorrect_test.py b/backend/app_sdk/autocorrect_test.py deleted file mode 100644 index a7624c76..00000000 --- a/backend/app_sdk/autocorrect_test.py +++ /dev/null @@ -1,187 +0,0 @@ -import re -import json - -input_data = """{ - "test4": $test, - "test5": , - "test6": "what" - } -""" - -input_data = """{ - "test0": {{ '' | default: [] }}, - "test": {{ | default: [] }}, - "test2": {{ $test.asd | default: [] }}, - "test3": {{ {"key": "val} | default: [] }}, - "test4": $test, - "test5": , - "test6": "what" - } -""" - - -liquiddata = "{{ $test.asd | some other stuff {{ $test.xyz | more stuff" -pattern = r'\{\{\s*\$[^|}]+\s*\|' - -replaced_data = re.sub(pattern, "{{ '' |", liquiddata) -print(replaced_data) - - -def patternfix_string(liquiddata, patterns, regex_patterns, inputtype="liquid"): - if not inputtype or inputtype == "liquid": - if "{{" not in liquiddata or "}}" not in liquiddata: - return liquiddata - elif inputtype == "json": - liquiddata = liquiddata.strip() - - # Validating if it looks like json or not - if liquiddata[0] == "{" and liquiddata[len(liquiddata)-1] == "}": - pass - else: - if liquiddata[0] == "[" and liquiddata[len(liquiddata)-1] == "]": - pass - else: - return liquiddata - - # If it's already json, don't touch it - try: - json.loads(liquiddata) - return liquiddata - except Exception as e: - pass - else: - print("No replace handler for %s" % inputtype) - return liquiddata - - skipkeys = [" "] - newoutput = liquiddata[:] - for pattern in patterns: - keylocations = [] - parsedvalue = "" - record = False - index = -1 - for key in liquiddata: - - # Return instant if possible - if inputtype == "json": - try: - json.loads(newoutput) - return newoutput - except: - pass - - index += 1 - if not key: - if record: - keylocations.append(index) - parsedvalue += key - - continue - - if key in skipkeys: - if record: - keylocations.append(index) - parsedvalue += key - - continue - - if key == pattern[0] and not record: - record = True - - if key not in pattern: - keylocations = [] - parsedvalue = "" - record = False - - if record: - keylocations.append(index) - parsedvalue += key - - if len(parsedvalue) == 0: - continue - - evaluated_value = parsedvalue[:] - for skipkey in skipkeys: - evaluated_value = "".join(evaluated_value.split(skipkey)) - - if evaluated_value == pattern: - #print("Found matching: %s (%s)" % (parsedvalue, keylocations)) - #print("Should replace with: %s" % patterns[pattern]) - - newoutput = newoutput.replace(parsedvalue, patterns[pattern], -1) - - # Return instant if possible - if inputtype == "json": - try: - json.loads(newoutput) - return newoutput - except: - pass - - - for pattern in regex_patterns: - newlines = [] - for line in newoutput.split("\n"): - replaced_line = re.sub(pattern, regex_patterns[pattern], line) - newlines.append(replaced_line) - - newoutput = "\n".join(newlines) - - # Return instant if possible - if inputtype == "json": - try: - json.loads(newoutput) - return newoutput - except: - pass - - # Dont return json properly unless actually json - if inputtype == "json": - try: - json.loads(newoutput) - return newoutput - except: - # Returns original if json fixing didn't work - return liquiddata - - return newoutput - -print("Start:\n%s" % input_data) - -try: - newinput = patternfix_string(input_data, - { - "{{|": '{{ "" |', - }, - { - #r'\{\{\s*|': "{{ '' |", - r'\{\{\s*\$[^|}]+\s*\|': '{{ "" |', - } - , - inputtype="liquid" - ) -except Exception as e: - print("[ERROR} Failed liquid parsing fix: %s" % e) - newinput = input_data - -try: - newinput = patternfix_string(newinput, - { - }, - { - r'\"\s*\:\s*,': '\": "",', - r'\"\s*\:\s*\$[^,]+\w*\,': '\": "",', - } - , - inputtype="json" - ) - - try: - json.loads(newinput) - print("It's json! Override.") - except Exception as e: - print("Bad json. DONT use the value at all: %s" % e) -except Exception as e: - print("[ERROR} Failed json parsing fix: %s" % e) - -print("\nEnd:\n%s" % newinput) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh deleted file mode 100755 index dd540b94..00000000 --- a/backend/app_sdk/build.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -### DEFAULT -NAME=shuffle-app_sdk -VERSION=1.2.0 - -docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force -docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly -t shuffle/shuffle:app_sdk -t shuffle/$NAME:$VERSION -t docker.pkg.github.com/shuffle/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:nightly - -docker push frikky/shuffle:app_sdk -docker push ghcr.io/frikky/$NAME:$VERSION -docker push ghcr.io/frikky/$NAME:nightly -docker push ghcr.io/frikky/$NAME:latest - -docker push shuffle/shuffle:app_sdk -docker push ghcr.io/shuffle/$NAME:$VERSION -docker push ghcr.io/shuffle/$NAME:nightly -docker push ghcr.io/shuffle/$NAME:latest - - - - -#### UBUNTU -NAME=shuffle-app_sdk_ubuntu -docker build . -f Dockerfile_ubuntu -t frikky/shuffle:app_sdk_ubuntu -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -docker push frikky/shuffle:app_sdk_ubuntu -docker push ghcr.io/frikky/$NAME:$VERSION - -#### Alpine GRPC -NAME=shuffle-app_sdk_grpc -docker build . -f Dockerfile_alpine_grpc -t frikky/shuffle:app_sdk_grpc -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -docker push frikky/shuffle:app_sdk_grpc -docker push ghcr.io/frikky/$NAME:$VERSION - - - -#### KALI ### -#NAME=shuffle-app_sdk_kali -#docker build . -f Dockerfile_kali -t frikky/shuffle:app_sdk_kali -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -# -#docker push frikky/shuffle:app_sdk_kali -#docker push ghcr.io/frikky/$NAME:$VERSION -#docker push ghcr.io/frikky/$NAME:nightly - -### BLACKARCH ### -#NAME=shuffle-app_sdk_blackarch -#docker build . -f Dockerfile_blackarch -t frikky/shuffle:app_sdk_blackarch -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -# -#docker push frikky/shuffle:app_sdk_blackarch -#docker push ghcr.io/frikky/$NAME:$VERSION -#docker push ghcr.io/frikky/$NAME:nightly diff --git a/backend/app_sdk/recurse_test.py b/backend/app_sdk/recurse_test.py deleted file mode 100644 index 1cf2bfea..00000000 --- a/backend/app_sdk/recurse_test.py +++ /dev/null @@ -1,326 +0,0 @@ -## A test script of the recurse_json function -## to validate that it can handle the different types of data -## and follow the dot formation format - - -import re -import json - -def recurse_json(basejson, parsersplit): - match = "#([0-9a-z]+):?-?([0-9a-z]+)?#?" - try: - outercnt = 0 - - # Loops over split values - splitcnt = -1 - for value in parsersplit: - splitcnt += 1 - #if " " in value: - # value = value.replace(" ", "_", -1) - - actualitem = re.findall(match, value, re.MULTILINE) - # Goes here if loop - if value == "#": - newvalue = [] - - if basejson == None: - return "", False - - for innervalue in basejson: - # 1. Check the next item (message) - # 2. Call this function again - - try: - ret, is_loop = recurse_json(innervalue, parsersplit[outercnt+1:]) - except IndexError: - # Only in here if it's the last loop without anything in it? - ret, is_loop = recurse_json(innervalue, parsersplit[outercnt:]) - - newvalue.append(ret) - - # Magical way of returning which makes app sdk identify - # it as multi execution - return newvalue, True - - # Checks specific regex like #1-2 for index 1-2 in a loop - elif len(actualitem) > 0: - - is_loop = True - newvalue = [] - firstitem = actualitem[0][0] - seconditem = actualitem[0][1] - if isinstance(firstitem, int): - firstitem = str(firstitem) - if isinstance(seconditem, int): - seconditem = str(seconditem) - - #print("[DEBUG] ACTUAL PARSED: %s" % actualitem) - - # Means it's a single item -> continue - if seconditem == "": - #print("[INFO] In first - handling %s. Len: %d" % (firstitem, len(basejson))) - if str(firstitem).lower() == "max" or str(firstitem).lower() == "last" or str(firstitem).lower() == "end": - firstitem = len(basejson)-1 - elif str(firstitem).lower() == "min" or str(firstitem).lower() == "first": - firstitem = 0 - else: - firstitem = int(firstitem) - - #print(f"[DEBUG] Post lower checks with item {firstitem}") - tmpitem = basejson[int(firstitem)] - try: - newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:]) - except IndexError: - newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:]) - else: - #print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem)) - if isinstance(firstitem, str): - if firstitem.lower() == "max" or firstitem.lower() == "last" or firstitem.lower() == "end": - firstitem = len(basejson)-1 - elif firstitem.lower() == "min" or firstitem.lower() == "first": - firstitem = 0 - else: - firstitem = int(firstitem) - else: - firstitem = int(firstitem) - - if isinstance(seconditem, str): - if str(seconditem).lower() == "max" or str(seconditem).lower() == "last" or str(firstitem).lower() == "end": - seconditem = len(basejson)-1 - elif str(seconditem).lower() == "min" or str(seconditem).lower() == "first": - seconditem = 0 - else: - seconditem = int(seconditem) - else: - seconditem = int(seconditem) - - #print(f"[DEBUG] Post lower checks 2: {firstitem} AND {seconditem}") - newvalue = [] - if int(seconditem) > len(basejson): - seconditem = len(basejson) - - for i in range(int(firstitem), int(seconditem)+1): - # 1. Check the next item (message) - # 2. Call this function again - - try: - ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt+1:]) - except IndexError: - #print("[DEBUG] INDEXERROR (1): ", parsersplit[outercnt]) - #ret = innervalue - ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt:]) - - newvalue.append(ret) - - return newvalue, is_loop - - else: - if len(value) == 0: - return basejson, False - - try: - if isinstance(basejson, list): - #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) - return basejson, False - elif isinstance(basejson, bool): - #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value) - return basejson, False - elif isinstance(basejson, int): - #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value) - return basejson, False - elif isinstance(basejson[value], str): - try: - if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")): - basejson = json.loads(basejson[value]) - else: - # Should we sanitize here? - #print("[DEBUG] VALUE TO SANITIZE FOR KEY '%s'?: %s" % (value, basejson[value])) - - # Check if we are on the last item? - if outercnt == len(parsersplit)-1: - #print("[DEBUG] LAST KEY") - return str(basejson[value]), False - else: - #print("[DEBUG] NOT LAST KEY") - pass - - except json.decoder.JSONDecodeError as e: - return str(basejson[value]), False - else: - basejson = basejson[value] - except KeyError as e: - print("[WARNING] Running secondary value check with replacement of underscore in %s: %s" % (value, e)) - if "_" in value: - value = value.replace("_", " ", -1) - elif " " in value: - value = value.replace(" ", "_", -1) - - try: - if isinstance(basejson, list): - #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) - return basejson, False - elif isinstance(basejson, bool): - #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value) - return basejson, False - elif isinstance(basejson, int): - #print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value) - return basejson, False - elif isinstance(basejson[value], str): - #print(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value]) - try: - #print("[DEBUG] BASEJSON: %s" % basejson) - if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")): - basejson = json.loads(basejson[value]) - else: - - if outercnt == len(parsersplit)-1: - #print("LAST KEY (2)") - return str(basejson[value]), False - else: - #print("NOT LAST KEY (2)") - pass - - except json.decoder.JSONDecodeError as e: - #print("[DEBUG] RETURNING BECAUSE '%s' IS A NORMAL STRING (1)" % basejson[value]) - return str(basejson[value]), False - else: - basejson = basejson[value] - except KeyError as e: - # Check if previous key was handled or not - previouskey = parsersplit[outercnt-1] - #print("[DEBUG] PREVIOUS KEY: ", previouskey) - - tmpval = previouskey + "." + value - #print("\n\n[WARNING] Running third dot notation fix '%s' on data %s: %s" % (value, basejson, e)) - if tmpval in basejson: - return basejson[tmpval], False - - try: - currentsplitcnt = splitcnt - - recursed_value = value - handled = False - - #tmpbase = basejson - previouskey = value - while True: - #print("\n\n[DEBUG] CURRENTSPLITCNT: ", currentsplitcnt) - newvalue = parsersplit[currentsplitcnt+1] - if newvalue == "#" or newvalue == "": - break - - recursed_value += "." + newvalue - #print("\n\nRECURSED: ", recursed_value) - - found = False - for key, value in basejson.items(): - if recursed_value.lower() in key.lower(): - found = True - - if found == False: - #print("[INFO] DIDN'T FIND similar VALUE: ", recursed_value) - - # Check if we are on the last key or not - return "", False - #if outercnt == len(parsersplit)-1: - # print("[DEBUG] LAST KEY (3)") - # break - #else: - # print("[DEBUG] NOT LAST KEY (3)") - # return "", False - - if recursed_value in basejson: - #print("[INFO] FOUND RECURSED VALUE: ", recursed_value) - basejson = basejson[recursed_value] - - # Whether to dig deeper or not - if isinstance(basejson, bool) or isinstance(basejson, int) or isinstance(basejson, str): - handled = False - else: - handled = True - - break - - currentsplitcnt += 1 - - if handled: - continue - - break - except IndexError as e: - print("[DEBUG] INDEXERROR (2):", parsersplit[outercnt]) - return "", False - - outercnt += 1 - - except KeyError as e: - print("[INFO] Lower keyerror: %s" % e) - return "", False - except Exception as e: - print("[WARNING] Exception: %s" % e) - return "", False - - return basejson, False - -print("[INFO] Starting") - -#input_data = "test" -#input_data = "test2.data" - - - -# Matchwith -basejson = { - "test": "hello", - "test2": { - "test3": "hello2", - "test3.data": "hello3", - "test4.data.testing": { - "value": "hello4" - }, - "test5.data.hello": "wut", - }, - "test3": ["hello", "hello2", "hello3"], - "test4": [{ - "id": "1", - }] -} - -# Inputexamples (ALL should be True) -inputs = { - #"": "", - "badkey": "", - "test": "hello", - "test2.badkey": "", - "test2.test3": "hello2", - "test2.test3.data": "hello3", - "test2.test4.data.testing": "{'value': 'hello4'}", # FIXME: Doesn't work due to break vs return "", False in last exception - "test2.test4.data.testing.value": "hello4", # FIXME: Doesn't work due to break vs return "", False in last exception. Not fixed as we didn't find one of these yet. - "test2.test5.data.hello": "wut", - "test2.test5.data.badkey": "", - "test3.#1": "hello2", - "test4.#0.id": "1", - "test4.#1.id": "", -} - -outputs = [] -for key, value in inputs.items(): - parsersplit = key.split(".") - ret, is_loop = recurse_json(basejson, parsersplit) - print("\n\nOUTPUT RET (%s): %s" % (key, ret)) - - outputs.append("[%s]: %s = '%s' vs '%s'" % (str(ret) == str(value), key, ret, value)) - -print("\n\n%s" % "\n".join(outputs)) - -#input_data = "" -#input_data = "badkey" -#input_data = "test" -#input_data = "test2.data" -#input_data = "test2.test3.data" -#input_data = "test2.test4.data.testing.value.as" -#input_data = "test2.test5.data.hello" - - - - diff --git a/backend/app_sdk/requirements.txt b/backend/app_sdk/requirements.txt index 13eb2a0c..2dfbed0d 100644 --- a/backend/app_sdk/requirements.txt +++ b/backend/app_sdk/requirements.txt @@ -6,3 +6,4 @@ flask[async]==2.0.2 waitress==2.1.0 #flask==1.1.2 python-dateutil==2.8.1 + diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 03312201..a6dfc67d 100755 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -211,6 +211,7 @@ func fixTags(tags []string) []string { func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string, downloadIfFail bool) error { ctx := context.Background() client, err := client.NewEnvClient() + defer client.Close() if err != nil { log.Printf("Unable to create docker client: %s", err) return err @@ -349,7 +350,7 @@ func deleteJob(client *kubernetes.Clientset, jobName, namespace string) error { }) } -func buildImage(tags []string, dockerfileFolder string) error { +func buildImage(tags []string, dockerfileLocation string) error { isKubernetes := false if os.Getenv("IS_KUBERNETES") == "true" { @@ -369,10 +370,8 @@ func buildImage(tags []string, dockerfileFolder string) error { log.Printf("[INFO] registry name: %s", registryName) - contextDir := strings.Replace(dockerfileFolder, "Dockerfile", "", -1) - contextDir = "/app/" + contextDir + contextDir := filepath.Join("/app/", filepath.Dir(dockerfileLocation)) log.Print("contextDir: ", contextDir) - dockerFile := "./Dockerfile" client, err := getK8sClient() if err != nil { @@ -407,7 +406,7 @@ func buildImage(tags []string, dockerfileFolder string) error { Image: "gcr.io/kaniko-project/executor:latest", Args: []string{ "--verbosity=debug", - "--dockerfile=" + dockerFile, + "--dockerfile=Dockerfile", "--context=dir://" + contextDir, "--skip-tls-verify", "--destination=" + registryName + "/" + tags[1], @@ -420,9 +419,7 @@ func buildImage(tags []string, dockerfileFolder string) error { }, }, }, - NodeSelector: map[string]string{ - "node": backendNodeName, - }, + NodeName: backendNodeName, RestartPolicy: corev1.RestartPolicyNever, Volumes: []corev1.Volume{ { @@ -480,13 +477,14 @@ func buildImage(tags []string, dockerfileFolder string) error { ctx := context.Background() client, err := client.NewEnvClient() + defer client.Close() if err != nil { log.Printf("Unable to create docker client: %s", err) return err } log.Printf("[INFO] Docker Tags: %s", tags) - dockerfileSplit := strings.Split(dockerfileFolder, "/") + dockerfileSplit := strings.Split(dockerfileLocation, "/") // Create a buffer buf := new(bytes.Buffer) @@ -836,14 +834,23 @@ func handleRemoteDownloadApp(resp http.ResponseWriter, ctx context.Context, user type tmpapp struct { Success bool `json:"success"` OpenAPI string `json:"openapi"` + App string `json:"app"` } app := tmpapp{} err := json.Unmarshal(respBody, &app) if err != nil || app.Success == false || len(app.OpenAPI) == 0 { log.Printf("[ERROR] Failed app unmarshal during auto-download. Success: %#v. Applength: %d: %s", app.Success, len(app.OpenAPI), err) + resp.WriteHeader(401) + if len(app.App) > 0 { + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Not an OpenAPI app, but a Python app. Please download the app using the Remote Download system: https://shuffler.io/docs/apps#importing-remote-apps"}`))) + } else { + resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + } + resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + return } diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index a1a98d5f..11ecca3f 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module shuffle go 1.22.0 -// replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared toolchain go1.22.2 @@ -20,7 +20,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.6.50 + github.com/shuffle/shuffle-shared v0.6.90 golang.org/x/crypto v0.22.0 google.golang.org/api v0.176.1 google.golang.org/grpc v1.63.2 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index fa08e27f..eb35434b 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -334,8 +334,10 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdR github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.6.50 h1:MBeGAiBNkw9Eg+3YTJIlBOuskWntGvT0uefFUYOBhbY= -github.com/shuffle/shuffle-shared v0.6.50/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus= +github.com/shuffle/shuffle-shared v0.6.77 h1:KKtM50xW2DLuRHINxhp3uXrNH0AhiwkeiiU93a8fB3A= +github.com/shuffle/shuffle-shared v0.6.77/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus= +github.com/shuffle/shuffle-shared v0.6.90 h1:FzIYtEt44eWgEsW/9tj2ki7qq8FEm/HWXUok+THp72M= +github.com/shuffle/shuffle-shared v0.6.90/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index d24cbc64..53d0f498 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -35,6 +35,7 @@ import ( "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" + gitProxy "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/storage/memory" // Random @@ -256,7 +257,6 @@ type Hook struct { Environment string `json:"environment" datastore:"environment"` } - func GetUsersHandler(w http.ResponseWriter, r *http.Request) { data := map[string]interface{}{ "id": "12345", @@ -396,6 +396,52 @@ func checkUsername(Username string) error { return nil } +func isGitNoProxy(rawURL string) bool { + noProxy := os.Getenv("NO_PROXY") + if noProxy == "" { + return false + } + + if noProxy == "*" { + return true + } + + noProxyList := strings.Split(noProxy, ",") + parsedURL, err := url.Parse(rawURL) + if err != nil { + return false + } + host := parsedURL.Hostname() + + for _, value := range noProxyList { + value = strings.TrimSpace(value) + + if host == value { + return true + } + if strings.HasPrefix(value, "*.") && strings.HasSuffix(host, value[2:]) { + return true + } + } + return false +} + +func checkGitProxy(cloneOptions *git.CloneOptions) *git.CloneOptions { + if os.Getenv("HTTP_PROXY") != "" && !isGitNoProxy(cloneOptions.URL) { + cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ + URL: os.Getenv("HTTP_PROXY"), + } + } + + if os.Getenv("HTTPS_PROXY") != "" && !isGitNoProxy(cloneOptions.URL) { + cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ + URL: os.Getenv("HTTPS_PROXY"), + } + } + + return cloneOptions +} + func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) error { // Returns false if there is an issue // Use this for register @@ -450,6 +496,7 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) newUser.ActiveOrg = shuffle.OrgMini{ Id: org.Id, Name: org.Name, + Role: newUser.Role, } if len(apikey) > 0 { @@ -511,7 +558,6 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) } } - return nil } @@ -615,7 +661,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { Name: newOrg.Name, } - user.ActiveOrg = currentOrg + user.ActiveOrg = currentOrg } } } @@ -884,18 +930,58 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { log.Printf("[DEBUG] Failed to get org during getinfo: %s", err) } - //if err == nil { if len(org.Id) > 0 { + if userInfo.Role == "" { + //err = shuffle.SetUser(ctx, &userInfo, false) + for _, user := range org.Users { + if user.Id != userInfo.Id { + continue + } + + userInfo.ActiveOrg.Role = user.Role + } + } + userInfo.ActiveOrg = shuffle.OrgMini{ Id: org.Id, Name: org.Name, CreatorOrg: org.CreatorOrg, + ChildOrgs: org.ChildOrgs, Role: userInfo.ActiveOrg.Role, Image: org.Image, } + + if parsedAdmin == "false" { + // Validating admin user again just to make sure + // This is to avoid issues for the first org ever + for _, user := range org.Users { + if user.Id != userInfo.Id { + continue + } + + if user.Role == "admin" { + break + } + } + } } - //} + + orgPriorities := org.Priorities + if len(org.Priorities) < 10 { + //log.Printf("[WARNING] Should find and add priorities as length is less than 10 for org %s", userInfo.ActiveOrg.Id) + newPriorities, err := shuffle.GetPriorities(ctx, userInfo, org) + if err != nil { + log.Printf("[WARNING] Failed getting new priorities for org %s: %s", org.Id, err) + //orgPriorities = []shuffle.Priority{} + } else { + orgPriorities = newPriorities + + // A way to manage them over time + } + } + + orgInterests := org.Interests userInfo.ActiveOrg.Users = []shuffle.UserMini{} userOrgs := []shuffle.OrgMini{} @@ -942,19 +1028,6 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { } userOrgs = shuffle.SortOrgList(userOrgs) - orgPriorities := org.Priorities - if len(org.Priorities) < 10 { - //log.Printf("[WARNING] Should find and add priorities as length is less than 10 for org %s", userInfo.ActiveOrg.Id) - newPriorities, err := shuffle.GetPriorities(ctx, userInfo, org) - if err != nil { - log.Printf("[WARNING] Failed getting new priorities for org %s: %s", org.Id, err) - //orgPriorities = []shuffle.Priority{} - } else { - orgPriorities = newPriorities - - // A way to manage them over time - } - } tutorialsFinished := []shuffle.Tutorial{} for _, tutorial := range userInfo.PersonalInfo.Tutorials { @@ -993,8 +1066,9 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { ChatDisabled: chatDisabled, Tutorials: tutorialsFinished, + Interests: orgInterests, Priorities: orgPriorities, - Licensed: licensed, + Licensed: licensed, } returnData, err := json.Marshal(returnValue) @@ -1015,7 +1089,6 @@ type passwordReset struct { Reference string `json:"reference"` } - func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { cors := shuffle.HandleCors(resp, request) if cors { @@ -1054,9 +1127,9 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { } // No childorg setup, only parent org - if len(org.ManagerOrgs) > 0 || len(org.CreatorOrg) > 0 { - continue - } + // if len(org.ManagerOrgs) > 0 || len(org.CreatorOrg) > 0 { + // continue + // } // Should run calculations if len(org.SSOConfig.OpenIdAuthorization) > 0 { @@ -1978,7 +2051,6 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { } func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { - if request.Method != "POST" { request.Method = "POST" } @@ -1999,7 +2071,7 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { location := strings.Split(request.URL.String(), "/") var pipelineId string - + if location[1] == "api" { if len(location) <= 4 { log.Printf("[INFO] Couldn't handle location. Too short in pipeline: %d", len(location)) @@ -2013,7 +2085,7 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { userAgent := request.Header.Get("User-Agent") if strings.Contains(strings.ToLower(userAgent), "microsoftpreview") || strings.Contains(strings.ToLower(userAgent), "googlebot") { - log.Printf("[AUDIT] Blocking googlebot and microsoftbot for pielines. UA: '%s'", userAgent) + log.Printf("[AUDIT] Blocking googlebot and microsoftbot for pipelines. UA: '%s'", userAgent) resp.WriteHeader(400) resp.Write([]byte(`{"success": false, "reason": "Google/Microsoft preview bots not allowed. Please change the useragent."}`)) return @@ -2058,11 +2130,27 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { return } - parsedBody := shuffle.GetExecutionbody(body) + // Parse concatenated JSON logs + jsonList, err := parseConcatenatedJSONLogs(string(body)) + if err != nil { + log.Printf("[DEBUG] JSON parsing error: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + parsedBody, err := json.Marshal(jsonList) + if err != nil { + log.Printf("[ERROR] Failed to marshal jsonList: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false}`)) + return + } + newBody := shuffle.ExecutionStruct{ Start: pipeline.StartNode, ExecutionSource: "pipeline", - ExecutionArgument: parsedBody, + ExecutionArgument: string(parsedBody), } workflow, err := shuffle.GetWorkflow(ctx, pipeline.WorkflowId) @@ -2093,8 +2181,7 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { } if len(pipeline.StartNode) == 0 { - log.Printf("[WARNING] No start node for pipeline %s - running with workflow default.", pipeline.TriggerId) - + log.Printf("[WARNING] No start node for pipeline %s - running with workflow default.") } newRequest := &http.Request{ @@ -2108,6 +2195,9 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { if err == nil { resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s"}`, workflowExecution.ExecutionId))) + + // Track Sigma rules + trackSigmaRules(ctx, pipeline.OrgId, jsonList) return } @@ -2115,6 +2205,42 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, executionResp))) } +func parseConcatenatedJSONLogs(logs string) ([]map[string]interface{}, error) { + var jsonList []map[string]interface{} + decoder := json.NewDecoder(strings.NewReader(logs)) + + for decoder.More() { + var jsonObject map[string]interface{} + if err := decoder.Decode(&jsonObject); err != nil { + log.Printf("[WARNING] JSON decoding error: %s. Skipping this object.", err) + continue + } + jsonList = append(jsonList, jsonObject) + } + + if err := decoder.Decode(&struct{}{}); err != io.EOF { + return nil, fmt.Errorf("error after decoding all JSON objects: %v", err) + } + + return jsonList, nil +} + +func trackSigmaRules(ctx context.Context, orgId string, jsonList []map[string]interface{}) { + ruleCount := make(map[string]int) + for _, logEntry := range jsonList { + if rule, ok := logEntry["rule"].(map[string]interface{}); ok { + if ruleName, ok := rule["title"].(string); ok { + ruleCount[ruleName]++ + } + } + } + + for ruleName, count := range ruleCount { + shuffle.IncrementCache(ctx, orgId, ruleName, count) + log.Printf("[INFO] Rule %s incremented by %d", ruleName, count) + } +} + func executeCloudAction(action shuffle.CloudSyncJob, apikey string) error { data, err := json.Marshal(action) if err != nil { @@ -3208,7 +3334,6 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User, s } } - log.Printf("[DEBUG] Successfully built app %s (%s)", api.Name, api.ID) if len(user.Id) > 0 { resp.WriteHeader(200) @@ -3249,8 +3374,6 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { buildSwaggerApp(resp, body, user, false) } - - // Hotloads new apps from a folder func handleAppHotload(ctx context.Context, location string, forceUpdate bool) error { @@ -3597,11 +3720,10 @@ func remoteOrgJobController(org shuffle.Org, body []byte) error { return nil } - func remoteOrgJobHandler(org shuffle.Org, interval int) error { // Check if it's 1 in 10 (10% chance random) - backupJob := shuffle.BackupJob{} + backupJob := shuffle.BackupJob{} // Check if workflow backup is active // Check if app backup is active @@ -3647,7 +3769,6 @@ func remoteOrgJobHandler(org shuffle.Org, interval int) error { backupJobData = []byte{} } - syncUrl := fmt.Sprintf("%s/api/v1/cloud/sync", syncUrl) client := shuffle.GetExternalClient(syncUrl) req, err := http.NewRequest( @@ -3809,7 +3930,7 @@ func runInitEs(ctx context.Context) { } if strings.Contains(os.Getenv("SHUFFLE_OPENSEARCH_URL"), "https") { - log.Printf("[INFO] Waiting during init to make sure the opensearch instance is up and running with security features properly") + log.Printf("[INFO] Waiting 30 seconds during init to make sure the opensearch instance is up and running with security features enabled") time.Sleep(30 * time.Second) } @@ -3853,7 +3974,7 @@ func runInitEs(ctx context.Context) { } // FIXME: Add a randomized timer to avoid all schedules running at the same time - // Many are at 5 minutes / 1 hour. The point is to spread these out + // Many are at 5 minutes / 1 hour. The point is to spread these out // a bit instead of all of them starting at the exact same time //log.Printf("Schedule: %#v", schedule) @@ -3888,22 +4009,32 @@ func runInitEs(ctx context.Context) { log.Printf("[DEBUG] Creating org for default user %s", username) orgId := uuid.NewV4().String() orgSetupName := "default" + tmpOrg := shuffle.OrgMini{ + Name: orgSetupName, + Id: orgId, + } + err = createNewUser(username, password, "admin", apikey, tmpOrg) + if err != nil { + log.Printf("[ERROR] Failed to create default user %s: %s", username, err) + } else { + log.Printf("[INFO] Successfully created user %s", username) + } + + user, err := shuffle.GetUser(ctx, username) newOrg := shuffle.Org{ Name: orgSetupName, Id: orgId, Org: orgSetupName, - Users: []shuffle.User{}, + Users: []shuffle.User{*user}, Roles: []string{"admin", "user"}, CloudSync: false, } err = shuffle.SetOrg(ctx, newOrg, newOrg.Id) - setUsers := false if err != nil { - log.Printf("[WARNING] Failed setting organization when creating original user: %s", err) + log.Printf("[ERROR] Failed setting organization when creating original user: %s", err) } else { log.Printf("[DEBUG] Successfully created the default org with id %s!", orgId) - setUsers = true item := shuffle.Environment{ Name: defaultEnv, @@ -3918,20 +4049,6 @@ func runInitEs(ctx context.Context) { log.Printf("[WARNING] Failed setting up new environment") } } - - if setUsers { - tmpOrg := shuffle.OrgMini{ - Name: orgSetupName, - Id: orgId, - } - - err = createNewUser(username, password, "admin", apikey, tmpOrg) - if err != nil { - log.Printf("[INFO] Failed to create default user %s: %s", username, err) - } else { - log.Printf("[INFO] Successfully created user %s", username) - } - } } } else { for _, user := range users { @@ -4145,6 +4262,8 @@ func runInitEs(ctx context.Context) { } } + cloneOptions = checkGitProxy(cloneOptions) + branch := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_BRANCH") if len(branch) > 0 && branch != "master" && branch != "main" { cloneOptions.ReferenceName = plumbing.ReferenceName(branch) @@ -4189,6 +4308,9 @@ func runInitEs(ctx context.Context) { cloneOptions := &git.CloneOptions{ URL: apis, } + + cloneOptions = checkGitProxy(cloneOptions) + _, err = git.Clone(storer, fs, cloneOptions) if err != nil { log.Printf("[ERROR] Failed loading repo %s into memory: %s", apis, err) @@ -4205,17 +4327,16 @@ func runInitEs(ctx context.Context) { log.Printf("[INFO] Skipping download of extra API samples as %d were found", len(workflowapps)) } - if os.Getenv("SHUFFLE_HEALTHCHECK_DISABLED") != "true" { - healthcheckInterval := 30 + healthcheckInterval := 30 log.Printf("[INFO] Starting healthcheck job every %d minute. Stats available on /api/v1/health/stats. Disable with SHUFFLE_HEALTHCHECK_DISABLED=true", healthcheckInterval) job := func() { - // Prepare a fake http.responsewriter + // Prepare a fake http.responsewriter resp := httptest.NewRecorder() request := http.Request{} // Add the "force=true" query to the fake request - request.URL, err = url.Parse("/api/v1/health/stats?force=true") + request.URL, err = url.Parse("/api/v1/health/stats?force=true") if err != nil { log.Printf("[ERROR] Failed to parse test url for healthstats: %s", err) } @@ -4234,7 +4355,6 @@ func runInitEs(ctx context.Context) { log.Printf("[INFO] Finished INIT (ES)") } - func handleVerifyCloudsync(orgId string) (shuffle.SyncFeatures, error) { ctx := context.Background() org, err := shuffle.GetOrg(ctx, orgId) @@ -4813,8 +4933,6 @@ func makeWorkflowPublic(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } - - func handleAppZipUpload(resp http.ResponseWriter, request *http.Request) { cors := shuffle.HandleCors(resp, request) if cors { @@ -4873,8 +4991,6 @@ func handleAppZipUpload(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte("OK")) } - - func initHandlers() { var err error ctx := context.Background() @@ -4906,7 +5022,7 @@ func initHandlers() { go runInitEs(ctx) } else { //go shuffle.runInit(ctx) - log.Printf("[ERROR] Opensearch is the only viable option. Please set SHUFFLE_ELASTIC=true") + log.Printf("[ERROR] Opensearch is the only viable option. Please set SHUFFLE_ELASTIC=true") os.Exit(1) } @@ -4921,7 +5037,7 @@ func initHandlers() { r.HandleFunc("/api/v1/users/register", handleRegister).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/users/checkusers", checkAdminLogin).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/users/getinfo", handleInfo).Methods("GET", "OPTIONS") - + r.HandleFunc("/api/v1/users/{userId}/apps", shuffle.HandleGetUserApps).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/users/apps", shuffle.HandleGetUserApps).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/users/generateapikey", shuffle.HandleApiGeneration).Methods("GET", "POST", "OPTIONS") r.HandleFunc("/api/v1/users/logout", shuffle.HandleLogout).Methods("POST", "OPTIONS") @@ -4942,6 +5058,7 @@ func initHandlers() { r.HandleFunc("/api/v1/register", handleRegister).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/checkusers", checkAdminLogin).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/getinfo", handleInfo).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/me", handleInfo).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/getsettings", shuffle.HandleSettings).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/generateapikey", shuffle.HandleApiGeneration).Methods("GET", "POST", "OPTIONS") r.HandleFunc("/api/v1/passwordchange", shuffle.HandlePasswordChange).Methods("POST", "OPTIONS") @@ -4976,7 +5093,8 @@ func initHandlers() { r.HandleFunc("/api/v1/apps/{appId}", shuffle.UpdateWorkflowAppConfig).Methods("PATCH", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}", shuffle.DeleteWorkflowApp).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}/config", shuffle.GetWorkflowAppConfig).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "POST", "OPTIONS") + r.HandleFunc("/api/v1/apps/{appName}/run_hotload", handleSingleAppHotloadRequest).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/get_existing", LoadSpecificApps).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/download_remote", LoadSpecificApps).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/validate", validateAppInput).Methods("POST", "OPTIONS") @@ -5055,7 +5173,7 @@ func initHandlers() { r.HandleFunc("/api/v1/triggers/gmail/register", shuffle.HandleNewGmailRegister).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/triggers/gmail/getFolders", shuffle.HandleGetGmailFolders).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/triggers/pipeline", shuffle.HandleNewPipelineRegister).Methods("POST", "OPTIONS") - //r.HandleFunc("/api/v1/triggers/pipeline/save", shuffle.HandleSavePipelineInfo).Methods("PUT", "OPTIONS") + //r.HandleFunc("/api/v1/triggers/pipeline/save", shuffle.HandleSavePipelineInfo).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/pipelines/{key}", handlePipelineCallback).Methods("POST", "GET", "PATCH", "PUT", "DELETE", "OPTIONS") r.HandleFunc("/api/v1/triggers", shuffle.HandleGetTriggers).Methods("GET", "OPTIONS") //r.HandleFunc("/api/v1/triggers/gmail/routing", handleGmailRouting).Methods("POST", "OPTIONS") @@ -5077,12 +5195,13 @@ func initHandlers() { //r.HandleFunc("/api/v1/orgs/", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleGetOrg).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleEditOrg).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/orgs/{orgid}/forms", shuffle.HandleGetOrgForms).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/create_sub_org", shuffle.HandleCreateSubOrg).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/change", shuffle.HandleChangeUserOrg).Methods("POST", "OPTIONS") // Swaps to the org r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleDeleteOrg).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/suborgs", shuffle.HandleGetSubOrgs).Methods("GET", "OPTIONS") - + // This is a new API that validates if a key has been seen before. // Not sure what the best course of action is for it. r.HandleFunc("/api/v1/environments/{key}/stop", shuffle.HandleStopExecutions).Methods("GET", "POST", "OPTIONS") @@ -5090,11 +5209,13 @@ func initHandlers() { r.HandleFunc("/api/v1/orgs/{orgId}/validate_app_values", shuffle.HandleKeyValueCheck).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/list_cache", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/orgs/{orgId}/cache/{cache_key}", shuffle.HandleGetCacheKey).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/get_cache", shuffle.HandleGetCacheKey).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/set_cache", shuffle.HandleSetCacheKey).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/delete_cache", shuffle.HandleDeleteCacheKeyPost).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/cache/{cache_key}", shuffle.HandleDeleteCacheKey).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/stats", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/orgs/{orgId}/stats", shuffle.HandleAppendStatistics).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/statistics", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/cache", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS") @@ -5104,7 +5225,6 @@ func initHandlers() { r.HandleFunc("/api/v1/orgs/{orgId}/datastore", shuffle.HandleSetCacheKey).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/datastore/{cache_key}", shuffle.HandleDeleteCacheKey).Methods("DELETE", "OPTIONS") - // Docker orborus specific - downloads an image r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/login_sso", shuffle.HandleSSO).Methods("GET", "POST", "OPTIONS") @@ -5123,6 +5243,17 @@ func initHandlers() { r.HandleFunc("/api/v1/files/{fileId}", shuffle.HandleDeleteFile).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/files", shuffle.HandleGetFiles).Methods("GET", "OPTIONS") + // This structure is horrendous. Needs fixing after we got the prototype up + r.HandleFunc("/api/v1/detections/{detectionType}/connect", shuffle.HandleDetectionAutoConnect).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/detections/{detection_type}", shuffle.HandleGetDetectionRules).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/detections/{triggerId}/selected_rules", shuffle.HandleGetSelectedRules).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/detections/{triggerId}/selected_rules/save", shuffle.HandleSaveSelectedRules).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/detections/{action}", shuffle.HandleFolderToggle).Methods("PUT", "OPTIONS") + + // This is weird. + r.HandleFunc("/api/v1/detections/{fileId}/{action}", shuffle.HandleToggleRule).Methods("PUT", "OPTIONS") + //r.HandleFunc("/api/v1/detections/siem/node_health", shuffle.HandleTenzirHealthUpdate).Methods("POST","OPTIONS") + // Introduced in 0.9.21 to handle notifications for e.g. failed Workflow r.HandleFunc("/api/v1/notifications", shuffle.HandleCreateNotification).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/notifications", shuffle.HandleGetNotifications).Methods("GET", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 7ff135e3..d6cac1fe 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -106,7 +106,6 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode } log.Printf("[INFO] Starting frequency for execution: %d", newfrequency) - //jobret, err := newscheduler.Every(newfrequency).Seconds().NotImmediately().Run(job) jobret, err := newscheduler.Every(newfrequency).Seconds().Run(job) @@ -292,29 +291,44 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { ctx := shuffle.GetContext(request) env, err := shuffle.GetEnvironment(ctx, orgId, "") timeNow := time.Now().Unix() - if err == nil && len(env.Id) > 0 && len(env.Name) > 0 { + if err == nil && len(env.Id) > 0 && len(env.Name) > 0 && request.Method == "POST" { // Updates every 60 seconds~ if time.Now().Unix() > env.Edited+60 { env.RunningIp = shuffle.GetRequestIp(request) + + // Orborus label = custom label for Orborus if len(orborusLabel) > 0 { env.RunningIp = orborusLabel } - if request.Method == "POST" { - body, err := ioutil.ReadAll(request.Body) - if err == nil { - var envData shuffle.OrborusStats - err = json.Unmarshal(body, &envData) - if err == nil { - if envData.Swarm { - env.Licensed = true - env.RunType = "docker" - } + // Set the checkin cache - if envData.Kubernetes { - env.RunType = "k8s" - } + + body, err := ioutil.ReadAll(request.Body) + if err == nil { + var envData shuffle.OrborusStats + err = json.Unmarshal(body, &envData) + if err == nil { + envData.RunningIp = env.RunningIp + + marshalled, err := json.Marshal(envData) + if err == nil { + cacheKey := fmt.Sprintf("queueconfig-%s-%s", env.Name, env.OrgId) + go shuffle.SetCache(context.Background(), cacheKey, marshalled, 2) } + + + + if envData.Swarm { + env.Licensed = true + env.RunType = "docker" + } + + if envData.Kubernetes { + env.RunType = "k8s" + } + + envData.DataLake = env.DataLake } } @@ -572,14 +586,20 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { //return } + if len(actionResult.ExecutionId) == 0 { + resp.WriteHeader(400) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Provide execution_id and authorization"}`))) + return + } + ctx := context.Background() workflowExecution, err := shuffle.GetWorkflowExecution(ctx, actionResult.ExecutionId) - if err != nil { + if err != nil || workflowExecution.ExecutionId != actionResult.ExecutionId { if len(actionResult.ExecutionId) > 0 { log.Printf("[WARNING][%s] Failed getting execution (streamresult): %s", actionResult.ExecutionId, err) } - resp.WriteHeader(401) + resp.WriteHeader(400) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key or execution_id might not exist."}`))) return } @@ -638,9 +658,27 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { } } + if workflowExecution.Workflow.Sharing == "form" { + newWorkflow := shuffle.Workflow{ + Name: workflowExecution.Workflow.Name, + ID: workflowExecution.Workflow.ID, + Owner: workflowExecution.Workflow.Owner, + OrgId: workflowExecution.Workflow.OrgId, + + Sharing: workflowExecution.Workflow.Sharing, + Description: workflowExecution.Workflow.Description, + InputQuestions: workflowExecution.Workflow.InputQuestions, + + FormControl: workflowExecution.Workflow.FormControl, + } + + workflowExecution.Results = []shuffle.ActionResult{} + workflowExecution.Workflow = newWorkflow + } + newjson, err := json.Marshal(workflowExecution) if err != nil { - resp.WriteHeader(401) + resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflow execution"}`))) return } @@ -670,7 +708,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { } //log.Printf("Actionresult unmarshal: %s", string(body)) - log.Printf("[DEBUG] Got workflow result from %s of length %d", request.RemoteAddr, len(body)) + //log.Printf("[DEBUG] Got workflow result from %s of length %d", request.RemoteAddr, len(body)) ctx := context.Background() err = shuffle.ValidateNewWorkerExecution(ctx, body) if err == nil { @@ -681,7 +719,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { log.Printf("[DEBUG] Handling other execution variant (subflow?): %s", err) } - log.Printf("[DEBUG] Got workflow result from %s of length %d.", request.RemoteAddr, len(body)) + //log.Printf("[DEBUG] Got workflow result from %s of length %d.", request.RemoteAddr, len(body)) var actionResult shuffle.ActionResult err = json.Unmarshal(body, &actionResult) @@ -739,8 +777,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { // Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult shuffle.ActionResult, resp http.ResponseWriter) { - log.Printf("[DEBUG][%s] Running workflow execution update", workflowExecutionId) - + log.Printf("[DEBUG][%s] Running workflow execution update with result from %s (%s) of status %s", workflowExecutionId, actionResult.Action.Label, actionResult.Action.ID, actionResult.Status) // Should start a tx for the execution here workflowExecution, err := shuffle.GetWorkflowExecution(ctx, workflowExecutionId) @@ -771,7 +808,6 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl setExecution := true if setExecution || workflowExecution.Status == "FINISHED" || workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" { err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) - //err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, dbSave) if err != nil { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution actionresult: %s"}`, err))) @@ -925,7 +961,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { if len(workflow.ParentWorkflowId) > 0 { resp.WriteHeader(403) resp.Write([]byte(`{"success": false, "reason": "Can't delete a workflow distributed from your parent org"}`)) - return + return } if user.Id != workflow.Owner || len(user.Id) == 0 { @@ -939,6 +975,27 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { } } + // Look for Child workflows and delete them + if workflow.ParentWorkflowId == "" { + log.Printf("[DEBUG] Looking for child workflows for workflow %s to delete. User %s (%s) in org %s (%s)", workflow.ID, user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id) + + childWorkflows, err := shuffle.ListChildWorkflows(ctx, workflow.ID) + if err != nil { + log.Printf("[ERROR] Failed to list child workflows: %s", err) + } else { + log.Printf("\n\n[DEBUG] Found %d child workflows for workflow %s\n\n", len(childWorkflows), workflow.ID) + + // Find cookies and append them to request.Header to replicate current request as closely as possible + for _, childWorkflow := range childWorkflows { + if childWorkflow.ID == workflow.ID { + continue + } + + go shuffle.SendDeleteWorkflowRequest(childWorkflow, request) + } + } + } + // Clean up triggers and executions for _, item := range workflow.Triggers { if item.TriggerType == "SCHEDULE" && item.Status != "uninitialized" { @@ -984,8 +1041,6 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": true}`)) } - - func handleExecution(id string, workflow shuffle.Workflow, request *http.Request, orgId string) (shuffle.WorkflowExecution, string, error) { //go func() { // log.Printf("\n\nPRE TIME: %s\n\n", time.Now().Format("2006-01-02 15:04:05")) @@ -1004,17 +1059,6 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request workflow = *tmpworkflow } - /* - if len(workflow.ExecutingOrg.Id) == 0 { - if len(orgId) > 0 { - workflow.ExecutingOrg.Id = orgId - } else { - log.Printf("[INFO] Stopped execution because there is no executing org for workflow %s", workflow.ID) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow has no executing org defined"), errors.New("Workflow has no executing org defined") - } - } - */ - if len(workflow.Actions) == 0 { workflow.Actions = []shuffle.Action{} } else { @@ -1065,28 +1109,31 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request workflowExecution, execInfo, _, workflowExecErr := shuffle.PrepareWorkflowExecution(ctx, workflow, request, int64(maxExecutionDepth)) if workflowExecErr != nil { - err := shuffle.SetWorkflowExecution(ctx, workflowExecution, true) - if err != nil { - log.Printf("[ERROR] Failed setting workflow execution during init (2): %s", err) + if len(workflowExecution.Workflow.Actions) > 0 && len(workflowExecution.Results) > 0 && len(workflowExecution.ExecutionId) > 0 { + err := shuffle.SetWorkflowExecution(ctx, workflowExecution, true) + if err != nil { + log.Printf("[ERROR] Failed setting workflow execution during init (2): %s", err) + } } if strings.Contains(fmt.Sprintf("%s", workflowExecErr), "User Input") { // Special for user input callbacks - log.Printf("[INFO] User input callback: %s", workflowExecErr) // return workflowExecution, fmt.Sprintf("%s", err), nil + //log.Printf("[INFO] User input callback: %s", workflowExecErr) + return shuffle.WorkflowExecution{}, "", nil } else { - log.Printf("[ERROR] Failed in prepareExecution: '%s'", err) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed running: %s", err), err + log.Printf("[ERROR] Failed in prepareExecution: '%s'", workflowExecErr) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed running: %s", workflowExecErr), workflowExecErr } } - err := imageCheckBuilder(execInfo.ImageNames) if err != nil { log.Printf("[ERROR] Failed building the required images from %#v: %s", execInfo.ImageNames, err) return shuffle.WorkflowExecution{}, "Failed unmarshal during execution", err } + /* makeNew := true start, startok := request.URL.Query()["start"] if request.Method == "POST" { @@ -1206,7 +1253,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request answer, answerok := request.URL.Query()["answer"] referenceId, referenceok := request.URL.Query()["reference_execution"] - if answerok && referenceok { + if answerok && referenceok && len(answer) > 0 && len(referenceId) > 0 { // If answer is false, reference execution with result log.Printf("[INFO] Answer is OK AND reference is OK!") if answer[0] == "false" { @@ -1230,7 +1277,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request log.Printf("%s - %s", result.Action.ID, start[0]) if result.Action.ID == start[0] { note, noteok := request.URL.Query()["note"] - if noteok { + if noteok && len(note) > 0 { result.Result = fmt.Sprintf("User note: %s", note[0]) } else { result.Result = fmt.Sprintf("User clicked %s", answer[0]) @@ -1354,7 +1401,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request } } - childNodes := shuffle.FindChildNodes(workflowExecution, workflowExecution.Start, []string{}, []string{}) + childNodes := shuffle.FindChildNodes(workflowExecution.Workflow, workflowExecution.Start, []string{}, []string{}) startFound := false newActions := []shuffle.Action{} @@ -1560,7 +1607,6 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request // newTriggers = append(newTriggers, trigger) //} //workflowExecution.Workflow.Triggers = newTriggers - _ = removeTriggers if !startFound { if len(workflowExecution.Start) == 0 && len(workflowExecution.Workflow.Start) > 0 { @@ -1585,12 +1631,17 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request if len(workflowExecution.ExecutionOrg) == 0 && len(workflow.ExecutingOrg.Id) > 0 { workflowExecution.ExecutionOrg = workflow.ExecutingOrg.Id } + */ + //workflowExecution, execInfo, _, workflowExecErr := shuffle.PrepareWorkflowExecution(ctx, workflow, request, int64(maxExecutionDepth)) err = shuffle.SetWorkflowExecution(ctx, workflowExecution, true) if err != nil { log.Printf("[ERROR] Failed setting workflow execution during init (2): %s", err) } + onpremExecution := execInfo.OnpremExecution + _ = onpremExecution + environments := execInfo.Environments var allEnvs []shuffle.Environment if len(workflowExecution.ExecutionOrg) > 0 { //log.Printf("[INFO] Executing ORG: %s", workflowExecution.ExecutionOrg) @@ -1854,7 +1905,6 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { } log.Printf("[INFO] Inside execute workflow for ID %s", fileId) - ctx := context.Background() workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil && workflow.ID == "" { @@ -2408,7 +2458,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { return } - workflow.Schedules = append(workflow.Schedules, schedule) + //workflow.Schedules = append(workflow.Schedules, schedule) err = shuffle.SetWorkflow(ctx, *workflow, workflow.ID) if err != nil { log.Printf("Failed setting workflow for schedule: %s", err) @@ -2665,6 +2715,8 @@ func loadGithubWorkflows(url, username, password, userId, branch, orgId string) cloneOptions.ReferenceName = plumbing.ReferenceName(branch) } + cloneOptions = checkGitProxy(cloneOptions) + storer := memory.NewStorage() r, err := git.Clone(storer, fs, cloneOptions) if err != nil { @@ -2766,6 +2818,70 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } +func handleSingleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { + cors := shuffle.HandleCors(resp, request) + if cors { + return + } + ctx := context.Background() + cacheKey := fmt.Sprintf("workflowapps-sorted-1000") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-0") + shuffle.DeleteCache(ctx, cacheKey) + // Just need to be logged in + // FIXME - should have some permissions? + user, err := shuffle.HandleApiAuthentication(resp, request) + if err != nil { + log.Printf("Api authentication failed in app hotload: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + if user.Role != "admin" { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Must be admin to hotload apps"}`)) + return + } + location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER") + if len(location) == 0 { + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "SHUFFLE_APP_HOTLOAD_FOLDER not specified in .env"}`))) + return + } + requestUrlFields := strings.Split(request.URL.String(), "/") + var appName string + if requestUrlFields[1] == "api" { + if len(requestUrlFields) <= 4 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + appName = requestUrlFields[4] + if strings.Contains(appName, "?") { + appName = strings.Split(appName, "?")[0] + } + } + location = location + "/" + appName + log.Printf("[INFO] Starting hotloading from %s", location) + err = handleAppHotload(ctx, location, true) + if err != nil { + log.Printf("[WARNING] Failed app hotload: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + cacheKey = fmt.Sprintf("workflowapps-sorted-100") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-1000") + shuffle.DeleteCache(ctx, cacheKey) + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) +} + func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { cors := shuffle.HandleCors(resp, request) if cors { @@ -3374,7 +3490,15 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflowExecution, err := shuffle.PrepareSingleAction(ctx, user, fileId, body) + + runValidationAction := false + query := request.URL.Query() + validation, ok := query["validation"] + if ok && validation[0] == "true" { + runValidationAction = true + } + + workflowExecution, err := shuffle.PrepareSingleAction(ctx, user, fileId, body, runValidationAction) if err != nil { log.Printf("[INFO] Failed workflowrequest POST read: %s", err) resp.WriteHeader(401) @@ -3410,7 +3534,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { // FIXME: Should use environment that is in the source workflow if it exists for i, _ := range workflowExecution.Workflow.Actions { workflowExecution.Workflow.Actions[i].Environment = environment - workflowExecution.Workflow.Actions[i].Label = "TMP" + workflowExecution.Workflow.Actions[i].Label = "TMP" } shuffle.SetWorkflowExecution(ctx, workflowExecution, false) @@ -3940,6 +4064,8 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { } } + cloneOptions = checkGitProxy(cloneOptions) + storer := memory.NewStorage() r, err := git.Clone(storer, fs, cloneOptions) if err != nil { @@ -4193,7 +4319,6 @@ func checkUnfinishedExecution(resp http.ResponseWriter, request *http.Request) { log.Printf("[ERROR] Failed adding execution to db: %s", err) } - resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Reran workflow in %s"}`, parsedEnv))) diff --git a/docker-compose.yml b/docker-compose.yml index 4b50c8de..1299bf8b 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,6 @@ -version: '3' services: frontend: - image: ghcr.io/shuffle/shuffle-frontend:latest + image: ghcr.io/shuffle/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -15,7 +14,7 @@ services: depends_on: - backend backend: - image: ghcr.io/shuffle/shuffle-backend:latest + image: ghcr.io/shuffle/shuffle-backend:nightly container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -34,7 +33,7 @@ services: - SHUFFLE_FILE_LOCATION=/shuffle-files restart: unless-stopped orborus: - image: ghcr.io/shuffle/shuffle-orborus:latest + image: ghcr.io/shuffle/shuffle-orborus:nightly container_name: shuffle-orborus hostname: shuffle-orborus networks: @@ -45,12 +44,10 @@ services: - SHUFFLE_APP_SDK_TIMEOUT=300 - SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=7 # The amount of concurrent executions Orborus can handle. #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} + - ENVIRONMENT_NAME=Shuffle + - ORG_ID=Shuffle - BASE_URL=http://${OUTER_HOSTNAME}:5001 - DOCKER_API_VERSION=1.40 - - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} - - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} - - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} - HTTP_PROXY=${HTTP_PROXY} - HTTPS_PROXY=${HTTPS_PROXY} - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} @@ -58,7 +55,8 @@ services: - SHUFFLE_STATS_DISABLED=true - SHUFFLE_SWARM_CONFIG=run - SHUFFLE_LOGS_DISABLED=true - - SHUFFLE_WORKER_IMAGE=ghcr.io/shuffle/shuffle-worker:latest + - SHUFFLE_WORKER_IMAGE=ghcr.io/shuffle/shuffle-worker:nightly + env_file: .env restart: unless-stopped security_opt: - seccomp:unconfined @@ -66,7 +64,6 @@ services: image: opensearchproject/opensearch:2.14.0 hostname: shuffle-opensearch container_name: shuffle-opensearch - env_file: .env environment: - "OPENSEARCH_JAVA_OPTS=-Xms2048m -Xmx2048m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - bootstrap.memory_lock=true @@ -86,7 +83,7 @@ services: soft: 65536 hard: 65536 volumes: - - ${DB_LOCATION}:/usr/share/opensearch/data:z + - shuffle-database:/usr/share/opensearch/data:z ports: - 9200:9200 networks: @@ -132,13 +129,18 @@ services: # networks: # - shuffle # + +volumes: + shuffle-database: + driver: local + driver_opts: + type: none + device: ${DB_LOCATION} + o: bind + networks: shuffle: driver: bridge - - # uncomment to set MTU for swarm mode. - # MTU should be whatever is your host's preferred MTU is. - # Refer to this doc to figure out what your host's MTU is: - # https://shuffler.io/docs/troubleshooting#TLS_timeout_error/Timeout_Errors/EOF_Errors # driver_opts: # com.docker.network.driver.mtu: 1460 + # uncomment to set MTU for swarm mode. MTU should be whatever is your host's preferred MTU is: https://shuffler.io/docs/troubleshooting#TLS_timeout_error/Timeout_Errors/EOF_Errors diff --git a/frontend/Dockerfile b/frontend/Dockerfile index e9d6d683..807ca56d 100755 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,6 +1,8 @@ # Build environment FROM node:21 as builder +ENV NODE_OPTIONS="--max-old-space-size=4096" + RUN mkdir /usr/src/app WORKDIR /usr/src/app ENV PATH /usr/src/app/node_modules/.bin:$PATH @@ -11,7 +13,7 @@ COPY package.json /usr/src/app/package.json #RUN yarn config set "strict-ssl" false -g #RUN yarn install --network-timeout 1000000 -RUN npm install --legacy-peer-deps +RUN npm install --timeout=60000 --legacy-peer-deps # copy only required files to not trigger rebuilding every time COPY ./certs /usr/src/app/certs/ @@ -25,28 +27,33 @@ COPY ./*.json /usr/src/app/ RUN npm run build --loglevel verbose 2>&1 # Production environment -FROM nginx:1.21.5 +FROM nginx:1.26.0 RUN mkdir -p /usr/share/nginx/html/build RUN mkdir -p /usr/share/nginx/html/css RUN mkdir -p /usr/share/nginx/html/js RUN mkdir -p /usr/share/nginx/html/img -COPY --from=builder /usr/src/app/build /usr/share/nginx/html -#Localhost certificate challenge: Y#XwrJ#DoZGz2w6x +# Localhost certificate challenge: Y#XwrJ#DoZGz2w6x +# Cert challenge doesn't matter to be here or not, as ALL production setups should be using their own certificates + reverse proxy: https://shuffler.io/docs/configuration#using-the-nginx-reverse-proxy-for-tls/ssl +COPY --from=builder /usr/src/app/build /usr/share/nginx/html COPY --from=builder /usr/src/app/certs/fullchain.pem /etc/nginx/fullchain.cert.pem COPY --from=builder /usr/src/app/certs/privkey.pem /etc/nginx/privkey.pem # install CONFD -ENV CONFD_VERSION 0.16.0 RUN apt-get update && apt-get install -y curl && apt-get clean -RUN curl -sSL https://github.com/kelseyhightower/confd/releases/download/v${CONFD_VERSION}/confd-${CONFD_VERSION}-linux-amd64 -o /usr/local/bin/confd && \ - chmod +x /usr/local/bin/confd -COPY ./confd /etc/confd +COPY ./confd/templates/nginx.conf /etc/nginx/nginx.conf.tmpl +## OLD CONFD THINGS (not compatible with arm) +#ENV CONFD_VERSION 0.16.0 +#RUN curl -sSL https://github.com/kelseyhightower/confd/releases/download/v${CONFD_VERSION}/confd-${CONFD_VERSION}-linux-amd64 -o /usr/local/bin/confd && \ +# chmod +x /usr/local/bin/confd +#COPY ./confd /etc/confd # rewrite command & entrypoint with ours + COPY ./entrypoint.sh / +ENV BACKEND_HOSTNAME="shuffle-backend" ENTRYPOINT [ "/entrypoint.sh" ] CMD ["nginx", "-g", "daemon off;"] diff --git a/frontend/README.md b/frontend/README.md index 571c5625..92b4ab6b 100755 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,3 +1,5 @@ +## Lalits frontend magic + ## Localhost Certificate info: diff --git a/frontend/confd/templates/nginx.conf b/frontend/confd/templates/nginx.conf index 3bb02c27..2c9df91e 100755 --- a/frontend/confd/templates/nginx.conf +++ b/frontend/confd/templates/nginx.conf @@ -71,7 +71,7 @@ http { } location ~ /api/v(1|2) { - proxy_pass http://{{ getenv "BACKEND_HOSTNAME" "shuffle-backend" }}:5001; + proxy_pass http://${BACKEND_HOSTNAME}:5001; proxy_buffering off; proxy_http_version 1.1; @@ -113,7 +113,8 @@ http { # Get the hostname from environment here? location ~ /api/v(1|2) { - proxy_pass http://{{ getenv "BACKEND_HOSTNAME" "shuffle-backend" }}:5001; + proxy_pass http://${BACKEND_HOSTNAME}:5001; + proxy_buffering off; proxy_http_version 1.1; diff --git a/frontend/entrypoint.sh b/frontend/entrypoint.sh index 09be2558..af1d0a43 100755 --- a/frontend/entrypoint.sh +++ b/frontend/entrypoint.sh @@ -1,7 +1,6 @@ -#!/bin/bash +#!/usr/bin/env sh +set -eu -# generate configs -/usr/local/bin/confd -backend="env" -confdir="/etc/confd" -onetime +envsubst '${BACKEND_HOSTNAME}' < /etc/nginx/nginx.conf.tmpl > /etc/nginx/nginx.conf -# run main command exec "$@" diff --git a/frontend/package.json b/frontend/package.json index 635c079f..7426ee23 100755 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,10 +1,9 @@ { "name": "shuffler", "homepage": "https://shuffler.io", - "version": "1.4.0", + "version": "2.0.0", "private": true, "dependencies": { - "@babel/plugin-proposal-class-properties": "^7.18.6", "@codemirror/commands": "^6.2.4", "@codemirror/lang-python": "^6.1.3", "@emotion/react": "^11.11.1", @@ -13,7 +12,7 @@ "@metamask/detect-provider": "^1.2.0", "@mui/icons-material": "^5.14.0", "@mui/material": "^5.14.0", - "@mui/styles": "^5.14.0", + "@mui/styles": "^6.1.4", "@mui/x-data-grid": "^5.17.11", "@mui/x-date-pickers": "^6.11.1", "@types/algoliasearch": "^3.34.11", @@ -21,7 +20,6 @@ "@uiw/codemirror-theme-vscode": "^4.21.20", "@uiw/codemirror-themes": "^4.21.9", "@uiw/react-codemirror": "^4.21.21", - "@use-it/interval": "^0.1.3", "algoliasearch": "^4.8.3", "class-transformer": "^0.2.0", "codemirror": "^6.0.1", @@ -49,7 +47,6 @@ "i18next-localstorage-backend": "^4.1.0", "i18next-xhr-backend": "^3.2.2", "import": "0.0.6", - "interweave": "^11.2.0", "is-plain-obj": "^4.1.0", "json-bigint": "^1.0.0", "match-sorter": "^6.3.1", @@ -58,12 +55,12 @@ "moment": "~2.29.4", "mui-chips-input": "^2.1.3", "mui-nested-menu": "^3.2.1", - "react": "^18.2.0", + "react": "^18.3.1", "react-ace": "^10.1.0", "react-alice-carousel": "^2.6.4", "react-avatar-editor": "^11.1.0", "react-beforeunload": "^2.2.1", - "react-chartjs-2": "^2.11.1", + "react-chartjs-2": "^2.11.2", "react-cookie": "^4.0.1", "react-cytoscapejs": "^2.0.0", "react-device-detect": "^2.2.3", @@ -73,21 +70,18 @@ "react-dropzone": "^14.2.3", "react-ga4": "^2.0.0", "react-hotkeys": "^2.0.0", - "react-i18next": "^13.1.2", "react-instantsearch-dom": "^6.28.0", "react-json-pretty": "^2.2.0", - "react-json-view": "^1.21.3", "react-json-view-ssr": "^1.19.1", "react-markdown": "^8.0.7", - "react-markdown-github": "^3.3.1", "react-powerhooks": "^0.0.7", "react-router": "^6.14.1", "react-router-dom": "^6.14.1", "react-scripts": "^5.0.1", "react-social-icons": "^5.15.0", - "react-stripe-elements": "^6.1.2", "react-toastify": "^9.1.3", "reaviz": "^14.9.7", + "rehype-raw": "^7.0.0", "remark-gfm": "^3.0.1", "remark-html": "^16.0.1", "remark-images": "^4.0.0", @@ -134,7 +128,6 @@ "babel-preset-es2015": "^6.24.1", "postcss": "^8.4.38", "promise-window": "^1.2.1", - "react-hot-loader": "^4.13.0", "webpack-cli": "^5.1.4" } } diff --git a/frontend/public/icons/copyIcon.svg b/frontend/public/icons/copyIcon.svg new file mode 100644 index 00000000..3efb3e78 --- /dev/null +++ b/frontend/public/icons/copyIcon.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/frontend/public/icons/deleteIcon.svg b/frontend/public/icons/deleteIcon.svg new file mode 100644 index 00000000..41e0cce0 --- /dev/null +++ b/frontend/public/icons/deleteIcon.svg @@ -0,0 +1,4 @@ + + + + diff --git a/frontend/public/icons/detection.svg b/frontend/public/icons/detection.svg new file mode 100644 index 00000000..a751f83d --- /dev/null +++ b/frontend/public/icons/detection.svg @@ -0,0 +1,4 @@ + + + + diff --git a/frontend/public/icons/docker copy.svg b/frontend/public/icons/docker copy.svg new file mode 100644 index 00000000..297bb83f --- /dev/null +++ b/frontend/public/icons/docker copy.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/frontend/public/icons/documentation.svg b/frontend/public/icons/documentation.svg new file mode 100644 index 00000000..28242959 --- /dev/null +++ b/frontend/public/icons/documentation.svg @@ -0,0 +1,4 @@ + + + + diff --git a/frontend/public/icons/downloadIcon.svg b/frontend/public/icons/downloadIcon.svg new file mode 100644 index 00000000..d9ed0beb --- /dev/null +++ b/frontend/public/icons/downloadIcon.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/frontend/public/icons/editIcon.svg b/frontend/public/icons/editIcon.svg new file mode 100644 index 00000000..e2eb0660 --- /dev/null +++ b/frontend/public/icons/editIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/frontend/public/icons/expandMoreIcon.svg b/frontend/public/icons/expandMoreIcon.svg new file mode 100644 index 00000000..9bef6b01 --- /dev/null +++ b/frontend/public/icons/expandMoreIcon.svg @@ -0,0 +1,4 @@ + + + + diff --git a/frontend/public/images/workflows/pulse.svg b/frontend/public/images/workflows/pulse.svg new file mode 100644 index 00000000..18c03f98 --- /dev/null +++ b/frontend/public/images/workflows/pulse.svg @@ -0,0 +1,3 @@ + + + diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 639b80bd..8f7ec2e1 100755 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -14,7 +14,9 @@ import HealthPage from "./components/HealthPage.jsx"; //import Header from "./components/Header.jsx"; import theme from "./theme"; import Apps from "./views/Apps"; +import Apps2 from "./views/Apps2.jsx"; import AppCreator from "./views/AppCreator"; +import DetectionDashBoard from "./views/DetectionDashboard.jsx"; import Welcome from "./views/Welcome.jsx"; import Dashboard from "./views/Dashboard.jsx"; @@ -22,6 +24,7 @@ import DashboardView from "./views/DashboardViews.jsx"; import AdminSetup from "./views/AdminSetup"; import Admin from "./views/Admin"; import Docs from "./views/Docs.jsx"; +import Usecases2 from "./views/Usecases2.jsx"; //import Introduction from "./views/Introduction"; import SetAuthentication from "./views/SetAuthentication"; import SetAuthenticationSSO from "./views/SetAuthenticationSSO"; @@ -42,11 +45,20 @@ import AlertTemplate from "./components/AlertTemplate"; import { isMobile } from "react-device-detect"; import RuntimeDebugger from "./components/RuntimeDebugger.jsx" +import MFASetUp from './components/MFASetUP.jsx'; +import ApiExplorerWrapper from './views/ApiExplorerWrapper.jsx'; +import LeftSideBar from './components/LeftSideBar.jsx'; +import CodeWorkflow from './views/CodeWorkflow.jsx'; +import NotFound from './views/404.jsx'; + import { ToastContainer, toast } from 'react-toastify'; import 'react-toastify/dist/ReactToastify.css'; import Drift from "react-driftjs"; +import { AppContext } from './context/ContextApi.jsx'; +import Workflows2 from "./views/Workflows2.jsx"; + // Production - backend proxy forwarding in nginx var globalUrl = window.location.origin; @@ -194,28 +206,38 @@ const App = (message, props) => { /> } -
-
+ : + isLoggedIn ? +
+ +
+ : +
+
-
+ {...props} + /> +
+ } {/*
@@ -375,6 +397,19 @@ const App = (message, props) => { {...props} /> } + /> + + } /> { {...props} /> } + /> + + } /> { /> } /> + } /> + } + /> { {...props} /> } + /> + + } /> { /> } /> + } /> } /> } /> + + } /> + } /> + } /> + { /> } /> + } /> { /> } /> - + + + } + /> +
return ( - - - - - {includedData} - - - - + + + + + + {includedData} + + + + + ); }; diff --git a/frontend/src/components/ApiExplorer.jsx b/frontend/src/components/ApiExplorer.jsx new file mode 100644 index 00000000..c2dfaeb3 --- /dev/null +++ b/frontend/src/components/ApiExplorer.jsx @@ -0,0 +1,3053 @@ +import ReactJson from "react-json-view-ssr"; +import React, { useState, useEffect, useRef, useCallback, memo, useContext} from "react"; +import { toast } from "react-toastify"; +import { + Search as SearchIcon, +} from "@mui/icons-material"; +import AceEditor from "react-ace"; +import "ace-builds/src-noconflict/mode-json"; +import "ace-builds/src-noconflict/theme-gruvbox"; +import { + Paper, + Button, + Box, + MenuItem, + TextField, + Tabs, + Tab, + Table, + TableBody, + TableCell, + TableContainer, + TableHead, + Select, + Typography, + TableRow, + InputAdornment, + Divider, + LinearProgress +} from "@mui/material"; +import throttle from "lodash/throttle"; +import theme from "../theme.jsx"; +import { validateJson, collapseField, } from "../views/Workflows.jsx"; + +import DeleteIcon from "@mui/icons-material/Delete"; +import { Context } from "../context/ContextApi.jsx"; + +function CustomTabPanel(props) { + const { children, value, index, ...other } = props; + + return ( + + ); +} + +function a11yProps(index) { + return { + id: `simple-tab-${index}`, + "aria-controls": `simple-tabpanel-${index}`, + }; +} + +const RequestMethods = [ + { + value: "GET", + color: "#61afee", + }, + { + value: "POST", + color: "#49cc90", + }, + { + value: "DELETE", + color: "#f93e3e", + }, + { + value: "PUT", + color: "#fca130", + }, + { + value: "PATCH", + color: "#50e3c2", + }, + { + value: "CONNECT", + color: "#ff69b4", + }, + { + value: "HEAD", + color: "#9012fe", + }, +]; + +const ApiExplorer = memo(({ openapi, globalUrl, userdata, HandleApiExecution, selectedAppData, ConfigurationTab, isLoggedIn, isLoaded }) => { + const [actions, setActions] = useState([]); + const [info, setInfo] = useState({}); + const [serverurl, setServerUrl] = useState(""); + const [selectedActionIndex, setSelectedActionIndex] = useState(0); + const [ExampleBody, setExampleBody] = useState({}); + const [filteredActions, setFilteredActions] = useState([]); + + const getJsonObject = (properties) => { + + let jsonObject = {}; + for (let key in properties) { + const property = properties[key]; + + let subloop = false; + if (property.hasOwnProperty("type")) { + if (property.type === "object" || property.type === "array") { + subloop = true; + } + } + + if (subloop) { + if ( + property.hasOwnProperty("items") && + property.items.hasOwnProperty("properties") + ) { + const jsonret = getJsonObject(property.items.properties); + if (property.type === "array") { + jsonObject[key] = [jsonret]; + } else { + jsonObject[key] = jsonret; + } + } else { + if (property.hasOwnProperty("properties")) { + const jsonret = getJsonObject(property.properties); + if (property.type === "array") { + jsonObject[key] = [jsonret]; + } else { + jsonObject[key] = jsonret; + } + } else { + } + } + } else { + if (property.hasOwnProperty("example")) { + jsonObject[key] = property.example; + } else if ( + property.hasOwnProperty("enum") && + property.enum.length > 0 + ) { + jsonObject[key] = property.enum[0]; + } else if (property.hasOwnProperty("default")) { + jsonObject[key] = property.default; + } else if (property.hasOwnProperty("maximum")) { + jsonObject[key] = property.maximum; + } else if (property.hasOwnProperty("minimum")) { + jsonObject[key] = property.minimum; + } else if (property.hasOwnProperty("type")) { + if (property.type === "integer" || property.type === "number") { + jsonObject[key] = 0; + } else if (property.type === "boolean") { + jsonObject[key] = false; + } else if (property.type === "string") { + jsonObject[key] = ""; + } else { + } + } else { + } + } + } + + return jsonObject; + }; + + const handleGetRef = (parameter, data) => { + try { + if (parameter === null || parameter["$ref"] === undefined) { + return parameter; + } + } catch (e) { + return parameter; + } + + const paramsplit = parameter["$ref"].split("/"); + if (paramsplit[0] !== "#") { + return parameter; + } + + var newitem = data; + for (let paramkey in paramsplit) { + var tmpparam = paramsplit[paramkey]; + if (tmpparam === "#") { + continue; + } + + if (newitem[tmpparam] === undefined) { + return parameter; + } + + newitem = newitem[tmpparam]; + } + return newitem; + }; + + useEffect(() => { + if (openapi !== undefined && openapi !== null) { + parseIncomingOpenapiData(openapi); + } + }, [openapi]); + + const parseIncomingOpenapiData = useCallback((data) => { + if (data.info !== null && data.info !== undefined) { + setInfo(data.info); + } + + try { + if (data.info !== null && data.info !== undefined) { + if (data.info.title !== undefined && data.info.title !== null) { + if (data.info.title.endsWith(" API")) { + data.info.title = data.info.title.substring( + 0, + data.info.title.length - 4 + ); + } else if (data.info.title.endsWith("API")) { + data.info.title = data.info.title.substring( + 0, + data.info.title.length - 3 + ); + } + } + + document.title = data.info.title + " Rest API" + + if ( + data.info["x-catefies"] !== undefined && + data.info["x-categories"].length > 0 + ) { + if (Array.isArray(data.info["x-categories"])) { + } else { + } + } + } + } catch (e) {} + + try { + if (data.tags !== undefined && data.tags.length > 0) { + var newtags = []; + for (let tagkey in data.tags) { + if (data.tags[tagkey]?.name.length > 50) { + continue; + } + + newtags.push(data.tags[tagkey]?.name); + } + + if (newtags.length > 10) { + newtags = newtags.slice(0, 9); + } + } + } catch (e) {} + + // This is annoying (: + // Weird generator problems to be handle + var securitySchemes = undefined; + try { + if (data.securitySchemes !== undefined) { + securitySchemes = data.securitySchemes; + if (securitySchemes === undefined) { + securitySchemes = data.securityDefinitions; + } + } + + if (securitySchemes === undefined && data.components !== undefined) { + securitySchemes = data.components.securitySchemes; + if (securitySchemes === undefined) { + securitySchemes = data.components.securityDefinitions; + } + } + } catch (e) {} + + const allowedfunctions = [ + "GET", + "CONNECT", + "HEAD", + "DELETE", + "POST", + "PATCH", + "PUT", + ]; + + var newActions = []; + var wordlist = {}; + var all_categories = []; + var parentUrl = ""; + + if (data.paths !== null && data.paths !== undefined) { + for (let [path, pathvalue] of Object.entries(data.paths)) { + for (let [method, methodvalue] of Object.entries(pathvalue)) { + if (methodvalue === null) { + continue; + } + + if (!allowedfunctions.includes(method.toUpperCase())) { + // Typical YAML issue + if (method !== "parameters") { + //toast("Skipped method (not allowed): " + method); + } + continue; + } + + var tmpname = methodvalue.summary; + if ( + methodvalue.operationId !== undefined && + methodvalue.operationId !== null && + methodvalue.operationId.length > 0 && + (tmpname === undefined || tmpname.length === 0) + ) { + tmpname = methodvalue.operationId; + } + + if (tmpname !== undefined && tmpname !== null) { + tmpname = tmpname.replaceAll(".", " "); + } + + if ( + (tmpname === undefined || tmpname === null) && + methodvalue.description !== undefined && + methodvalue.description !== null && + methodvalue.description.length > 0 + ) { + tmpname = methodvalue.description + .replaceAll(".", " ") + .replaceAll("_", " "); + } + + var newaction = { + name: tmpname, + description: methodvalue.description, + url: path, + file_field: "", + method: method.toUpperCase(), + headers: "", + queries: [], + paths: [], + body: "", + errors: [], + example_response: "", + action_label: "No Label", + required_bodyfields: [], + }; + + if ( + methodvalue["x-label"] !== undefined && + methodvalue["x-label"] !== null + ) { + // FIX: Map labels only if they're actually in the category list + newaction.action_label = methodvalue["x-label"]; + } + + if ( + methodvalue["x-required-fields"] !== undefined && + methodvalue["x-required-fields"] !== null + ) { + newaction.required_bodyfields = methodvalue["x-required-fields"]; + } + + if ( + newaction.url !== undefined && + newaction.url !== null && + newaction.url.includes("_shuffle_replace_") + ) { + //const regex = /_shuffle_replace_\d/i; + const regex = /_shuffle_replace_\d+/i; + + newaction.url = newaction.url.replaceAll( + new RegExp(regex, "g"), + "" + ); + } + + // Finding category + if (path.includes("/")) { + const pathsplit = path.split("/"); + // Stupid way of finding a category/grouping + for (let splitkey in pathsplit) { + if (pathsplit[splitkey].includes("_shuffle_replace_")) { + //const regex = /_shuffle_replace_\d/i; + const regex = /_shuffle_replace_\d+/i; + pathsplit[splitkey] = pathsplit[splitkey].replaceAll( + new RegExp(regex, "g"), + "" + ); + } + + if ( + pathsplit[splitkey].length > 0 && + pathsplit[splitkey] !== "v1" && + pathsplit[splitkey] !== "v2" && + pathsplit[splitkey] !== "api" && + pathsplit[splitkey] !== "1.0" && + pathsplit[splitkey] !== "apis" + ) { + newaction["category"] = pathsplit[splitkey]; + if (!all_categories.includes(pathsplit[splitkey])) { + all_categories.push(pathsplit[splitkey]); + } + break; + } + } + } + + if (path === "/files/{file_id}/content") { + } + + // Typescript? I think not ;) + if (methodvalue["requestBody"] !== undefined) { + if ( + methodvalue["requestBody"]["$ref"] !== undefined && + methodvalue["requestBody"]["$ref"] !== null + ) { + // Handle ref + const parameter = handleGetRef( + { $ref: methodvalue["requestBody"]["$ref"] }, + data + ); + if ( + parameter.content !== undefined && + parameter.content !== null + ) { + methodvalue["requestBody"]["content"] = parameter.content; + } + } + + if (methodvalue["requestBody"]["content"] !== undefined) { + // Handle content - XML or JSON + // + if ( + methodvalue["requestBody"]["content"]["application/json"] !== + undefined + ) { + if ( + methodvalue["requestBody"]["content"]["application/json"][ + "schema" + ] !== undefined && + methodvalue["requestBody"]["content"]["application/json"][ + "schema" + ] !== null + ) { + try { + if ( + methodvalue["requestBody"]["content"]["application/json"][ + "schema" + ]["properties"] !== undefined + ) { + // Read out properties from a JSON object + const jsonObject = getJsonObject( + methodvalue["requestBody"]["content"][ + "application/json" + ]["schema"]["properties"] + ); + if (jsonObject !== undefined && jsonObject !== null) { + try { + newaction["body"] = JSON.stringify( + jsonObject, + null, + 2 + ); + } catch (e) {} + } + + //newaction["body"] = JSON.stringify(jsonObject, null, 2); + + var tmpobject = {}; + for (let prop of methodvalue["requestBody"]["content"][ + "application/json" + ]["schema"]["properties"]) { + tmpobject[prop] = `\$\{${prop}\}`; + } + for (let subkey in methodvalue["requestBody"]["content"][ + "application/json" + ]["schema"]["required"]) { + const tmpitem = + methodvalue["requestBody"]["content"][ + "application/json" + ]["schema"]["required"][subkey]; + tmpobject[tmpitem] = `\$\{${tmpitem}\}`; + } + + newaction["body"] = JSON.stringify(tmpobject, null, 2); + } else if ( + methodvalue["requestBody"]["content"]["application/json"][ + "schema" + ]["$ref"] !== undefined && + methodvalue["requestBody"]["content"]["application/json"][ + "schema" + ]["$ref"] !== null + ) { + const retRef = handleGetRef( + methodvalue["requestBody"]["content"][ + "application/json" + ]["schema"], + data + ); + var newbody = {}; + for (let propkey in retRef.properties) { + const parsedkey = propkey + .replaceAll(" ", "_") + .toLowerCase(); + newbody[parsedkey] = "${" + parsedkey + "}"; + } + + newaction["body"] = JSON.stringify(newbody, null, 2); + } + } catch (e) {} + } + } else if ( + methodvalue["requestBody"]["content"]["application/xml"] !== + undefined + ) { + //newaction["headers"] = "" + //"Content-Type=application/xml\nAccept=application/xml"; + if ( + methodvalue["requestBody"]["content"]["application/xml"][ + "schema" + ] !== undefined && + methodvalue["requestBody"]["content"]["application/xml"][ + "schema" + ] !== null + ) { + try { + if ( + methodvalue["requestBody"]["content"]["application/xml"][ + "schema" + ]["properties"] !== undefined + ) { + for (let [prop, propvalue] of Object.entries( + methodvalue["requestBody"]["content"][ + "application/xml" + ]["schema"]["properties"] + )) { + tmpobject[prop] = `\$\{${prop}\}`; + } + + for (let [subkey, subkeyval] in Object.entries( + methodvalue["requestBody"]["content"][ + "application/xml" + ]["schema"]["required"] + )) { + const tmpitem = + methodvalue["requestBody"]["content"][ + "application/xml" + ]["schema"]["required"][subkey]; + tmpobject[tmpitem] = `\$\{${tmpitem}\}`; + } + + //newaction["body"] = XML.stringify(tmpobject, null, 2) + } + } catch (e) {} + } + } else { + if ( + methodvalue["requestBody"]["content"]["example"] !== undefined + ) { + if ( + methodvalue["requestBody"]["content"]["example"][ + "example" + ] !== undefined + ) { + newaction["body"] = + methodvalue["requestBody"]["content"]["example"][ + "example" + ]; + } + } + + if ( + methodvalue["requestBody"]["content"][ + "multipart/form-data" + ] !== undefined + ) { + if ( + methodvalue["requestBody"]["content"][ + "multipart/form-data" + ]["schema"] !== undefined && + methodvalue["requestBody"]["content"][ + "multipart/form-data" + ]["schema"] !== null + ) { + try { + if ( + methodvalue["requestBody"]["content"][ + "multipart/form-data" + ]["schema"]["type"] === "object" + ) { + const fieldname = + methodvalue["requestBody"]["content"][ + "multipart/form-data" + ]["schema"]["properties"]["fieldname"]; + + if (fieldname !== undefined) { + newaction.file_field = fieldname["value"]; + } else { + for (const [subkey, subvalue] of Object.entries( + methodvalue["requestBody"]["content"][ + "multipart/form-data" + ]["schema"]["properties"] + )) { + if (subkey.includes("file")) { + newaction.file_field = subkey; + break; + } + } + + if ( + newaction.file_field === undefined || + newaction.file_field === null || + newaction.file_field.length === 0 + ) { + } + } + } else { + } + } catch (e) {} + } + } else { + var schemas = []; + const content = methodvalue["requestBody"]["content"]; + if (content !== undefined && content !== null) { + for (const [subkey, subvalue] of Object.entries(content)) { + if ( + subvalue["schema"] !== undefined && + subvalue["schema"] !== null + ) { + if ( + subvalue["schema"]["$ref"] !== undefined && + subvalue["schema"]["$ref"] !== null + ) { + if (!schemas.includes(subvalue["schema"]["$ref"])) { + schemas.push(subvalue["schema"]["$ref"]); + } + } + } else { + if ( + subvalue["example"] !== undefined && + subvalue["example"] !== null + ) { + newaction["body"] = subvalue["example"]; + } else { + } + } + } + } + + try { + if (schemas.length === 1) { + const parameter = handleGetRef( + { $ref: schemas[0] }, + data + ); + if ( + parameter.properties !== undefined && + parameter["type"] === "object" + ) { + var newbody = {}; + for (let propkey in parameter.properties) { + const parsedkey = propkey + .replaceAll(" ", "_") + .toLowerCase(); + if ( + parameter.properties[propkey].type === undefined + ) { + continue; + } + + if (parameter.properties[propkey].type === "string") { + if ( + parameter.properties[propkey].description !== + undefined + ) { + newbody[parsedkey] = + parameter.properties[propkey].description; + } else { + newbody[parsedkey] = ""; + } + } else if ( + parameter.properties[propkey].type.includes( + "int" + ) || + parameter.properties[propkey].type.includes( + "uint64" + ) + ) { + newbody[parsedkey] = 0; + } else if ( + parameter.properties[propkey].type.includes( + "boolean" + ) + ) { + newbody[parsedkey] = false; + } else if ( + parameter.properties[propkey].type.includes("array") + ) { + newbody[parsedkey] = []; + } else { + newbody[parsedkey] = []; + } + } + + newaction["body"] = JSON.stringify(newbody, null, 2); + } else { + } + } + } catch (e) {} + } + } + } + } + + if ( + methodvalue.responses !== undefined && + methodvalue.responses !== null + ) { + if (methodvalue.responses.default !== undefined) { + if (methodvalue.responses.default.content !== undefined) { + if ( + methodvalue.responses.default.content["text/plain"] !== + undefined + ) { + if ( + methodvalue.responses.default.content["text/plain"][ + "schema" + ] !== undefined + ) { + if ( + methodvalue.responses.default.content["text/plain"][ + "schema" + ]["example"] !== undefined + ) { + newaction.example_response = + methodvalue.responses.default.content["text/plain"][ + "schema" + ]["example"]; + } + + if ( + methodvalue.responses.default.content["text/plain"][ + "schema" + ]["format"] === "binary" && + methodvalue.responses.default.content["text/plain"][ + "schema" + ]["type"] === "string" + ) { + newaction.example_response = "shuffle_file_download"; + } + } + } + } + } else { + var selectedReturn = ""; + if (methodvalue.responses["200"] !== undefined) { + selectedReturn = "200"; + } else if (methodvalue.responses["201"] !== undefined) { + selectedReturn = "201"; + } + + // Parsing examples. This should be standardized lol + if (methodvalue.responses[selectedReturn] !== undefined) { + const selectedExample = methodvalue.responses[selectedReturn]; + if (selectedExample["content"] !== undefined) { + if ( + selectedExample["content"]["application/json"] !== undefined + ) { + if ( + selectedExample["content"]["application/json"][ + "schema" + ] !== undefined && + selectedExample["content"]["application/json"][ + "schema" + ] !== null + ) { + if ( + selectedExample["content"]["application/json"][ + "schema" + ]["properties"] !== undefined && + selectedExample["content"]["application/json"][ + "schema" + ]["properties"] !== null + ) { + const jsonObject = getJsonObject( + selectedExample["content"]["application/json"][ + "schema" + ]["properties"] + ); + if (jsonObject !== undefined && jsonObject !== null) { + try { + newaction.example_response = JSON.stringify( + jsonObject, + null, + 2 + ); + } catch (e) {} + } + } + + if ( + selectedExample["content"]["application/json"][ + "schema" + ]["$ref"] !== undefined + ) { + const parameter = handleGetRef( + selectedExample["content"]["application/json"][ + "schema" + ], + data + ); + if ( + parameter.properties !== undefined && + parameter["type"] === "object" + ) { + var newbody = {}; + for (let propkey in parameter.properties) { + const parsedkey = propkey + .replaceAll(" ", "_") + .toLowerCase(); + if ( + parameter.properties[propkey].type === undefined + ) { + continue; + } + + if ( + parameter.properties[propkey].type === "string" + ) { + if ( + parameter.properties[propkey].description !== + undefined + ) { + newbody[parsedkey] = + parameter.properties[propkey].description; + } else { + newbody[parsedkey] = ""; + } + } else if ( + parameter.properties[propkey].type.includes("int") + ) { + newbody[parsedkey] = 0; + } else if ( + parameter.properties[propkey].type.includes( + "boolean" + ) + ) { + newbody[parsedkey] = false; + } else if ( + parameter.properties[propkey].type.includes( + "array" + ) + ) { + //const parameter = handleGetRef(selectedExample["content"]["application/json"]["schema"], data) + newbody[parsedkey] = []; + } else { + newbody[parsedkey] = []; + } + } + newaction.example_response = JSON.stringify( + newbody, + null, + 2 + ); + } else { + } + } else { + // Just selecting the first one. bleh. + if ( + selectedExample["content"]["application/json"][ + "schema" + ]["allOf"] !== undefined + ) { + var selectedComponent = + selectedExample["content"]["application/json"][ + "schema" + ]["allOf"]; + if (selectedComponent.length >= 1) { + selectedComponent = selectedComponent[0]; + + const parameter = handleGetRef( + selectedComponent, + data + ); + if ( + parameter.properties !== undefined && + parameter["type"] === "object" + ) { + var newbody = {}; + for (let propkey in parameter.properties) { + const parsedkey = propkey + .replaceAll(" ", "_") + .toLowerCase(); + if ( + parameter.properties[propkey].type === + undefined + ) { + continue; + } + + if ( + parameter.properties[propkey].type === + "string" + ) { + if ( + parameter.properties[propkey] + .description !== undefined + ) { + newbody[parsedkey] = + parameter.properties[propkey].description; + } else { + newbody[parsedkey] = ""; + } + } else if ( + parameter.properties[propkey].type.includes( + "int" + ) + ) { + newbody[parsedkey] = 0; + } else if ( + parameter.properties[propkey].type.includes( + "boolean" + ) + ) { + newbody[parsedkey] = false; + } else { + newbody[parsedkey] = []; + } + } + + newaction.example_response = JSON.stringify( + newbody, + null, + 2 + ); + //newaction.example_response = JSON.stringify(parameter.properties, null, 2) + } else { + //newaction.example_response = parameter.properties + } + } else { + } + } else if ( + selectedExample["content"]["application/json"][ + "schema" + ]["properties"] !== undefined + ) { + if ( + selectedExample["content"]["application/json"][ + "schema" + ]["properties"]["data"] !== undefined + ) { + const parameter = handleGetRef( + selectedExample["content"]["application/json"][ + "schema" + ]["properties"]["data"], + data + ); + if ( + parameter.properties !== undefined && + parameter["type"] === "object" + ) { + var newbody = {}; + for (let propkey in parameter.properties) { + const parsedkey = propkey + .replaceAll(" ", "_") + .toLowerCase(); + if ( + parameter.properties[propkey].type === + undefined + ) { + continue; + } + + if ( + parameter.properties[propkey].type === + "string" + ) { + if ( + parameter.properties[propkey] + .description !== undefined + ) { + newbody[parsedkey] = + parameter.properties[propkey].description; + } else { + newbody[parsedkey] = ""; + } + } else if ( + parameter.properties[propkey].type.includes( + "int" + ) + ) { + newbody[parsedkey] = 0; + } else { + newbody[parsedkey] = []; + } + } + + newaction.example_response = JSON.stringify( + newbody, + null, + 2 + ); + //newaction.example_response = JSON.stringify(parameter.properties, null, 2) + } else { + //newaction.example_response = parameter.properties + } + } + } + } + } + } + } + } + } + } + + for (let paramkey in methodvalue.parameters) { + const parameter = handleGetRef( + methodvalue.parameters[paramkey], + data + ); + + if (parameter.in === "query") { + var tmpaction = { + description: parameter.description, + name: parameter?.name, + required: parameter.required, + in: "query", + }; + + if ( + parameter.example !== undefined && + parameter.example !== null + ) { + tmpaction.example = parameter.example; + } + + if (parameter.required === undefined) { + tmpaction.required = false; + } + + newaction.queries.push(tmpaction); + } else if (parameter.in === "path") { + // FIXME - parse this to the URL too + newaction.paths.push(parameter?.name); + + // FIXME: This doesn't follow OpenAPI3 exactly. + // https://swagger.io/docs/specification/describing-request-body/ + // https://swagger.io/docs/specification/describing-parameters/ + // Need to split the data. + } else if (parameter.in === "body") { + // FIXME: Add tracking for components + // E.G: https://raw.githubusercontent.com/owentl/Shuffle/master/gosecure.yaml + if ( + parameter.example !== undefined && + parameter.example !== null + ) { + if ( + newaction.body === undefined || + newaction.body === null || + newaction.body.length < 5 + ) { + newaction.body = parameter.example; + } + } + } else if (parameter.in === "header") { + newaction.headers += `${parameter?.name}=${parameter.example}\n`; + } else { + } + } + + // Check if body is valid JSON. + if ( + newaction.body !== undefined && + newaction.body !== null && + newaction.body.length > 0 + ) { + // Trim starting / ending newlines, spaces and tabs + newaction.body = newaction.body.trim(); + } + + if (newaction?.name === "" || newaction?.name === undefined) { + // Find a unique part of the string + // FIXME: Looks for length between /, find the one where they differ + // Should find others with the same START to their path + // Make a list of reserved names? Aka things that show up only once + if (Object.getOwnPropertyNames(wordlist).length === 0) { + for (let [newpath, pathvalue] of Object.entries(data.paths)) { + const newpathsplit = newpath.split("/"); + + for (let splitkey in newpathsplit) { + const pathitem = newpathsplit[splitkey].toLowerCase(); + if (wordlist[pathitem] === undefined) { + wordlist[pathitem] = 1; + } else { + wordlist[pathitem] += 1; + } + } + } + } + + // Remove underscores and make it normal with upper case etc + const urlsplit = path.split("/"); + if (urlsplit.length > 0) { + var curname = ""; + for (let urlkey in urlsplit) { + var subpath = urlsplit[urlkey]; + if (wordlist[subpath] > 2 || subpath.length < 1) { + continue; + } + + curname = subpath; + break; + } + + // FIXME: If name exists, + // FIXME: Check if first part of parsedname is verb, otherwise use method + const parsedname = curname + .split("_") + .join(" ") + .split("-") + .join(" ") + .split("{") + .join(" ") + .split("}") + .join(" ") + .trim(); + if (parsedname.length === 0) { + newaction.errors.push("Missing name"); + } else { + const newname = + method.charAt(0).toUpperCase() + + method.slice(1) + + " " + + parsedname; + const searchactions = newActions.find( + (data) => data?.name === newname + ); + + if (searchactions !== undefined) { + newaction.errors.push("Missing name"); + } else { + newaction.name = newname; + } + } + } else { + newaction.errors.push("Missing name"); + } + } + + //newaction.action_label = "No Label" + newActions.push(newaction); + } + } + + if (data.servers !== undefined && data.servers.length > 0) { + var firstUrl = data.servers[0].url; + if ( + firstUrl.includes("{") && + firstUrl.includes("}") && + data.servers[0].variables !== undefined + ) { + const regex = /{\w+}/g; + const found = firstUrl.match(regex); + if (found !== null) { + for (let foundkey in found) { + const item = found[foundkey].slice(1, found[foundkey].length - 1); + const foundVar = data.servers[0].variables[item]; + if (foundVar["default"] !== undefined) { + firstUrl = firstUrl.replace( + found[foundkey], + foundVar["default"] + ); + } + } + } + } + + if (firstUrl.endsWith("/")) { + parentUrl = firstUrl.slice(0, firstUrl.length - 1); + } else { + parentUrl = firstUrl; + } + } + } + var prefixCheck = "/v1"; + if (parentUrl.includes("/")) { + const urlsplit = parentUrl.split("/"); + if (urlsplit.length > 2) { + // Skip if http:// in it too + prefixCheck = "/" + urlsplit.slice(3).join("/"); + } + + if ( + prefixCheck.length > 0 && + prefixCheck !== "/" && + prefixCheck.startsWith("/") + ) { + for (var actionKey in newActions) { + const action = newActions[actionKey]; + + if ( + action.url !== undefined && + action.url !== null && + action.url.startsWith(prefixCheck) + ) { + newActions[actionKey].url = action.url.slice( + prefixCheck.length, + action.url.length + ); + } + } + } + } + + setServerUrl(parentUrl); + var newActions2 = []; + // Remove with duplicate action URLs + for (var actionKey in newActions) { + const action = newActions[actionKey]; + if (action.url === undefined || action.url === null) { + continue; + } + + var found = false; + for (var actionKey2 in newActions2) { + const action2 = newActions2[actionKey2]; + if (action2.url === undefined || action2.url === null) { + continue; + } + + if (action.url === action2.url) { + found = true; + break; + } + } + + if (!found) { + newActions2.push(action); + } else { + newActions2.push(action); + } + } + + newActions = newActions2; + + // Rearrange them by which has action_label + const firstActions = newActions.filter( + (data) => + data.action_label !== undefined && + data.action_label !== null && + data.action_label !== "No Label" + ); + const secondActions = newActions.filter( + (data) => + data.action_label === undefined || + data.action_label === null || + data.action_label === "No Label" + ); + newActions = firstActions.concat(secondActions); + setActions(newActions); + setExampleBody(newActions[0]?.body); + }, [openapi]); + + return ( +
+ + + + +
+ ); +}); + +export default ApiExplorer; + + +const ActionResponseAndRequest = memo(({ isLoggedIn, isLoaded, ConfigurationTab, selectedAppData,actions, info, HandleApiExecution, userdata, filteredActions, setFilteredActions, serverurl, globalUrl, setSelectedActionIndex, ExampleBody, setExampleBody, selectedActionIndex}) => { + const [apiResponse, setApiResponse] = useState({}); + const [isLoading, setIsLoading] = useState(false); + const loadAction = 10; + const loadedAction = useRef(null); + + const loadMoreActions = useCallback(() => { + if (isLoading || filteredActions.length >= actions.length) return; + + setIsLoading(true); + + setFilteredActions((prevActions) => { + const newActions = actions.slice(prevActions.length, prevActions.length + loadAction); + setIsLoading(false); + return [...prevActions, ...newActions]; + }); + }, [isLoading, actions.length, filteredActions.length, loadAction]); + + // Scroll position reference + const scrollPosition = useRef(0); + + // Handle scroll event with debounce + const handleScroll = useCallback(() => { + const actionContainer = loadedAction.current; + if ( + actionContainer && + actionContainer.scrollTop + actionContainer.clientHeight >= actionContainer.scrollHeight - 10 + ) { + loadMoreActions(); + } + scrollPosition.current = actionContainer?.scrollTop || 0; + }, [loadMoreActions]); + + // Add scroll event listener on mount and remove on unmount + useEffect(() => { + const actionContainer = loadedAction.current; + if (actionContainer) { + actionContainer.addEventListener("scroll", handleScroll); + } + return () => { + if (actionContainer) { + actionContainer.removeEventListener("scroll", handleScroll); + } + }; + }, [handleScroll]); + + + // Restore scroll position when the component rerenders or new items are added + useEffect(() => { + const actionContainer = loadedAction.current; + if (actionContainer) { + actionContainer.scrollTop = scrollPosition.current; + } + }, [actions, filteredActions]); + + useEffect(() => { + if (actions?.length > 0 && filteredActions?.length === 0) { + setFilteredActions(actions.slice(0, loadAction)); + } + }, [actions?.length]); + + return ( + +
+
+ {filteredActions.map((action, index) => ( +
+ +
+ ))} +
+ + +
+ )}) + + +const ActionsList = memo(({ + actions, + selectedActionIndex, + setSelectedActionIndex, + setExampleBody, + setFilteredActions, + filteredActions, + userdata, + info, + openapi, + isLoggedIn, + isLoaded +}) => { + + const [searchQuery, setSearchQuery] = useState(""); + const [visibleActions, setVisibleActions] = useState([]); + + + useEffect(() => { + if (visibleActions?.length === 0 && actions?.length > 0) { + setVisibleActions(actions) + } + }, [actions?.length]) + + const handleActionClick = (index, action) => { + const actionId = action.name.replace(/ /g, "-").replace(/_/g, "-"); + setSelectedActionIndex(index); + setExampleBody(action.example_response); + + const actionIndex = actions.findIndex((act) => { + const id = act.name.replace(/ /g, "-").replace(/_/g, "-"); + return id === actionId; + }); + + if (actionIndex !== -1 && !filteredActions.some((act) => { + const id = act.name.replace(/ /g, "-").replace(/_/g, "-"); + return id === actionId; + })) { + const newActionToLoad = [ + ...filteredActions, + ...actions.slice(filteredActions.length, actionIndex + 1) + ]; + setFilteredActions(newActionToLoad); + } + + // Update URL hash and scroll to action + window.history.pushState(null, "", `#${actionId}`); + const actionElement = document.getElementById(actionId); + if (actionElement) { + actionElement.scrollIntoView({ behavior: "smooth", block: "start" }); + } +}; + + const handleSearch = (e) => { + const query = e.target.value; + setSearchQuery(query); + if (query.length === 0) { + setVisibleActions(actions); + } else { + setVisibleActions( + actions.filter((action) => + action.name.toLowerCase().includes(searchQuery.toLowerCase()) + ) + ); + } + }; + return ( +
+
+
+ {info?.title ? ( +
+ app logo + + {info.title} + +
+ ) : ( + + Api Explorer + + )} +
+
+ + + + ), + style: { height: "100%", marginTop: 10, width: '90%', }, + }} + sx={{ + marginLeft: 2, + width:'100%', + "& .MuiOutlinedInput-root fieldset": { + border: "1px solid rgba(73, 73, 73, 1)", + }, + }} + /> +
+ {visibleActions.length > 0 ? ( + visibleActions.map((action, actionIndex) => ( + + )) + ) : ( +
+ No actions found +
+ )} +
+
+ ); +}); + + + +const Action = memo(( + { + action, + index, + serverurl, + setApiResponse, + setExampleBody, + globalUrl, + info, + setSelectedActionIndex, + selectedActionIndex, + HandleApiExecution, + ConfigurationTab, + }, + ) => + { + const [RequestHeader, setRequestHeader] = useState([{ key: "Content-Type", value: "application/json" }]); + const [RequestBody, setRequestBody] = useState(action?.body); + const editorRef = useRef(null); + const [AceEditorHeight, setAceEditorHeight] = useState(275) + const [baseUrl, setBaseUrl] = useState(serverurl) + const [path, setPath] = useState(action?.url) + const inputRef = useRef(null); + const [shouldChageInputFocus, setShouldChangeInputFocus] = useState(true); + const [disableExecuteButton, setDisableExecuteButton] = useState(false); + const [showResponseLoader, setShowResponseLoader] = useState(false); + const [appAuthentication, setAppAuthentication] = useState([]) + const parseHeaders = (headersString) => { + if (headersString?.length > 0) { + const headersArray = headersString.split("\n"); + const parsedHeaders = headersArray + .map((header) => { + const [key, value] = header.split("="); // Split by '=' to get key-value pairs + + // Only proceed if both key and value exist, and neither is undefined + if (key && value) { + return { key: key.trim(), value: value.trim() }; + } + return null; // Return null if the header is invalid + }) + .filter(Boolean); // Filter out any null values + + setRequestHeader(parsedHeaders); + } + }; + + useEffect(() => { + if (action?.headers) { + parseHeaders(action.headers); + } + }, []); + + const [RequestParams, setRequestParams] = useState([ + { + key: "", + value: "", + }, + ]); + + const [curTab, setCurTab] = useState(0) + const [actionUrl, setActionUrl] = useState(action?.url) + + const [selectedMethod, setSelectedMethod] = useState(action?.method) + + const fix_url = (newUrl) => { + if (newUrl.includes("hhttp")) { + newUrl = newUrl.replace("hhttp", "http"); + } + + if (newUrl.includes("http:/") && !newUrl.includes("http://")) { + newUrl = newUrl.replace("http:/", "http://"); + } + if (newUrl.includes("https:/") && !newUrl.includes("https://")) { + newUrl = newUrl.replace("https:/", "https://"); + } + if (newUrl.includes("http:///")) { + newUrl = newUrl.replace("http:///", "http://"); + } + if (newUrl.includes("https:///")) { + newUrl = newUrl.replace("https:///", "https://"); + } + if (!newUrl.includes("http://") && !newUrl.includes("https://")) { + newUrl = `http://${newUrl}`; + } + return newUrl; + }; + + function isValidMethod(method) { + const validMethods = ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"]; + method = method.toUpperCase(); + + if (validMethods.includes(method)) { + return method; + } else { + throw new Error(`Invalid HTTP method: ${method}`); + } + } + + function fixHeader(headers) { + if (Array.isArray(headers)) { + return headers.reduce((acc, header) => { + if (header.key.trim() !== "" || header.value.trim() !== "") { + acc[header.key.trim()] = header.value.trim(); + } + return acc; + }, {}); + } + + const parsedHeaders = {}; + + if (typeof headers === 'string' && headers) { + const splitHeaders = headers.split("\n"); + + splitHeaders.forEach(header => { + let splitItem; + if (header.includes(":")) { + splitItem = ":"; + } else if (header.includes("=")) { + splitItem = "="; + } else { + return; + } + + const splitHeader = header.split(splitItem); + if (splitHeader.length >= 2) { + const key = splitHeader[0].trim(); + const value = splitHeader.slice(1).join(splitItem).trim(); + parsedHeaders[key] = value; + } + }); + } + + return parsedHeaders; + } + + function fixParams(queries) { + if (Array.isArray(queries)) { + return queries + .filter(query => query.key.trim() !== "" || query.value.trim() !== "") + .map(query => ({ key: query.key.trim(), value: query.value.trim() })); + } + + const parsedQueries = []; + if (typeof queries === 'string') { + if (!queries.trim()) return parsedQueries; + const cleanedQueries = queries.trim().replace(/\s+/g, " "); + const splittedQueries = cleanedQueries.split("&"); + splittedQueries.forEach(query => { + if (!query.includes("=")) { + console.info("Skipping as there is no '=' in the query"); + return; + } + const [key, value] = query.split("="); + if (!key.trim() || !value.trim()) { + console.info("Skipping because either key or value is not present in query"); + return; + } + parsedQueries.push({ key: key.trim(), value: value.trim() }); + }); + } + + return parsedQueries; + } + + async function prepareResponse(response) { + try { + const parsedHeaders = {}; + response.headers.forEach((value, key) => { + parsedHeaders[key] = value; + }); + + const cookies = {}; + if (response.headers.has("set-cookie")) { + const cookieHeader = response.headers.get("set-cookie").split(";"); + cookieHeader.forEach(cookie => { + const [key, value] = cookie.split("="); + if (key && value) { + cookies[key.trim()] = value.trim(); + } + }); + } + + const textData = await response.text(); + + let parsedBody; + try { + parsedBody = JSON.parse(textData); + } catch (error) { + console.error("Error parsing JSON response:", error); + parsedBody = textData; + } + + return { + success: true, + status: response.status, + url: response.url, + body: parsedBody, + headers: parsedHeaders, + cookies: cookies, + }; + } catch (error) { + console.error("Error preparing response:", error); + return { + success: false, + status: response?.status, + error: error.message, + }; + } + } + + const handleRequestWithCustomAction = async (selectedMethod, baseUrl, path, RequestHeader, RequestBody, RequestParams, info, action) => { + + if((HandleApiExecution !== undefined || HandleApiExecution !== null) && typeof HandleApiExecution === 'function'){ + + try { + if (baseUrl.length === 0) { + setBaseUrl(serverurl) + } + if (path.length === 0) { + setPath(action.url) + } + + const apiResponse = await HandleApiExecution( + selectedMethod, baseUrl, path, RequestHeader, RequestBody, RequestParams, info, action, setCurTab, + ) + + const response = { + "action name" : action.name.replaceAll("_", " "), + ...apiResponse + }; + + if (typeof response.result === "string") { + try { + response.result = JSON.parse(response.result); + } catch (parseError) { + console.error("Error parsing result:", parseError); + toast.error("Error parsing response result."); + } + } + + setDisableExecuteButton(false); + setApiResponse(response); // Set the response with the action name added + setShowResponseLoader(false); + + return apiResponse; + + } catch (error) { + console.error("Error during HandleApiExecution:", error); + toast.error(`Error: ${error.message}`); + return { error: error.message }; + } + } else{ + + const newUrl = fix_url(baseUrl); + let validMethod; + try { + validMethod = isValidMethod(selectedMethod); + } catch (error) { + console.error(error); + toast.error(error.message); + return { error: error.message }; + } + try { + + if (path && !path.startsWith('/')) { + path = '/' + path; + } + + const finalUrl = newUrl + path; + const newHeader = fixHeader(RequestHeader); + const newParams = fixParams(RequestParams); + + if (typeof RequestBody === 'object') { + try { + RequestBody = JSON.stringify(RequestBody); + } catch (error) { + console.error(`Error: ${error}`); + toast.error("Invalid JSON format for request body: ", error); + return { error: "Invalid JSON format for request body" }; + } + } + const queryString = new URLSearchParams(newParams.map(param => [param.key, param.value])).toString(); + const fullUrl = queryString ? `${finalUrl}?${queryString}` : finalUrl; + const response = await fetch(fullUrl, { + method: validMethod, + headers: newHeader, + body: validMethod !== 'GET' ? RequestBody : undefined, + }); + + const preparedResponse = await prepareResponse(response); + + setApiResponse(preparedResponse); + + return preparedResponse; + + } catch (error) { + console.error("Error:", error); + toast.error(`${error.message} Please ensure all fields are filled out correctly and try again.`); + return { error: error.message }; + } + } + }; + + const addRequestParamsRow = () => { + setRequestParams((prevRows) => { + const updatedRows = [...RequestParams, { key: "", value: "" }]; + return updatedRows; + }); + }; + + const handleRequestParamsChange = (rowIndex, field, value) => { + setRequestParams( + RequestParams.map((row, index) => { + return { + ...row, + [field]: index === rowIndex ? value : row[field], + }; + }) + ); + }; + + const addRow = () => { + setRequestHeader((prevRows) => { + const updatedRows = [...RequestHeader, { key: "", value: "" }]; + return updatedRows; + }); + }; + + const handleInputChange = (rowIndex, field, value) => { + setRequestHeader( + RequestHeader.map((row, i) => { + return { + ...row, + [field]: i === rowIndex ? value : row[field], + }; + }) + ); + }; + const handleChangeTab = (actionIndex, newValue) => { + setCurTab(newValue); + }; + + const shouldShowBodyTab = ![ + "GET", + "CONNECT", + "OPTIONS", + "TRACE", + "HEAD", + ].includes(selectedMethod); + + const extractParamsFromText = (text) => { + const params = []; + const queryString = text.split("?")[1]; + + if (queryString) { + const pairs = queryString.split("&"); + pairs.forEach((pair) => { + const [key, value] = pair.split("="); + if (key && value) { + params.push({ key, value }); + } + }); + } + + return params.length > 0 ? params : [{ key: "", value: "" }]; + }; + + const actionRef = useRef(null); + const scrollTimeoutRef = useRef(null); + const [isUserInteracting, setIsUserInteracting] = useState(false); + + useEffect(() => { + const observer = new IntersectionObserver( + throttle((entries) => { + if (!isUserInteracting) return; + let nextSelectedActionIndex = null; + + entries.forEach((entry) => { + if (entry.isIntersecting && selectedActionIndex !== index) { + nextSelectedActionIndex = index; + } + }); + + if (scrollTimeoutRef.current) { + clearTimeout(scrollTimeoutRef.current); + } + + if (nextSelectedActionIndex !== null) { + scrollTimeoutRef.current = setTimeout(() => { + if (selectedActionIndex !== nextSelectedActionIndex) { + setSelectedActionIndex(nextSelectedActionIndex); + const actionId = action.name.replace(/ /g, "-").replace(/_/g, "-"); + window.history.pushState(null, "", `#${actionId}`); + setExampleBody(action?.example_response); + document.getElementById(`action-list-${nextSelectedActionIndex}`).scrollIntoView({ behavior: "smooth", block: "center" }); + } + }, 300); + } + }, 200), + { threshold: 0.5 } + ); + + if (actionRef.current) { + observer.observe(actionRef.current); + } + + return () => { + if (actionRef.current) { + observer.unobserve(actionRef.current); + } + if (scrollTimeoutRef.current) { + clearTimeout(scrollTimeoutRef.current); + } + }; + }, [isUserInteracting]); + + const handleAceEditorChange = (value) => { + setRequestBody(value); + if (editorRef.current) { + const editor = editorRef.current.editor; + const lineHeight = editor.renderer.lineHeight; + const minHeight = 100; + const maxHeight = 300; + const session = editor.getSession(); + const screenLength = session.getScreenLength(); + const contentHeight = screenLength * lineHeight; + const padding = 20; + + // Calculate new height + let newHeight = Math.min( + Math.max( + minHeight, + contentHeight + padding + ), + maxHeight + ); + + if (newHeight !== AceEditorHeight) { + if (value.length < (editorRef.current._lastValue || '').length) { + if (contentHeight + padding < AceEditorHeight) { + setAceEditorHeight(newHeight); + } + } else { + if (contentHeight + padding > AceEditorHeight) { + setAceEditorHeight(newHeight); + } + } + } + editorRef.current._lastValue = value; + } + }; + + useEffect(() => { + if (editorRef.current) { + const editor = editorRef.current.editor; + editor.commands.addCommand({ + name: "executeOnCtrlEnter", + bindKey: { win: "Ctrl-Enter", mac: "Command-Enter" }, + exec: () => { + setShowResponseLoader(true); + setDisableExecuteButton(true); + handleRequestWithCustomAction( + selectedMethod, + baseUrl, + path, + RequestHeader, + RequestBody, + RequestParams, + info, + action + ); + }, + }); + } + }, [editorRef.current]); + + + return ( +
setIsUserInteracting(true)} + > +
+ + {action.name} + +
+ + { + if (e.key === "Enter") { + if (actionUrl.length === 0) { + toast.error("URL cannot be empty"); + return; + }else{ + setShowResponseLoader(true); + setDisableExecuteButton(true) + handleRequestWithCustomAction(selectedMethod, baseUrl, path, RequestHeader, RequestBody, RequestParams, info, action); + } + }} + } + onClick={(e) => { + const clickPosition = e.target.selectionStart; + const baseUrlLength = baseUrl.length; + if (shouldChageInputFocus) { + e.target.setSelectionRange(baseUrlLength + clickPosition, baseUrlLength + clickPosition); + setShouldChangeInputFocus(false); + } + }} + onFocus={(e) => { + const validParams = RequestParams.filter(param => param.key.trim().length > 0 && param.value.trim().length > 0); + const fullUrl = validParams.length > 0 ? `${baseUrl}${path}?${validParams.map(param => `${param.key}=${param.value}`).join("&")}` : `${baseUrl}${path}`; + if (fullUrl.length === 0) { + setActionUrl(serverurl + action?.url); + }else{ + setActionUrl(fullUrl); + } + if(!shouldChageInputFocus){ + setShouldChangeInputFocus(true); + } + }} + + onBlur={(e) => { + if(e.target.value.trim().length === 0) { + setActionUrl(path); + }else if(path.length === 0){ + setActionUrl(action?.url); + }else if(baseUrl.length === 0){ + setBaseUrl(serverurl) + setActionUrl(path) + }else{ + const validParams = RequestParams.filter(param => param.key.trim().length > 0 && param.value.trim().length > 0); + const validPath = validParams?.length > 0 ? `${path}?${validParams.map(param => `${param.key}=${param.value}`).join("&")}` : path; + setActionUrl(validPath) + } + setShouldChangeInputFocus(false); + }} + + onChange={(e) => { + const newUrl = e.target.value; + setActionUrl(newUrl); + const params = extractParamsFromText(newUrl); + setRequestParams(params); + + if (newUrl.startsWith("http://") || newUrl.startsWith("https://")) { + try { + const url = new URL(newUrl); + setBaseUrl(url.origin); + const newPath = decodeURIComponent(url.pathname);; + + setPath(newPath); + } catch (error) { + console.error("Invalid URL:", error); + } + } + }} + /> + + +
+ {showResponseLoader? ( + + ) : null} + +
+ + handleChangeTab(index, newValue) + } + aria-label="basic tabs example" + > + + Headers + + {...a11yProps(0)} + /> + {shouldShowBodyTab && ( + + Body + + {...a11yProps(1)} + /> + )} + + Params + + {...a11yProps(shouldShowBodyTab ? 2 : 1)} + /> + {ConfigurationTab ? ( + + Configuration + + {...a11yProps(shouldShowBodyTab ? 3 : 2)} + /> + ) : null} + +
+ + + + + + + Key + + + Value + + + + + {RequestHeader.map((row, rowIndex) => ( + + + + handleInputChange( + rowIndex, + "key", + e.target.value + ) + } + inputProps={{ + style: { + backgroundColor: "rgba(33, 33, 33, 1)", + padding: "4px 8px", + }, + }} + sx={{ + "& .MuiOutlinedInput-root": { + "& fieldset": { + border: "none", + }, + "&:hover fieldset": { + border: "none", + }, + "&.Mui-focused fieldset": { + border: "none", + }, + }, + }} + onKeyDown={(e) => { + if (e.key === "Enter") { + document.getElementById( + `header-value-${index}-${rowIndex}` + ).focus(); + } + if (e.key === "Backspace" && row.key.length === 0 && rowIndex !== 0) { + setRequestHeader(RequestHeader.filter((header, i) => i !== rowIndex)); + document.getElementById( + `header-value-${index}-${rowIndex - 1}` + ).focus(); + e.preventDefault() + } + }} + /> + + + + handleInputChange( + rowIndex, + "value", + e.target.value + ) + } + inputProps={{ + endAdornment: ( + + + + ), + style: { + backgroundColor: "rgba(33, 33, 33, 1)", + padding: "4px 8px", + }, + }} + sx={{ + "& .MuiOutlinedInput-root": { + "& fieldset": { + border: "none", + }, + "&:hover fieldset": { + border: "none", + }, + "&.Mui-focused fieldset": { + border: "none", + }, + }, + }} + onKeyDown={(e) => { + if (e.key === "Enter" && !e.ctrlKey) { + addRow(); + setTimeout(() => { + document.getElementById( + `header-key-${index}-${rowIndex + 1}` + ).focus(); + }, 0); + } + if (e.ctrlKey && e.key === "Enter") { + setShowResponseLoader(true); + setDisableExecuteButton(true) + handleRequestWithCustomAction(selectedMethod, baseUrl, path, RequestHeader, RequestBody, RequestParams, info, action); + } + if (e.key === "Backspace" && row.value.length === 0 && rowIndex !== 0) { + setRequestHeader(RequestHeader.filter((header, i) => i !== rowIndex)); + document.getElementById( + `header-value-${index}-${rowIndex - 1}` + ).focus(); + e.preventDefault() + }} + } + /> + + + ))} + +
+ + + + {shouldShowBodyTab && ( + + + + )} + + + + + + + Key + + + Value + + + + + {RequestParams.map((row, rowIndex) => ( + + + + handleRequestParamsChange( + rowIndex, + "key", + e.target.value + ) + } + onKeyDown={(e) => { + if (e.key === "Enter") { + document.getElementById( + `param-value-${index}-${rowIndex}` + ).focus(); + } + if (e.key === "Backspace" && row.value.length === 0 && rowIndex !== 0) { + setRequestParams(RequestParams.filter((param, i) => i !== rowIndex)); + document.getElementById( + `param-value-${index}-${rowIndex - 1}` + ).focus(); + } + }} + /> + + + + handleRequestParamsChange( + rowIndex, + "value", + e.target.value + ) + } + onKeyDown={(e) => { + if (e.key === "Enter" && !e.ctrlKey) { + addRequestParamsRow(); + setTimeout(() => { + document.getElementById( + `param-key-${index}-${rowIndex + 1}` + ).focus(); + }, 0); + } + if (e.ctrlKey && e.key === "Enter") { + setShowResponseLoader(true); + setDisableExecuteButton(true) + handleRequestWithCustomAction(selectedMethod, baseUrl, path, RequestHeader, RequestBody, RequestParams, info, action); + } + + if (e.key === "Backspace" && row.value.length === 0 && rowIndex !== 0) { + setRequestParams(RequestParams.filter((param, i) => i !== rowIndex)); + document.getElementById( + `param-value-${index}-${rowIndex - 1}` + ).focus(); + e.preventDefault() + } + } + } + /> + + + ))} + +
+ + + + {(ConfigurationTab && ((shouldShowBodyTab && curTab === 3 ) || (!shouldShowBodyTab && curTab === 2)))? ( + + ) : null} + +
+
+
+ + {action.name.replaceAll("_", " ")} + +

+ {action.description + ? action.description + : ""} +

+
+
+
+ ); +}) + +const ActionResponse = memo(({ apiResponse, ExampleBody, isLoggedIn, isLoaded }) => { + const [height, setHeight] = useState("14vh") + const [responseTabIndex, setResponseTabIndex] = useState(0) + const [oldResponse, setOldResponse] = useState(apiResponse) + const [highlight, setHighlight] = useState(false) + + const MIN_HEIGHT = 50 + + useEffect(() => { + var apiResp = apiResponse + var oldResp = oldResponse + try { + apiResp = JSON.stringify(apiResponse) + } catch (error) { + //console.error("Error parsing JSON response:", error); + } + + try { + oldResp = JSON.stringify(oldResponse) + } catch (error) { + //console.error("Error parsing JSON response:", error); + } + + if (apiResp === oldResp) { + return + } + + setOldResponse(apiResponse) + + //console.log("CHANGES MADE: ", apiResponse, oldResponse) + //toast("CHANGES!") + //console.log("HEIGHT: ", height) + + if (height === "14vh") { + setHeight("30vh") + } else { + // Check if height is less than 250px + var heightNum = 0 + try { + heightNum = parseInt(height.slice(0, -2)) + } catch (error) { + } + + if (heightNum < 350) { + setHeight("350px") + } + } + + setHighlight(true) + setTimeout(() => { + setHighlight(false) + }, 2000) + }, [apiResponse, ExampleBody]) + + const handleReactJsonClipboard = (copy) => { + const elementName = "copy_element_shuffle"; + let copyText = document.getElementById(elementName); + + if (copyText) { + if (copy.namespace && copy.name && copy.src) { + copy = copy.src; + } + + const clipboard = navigator.clipboard; + if (!clipboard) { + toast("Can only copy over HTTPS (port 3443)"); + return; + } + + let stringified = JSON.stringify(copy); + if (stringified.startsWith('"') && stringified.endsWith('"')) { + stringified = stringified.slice(1, -1); + } + + navigator.clipboard.writeText(stringified); + toast("Copied value to clipboard, NOT json path."); + } else { + console.log("Failed to copy from " + elementName + ": ", copyText); + } + }; + + const stopResizing = () => { + window.removeEventListener("mousemove", startResizing); + window.removeEventListener("mouseup", stopResizing); + }; + + const initResize = (e) => { + e.preventDefault(); + window.addEventListener("mousemove", startResizing); + window.addEventListener("mouseup", stopResizing); + }; + + const startResizing = useCallback((e) => { + const newHeight = window.innerHeight - e.clientY; + if (newHeight >= MIN_HEIGHT) { + setHeight(`${newHeight}px`); + } + }, []); + + const formData = (exampleBody) => { + try { + return exampleBody ? JSON.parse(exampleBody) : {}; + } catch (error) { + console.error("Error parsing the example string:", error); + return {}; + } + }; + + useEffect(() => { + const handleResize = () => { + const newHeight = window.innerHeight * 0.1; + setHeight(`${newHeight}px`); + }; + + window.addEventListener('resize', handleResize); + return () => { + window.removeEventListener('resize', handleResize); + }; + }, []); + + return ( + +
+ {highlight === true ? + + : null + } +
+ setResponseTabIndex(newValue)} + > + Response} + {...a11yProps(0)} + /> + Example Response} + disabled={ExampleBody === undefined || ExampleBody === null || ExampleBody === ""} + {...a11yProps(1)} + /> + History} + disabled={true} + {...a11yProps(2)} + /> + +
+
+ + + + + + + + +
+
+ + ); +}); + +const ResponseTabWrapper = memo(({ apiResponse }) => { + const handleReactJsonClipboard = (copy) => { + const elementName = "copy_element_shuffle"; + let copyText = document.getElementById(elementName); + + if (copyText) { + if (copy.namespace && copy.name && copy.src) { + copy = copy.src; + } + + const clipboard = navigator.clipboard; + if (!clipboard) { + toast("Can only copy over HTTPS (port 3443)"); + return; + } + + let stringified = JSON.stringify(copy); + if (stringified.startsWith('"') && stringified.endsWith('"')) { + stringified = stringified.slice(1, -1); + } + + navigator.clipboard.writeText(stringified); + toast("Copied value to clipboard, NOT json path."); + } else { + console.log("Failed to copy from " + elementName + ": ", copyText); + } + }; + + return( + { + return collapseField(jsonField) + }} + iconStyle={theme.palette.jsonIconStyle} + collapseStringsAfterLength={theme.palette.jsonCollapseStringsAfterLength} + enableClipboard={handleReactJsonClipboard} + displayDataTypes={false} + name={false} + /> + )}) + +const PaddingWrapper = memo(({ isLoggedIn, isLoaded, children }) => { + const { leftSideBarOpenByClick, windowWidth } = useContext(Context); + return ( +
= 1920 ? "calc(100% - 630px)" : "calc(100% - 570px)" + : windowWidth >= 1920 ? "calc(100vw - 460px)": "calc(100% - 410px)" + : windowWidth >= 1920 ? "calc(100% - 370px)" : "calc(100% - 320px)", + backgroundColor: "#1a1a1a", + position: "fixed", + bottom: 0, + right: 0, + display: "flex", + flexDirection: "column", + borderTop: "1px solid #212121", + transition: "width 0.3s ease", + minHeight: "10%", + }} + > + {children} +
+ ); +}); + +const ApiResponseWrapper = memo(({ children, isLoaded, isLoggedIn }) => { + return ( + + {children} + + ); +}); diff --git a/frontend/src/components/AppFramework.jsx b/frontend/src/components/AppFramework.jsx index 00ef41c0..c65cf1cf 100644 --- a/frontend/src/components/AppFramework.jsx +++ b/frontend/src/components/AppFramework.jsx @@ -36,6 +36,7 @@ import edgehandles from "cytoscape-edgehandles"; import cytoscape from "cytoscape"; import { toast } from 'react-toastify'; +import { isMobile } from 'react-device-detect'; cytoscape.use(edgehandles) @@ -52,13 +53,15 @@ export const findSpecificApp = (framework, inputcategory) => { } const category = inputcategory.toLowerCase().split(":")[0].trim() - - //console.log("findSpecificApp: ", category, framework) if (category === "edr" || category === "eradication" || category === "edr & av") { - if (framework["EDR & AV"] !== undefined && framework["EDR & AV"].name !== undefined) { + if (framework["EDR & AV"] !== undefined && framework["EDR & AV"].name !== undefined && framework["EDR & AV"].name !== "") { return framework["EDR & AV"] } + if (framework["edr"] !== undefined && framework["edr"].name !== undefined && framework["edr"].name !== "") { + return framework["edr"] + } + return { name: "EDR :default", large_image: parsedDatatypeImages()["EDR & AV"], @@ -67,10 +70,14 @@ export const findSpecificApp = (framework, inputcategory) => { id: "", } } else if (category === "communication" || category === "comms") { - if (framework["Comms"] !== undefined && framework["Comms"].name !== undefined) { + if (framework["Comms"] !== undefined && framework["Comms"].name !== undefined && framework["Comms"].name !== "") { return framework["Comms"] } + if (framework["communication"] !== undefined && framework["communication"].name !== undefined && framework["communication"].name !== "") { + return framework["communication"] + } + return { name: "COMMS :default", large_image: parsedDatatypeImages()["COMMS"], @@ -79,10 +86,14 @@ export const findSpecificApp = (framework, inputcategory) => { id: "", } } else if (category === "email") { - if (framework["Email"] !== undefined && framework["Email"].name !== undefined) { + if (framework["Email"] !== undefined && framework["Email"].name !== undefined && framework["Email"].name !== "") { return framework["Email"] } + if (framework["email"] !== undefined && framework["email"].name !== undefined && framework["email"].name !== "") { + return framework["email"] + } + return { name: "COMMS :default", large_image: parsedDatatypeImages()["COMMS"], @@ -91,10 +102,14 @@ export const findSpecificApp = (framework, inputcategory) => { id: "", } } else if (category === "assets") { - if (framework["Assets"] !== undefined && framework["Assets"].name !== undefined) { + if (framework["Assets"] !== undefined && framework["Assets"].name !== undefined && framework["Assets"].name !== "") { return framework["Assets"] } + if (framework["assets"] !== undefined && framework["assets"].name !== undefined && framework["assets"].name !== "") { + return framework["assets"] + } + return { name: "ASSETS :default", large_image: parsedDatatypeImages()["ASSETS"], @@ -103,10 +118,14 @@ export const findSpecificApp = (framework, inputcategory) => { id: "", } } else if (category === "cases") { - if (framework["Cases"] !== undefined && framework["Cases"].name !== undefined) { + if (framework["Cases"] !== undefined && framework["Cases"].name !== undefined && framework["Cases"].name !== "") { return framework["Cases"] } + if (framework["cases"] !== undefined && framework["cases"].name !== undefined && framework["cases"].name !== "") { + return framework["cases"] + } + return { name: "CASES :default", large_image: parsedDatatypeImages()["CASES"], @@ -115,10 +134,14 @@ export const findSpecificApp = (framework, inputcategory) => { id: "", } } else if (category === "iam") { - if (framework["IAM"] !== undefined && framework["IAM"].name !== undefined) { + if (framework["IAM"] !== undefined && framework["IAM"].name !== undefined && framework["IAM"].name !== "") { return framework["IAM"] } + if (framework["iam"] !== undefined && framework["iam"].name !== undefined && framework["iam"].name !== "") { + return framework["iam"] + } + return { name: "IAM :default", large_image: parsedDatatypeImages()["IAM"], @@ -127,10 +150,14 @@ export const findSpecificApp = (framework, inputcategory) => { id: "", } } else if (category === "network") { - if (framework["Network"] !== undefined && framework["Network"].name !== undefined) { + if (framework["Network"] !== undefined && framework["Network"].name !== undefined && framework["Network"].name !== "") { return framework["Network"] } + if (framework["network"] !== undefined && framework["network"].name !== undefined && framework["network"].name !== "") { + return framework["network"] + } + return { name: "Network :default", large_image: parsedDatatypeImages()["NETWORK"], @@ -139,10 +166,14 @@ export const findSpecificApp = (framework, inputcategory) => { id: "", } } else if (category === "intel") { - if (framework["Intel"] !== undefined && framework["Intel"].name !== undefined) { + if (framework["Intel"] !== undefined && framework["Intel"].name !== undefined && framework["Intel"].name !== "") { return framework["Intel"] } + if (framework["intel"] !== undefined && framework["intel"].name !== undefined && framework["intel"].name !== "") { + return framework["intel"] + } + return { name: "INTEL :default", large_image: parsedDatatypeImages()["INTEL"], @@ -151,9 +182,13 @@ export const findSpecificApp = (framework, inputcategory) => { id: "", } } else if (category === "siem") { - if (framework["SIEM"] !== undefined && framework["SIEM"].name !== undefined) { + if (framework["SIEM"] !== undefined && framework["SIEM"].name !== undefined && framework["SIEM"].name !== "") { return framework["SIEM"] } + + if (framework["siem"] !== undefined && framework["siem"].name !== undefined && framework["siem"].name !== "") { + return framework["siem"] + } return { name: "SIEM :default", @@ -1094,7 +1129,7 @@ const AppFramework = (props) => { }, [newSelectedApp]) - const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + const isCloud = (window.location.host === "localhost:3002" || window.location.host === "shuffler.io") ? true : (process.env.IS_SSR === "true"); const imgSize = 50; var parsedFrameworkData = frameworkData === undefined ? {} : frameworkData @@ -1937,14 +1972,14 @@ const AppFramework = (props) => { {data.name}
-
+
{parsedLeftImage} {parsedLeftText}
-
+
{svgIcon}
-
+
{parsedRightImage} {parsedRightText}
@@ -2152,7 +2187,7 @@ const AppFramework = (props) => { { Object.getOwnPropertyNames(discoveryData).length > 0 ? - + {paperTitle.length > 0 ? @@ -2321,7 +2356,7 @@ const AppFramework = (props) => { elements={elements} minZoom={0.35} maxZoom={2.00} - style={{width: 560*scale, height: 560*scale, backgroundColor: theme.palette.backgroundColor, margin: "auto",}} + style={{width: isMobile?null:560*scale, height: 560*scale, backgroundColor: theme.palette.backgroundColor, margin: isMobile?null:"auto",}} stylesheet={frameworkStyle} boxSelectionEnabled={false} panningEnabled={false} diff --git a/frontend/src/components/AppGrid.jsx b/frontend/src/components/AppGrid.jsx index 0f3fedef..b0577937 100644 --- a/frontend/src/components/AppGrid.jsx +++ b/frontend/src/components/AppGrid.jsx @@ -79,6 +79,7 @@ const AppGrid = (props) => { const [formMail, setFormMail] = React.useState(""); const [message, setMessage] = React.useState(""); const [formMessage, setFormMessage] = React.useState(""); + const [deactivatedIndexes, setDeactivatedIndexes] = React.useState([]); const buttonStyle = { borderRadius: 30, @@ -232,6 +233,11 @@ const AppGrid = (props) => { removeQuery("q"); refine(event.currentTarget.value); }} + onKeyDown={(event) => { + if(event.key === "Enter") { + event.preventDefault(); + } + }} limit={5} /> {/*isSearchStalled ? 'My search is stalled' : ''*/} @@ -293,7 +299,7 @@ const AppGrid = (props) => { useEffect(() => { var baseurl = globalUrl; - fetch(baseurl + "/api/v1/getinfo", { + fetch(baseurl + "/api/v1/me", { credentials: "include", headers: { 'Content-Type': 'application/json', @@ -347,7 +353,7 @@ const AppGrid = (props) => { if (responseJson.success === false) { toast.error(responseJson.reason); } else { - toast.success(`App ${type}d Successfully!`); + //toast.success(`App ${type}d Successfully!`); if (type === 'activate') { setAllActivatedAppIds(prev => [...prev, data.objectID]); setIsAnyAppActivated(true); @@ -390,7 +396,7 @@ const AppGrid = (props) => { {!isLoading ? (
{hits.length === 0 && searchQuery.length >= 0 && showNoAppFound ? ( - No App Found + No Apps Found ) : (
{ }} > {data.name} { + // Replace the image with the default image + const foundImage = document.getElementById(`image_${index}`) + if (foundImage !== undefined && foundImage !== null) { + foundImage.src = theme.palette.defaultImage + data.image_url = theme.palette.defaultImage + } + }} style={{ width: 80, height: 80, @@ -989,11 +1004,16 @@ const AppGrid = (props) => { }} autoComplete="off" color="primary" - placeholder="Search your Activated or Self-built apps" + placeholder="Search your Activated or self-built apps" id="shuffle_search_field" onChange={(event) => { setSearchQuery(event.currentTarget.value); }} + onKeyDown={(event) => { + if(event.key === "Enter") { + event.preventDefault(); + } + }} limit={5} /> {/*isSearchStalled ? 'My search is stalled' : ''*/} @@ -1541,9 +1561,13 @@ const AppGrid = (props) => { Filter By + + + +
)} @@ -1712,6 +1736,10 @@ const AppGrid = (props) => { ? `/apps/${data.id}` : `https://shuffler.io/apps/${data.id}`; + if (data.name === "" && data.id === "") { + return null + } + return ( { width: 230, textAlign: 'start', marginLeft: 8, - color: "rgba(158, 158, 158, 1)" + color: "rgba(158, 158, 158, 1)", + display: "flex", }} > - {data.tags && - data.tags.map((tag, tagIndex) => ( - - {normalizedString(tag)} - {tagIndex < data.tags.length - 1 ? ", " : ""} - - ))} -
+
+ {data.generated !== true ? +
+ {data.tags && + data.tags.slice(0,2).map((tag, tagIndex) => ( + + {normalizedString(tag)} + {tagIndex < data.tags.length - 1 ? ", " : ""} + + )) + } +
+ : null} +
+ {currTab === 1 && !deactivatedIndexes.includes(index) && mouseHoverIndex === index && data.generated === true ? + + : null} +
+ {/* )} */}
@@ -1936,6 +2018,7 @@ const AppGrid = (props) => { selectedOptionOfCreatedWith={selectedOptionOfCreatedWith} /> )} + { setSelectedTagsForUserAndOrgApps={setSelectedTagsForUserAndOrgApps} setSelectedOptionOfCreatedWith={setSelectedOptionOfCreatedWith} /> +
diff --git a/frontend/src/components/AppModal.jsx b/frontend/src/components/AppModal.jsx new file mode 100644 index 00000000..4fcc5d07 --- /dev/null +++ b/frontend/src/components/AppModal.jsx @@ -0,0 +1,759 @@ +import React, { useEffect, useState } from 'react'; +import { useNavigate } from 'react-router'; + +import { + Dialog, + DialogTitle, + DialogContent, + IconButton, + Typography, + Box, + Button, + Stack, + Avatar, +} from '@mui/material'; + +import CloseIcon from '@mui/icons-material/Close'; +import EditIcon from '@mui/icons-material/Edit'; +import Search from '@mui/icons-material/Search'; +import AddIcon from '@mui/icons-material/Add'; +import ForkRightIcon from '@mui/icons-material/ForkRight'; +import OpenInNewIcon from '@mui/icons-material/OpenInNew'; +import LaunchIcon from '@mui/icons-material/Launch'; +import CheckCircleIcon from '@mui/icons-material/CheckCircle'; +import { CloudDownloadOutlined } from '@mui/icons-material'; +import { findSpecificApp } from './AppFramework'; +import theme from "../theme"; +import YAML from 'yaml'; +import { toast } from 'react-toastify'; +import { Link } from 'react-router-dom'; + +const AppModal = ({ open, onClose, app, globalUrl }) => { + + const [frameworkData, setFrameworkData] = useState({}) + const [userdata, setUserdata] = useState({}) + const [usecases, setUsecases] = useState([]) + const [workflows, setWorkflows] = useState([]) + const [prevSubcase, setPrevSubcase] = useState({}) + const [inputUsecase, setInputUsecase] = useState({}) + const [latestUsecase, setLatestUsecase] = useState([]) + const [foundAppUsecase, setFoundAppUsecase] = useState({}) + const navigate = useNavigate(); + const parseUsecase = (subcase) => { + const srcdata = findSpecificApp(frameworkData, subcase.type) + const dstdata = findSpecificApp(frameworkData, subcase.last) + + if (srcdata !== undefined && srcdata !== null) { + subcase.srcimg = srcdata.large_image + subcase.srcapp = srcdata.name + } + + if (dstdata !== undefined && dstdata !== null) { + subcase.dstimg = dstdata.large_image + subcase.dstapp = dstdata.name + } + return subcase + } + + useEffect(() => { + var baseurl = globalUrl; + fetch(baseurl + "/api/v1/me", { + credentials: "include", + headers: { + 'Content-Type': 'application/json', + }, + }) + .then(response => response.json()) + .then(responseJson => { + if (responseJson.success) { + setUserdata(responseJson) + } + }) + .catch(error => { + console.log("Failed login check: ", error); + }); + }, [app]); + + + const getFramework = () => { + fetch(globalUrl + "/api/v1/apps/frameworkConfiguration", { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for framework!"); + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success === false) { + const preparedData = { + "siem": findSpecificApp({}, "SIEM"), + "communication": findSpecificApp({}, "COMMUNICATION"), + "assets": findSpecificApp({}, "ASSETS"), + "cases": findSpecificApp({}, "CASES"), + "network": findSpecificApp({}, "NETWORK"), + "intel": findSpecificApp({}, "INTEL"), + "edr": findSpecificApp({}, "EDR"), + "iam": findSpecificApp({}, "IAM"), + "email": findSpecificApp({}, "EMAIL"), + } + + setFrameworkData(preparedData) + } else { + setFrameworkData(responseJson) + } + }) + .catch((error) => { + console.log("Error getting framework: ", error) + }) + } + + const fetchUsecases = (workflows) => { + fetch(globalUrl + "/api/v1/workflows/usecases", { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for usecases"); + } + + return response.json(); + }) + .then((responseJson) => { + + + const newUsecases = [...usecases] + newUsecases.forEach((category, index) => { + category.list.forEach((subcase, subindex) => { + getUsecase(subcase, index, subindex) + }) + }) + + setLatestUsecase(newUsecases) + // Matching workflows with usecases + if (responseJson.success !== false) { + if (workflows !== undefined && workflows !== null && workflows.length > 0) { + var categorydata = responseJson + + var newcategories = [] + for (var key in categorydata) { + var category = categorydata[key] + category.matches = [] + + for (var subcategorykey in category.list) { + var subcategory = category.list[subcategorykey] + subcategory.matches = [] + + for (var workflowkey in workflows) { + const workflow = workflows[workflowkey] + + if (workflow.usecase_ids !== undefined && workflow.usecase_ids !== null) { + for (var usecasekey in workflow.usecase_ids) { + if (workflow.usecase_ids[usecasekey].toLowerCase() === subcategory?.name?.toLowerCase()) { + + category.matches.push({ + "workflow": workflow.id, + "category": subcategory?.name, + }) + + subcategory.matches.push(workflow) + break + } + } + } + + if (subcategory.matches.length > 0) { + break + } + } + } + + newcategories.push(category) + } + + if (newcategories !== undefined && newcategories !== null && newcategories.length > 0) { + setUsecases(newcategories) + } else { + setUsecases(responseJson) + } + } else { + setUsecases(responseJson) + } + } + }) + .catch((error) => { + //toast("ERROR: " + error.toString()); + console.log("ERROR: " + error.toString()); + }); + }; + + const getAvailableWorkflows = () => { + fetch(globalUrl + "/api/v1/workflows", { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + fetchUsecases() + console.log("Status not 200 for workflows :O!: ", response.status); + return; + } + + return response.json(); + }) + .then((responseJson) => { + fetchUsecases(responseJson) + + if (responseJson !== undefined) { + setWorkflows(responseJson); + } + }) + .catch((error) => { + fetchUsecases() + //toast(error.toString()); + }); + } + + const getUsecase = (subcase, index, subindex) => { + subcase = parseUsecase(subcase) + setPrevSubcase(subcase) + + fetch(`${globalUrl}/api/v1/workflows/usecases/${escape(subcase?.name?.replaceAll(" ", "_"))}`, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for framework!"); + } + + return response.json(); + }) + .then((responseJson) => { + var parsedUsecase = responseJson + + if (responseJson.success === false) { + parsedUsecase = subcase + } else { + parsedUsecase = responseJson + + parsedUsecase.srcimg = subcase.srcimg + parsedUsecase.srcapp = subcase.srcapp + parsedUsecase.dstimg = subcase.dstimg + parsedUsecase.dstapp = subcase.dstapp + } + // Look for the type of app and fill in img1, srcapp... + setInputUsecase(parsedUsecase) + }) + .catch((error) => { + //toast(error.toString()); + setInputUsecase(subcase) + console.log("Error getting usecase: ", error) + }) + } + + + + useEffect(() => { + getAvailableWorkflows() + getFramework() + }, [app]) + + + useEffect(() => { + const foundCategory = latestUsecase?.find((category) => + category?.list?.some((subcase) => subcase?.srcapp === app?.name || subcase?.dstapp === app?.name) + ); + + const foundSubcase = foundCategory?.list?.find( + (subcase) => subcase?.srcapp === app?.name || subcase?.dstapp === app?.name + ); + + setFoundAppUsecase(foundSubcase); + }, [latestUsecase]) + + const downloadApp = (inputdata) => { + const id = inputdata.id; + + toast("Downloading.."); + fetch(globalUrl + "/api/v1/apps/" + id + "/config", { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + window.location.pathname = "/apps"; + } + + return response.json(); + }) + .then((responseJson) => { + if (!responseJson.success) { + toast("Failed to download file"); + } else { + console.log(responseJson); + const basedata = atob(responseJson.openapi); + console.log("BASE: ", basedata); + var inputdata = JSON.parse(basedata); + console.log("POST INPUT: ", inputdata); + inputdata = JSON.parse(inputdata.body); + + const newpaths = {}; + if (inputdata["paths"] !== undefined) { + Object.keys(inputdata["paths"]).forEach(function (key) { + newpaths[key.split("?")[0]] = inputdata.paths[key]; + }); + } + + inputdata.paths = newpaths; + console.log("INPUT: ", inputdata); + var name = inputdata.info.title; + name = name.replace(/ /g, "_", -1); + name = name.toLowerCase(); + + delete inputdata.id; + delete inputdata.editing; + + const data = YAML.stringify(inputdata); + var blob = new Blob([data], { + type: "application/octet-stream", + }); + + var url = URL.createObjectURL(blob); + var link = document.createElement("a"); + link.setAttribute("href", url); + link.setAttribute("download", `${name}.yaml`); + var event = document.createEvent("MouseEvents"); + event.initMouseEvent( + "click", + true, + true, + window, + 1, + 0, + 0, + 0, + 0, + false, + false, + false, + false, + 0, + null + ); + link.dispatchEvent(event); + //link.parentNode.removeChild(link) + } + }) + .catch((error) => { + console.log(error); + toast(error.toString()); + }); + }; + + const isCloud = + window.location.host === "localhost:3002" || + window.location.host === "shuffler.io" || window.location.host === "localhost:3000" + ? true + : false; + + var newAppname = app?.name; + if (newAppname === undefined) { + newAppname = "Undefined"; + } else { + newAppname = newAppname.charAt(0).toUpperCase() + newAppname.substring(1); + newAppname = newAppname?.replaceAll("_", " "); + } + + var canEditApp = userdata.admin === "true" || userdata?.id === app?.owner || app?.owner === "" || (userdata.admin === "true" && userdata.active_org.id === app?.reference_org) || !app?.generated + + + + return ( + + + + About {app?.name.replace(/_/g, ' ').replace(/\b\w/g, char => char.toUpperCase())} + + + + + + + + + +
+ {app?.name} +
+
+ + {newAppname} + + { + isCloud && ( + + + + + + ) + } +
+ + {app?.categories ? app.categories.join(", ") : "Communication"} + +
+
+
+ + {app?.activated && + app?.private_id !== undefined && + app?.private_id?.length > 0 && + app?.generated ? ( + ) : null} + +
+ + +
+
+ + 20 + + + Public Workflow + +
+
+ + {Array.isArray(app?.actions) ? app.actions.length : app?.actions} + + + Actions + +
+
+
+ { + app?.collection ? ( + <> + + + app.collection + + + + ) : ( + + No collection yet + + ) + } + +
+ + Part of a collection + +
+
+ +
+
+ { + (foundAppUsecase?.srcapp !== undefined && foundAppUsecase?.dstapp !== undefined) ? ( + "Connect " + foundAppUsecase?.srcapp?.replaceAll("_", " ") + " to " + foundAppUsecase?.dstapp?.replaceAll("_", " ") + ) : ( + "Connect " + app?.name + " to any tool" + ) + } +
+ + + + { + foundAppUsecase === undefined ? ( + + + + ) : ( + + ) + } + { + foundAppUsecase === undefined ? ( + + + + ) : ( + + ) + } + + + {foundAppUsecase?.name || "Search for a Usecase"} + + +
+ +
+ +
+ + + ); +}; + +export default AppModal; diff --git a/frontend/src/components/AppSearchButtons.jsx b/frontend/src/components/AppSearchButtons.jsx index 262843ad..4a4eef4e 100644 --- a/frontend/src/components/AppSearchButtons.jsx +++ b/frontend/src/components/AppSearchButtons.jsx @@ -2,7 +2,7 @@ import React, { useState, useEffect, useRef } from "react"; import theme from '../theme.jsx'; import ReactGA from 'react-ga4'; import { useNavigate, Link } from 'react-router-dom'; - +import { isMobile } from 'react-device-detect'; import { Search as Searchicon, CloudQueue as CloudQueueicon, Code as Codeicon, Close as Closeicon, Folder as Foldericon, LibraryBooks as LibraryBooksicon, Delete as DeleteIcon, Close as CloseIcon, } from '@mui/icons-material'; import aa from 'search-insights' import Deleteicon from '@mui/icons-material/Delete'; @@ -47,6 +47,8 @@ const AppSearchButtons = (props) => { const [newSelectedApp, setNewSelectedApp] = useState(undefined) useEffect(() => { + console.log("UPDATED APP: ", newSelectedApp) + if (newSelectedApp !== undefined && setMissing != undefined) { const submitAppFramework = { "description": newSelectedApp.description, @@ -136,8 +138,14 @@ const AppSearchButtons = (props) => { const icon = foundApp.large_image var foundAppImage = AppImage - if (foundApp.name !== undefined && foundApp.name !== null && !foundApp.name.includes(":default")) { - foundAppImage = foundApp.large_image + if (foundApp.name !== undefined && foundApp.name !== null && foundApp.name.length > 0 && !foundApp.name.includes(":default")) { + + if (AppImage === undefined || AppImage === null || AppImage.length < 10) { + foundAppImage = foundApp.large_image + } + } else { + const newapp = findSpecificApp(appFramework, appType) + // const { userdata, globalUrl, appFramework, moreButton, finishedApps, appType, totalApps, index, onNodeSelect, setDiscoveryData, appName, AppImage, setDefaultSearch, discoveryData, checkLogin, setMissing, getAppFramework, } = props } let xsValue = 12; @@ -181,7 +189,7 @@ const AppSearchButtons = (props) => { width: 319, height: 395, flexShrink: 0, - marginLeft: 70, + marginLeft: isMobile? null:70, marginTop: 68, position: "absolute", zIndex: 100, @@ -204,9 +212,14 @@ const AppSearchButtons = (props) => { { + {/* { + */}
{ userdata, globalUrl, appFramework, + setAppFramework, setActiveStep, defaultSearch, setDefaultSearch, checkLogin, + isAppPage=false + } = props; const [discoveryData, setDiscoveryData] = React.useState({}) const [selectionOpen, setSelectionOpen] = React.useState(false) @@ -57,9 +61,10 @@ const AppSelection = props => { const [moreButton, setMoreButton] = useState(false); // const [mouseHoverIndex, setMouseHoverIndex] = useState(-1) + document.title = "Choose your apps" const ref = useRef() let navigate = useNavigate(); - const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + const isCloud = (window.location.host === "localhost:3002" || window.location.host === "shuffler.io") ? true : (process.env.IS_SSR === "true"); useEffect(() => { if (newSelectedApp === undefined || newSelectedApp.objectID === undefined || newSelectedApp.objectID === undefined || newSelectedApp.objectID.length === 0) { @@ -98,74 +103,75 @@ const AppSelection = props => { else if (discoveryData === "IAM") { appFramework.iam = submitNewApp } - setFrameworkItem(submitNewApp); - setSelectionOpen(false); - console.log("Selected app changed (effect)"); + + setFrameworkItem(submitNewApp) + setSelectionOpen(false) + + if (setAppFramework !== undefined) { + setAppFramework(appFramework) + } + GetApps() }, [newSelectedApp]); + const reloadAppButtons = (framework) => { + var tempApps = [] + const lastApps = {} + let endTypes = ["network", "assets", "iam"] + + if (framework === undefined || framework === null || Object.keys(framework).length === 0) { + //window.location.href = "/welcome" + return + } + + Object.entries(framework).forEach(([key, value]) => { + // Overwrrite email properly + if (key.toLowerCase() === "communication") { + value["type"] = "email" + framework["email"] = value + return + } + + if (key.toLowerCase() === "other") { + return + } + + value.type = key; + if (endTypes.includes(value.type.toLowerCase())) { + lastApps[value.type] = value + return + } + + if (lastPosted.type === value.type) { + value = lastPosted + } + + tempApps.push(JSON.parse(JSON.stringify(value))); + }); + + tempApps.sort((a, b) => { + if (a.type.length > b.type.length) { + return -1; + } else if (a.type.length < b.type.length) { + return 1; + } + }); + + let lastType = lastPosted.type === undefined ? "" : lastPosted.type.toLowerCase() + if (endTypes.includes(lastType)) { + lastApps[lastPosted.type] = lastPosted + } + + if (moreButton) { + tempApps.push(JSON.parse(JSON.stringify(lastApps["network"]))) + tempApps.push(JSON.parse(JSON.stringify(lastApps["assets"]))) + tempApps.push(JSON.parse(JSON.stringify(lastApps["iam"]))) + } + + setAppButtons(tempApps) + } + useEffect(() => { - var tempApps = [] - if (tempApps.length === 0) { - // Object.entries(appFramework).forEach(([key, value]) => { - // value.type = key; - // tempApps.push(value); - // }); - - // // Define the custom sorting order - // const customSortingOrder = ["CASES", "SIEM", "ENDPOINT", "INTEL", "EMAIL"]; - - const lastApps = {} - let endTypes = ["network", "assets", "iam"] - - if (appFramework === undefined || appFramework === null || Object.keys(appFramework).length === 0) { - //window.location.href = "/welcome" - return - } - - - Object.entries(appFramework).forEach(([key, value]) => { - if (key.toLowerCase() === "other" || key.toLowerCase() === "communication") { - return - } - - value.type = key; - - if (endTypes.includes(value.type.toLowerCase())) { - lastApps[value.type] = value - return - } - - if (lastPosted.type === value.type) { - value = lastPosted - } - - tempApps.push(JSON.parse(JSON.stringify(value))); - }); - - tempApps.sort((a, b) => { - if (a.type.length > b.type.length) { - return -1; - } else if (a.type.length < b.type.length) { - return 1; - } - }); - - let lastType = lastPosted.type === undefined ? "" : lastPosted.type.toLowerCase() - - if (endTypes.includes(lastType)) { - lastApps[lastPosted.type] = lastPosted - } - - if (moreButton) { - tempApps.push(JSON.parse(JSON.stringify(lastApps["network"]))) - tempApps.push(JSON.parse(JSON.stringify(lastApps["assets"]))) - tempApps.push(JSON.parse(JSON.stringify(lastApps["iam"]))) - } - - setAppButtons(tempApps) - console.log("Updated appButtons: ", appButtons) - GetApps() - } + reloadAppButtons(appFramework) }, [lastPosted, moreButton]) if (appFramework === undefined || appFramework === null || Object.keys(appFramework).length === 0) { @@ -174,11 +180,6 @@ const AppSelection = props => { } const setFrameworkItem = (data) => { - console.log("Setting framework item: ", data, isCloud) - // if (!isCloud) { - // activateApp(data.id) - // } - fetch(globalUrl + "/api/v1/apps/frameworkConfiguration", { method: "POST", headers: { @@ -241,31 +242,39 @@ const AppSelection = props => { body: JSON.stringify(data), credentials: "include", }) - .then((responseJson) => { - if (responseJson === null) { - console.log("null-response from server") - const pretend_apps = [{ - "description": "TBD", - "id": "TBD", - "large_image": "", - "name": "TBD", - "type": "TBD" - }] + .then((response) => { + return response.json() + }) + .then((responseJson) => { + if (responseJson === null || responseJson === undefined) { + console.log("null-response from server") + const pretend_apps = [{ + "description": "TBD", + "id": "TBD", + "large_image": "", + "name": "TBD", + "type": "TBD" + }] - setApps(pretend_apps) - return - } + setApps(pretend_apps) + return + } - if (responseJson.success === false) { - console.log("error loading apps: ", responseJson) - return - } + if (responseJson.success === false) { + console.log("error loading apps: ", responseJson) + return + } - setApps(responseJson); - }) - .catch((error) => { - console.log("App loading error: " + error.toString()); - }) + if (setAppFramework !== undefined) { + setAppFramework(responseJson) + } + + setApps(responseJson) + reloadAppButtons(responseJson) + }) + .catch((error) => { + console.log("App loading error: " + error.toString()); + }) } const onNodeSelect = (label) => { @@ -277,8 +286,6 @@ const AppSelection = props => { }); } - console.log("NODESELECT: ", label) - setDiscoveryData(label) setSelectionOpen(true) setDefaultSearch(label.charAt(0).toUpperCase() + (label.substring(1)).toLowerCase()) @@ -304,209 +311,256 @@ const AppSelection = props => { }; return ( - - - { - navigate('/welcome'); - window.location.reload(); - }} - > - - Back - - -
- {selectionOpen ? ( -
-
-
- {discoveryData} -
-
- - { - setSelectionOpen(false) - }} - > - - - - - { - e.preventDefault(); - setSelectionOpen(false) - setDefaultSearch("") - const submitDeletedApp = { - "description": "", - "id": "remove", - "name": "", - "type": discoveryData - } - setFrameworkItem(submitDeletedApp) - setNewSelectedApp({}) - setTimeout(() => { - setDiscoveryData({}) - setFrameworkItem(submitDeletedApp) - setNewSelectedApp({}) - }, 1000) - //setAppName(discoveryData.cases.name) - }} - > - - - -
-
-
- -
- ) : null} - - Find your apps - - - Select the apps you work with and we will connect them for you. - - - {appButtons.map((appData, index) => { - // This is here due to a memory issue with setting apps properly - if (appData.id === "remove") { - console.log("Removed as appdata is overridden: ", appData) + +
+ {/* + + { + navigate('/welcome'); + window.location.reload(); + }} + > + + Back + + + */} +
+ {selectionOpen ? ( +
+
+
+ {discoveryData} +
+
+ + { + setSelectionOpen(false) + }} + > + + + + + { + e.preventDefault(); + setSelectionOpen(false) + setDefaultSearch("") + const submitDeletedApp = { + "description": "", + "id": "remove", + "name": "", + "type": discoveryData + } - const appName = appData.name - const AppImage = appData.large_image - const appType = appData.type + setFrameworkItem(submitDeletedApp) + setNewSelectedApp({}) + setTimeout(() => { + setDiscoveryData({}) + setFrameworkItem(submitDeletedApp) + setNewSelectedApp({}) + }, 200) + //setAppName(discoveryData.cases.name) + }} + > + + + +
+
+
+ +
+ ) : null} + { + !isAppPage && ( + <> + + Find your apps + + + Select the apps you work with and we will connect them for you. + + + ) + } + { + isAppPage && ( +
+ + Your organization has no apps yet, select your starting apps here + or discover more apps using the { + navigate("/apps2?tab=all_apps") + }} + style={{ color: "#FF8444", fontWeight: "medium", fontSize: 16, cursor:"pointer" }}>App Library + +
+ ) + } + + {appButtons.map((appData, index) => { + // This is here due to a memory issue with setting apps properly + if (appData.id === "remove") { + appData = { + "count": 0, + "description": "", + "id": "", + "large_image": "", + "name": "", + "type": appData.type, + } + } - return ( - - ) - })} - -
- {!moreButton ? ( -
- { - setMoreButton(true) - setTimeout(() => { - navigate("/welcome?tab=2") - }, 250) - }} - >See More Apps -
) : ""} -
- -
- + if (appData === undefined || appData === null || appData.name === undefined || appData.name === "") { + appData = { + "count": 0, + "description": "", + "id": "", + "large_image": "", + "name": "", + "type": appData.type, + } + } + + //console.log("APP: ", appData) + + const appName = appData.name + const AppImage = appData.large_image + const appType = appData.type + + return ( + + ) + })} + +
+ { + !isAppPage && ( + <> + {!moreButton ? ( +
+ { + setMoreButton(true) + setTimeout(() => { + navigate("/welcome?tab=2") + }, 250) + }} + >See More Apps +
) : ""} + +
+ +
+ + ) + } +
+ ) } diff --git a/frontend/src/components/Appsearch.jsx b/frontend/src/components/Appsearch.jsx index e4f1782f..dedacfe6 100644 --- a/frontend/src/components/Appsearch.jsx +++ b/frontend/src/components/Appsearch.jsx @@ -24,12 +24,9 @@ const searchClient = algoliasearch("JNSS5CFDZZ", "db08e40265e2941b9a7d8f644b6e52 const Appsearch = props => { const { maxRows, showName, showSuggestion, isMobile, globalUrl, parsedXs, newSelectedApp, setNewSelectedApp, defaultSearch, showSearch, ConfiguredHits, userdata, cy, isCreatorPage, actionImageList, setActionImageList, setUserSpecialzedApp } = props - const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + const isCloud = (window.location.host === "localhost:3002" || window.location.host === "shuffler.io") ? true : (process.env.IS_SSR === "true"); const rowHandler = maxRows === undefined || maxRows === null ? 50 : maxRows const xs = parsedXs === undefined || parsedXs === null ? 12 : parsedXs - //const theme = useTheme(); - //const [apps, setApps] = React.useState([]); - //const [filteredApps, setFilteredApps] = React.useState([]); const [formMail, setFormMail] = React.useState(""); const [message, setMessage] = React.useState(""); const [formMessage, setFormMessage] = React.useState(""); diff --git a/frontend/src/components/AuthenticationItem.jsx b/frontend/src/components/AuthenticationItem.jsx index eae08e69..604418b2 100644 --- a/frontend/src/components/AuthenticationItem.jsx +++ b/frontend/src/components/AuthenticationItem.jsx @@ -45,21 +45,21 @@ const AuthenticationItem = (props) => { data.fields = [ { key: "url", - value: "Secret. Replaced during app execution!", + value: "URL Secret. Replaced during runtime", }, { key: "client_id", - value: "Secret. Replaced during app execution!", + value: "ClientID Secret. Replaced during runtime.", }, { key: "client_secret", - value: "Secret. Replaced during app execution!", + value: "Client Secret. Replaced during runtime.", }, { key: "scope", - value: "Secret. Replaced during app execution!", + value: "Scope Secret. Replaced during runtime.", }, - ]; + ] } const deleteAuthentication = (data) => { @@ -152,7 +152,7 @@ const AuthenticationItem = (props) => { src={data.app.large_image} style={{ maxWidth: 50, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} /> style={{ minWidth: 75, maxWidth: 75 }} diff --git a/frontend/src/components/AuthenticationNormal.jsx b/frontend/src/components/AuthenticationNormal.jsx index 271dc59a..96569b41 100644 --- a/frontend/src/components/AuthenticationNormal.jsx +++ b/frontend/src/components/AuthenticationNormal.jsx @@ -222,7 +222,7 @@ const AuthenticationData = (props) => { { { onClick={() => { setAuthenticationModalOpen(false); }} - color="primary" + color="secondary" > Cancel
-
+
{top_text === "Base Cloud Access" && userdata.has_card_available === true ? { @@ -436,80 +563,80 @@ const Billing = (props) => { }} variant="outlined" color="primary" - /> + /> : null} {top_text} {top_text === "Base Cloud Access" && userdata.has_card_available === false ? - - : null} + : null} {isCloud && highlight === true && top_text !== "Base Cloud Access" ? - { setSignatureOpen(true) }} > - + - : null} + : null}
- -
- - {subscription.name} - + +
+ + {subscription.name} + - {subscription.currency_text !== undefined ? -
- - {subscription.currency_text}{subscription.price} - - - / {subscription.interval} - -
+ {subscription.currency_text !== undefined ? +
+ + {subscription.currency_text}{subscription.price} + + + / {subscription.interval} + +
: null} - - Features - -
    + + Features + +
      {subscription.features !== undefined && subscription.features !== null ? subscription.features.map((feature, index) => { var parsedFeature = feature if (feature.includes("Documentation: ")) { - parsedFeature = - Documentation to get started } if (feature.includes("Worker License: ")) { - const fieldId = "webhook_uri_field_"+index + const fieldId = "webhook_uri_field_" + index parsedFeature = - + @@ -517,47 +644,47 @@ const Billing = (props) => { {}} - InputProps={{ - endAdornment: - - { - var copyText = document.getElementById(fieldId); - if (copyText !== undefined && copyText !== null) { - console.log("NAVIGATOR: ", navigator); - const clipboard = navigator.clipboard; - if (clipboard === undefined) { - toast("Can only copy over HTTPS (port 3443)"); - return; - } + style={{ + backgroundColor: theme.palette.inputColor, + borderRadius: theme.palette?.borderRadius, + }} + id={fieldId} + onClick={() => { }} + InputProps={{ + endAdornment: + + { + var copyText = document.getElementById(fieldId); + if (copyText !== undefined && copyText !== null) { + console.log("NAVIGATOR: ", navigator); + const clipboard = navigator.clipboard; + if (clipboard === undefined) { + toast("Can only copy over HTTPS (port 3443)"); + return; + } - navigator.clipboard.writeText(copyText.value); - copyText.select(); - copyText.setSelectionRange( - 0, - 99999 - ); /* For mobile devices */ + navigator.clipboard.writeText(copyText.value); + copyText.select(); + copyText.setSelectionRange( + 0, + 99999 + ); /* For mobile devices */ - /* Copy the text inside the text field */ - document.execCommand("copy"); - toast("Copied Webhook URL"); - } else { - console.log("Couldn't find webhook URI field: ", copyText); - } - }} - edge="end" - > - - - - }} + /* Copy the text inside the text field */ + document.execCommand("copy"); + toast("Copied Webhook URL"); + } else { + console.log("Couldn't find webhook URI field: ", copyText); + } + }} + edge="end" + > + + + + }} fullWidth /> @@ -565,30 +692,105 @@ const Billing = (props) => { return (
    • - + {parsedFeature}
      • ) }) : null} -
    -
- {isCloud && (highlight === true && (subscription.name === "Pay as you go" && subscription.limit <= 10000) || subscription.name === "Open Source") ? - - - {subscription.name.includes("Scale") ? - "" - : + +
+ {isCloud && (highlight === true && (subscription.name === "Pay as you go" && subscription.limit <= 10000) || subscription.name === "Open Source") ? + + + {subscription.name.includes("Scale") ? + "" + : - userdata.has_card_available === true ? - "While you have a card attached to your account, Shuffle will no longer prevent workflows from running. Billing will occur at the start of each month." - : + userdata.has_card_available === true ? + "While you have a card attached to your account, Shuffle will no longer prevent workflows from running. Billing will occur at the start of each month." + : + isCloud ? `You are not subscribed to any plan and are using the free plan with max 10,000 app runs per month. Upgrade to deactivate this limit.` - } + : + `You are not subscribed to any plan and are using the free, open source plan. This plan has no enforced limits, but scale issues may occur due to CPU congestion.` + } + +
+ + {BillingEmail?.length > 0 ? `Billing email: ${BillingEmail}` : null} - Billing email: {selectedOrganization.org} - {/*isCloud ? + {userdata.has_card_available === true && ( + + )} + + Change Billing Email + + + Enter the new billing email address. + + { if (event.key === 'Enter') HandleChangeBillingEmail(selectedOrganization.id) }} + onChange={(e) => setNewBillingEmail(e.target.value)} + /> + + + + + + +
+ {/*isCloud ? : null*/} - + {userdata.has_card_available === true ? + - {userdata.has_card_available === true ? - : null} - - + + : null} - {showSupport ? + {showSupport ? - : null } - + : null} +
+ ) + } + const ConsultationManagement = (props) => { + const { globalUrl, userdata, selectedOrganization, } = props; + + const [inputHour, setInputHour] = React.useState( + selectedOrganization.Billing && + selectedOrganization.Billing.Consultation && + selectedOrganization.Billing.Consultation.hours !== undefined && + selectedOrganization.Billing.Consultation.hours !== "" + ? selectedOrganization.Billing.Consultation.hours + : 0 + ); + + const [inputMinutes, setInputMinutes] = React.useState( + selectedOrganization.Billing && + selectedOrganization.Billing.Consultation && + selectedOrganization.Billing.Consultation.minutes !== undefined && + selectedOrganization.Billing.Consultation.minutes !== "" + ? selectedOrganization.Billing.Consultation.minutes + : 0 + ); + + const [editConsultation, setEditConsultation] = React.useState(false); + const [openUpgradePlan, setOpenUpgradePlan] = React.useState(false); + const [consultationHours, setConsultationHours] = React.useState(1); + const [message, setMessage] = React.useState(""); + const [hovered, setHovered] = React.useState(false) + const [getProfessionalServices, setGetProfessionalServices] = React.useState(false) + const [clickOnBuy, setClickOnBuy] = React.useState(false) + + const formatedHours = String(inputHour).padStart(2, "0") + const formatedMinutes = String(inputMinutes).padStart(2, "0") + + const handleHourChange = (event) => { + setInputHour(parseInt(event.target.value, 10)); + }; + + const handleMinuteChange = (event) => { + setInputMinutes(parseInt(event.target.value, 10)); + }; + const toggleEditMode = () => { + setEditConsultation(!editConsultation); + }; + + const handleCancel = () => { + setEditConsultation(false); + }; + + const handleSave = () => { + + toast("Saving consultation hours. Please wait.") + + const url = `${globalUrl}/api/v1/orgs/${selectedOrganization.id}`; + const data = { + org_id: selectedOrganization.id, + Billing: { + Consultation: { + hours: String(inputHour), + minutes: String(inputMinutes), + }, + } + }; + + fetch(url, { + body: JSON.stringify(data), + mode: "cors", + method: "POST", + credentials: "include", + crossDomain: true, + withCredentials: true, + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + }) + .then((response) => { + if (response.status !== 200) { + console.log("Error in response"); + } + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success === true) { + toast.success("Consultation hours saved successfully"); + setEditConsultation(false); + } else { + toast.error("Failed saving consultation hours."); + } + if (inputHour > 0 || inputMinutes > 0) { + setGetProfessionalServices(true) + } else { + setGetProfessionalServices(false) + } + }) + .catch((error) => { + console.log("Error: ", error); + }); + } + + const handleUpgradeConsultation = () => { + + toast("Sending request for consultation hours. Please wait.") + + const url = `${globalUrl}/api/v1/orgs/${selectedOrganization.id}/consultation`; + const data = { + org_id: selectedOrganization.id, + consultationHours: String(consultationHours), + message: message, + }; + + fetch(url, { + body: JSON.stringify(data), + mode: "cors", + method: "POST", + credentials: "include", + crossDomain: true, + withCredentials: true, + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + }) + .then((response) => { + if (response.status !== 200) { + console.log("Error in response"); + } + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success === true) { + toast.success("Thank you for your request. We will get back to you soon."); + setOpenUpgradePlan(false); + setEditConsultation(false); + } else { + toast.error("Failed sending consultation hours request. Please try again later."); + } + }) + .catch((error) => { + console.log("Error: ", error); + }); + } + + useEffect(() => { + if (inputHour !== undefined && inputMinutes !== undefined && inputHour > 0 || inputMinutes > 0) { + setGetProfessionalServices(true) + } else { + setGetProfessionalServices(false) + } + }) + + return ( +
setHovered(true)} + onMouseLeave={() => setHovered(false)}> + + Professional Services + + + + Consultation & Management + +
+ + You currently have a total of {inputHour} hours and {inputMinutes} minutes of professional services available by our experts. + +
+ {editConsultation ? + <> + + : + + + : + + {`${formatedHours}h:${formatedMinutes}m`} + } +
+ {userdata.support === true ? +
+ {editConsultation ? ( + + ) : ( + + )} + {editConsultation && } +
+ : null} + + Features + +
    +
  • + + Build custom apps, integrations, and worklows for your specific use cases or applications + +
    • +
  • + + Help solve / debug / update / add features and capabilities of the platform + +
    • +
+
+
+ + { setClickOnBuy(false) }} PaperProps={{style: {backgroundColor: "rgb(26, 26, 26)"}}}> + + You will be taken to Stripe to book professional service hours. You can adjust the number of hours on the left side of the Stripe page. + + + + + + + + + + + +
+ setOpenUpgradePlan(false)} + fullWidth + style={{ display: 'flex', justifyContent: 'center', alignItems: 'center' }} + PaperProps={{ + style: { + width: 500, + margin: 0, + } + }} + > + + Upgrade Consultation Plan + + + + Enter the total hours of consultation you want. + +
+ setConsultationHours(val)} + aria-labelledby="continuous-slider" + step={1} + min={1} + max={(inputHour === "0" && inputMinutes > 0) ? 1 : inputHour} + style={{ width: '80%', color: theme.palette.primary.main }} + marks + valueLabelDisplay="auto" + /> + +
+ + If you have any additional requirements or questions, please leave a message below. + + setMessage(e.target.value)} + /> + + + +
+ ) + } + + const TrainingService = () => { + + const [hovered, setHovered] = React.useState(false) + const [openPrivateTraining, setOpenPrivateTraining] = React.useState(false) + const [PrivateTrainingMember, setPrivateTrainingMember] = React.useState(5) + const [message, setMessage] = React.useState(""); + + const handlePrivateTraining = () => { + + toast("Submitting your request for private training. Please wait...") + + const data = { + org_id: selectedOrganization.id, + trainingMembers: String(PrivateTrainingMember), + message: message + } + + const url = `${globalUrl}/api/v1/orgs/${selectedOrganization.id}/privateTraining` + + fetch(url, { + body: JSON.stringify(data), + mode: "cors", + method: "POST", + credentials: "include", + crossDomain: true, + withCredentials: true, + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + }).then((response) => { + if (response.status !== 200) { + console.log("Error in response"); + } + return response.json(); + }).then((responseJson) => { + if (responseJson.success === true) { + toast.success("Your request for private training has been submitted successfully. We will get back to you soon.") + setOpenPrivateTraining(false) + } else { + toast.error("Failed sending request for private training. Please try again later or contact support@shuffler.io for help.") + } + }) + } + + return ( +
setHovered(true)} + onMouseLeave={() => setHovered(false)} + > + + Training + + + + Become a Shuffle Expert + +
+ + Public Training + +
    +
  • + + Public course on Automation for Security Professionals + +
    • +
  • + + Covers Shuffle Platform, Apps, Workflows, Usecases, JSON, Liquid Formatting, and more. + +
    • +
+ + Private Training + +
    +
  • + + Everything from Public Training + +
    • +
  • + + Customized for your team’s usecases, date and time, location, and more. + +
    • +
+
+
+ + + + setOpenPrivateTraining(false)} + fullWidth + style={{ display: 'flex', justifyContent: 'center', alignItems: 'center' }} + PaperProps={{ + style: { + width: 500, + margin: 0, + backgroundColor: "rgb(26, 26, 26)", + } + }} + > + + Private Training + + + + Enter the total members for private training. Minimum 5 members required. + +
+ setPrivateTrainingMember(val)} + aria-labelledby="continuous-slider" + step={1} + min={5} + max={50} + style={{ width: '80%', color: theme.palette.primary.main }} + marks + valueLabelDisplay="auto" + /> + +
+ + If you have any additional requirements or questions, please leave a message below. + + setMessage(e.target.value)} + /> + + + +
+
) } const addDealModal = ( - { - setSelectedDealModalOpen(false); - }} - PaperProps={{ - style: { - backgroundColor: theme.palette.surfaceColor, - color: "white", - minWidth: "800px", - minHeight: "320px", - }, - }} - > - - Register new deal - - -
- { - setDealName(e.target.value); - }} - /> - { - setDealAddress(e.target.value); - }} - /> -
-
- { - setDealValue(e.target.value); - }} - /> - option.label} - onChange={(event, newValue) => { - setDealCountry(newValue.label); - }} - renderOption={(props, option) => ( - img": { mr: 2, flexShrink: 0 } }} - {...props} - > - - {option.label} ({option.code}) +{option.phone} - - )} - renderInput={(params) => ( - - )} - /> - { - setDealType(newValue); - }} - getOptionLabel={(option) => option.label} - renderOption={(props, option) => ( - img": { mr: 2, flexShrink: 0 } }} - {...props} - > - {option.label} - - )} - renderInput={(params) => ( - - )} - /> -
- {dealerror.length > 0 ? ( - - error registering: {dealerror} - - ) : null} -
- - -
- - - ); + //setDealName("") + //setDealAddress("") + //setDealCountry("") + //setDealValue("") + }} + > + Cancel + + +
+ + + ); - const submitDeal = (dealName, dealAddress, dealCountry, dealValue) => { - if (dealerror.length > 0) { - setDealerror(""); - } + const submitDeal = (dealName, dealAddress, dealCountry, dealValue) => { + if (dealerror.length > 0) { + setDealerror(""); + } - const orgId = selectedOrganization.id; - const data = { - reseller_org: orgId, - name: dealName, - address: dealAddress, - country: dealCountry, - value: dealValue, - }; + const orgId = selectedOrganization.id; + const data = { + reseller_org: orgId, + name: dealName, + address: dealAddress, + country: dealCountry, + value: dealValue, + }; - const url = `${globalUrl}/api/v1/orgs/${orgId}/deals`; - fetch(url, { - mode: "cors", - method: "POST", - body: JSON.stringify(data), - credentials: "include", - crossDomain: true, - withCredentials: true, - headers: { - "Content-Type": "application/json; charset=utf-8", - }, - }) - .then(function (response) { - if (response.status !== 200) { - console.log("Error in response"); - } + const url = `${globalUrl}/api/v1/orgs/${orgId}/deals`; + fetch(url, { + mode: "cors", + method: "POST", + body: JSON.stringify(data), + credentials: "include", + crossDomain: true, + withCredentials: true, + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + }) + .then(function (response) { + if (response.status !== 200) { + console.log("Error in response"); + } - return response.json(); - }) - .then(function (responseJson) { - if (responseJson.success === true) { - setSelectedDealModalOpen(false); - toast( - "Added new deal! We will be in touch shortly with an update." - ); + return response.json(); + }) + .then(function (responseJson) { + if (responseJson.success === true) { + setSelectedDealModalOpen(false); + toast( + "Added new deal! We will be in touch shortly with an update." + ); - setDealName(""); - setDealAddress(""); - setDealValue(""); - setDealCountry("United States"); - setDealType("MSSP"); - } else { - setDealerror(responseJson.reason); - } - }) - .catch(function (error) { - //console.log("Error: ", error); - setDealerror(error.toString()); - toast("Failed adding deal reg: ", error); - }); - }; + setDealName(""); + setDealAddress(""); + setDealValue(""); + setDealCountry("United States"); + setDealType("MSSP"); + } else { + setDealerror(responseJson.reason); + } + }) + .catch(function (error) { + //console.log("Error: ", error); + setDealerror(error.toString()); + toast("Failed adding deal reg: ", error); + }); + }; + const addAlertThreshold = () => { + setAlertThresholds([...alertThresholds, { percentage: '', count: '', Email_send: false }]); + }; - const isChildOrg = userdata.active_org.creator_org !== "" && userdata.active_org.creator_org !== undefined && userdata.active_org.creator_org !== null + const updateAlertThreshold = (index, field, value) => { + + if (field === 'percentage') { + if (value > 100 || value < 0) { + value = 0 + toast("The percentage value should be between 0 and 100") + } + } else if (field === 'count') { + if (value < 0 || value >= userdata.app_execution_limit) { + value = 0 + toast("The count value should be greater than 0 and less than the total app execution limit") + } + } + + + const totalValue = userdata.app_execution_limit; + const newAlertThresholds = alertThresholds.map((threshold, i) => { + if (i === index) { + const newValue = parseFloat(value); + if (field === 'percentage') { + const newCount = (newValue / 100) * totalValue; + return { + ...threshold, + percentage: isNaN(newValue) ? '' : Math.round(newValue), + count: isNaN(newCount) ? '' : Math.round(newCount), + Email_send: false + }; + } else if (field === 'count') { + const newPercentage = (newValue / totalValue) * 100; + return { + ...threshold, + count: newValue, + percentage: isNaN(newPercentage) ? '' : Math.round(newPercentage), + Email_send: false + }; + } + } + return threshold; + }); + setAlertThresholds(newAlertThresholds); + }; + + + const handleDeleteAlertThreshold = (index) => { + const newAlertThresholds = alertThresholds.filter((_, i) => i !== index); + setAlertThresholds(newAlertThresholds); + + // Update currentIndex based on remaining elements + const findCurrentIndex = newAlertThresholds.some(threshold => threshold.Email_send === false); + setCurrentIndex(findCurrentIndex ? newAlertThresholds.findIndex(threshold => threshold.Email_send === false) : - 1); + toast.info("Alert Threshold deleted successfully. Don't forget to save your changes."); + }; + + const HandleEditOrgForAlertThreshold = (orgId) => { + + // Use the `some` method to check for invalid counts + const invalidCount = alertThresholds.some((threshold) => { + if (threshold.count === '') { + toast("Please enter a valid Count or Percentage value"); + return true; // Stop checking further and return true if invalid + } + return false; + }); + + // If any invalid count is found, return early + if (invalidCount) { + return; + } + + toast("Updating Email Alert Threshold. Please wait..."); + + const data = { + org_id: orgId, + billing: { + email: BillingEmail, + AlertThreshold: alertThresholds.map(threshold => ({ + ...threshold, + percentage: parseInt(threshold.percentage, 10), + count: parseInt(threshold.count, 10), + })), + }, + }; + + const url = `${globalUrl}/api/v1/orgs/${orgId}`; + fetch(url, { + method: "POST", + body: JSON.stringify(data), + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => { + if (response.status !== 200) { + console.log("Bad status code in get org:", response.status); + } + return response.json(); + }).then((responseJson) => { + console.log("Got org:", responseJson); + if (responseJson.success === true) { + toast.success("Successfully updated Email Alert Thresholds"); + const findCurrentIndex = alertThresholds.some(threshold => threshold.Email_send === false); + setCurrentIndex(findCurrentIndex ? alertThresholds.findIndex(threshold => threshold.Email_send === false) : - 1); + } else { + toast.error("Failed to update Email Alert Thresholds. Please try again."); + } + }) + .catch((error) => { + console.log("Error getting org:", error); + }); + }; + + const getSafeValue = (value) => { + + if (value === undefined || value === null || isNaN(value)) { + return 0; + } else { + return value; + } + }; + + const isChildOrg = userdata?.active_org?.creator_org !== "" && userdata?.active_org?.creator_org !== undefined && userdata?.active_org?.creator_org !== null + return ( -
- {addDealModal} - {clickedFromOrgTab? -

Billing & Licensing

: - - Billing & Licensing - } - {clickedFromOrgTab? - {isCloud ? - "Get more out of Shuffle by adding your credit card, such as no App Run limitations, and priority support from our team. We use Stripe to manage subscriptions and do not store any of your billing information. You can manage your subscription and billing information below." - : - "Shuffle is an Open Source automation platform, and no license is required. We do however offer a Scale license with HA guarantees, along with support hours. By buying a license on https://shuffler.io, you can get access to the license immediately, and if Cloud Syncronisation is enabled, the UI in your local instance will also update." - }: - - {isCloud ? + +
+
+ {addDealModal} + {clickedFromOrgTab ? + Billing & Licensing : + + Billing & Licensing + } + {clickedFromOrgTab ? + {isCloud ? "Get more out of Shuffle by adding your credit card, such as no App Run limitations, and priority support from our team. We use Stripe to manage subscriptions and do not store any of your billing information. You can manage your subscription and billing information below." : "Shuffle is an Open Source automation platform, and no license is required. We do however offer a Scale license with HA guarantees, along with support hours. By buying a license on https://shuffler.io, you can get access to the license immediately, and if Cloud Syncronisation is enabled, the UI in your local instance will also update." - } - } + } : + + {isCloud ? + "Get more out of Shuffle by adding your credit card, such as no App Run limitations, and priority support from our team. We use Stripe to manage subscriptions and do not store any of your billing information. You can manage your subscription and billing information below." + : + "Shuffle is an Open Source automation platform, and no license is required. We do however offer a Scale license with HA guarantees, along with support hours. By buying a license on https://shuffler.io, you can get access to the license immediately, and if Cloud Syncronisation is enabled, the UI in your local instance will also update." + } + } - {userdata.support === true ? -
+ {userdata.support === true ? + : null } - {isChildOrg ? - - Billing is handled by your parent organisation. Reach out to support@shuffler.io if you have questions about this. - + {isChildOrg ? + + Licensing is handled by your parent organisation. Reach out to support@shuffler.io if you have questions about this. + : null} -
- {isCloud && billingInfo.subscription !== undefined && billingInfo.subscription !== null ? isChildOrg ? null : +
+ {isCloud && billingInfo.subscription !== undefined && billingInfo.subscription !== null ? isChildOrg ? null : { subscription={billingInfo.subscription} highlight={selectedOrganization.subscriptions === undefined || selectedOrganization.subscriptions === null || selectedOrganization.subscriptions.length === 0} /> - : !isCloud ? - - - - - : null} + : !isCloud ? + + + + + : null} {isCloud && selectedOrganization.subscriptions !== undefined && selectedOrganization.subscriptions !== null && selectedOrganization.subscriptions.length > 0 && - !isChildOrg ? - selectedOrganization.subscriptions - .reverse() - .map((sub, index) => { - return ( - - ) - }) - : null} - {/* + !isChildOrg ? + selectedOrganization.subscriptions + .reverse() + .map((sub, index) => { + return ( + + ) + }) + : null} + {/* { */} -
+
- {/*isCloud && + {/*isCloud && selectedOrganization.partner_info !== undefined && selectedOrganization.partner_info.reseller === true ? (
@@ -1333,23 +2254,220 @@ const Billing = (props) => {
) : null*/} -
+ {!isChildOrg && isCloud && ( +
+ + Professional Services + + + We offer priority support through consultations and training to help you make the most of our product. If you have any questions, please reach out to us at support@shuffler.io. + +
+ {billingInfo.subscription !== undefined && billingInfo.subscription !== null ? ( + isChildOrg ? null : ( + + ) + ) : null} + +
+
+ )} +
- Utilization & Stats + Manage Billing -
- + Manage your billing and licensing information below. When you reach the certain thresholds of your subscription limit, you will be notified by email. + + Current Usage: + + + You have used {currentAppRunsInPercentage}% of total app execution limit or {userdata.app_execution_usage} app runs out of {userdata.app_execution_limit} app runs. + + +
+ + Set email alert thresholds for app runs + + + You will be notified by email when you reach the + {currentIndex !== -1 + ? " " + getSafeValue(alertThresholds[currentIndex].percentage) + '%' + " " + : " " + '0%' + " " + } + of your total app execution limit or + {currentIndex !== -1 + ? " " + getSafeValue(alertThresholds[currentIndex].count) + " " + : " " + 0 + " "} + app runs. + + + Please note: Once your app runs reach the set alert threshold, all admins in the organization will receive an email notification. + +
+ {alertThresholds.map((threshold, index) => ( +
+ updateAlertThreshold(index, 'percentage', e.target.value)} + margin="normal" + variant="outlined" + inputProps={{ + max: 100, + }} + /> + updateAlertThreshold(index, 'count', e.target.value)} + margin="normal" + variant="outlined" + /> + {alertThresholds[index].Email_send === true && } + { + alertThresholds.length > 1 && + ( + + ) + } + setDeleteAlertVerification(false)} sx={{ '& .MuiBackdrop-root': { backgroundColor: 'rgba(0, 0, 0, 0.3)', }, }}> + Are you sure you want to delete this threshold? + + + + + +
+ ))} +
+
+ + + +
+
+
+ + Utilization & Stats + +
+ -
+ /> +
+ ) -} +}) -export default Billing; +export default memo(Billing); + +const PaddingWrapper = memo(({ clickedFromOrgTab, children }) => { + + const wrapperStyle = useMemo(() => ({ + width: clickedFromOrgTab + ? "100%" + : "auto", + padding: "27px 10px 19px 27px", + backgroundColor: '#212121', + borderRadius: '16px', + height: '100%', + boxSizing: 'border-box', + borderLeft: '1px solid #494949', + overflow: 'hidden', + maxHeight: "1700px", overflowY: "auto",scrollbarColor: '#494949 transparent', scrollbarWidth: 'thin' + }), [clickedFromOrgTab]); + + return ( +
+ {children} +
+ ); + }); + + const Wrapper = memo(({ children, clickedFromOrgTab }) => { + return ( + + {children} + + ); + }); diff --git a/frontend/src/components/BillingStats.jsx b/frontend/src/components/BillingStats.jsx index ac18d39b..8b5d4a1c 100644 --- a/frontend/src/components/BillingStats.jsx +++ b/frontend/src/components/BillingStats.jsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect } from 'react'; +import React, { useState, useEffect, useContext, memo, useMemo } from 'react'; import theme from '../theme.jsx'; import classNames from "classnames"; @@ -37,13 +37,14 @@ import { } from 'reaviz'; import { typecost, typecost_single, } from "../views/HandlePaymentNew.jsx"; +import { Context } from '../context/ContextApi.jsx'; const LineChartWrapper = ({keys, inputname, height, width}) => { const [hovered, setHovered] = useState(""); const inputdata = keys.data === undefined ? keys : keys.data return ( -
+
{inputname} @@ -82,8 +83,8 @@ const AppStats = (defaultprops) => { const [workflows, setWorkflows] = useState(inputWorkflows === undefined ? [] : inputWorkflows) const [resultRows, setResultRows] = useState([]) const [resultLoading, setResultLoading] = useState(true) - - const includedExecutions = selectedOrganization.sync_features.app_executions !== undefined ? selectedOrganization.sync_features.app_executions.limit : 0 + + const includedExecutions = selectedOrganization?.sync_features?.app_executions !== undefined ? selectedOrganization?.sync_features?.app_executions?.limit : 0 useEffect(() => { if (workflows === undefined || workflows === null || workflows.length === 0) { @@ -92,7 +93,6 @@ const AppStats = (defaultprops) => { }, []) - const getWorkflowStats = async (workflow, startTime, endTime) => { if (!userdata.support) { return workflow @@ -134,6 +134,9 @@ const AppStats = (defaultprops) => { Accept: "application/json", }, credentials: "include", + }).catch((error) => { + console.log("Error getting workflow stats: " + error); + return workflow }) if (response.status !== 200) { @@ -159,8 +162,6 @@ const AppStats = (defaultprops) => { const loadWorkflowStats = (foundWorkflows, startTime, endTime) => { if (!userdata.support) { - console.log("Not support") - return } @@ -485,8 +486,13 @@ const AppStats = (defaultprops) => { setApprunCosts(appcostRuns) } - const getStats = () => { - fetch(`${globalUrl}/api/v1/orgs/${selectedOrganization.id}/stats`, { + const getStats = (orgid) => { + + if (orgid === undefined || orgid === null) { + return + } + + fetch(`${globalUrl}/api/v1/orgs/${orgid}/stats`, { method: "GET", headers: { "Content-Type": "application/json", @@ -516,8 +522,10 @@ const AppStats = (defaultprops) => { } useEffect(() => { - getStats() - }, []) + if(selectedOrganization?.id?.length > 0) { + getStats(selectedOrganization.id) + } + }, [selectedOrganization]) const paperStyle = { textAlign: "center", @@ -636,7 +644,7 @@ const AppStats = (defaultprops) => { const data = (
- + All shown statistics are gathered from {
: null} + {clickedFromOrgTab? ( + +
+
+ } + /> +
+
+ } + /> +
+
+ + ):(
{ />
+ )} +
diff --git a/frontend/src/components/Branding.jsx b/frontend/src/components/Branding.jsx index 48466476..e549d12e 100644 --- a/frontend/src/components/Branding.jsx +++ b/frontend/src/components/Branding.jsx @@ -1,17 +1,28 @@ -import React, { useState, useEffect } from "react"; +import React, { useState, useEffect, useContext } from "react"; import ReactGA from 'react-ga4'; import theme from "../theme.jsx"; import { ToastContainer, toast } from "react-toastify" +import { + CheckCircle as CheckCircleIcon, +} from "@mui/icons-material"; + import { Paper, Typography, Divider, Button, + Tooltip, Grid, Card, } from "@mui/material"; +import { + red, + green, +} from "../views/AngularWorkflow.jsx" +import { Context } from "../context/ContextApi.jsx"; + //import { useAlert const Branding = (props) => { @@ -20,7 +31,8 @@ const Branding = (props) => { const [publishingInfo, setPublishingInfo] = useState(""); const [publishRequirements, setPublishRequirements] = useState([]) - + const { leftSideBarOpenByClick } = useContext(Context) + const handleEditOrg = (joinStatus) => { const data = { "org_id": selectedOrganization.id, @@ -45,7 +57,7 @@ const Branding = (props) => { toast("Failed updating org: ", responseJson.reason); } else { if (joinStatus == "join") { - setPublishingInfo("Your organization is now part of the Creator Incentive Program. You can now create and publish content to your organization's page. You can also create a creator account to manage your organization's content.") + setPublishingInfo("Your organization is now part of the Partner Program. You can now create, publish and manage content for your organization's public page.") } else { setPublishingInfo("Your organization is no longer part of the Creator Incentive Program. You can still create a creator account to manage your organization's content.") } @@ -70,7 +82,15 @@ const Branding = (props) => { } const isOrganizationReady = () => { - console.log("Is organization ready?") + + // Check if it's a suborg + if (selectedOrganization.creator_org !== "") { + const comment = "Child orgs can't become creators" + if (!publishRequirements.includes(comment)) { + setPublishRequirements([...publishRequirements, comment]) + } + return false; + } // A simple checklist to ensure the button shows up properly if (selectedOrganization.name === selectedOrganization.org) { @@ -82,15 +102,6 @@ const Branding = (props) => { return false; } - // Check if it's a suborg - if (selectedOrganization.creator_org !== "") { - const comment = "Child orgs can't become creators" - if (!publishRequirements.includes(comment)) { - setPublishRequirements([...publishRequirements, comment]) - } - return false; - } - if (selectedOrganization.large_image === "" || selectedOrganization.large_image === theme.palette.defaultImage) { const comment = "Add a logo for your organization" if (!publishRequirements.includes(comment)) { @@ -102,38 +113,89 @@ const Branding = (props) => { return true } + const isPublished = selectedOrganization.creator_id === "" + const leadinfo = selectedOrganization.lead_info === undefined || selectedOrganization.lead_info === null || selectedOrganization.lead_info === "" ? "" : JSON.stringify(selectedOrganization.lead_info) + const isPartner = leadinfo.includes("partner") + + return ( -
-

- Branding -

- +
+
+
+ + Partner Status & Branding + + You can customize your organization's branding by uploading a logo, changing the color scheme and a lot more. + + {isPublished ? : } + {isPublished ? "Not Published" : "Published"} + + + + + {!isPartner ? : } + + {!isPartner? "Not Officially Partnered" : "Officially Partnered"} + + + + + {!isPublished ? ( + + + + ) : ( + + )} + -

- Creator Incentive Program -

-
+ + Partner Program + +
- - By changing publishing settings, you agree to our Terms of Service, and acknowledge that your organization's non-sensitive data will be added as a creator account. None of your existing workflows, apps, or other stored data will be published. Any admin in your organization can manage the creator configuration. Becoming a creator organization is reversible.
Support: support@shuffler.io + + By changing publishing settings, you agree to our Terms of Service, and acknowledge that your organization's non-sensitive data will be added as a creator account. None of your existing workflows, apps, or other stored data will be published. Any admin in your organization can manage the creator configuration. Becoming a creator organization IS reversible.
Support: support@shuffler.io {selectedOrganization.creator_id == "" ?   : - - - Modify your creator organization - + null } @@ -159,6 +221,8 @@ const Branding = (props) => {
+
+
) } diff --git a/frontend/src/components/CacheView.jsx b/frontend/src/components/CacheView.jsx index bd653e7d..4e8c6c24 100644 --- a/frontend/src/components/CacheView.jsx +++ b/frontend/src/components/CacheView.jsx @@ -1,7 +1,7 @@ -import React, { useState, useEffect } from "react"; +import React, { useState, useEffect, useContext, memo } from "react"; import theme from "../theme.jsx"; import { toast } from 'react-toastify'; -import ReactJson from "react-json-view"; +import ReactJson from "react-json-view-ssr"; import { Typography, @@ -19,6 +19,7 @@ import { Dialog, DialogTitle, DialogActions, + Skeleton, } from "@mui/material"; import { @@ -47,6 +48,7 @@ import { VisibilityOff as VisibilityOffIcon, } from "@mui/icons-material"; import { validateJson, } from "../views/Workflows.jsx"; +import { Context } from "../context/ContextApi.jsx"; const scrollStyle1 = { height: 100, @@ -65,7 +67,7 @@ const scrollStyle2 = { } -const CacheView = (props) => { +const CacheView = memo((props) => { const { globalUrl, userdata, serverside, orgId, isSelectedDataStore } = props; const [orgCache, setOrgCache] = React.useState(""); const [listCache, setListCache] = React.useState([]); @@ -78,11 +80,13 @@ const CacheView = (props) => { const [cacheCursor, setCacheCursor] = React.useState(""); const [dataValue, setDataValue] = React.useState({}); const [editCache, setEditCache] = React.useState(false); + const [cachedLoaded, setCachedLoaded] = React.useState(false); const [show, setShow] = useState({}); - useEffect(() => { - listOrgCache(orgId); - }, []); + if(orgId?.length >0){ + listOrgCache(orgId); + } + }, [orgId]); const listOrgCache = (orgId) => { fetch(globalUrl + `/api/v1/orgs/${orgId}/list_cache`, { @@ -104,6 +108,7 @@ const CacheView = (props) => { .then((responseJson) => { if (responseJson.success === true) { setListCache(responseJson.keys); + setCachedLoaded(true); } if (responseJson.cursor !== undefined && responseJson.cursor !== null && responseJson.cursor !== "") { @@ -232,6 +237,34 @@ const CacheView = (props) => { } } + const handleReactJsonClipboard = (copy) => { + const elementName = "copy_element_shuffle"; + let copyText = document.getElementById(elementName); + + if (copyText) { + if (copy.namespace && copy.name && copy.src) { + copy = copy.src; + } + + const clipboard = navigator.clipboard; + if (!clipboard) { + toast("Can only copy over HTTPS (port 3443)"); + return; + } + + let stringified = JSON.stringify(copy); + if (stringified.startsWith('"') && stringified.endsWith('"')) { + stringified = stringified.slice(1, -1); + } + + navigator.clipboard.writeText(stringified); + toast("Copied value to clipboard, NOT json path."); + } else { + console.log("Failed to copy from " + elementName + ": ", copyText); + } + }; + + const modalView = ( // console.log("key:", dataValue.key), //console.log("value:",dataValue.value), @@ -241,11 +274,23 @@ const CacheView = (props) => { setModalOpen(false); }} PaperProps={{ - style: { - backgroundColor: theme.palette.surfaceColor, - color: "white", + sx: { + borderRadius: theme?.palette?.DialogStyle?.borderRadius, + border: theme?.palette?.DialogStyle?.border, minWidth: "800px", minHeight: "320px", + fontFamily: theme?.typography?.fontFamily, + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + zIndex: 1000, + '& .MuiDialogContent-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + }, + '& .MuiDialogTitle-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + }, + '& .MuiDialogActions-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + }, }, }} > @@ -254,7 +299,7 @@ const CacheView = (props) => { { editCache ? "Edit Cache" : "Add Cache" } -
+
Key { onChange={(e) => setKey(e.target.value)} />
-
+
Value - ({isValidJson.valid === true ? "Valid" : "Invalid"} JSON) @@ -320,7 +365,7 @@ const CacheView = (props) => {
+
- ); -} -export default CacheView; +}); + +export default memo(CacheView); diff --git a/frontend/src/components/ConfigureWorkflow.jsx b/frontend/src/components/ConfigureWorkflow.jsx index 6a7b55b7..ea8af7bd 100755 --- a/frontend/src/components/ConfigureWorkflow.jsx +++ b/frontend/src/components/ConfigureWorkflow.jsx @@ -2,6 +2,7 @@ import React, { useState, useEffect } from "react"; import { useInterval } from "react-powerhooks"; import { toast } from 'react-toastify'; import theme from "../theme.jsx"; +import WorkflowValidationTimeline from "../components/WorkflowValidationTimeline.jsx" import { InputAdornment, @@ -79,7 +80,7 @@ const ConfigureWorkflow = (props) => { useEffect(() => { if (requiredActions.length === 0) { if (setConfigurationFinished !== undefined) { - setConfigurationFinished(true) + setConfigurationFinished(true) } } }, [requiredActions]) @@ -141,17 +142,18 @@ const ConfigureWorkflow = (props) => { // Where is this from? if (workflow === undefined || workflow === null || workflow.id === undefined) { - return null; + //console.log("Workflow is undefined or null: ", workflow) + return null } if (apps === undefined || apps === null) { - console.log("Apps is undefined or null: ", apps) - return null; + //console.log("Apps is undefined or null: ", apps) + return null } if (appAuthentication === undefined || appAuthentication === null) { - console.log("App authentication is undefined or null: ", appAuthentication) - return null; + //console.log("App authentication is undefined or null: ", appAuthentication) + return null } const getApp = (actionId, appId) => { @@ -310,7 +312,7 @@ const ConfigureWorkflow = (props) => { } } - if (action.authentication_id === "" && app.authentication.required === true && action.parameters !== undefined && action.parameters !== null) { + if (action?.authentication_id === "" && app?.authentication?.required === true && action.parameters !== undefined && action.parameters !== null) { // Check if configuration is filled or not var filled = true; for (let [key,keyval] in Object.entries(action.parameters)) { @@ -322,7 +324,7 @@ const ConfigureWorkflow = (props) => { } } - if (app.authentication.type === "oauth2" || app.authentication.type === "oauth2-app") { + if (app?.authentication?.type === "oauth2" || app?.authentication?.type === "oauth2-app") { filled = false action.auth_type = "oauth2" @@ -425,10 +427,8 @@ const ConfigureWorkflow = (props) => { trigger.index = key; if (trigger.trigger_type === "WEBHOOK") { - console.log("Found webhook: ", trigger) if (trigger.app_association !== undefined && trigger.app_association.name !== null && trigger.app_association.name !== "") { - console.log("Actions: ", newactions) const findapp = trigger.app_association.name.toLowerCase() const foundindex = newactions.findIndex(action => action.app_name.toLowerCase() === findapp) @@ -449,9 +449,6 @@ const ConfigureWorkflow = (props) => { newactions[foundindex].show_steps = true - console.log("CHANGED ACTION: ", newactions[foundindex]) - //console.log("Index: ", newactions[foundindex]) - continue } } @@ -601,7 +598,7 @@ const ConfigureWorkflow = (props) => { , @@ -796,15 +793,13 @@ const ConfigureWorkflow = (props) => { } parsedName = (parsedName.charAt(0).toUpperCase() + parsedName.slice(1)).replaceAll("_", " "); - - console.log("AUTH Action: ", action) return (
@@ -853,7 +848,7 @@ const ConfigureWorkflow = (props) => { {opened ?
- {action.app.authentication.type === "oauth2-app" || action.app.authentication.type === "oauth2" || action.auth_type === "oauth2" ? + {action.app?.authentication?.type === "oauth2-app" || action.app?.authentication?.type === "oauth2" || action.auth_type === "oauth2" ?
{ ) })} - {action.app.authentication.type !== "oauth2-app" && action.app.authentication.type !== "oauth2" ? + {action.app?.authentication?.type !== "oauth2-app" && action.app?.authentication?.type !== "oauth2" ? + + + + setSearchQuery(e.target.value)} + /> + {/* + { + uploadFiles(event.target.files); + }} + /> */} + + + + Global disable/enable + + + handleDirectoryChange(folderDisabled, setFolderDisabled, globalUrl, isTenzirActive) + } + disabled={!isTenzirActive} + /> + + + + {filteredRules?.length > 0 ? + filteredRules.map((card) => { + console.log("RULE CARD: ", card); + + return ( + + ) + }) + : null } + + + + ); +}; + +export default Detection; diff --git a/frontend/src/components/DetectionExplorer.jsx b/frontend/src/components/DetectionExplorer.jsx new file mode 100644 index 00000000..81eda8c0 --- /dev/null +++ b/frontend/src/components/DetectionExplorer.jsx @@ -0,0 +1,455 @@ +import React, { useState, useEffect, } from "react"; +import { + Container, + Box, + TextField, + Switch, + Typography, + Button, + CircularProgress, + Paper, + Divider, + IconButton, + Tooltip, +} from "@mui/material"; + +import { + OpenInNew as OpenInNewIcon, + FmdGood as FmdGoodIcon, +} from "@mui/icons-material" + +import { toast } from "react-toastify"; +import theme from '../theme.jsx'; +import DetectionRuleCard from "../components/DetectionRuleCard.jsx"; +import { + green, + red, + grey, +} from "../views/AngularWorkflow.jsx" + +import WorkflowValidationTimeline from "../components/WorkflowValidationTimeline.jsx" + +const handleDirectoryChange = (folderDisabled, setFolderDisabled, globalUrl, isDetectionActive) => { + if (!isDetectionActive) { + toast.warn("Connect to siem first for global enable/disable to work"); + return; + } + + const action = folderDisabled ? "enable_folder" : "disable_folder"; + const url = `${globalUrl}/api/v1/detections/${action}`; + + fetch(url, { + method: "PUT", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => + response.json().then((responseJson) => { + if (responseJson["success"] === true) { + if (action === "enable_folder") setFolderDisabled(false); + else setFolderDisabled(true); + } else { + //toast(`failed to disable rule`); + } + }) + ) + .catch((error) => { + console.log(`Error in ${action} the rule: `, error); + toast(`An error occurred while ${action} the rule`); + }); +}; + +const DetectionExplorer = (props) => { + const { globalUrl, userdata, ruleInfo, folderDisabled, setFolderDisabled, detectionInfo, importDetectionFromUrl, rulesLoading, isDetectionActive, setIsDetectionActive, ruleMapping, setRuleMapping, } = props; + const [searchQuery, setSearchQuery] = useState(""); + const [loading, setLoading] = useState(false); + + const [workflow, setWorkflow] = useState({}) + const [detectionWorkflowId, setDetectionWorkflowId] = useState("") + const [isDetectionValid, setIsDetectionValid] = useState(false) + const [availableDetection, setAvailableDetection] = React.useState([]); + const [environmentList, setEnvironmentList] = React.useState([]) + + const loadUsecases = () => { + const url = `${globalUrl}/api/v1/workflows/usecases` + fetch(url, { + method: "GET", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => + response.json().then((responseJson) => { + if (responseJson.success === false) { + return + } + + if (responseJson.length == 0) { + return + } + + for (var usecaseCategory in responseJson) { + const category = responseJson[usecaseCategory] + if (!category.name.toLowerCase().includes("respond") && !category.name.toLowerCase().includes("response")) { + continue + } + + setAvailableDetection(category.list) + break + } + }) + ) + .catch((error) => { + console.log(`Error in loading usecases: `, error); + //toast(`An error occurred while loading usecases`); + }) + } + + const loadWorkflow = (workflowId) => { + const url = `${globalUrl}/api/v1/workflows/${workflowId}` + fetch(url, { + method: "GET", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => + response.json().then((responseJson) => { + if (responseJson.id === workflowId) { + setWorkflow(responseJson) + } else { + toast(`Failed to load workflow ${workflowId}`); + } + })) + .catch((error) => { + console.log(`Error in loading workflow ${workflowId}: `, error); + toast(`An error occurred while loading workflow ${workflowId}`); + }) + } + + const handleConnectClick = () => { + if (detectionWorkflowId !== "") { + // FIXME: Show the Usecase UI for how to fix the workflow(s) + // Instead loading full workflow and showing it directly? Hmm + //toast.warn("Please reload the UI to load the detection status") + return + } + + if (isDetectionActive) { + return + } + + if (detectionInfo.category === undefined || detectionInfo.category === null) { + toast.warn("Detection category not found. Please try again or contact support@shuffler.io if you think this is a bug.") + return + } + + setLoading(true); + const url = `${globalUrl}/api/v1/detections/${detectionInfo?.category}/connect` + + fetch(url, { + method: "GET", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => + response.json().then((responseJson) => { + if (responseJson["success"] === true) { + setLoading(false) + + if (setIsDetectionActive !== undefined) { + setIsDetectionActive(true) + } + + if (responseJson.workflow_id !== undefined && responseJson.workflow_id !== null) { + setDetectionWorkflowId(responseJson.workflow_id) + + loadWorkflow(responseJson.workflow_id) + } + + if (responseJson.workflow_valid !== undefined && responseJson.workflow_valid !== null) { + setIsDetectionValid(responseJson.workflow_valid) + } + } else { + if (responseJson.reason !== undefined && responseJson.reason !== null) { + toast(responseJson.reason) + } else { + if (responseJson.workflow_id === "" && responseJson.workflow_valid === false) { + toast.info(`Sent job to generate a Detection Workflow and enable ${detectionInfo?.category}. Please wait a minute and reload this UI.`); + } else { + toast.error(`Failed to connect to ${detectionInfo?.category}`); + } + } + + if (responseJson.action !== undefined && responseJson.actio !== null && responseJson.action.length > 0) { + //if (responseJson.action === "environment_create") { + // navigate("/admin?tab=environments") + //} + } + + setLoading(false); + } + }) + ) + .catch((error) => { + setLoading(false); + console.log(`Error in connecting to ${detectionInfo?.category}: `, error); + toast.error(`An error occurred while connecting to ${detectionInfo?.category}`); + }); + } + + const loadEnvironments = () => { + const url = `${globalUrl}/api/v1/getenvironments` + fetch(url, { + method: "GET", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => { + return response.json() + }) + .then((responseJson) => { + if (responseJson.success === false) { + return + } + + if (responseJson.length == 0) { + return + } + + setEnvironmentList(responseJson) + }) + .catch((error) => { + console.log(`Error in loading environments: `, error); + }) + } + + useEffect(() => { + loadUsecases() + loadEnvironments() + }, []) + + useEffect(() => { + if (detectionInfo === undefined || detectionInfo === null) { + return + } + + if (detectionInfo.category === undefined || detectionInfo.category === null || detectionInfo.category === "") { + return + } + + handleConnectClick() + }, [detectionInfo]) + + const filteredRules = ruleInfo === "default" ? [] : ruleInfo?.filter((rule) => + rule.title.toLowerCase().includes(searchQuery.toLowerCase()) || + rule.description.toLowerCase().includes(searchQuery.toLowerCase()) + ) + + const lakeNodes = environmentList !== undefined && environmentList !== null ? environmentList.filter((env) => env?.data_lake?.enabled === true).length : 0 + + return ( + + + + + {detectionInfo?.title} {filteredRules === undefined || filteredRules === null ? null : `(${filteredRules?.length} rules)`} + + +
+ {workflow !== undefined && workflow !== null && workflow.id !== undefined && workflow.id !== null && workflow.id.length > 0 ? +
+
+ +
+ + { + window.open(`/workflows/${workflow.id}`, "_blank") + }} + > + + +
+ + : + + } + + {detectionInfo?.category === "SIGMA" || detectionInfo?.category === "SIEM" ? + + + 0 ? green : red}} /> + + + : null} +
+ + + {filteredRules?.length > 0 ? + + + setSearchQuery(e.target.value)} + /> + + + + Global disable/enable + + + handleDirectoryChange(folderDisabled, setFolderDisabled, globalUrl, isDetectionActive) + } + /> + + + : null} + + + + {filteredRules?.length > 0 ? + + ruleMapping !== undefined && ruleMapping !== null && ruleMapping.value !== undefined && ruleMapping.value !== null ? + filteredRules.map((rule, index) => { + return ( +
+ +
+ ) + }) + : null + : +
+ {rulesLoading === true ? + +
+ + Downloading rules, please wait... +
+ + : +
+ + No rules loaded yet + + +
+ } +
+ } + + + + ); +}; + +export default DetectionExplorer; diff --git a/frontend/src/components/DetectionRuleCard.jsx b/frontend/src/components/DetectionRuleCard.jsx new file mode 100644 index 00000000..986b227f --- /dev/null +++ b/frontend/src/components/DetectionRuleCard.jsx @@ -0,0 +1,299 @@ +import React, { useState, useEffect, } from "react"; +import { + Card, + CardContent, + IconButton, + Typography, + Switch, + Tooltip, + Select, + MenuItem, + Divider, + FormLabel, +} from "@mui/material"; + +import DashboardBarchart, { LoadStats } from '../components/DashboardBarchart.jsx'; +import { + Edit as EditIcon, +} from "@mui/icons-material"; +import { toast } from "react-toastify"; +import ShuffleCodeEditor from "../components/ShuffleCodeEditor1.jsx"; +import theme from '../theme.jsx'; + + +const RuleCard = ({ ruleName, description, file_id, globalUrl, folderDisabled, isTenzirActive, availableDetection, ruleMapping, setRuleMapping, ...otherProps }) => { + const [openCodeEditor, setOpenCodeEditor] = React.useState(false); + const [fileData, setFileData] = React.useState(""); + const [isEnabled, setIsEnabled] = React.useState(otherProps.is_enabled); + const [filteredBarchart, setFilteredBarchart] = React.useState(null) + + const [responseValue, setResponseValue] = React.useState("No response action") + const isCloud = ["localhost:3002", "shuffler.io"].includes(window.location.host); + + console.log("Rulemapping: ", ruleMapping) + useEffect(() => { + + //const url = `${globalUrl}/api/v1/stats/app_executions_test2` + //const resp = LoadStats(globalUrl, ruleName) + //const resp = LoadStats(globalUrl, "app_executions_test2") + const resp = LoadStats(globalUrl, "app_executions_cloud") + resp.then((data) => { + if (data === undefined) { + setFilteredBarchart([]) + } else { + setFilteredBarchart(data) + } + }) + + if (ruleMapping !== undefined && ruleMapping !== null && ruleMapping.value !== undefined && ruleMapping.value !== null) { + console.log("FIX MAPPING FROM ruleMapping.value: ", ruleMapping) + } + }, []) + + console.log("Response Value: ", responseValue) + + const handleSwitchChange = (event) => { + if (folderDisabled) { + toast.warn("Enable the directory to enable individual rules"); + return; + } + + if (!isTenzirActive) { + toast.warn("Connect to the siem first to enable/disable the rule"); + return; + } + + const newIsEnabled = event.target.checked; + toggleRule(file_id, !newIsEnabled, globalUrl, () => { + setIsEnabled(newIsEnabled); + }) + } + + + const UpdateText = (text) => { + fetch(`${globalUrl}/api/v1/files/${file_id}/edit`, { + method: "PUT", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: text, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Can't update file"); + } + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success === true) { + toast("Successfully updated rule"); + } + }) + .catch((error) => { + toast("Error updating file: " + error.toString()); + }); + }; + + return ( + + +
+ {ruleName.replaceAll("_", " ")} ({filteredBarchart === null || filteredBarchart.total === undefined ? 0 : filteredBarchart.total}) +
+ + + + + + openEditBar(file_id, setOpenCodeEditor, setFileData, globalUrl)}> + + + + + + +
+
+ +
+ {filteredBarchart === null ? null : + + } +
+ + {/* + + {description} + + */} + + + + + ); +} + +const toggleRule = (fileId, isCurrentlyEnabled, globalUrl, callback) => { + const action = isCurrentlyEnabled ? "disable" : "enable"; + const url = `${globalUrl}/api/v1/detections/${fileId}/${action}_rule`; + + fetch(url, { + method: "PUT", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => + response.json().then((responseJson) => { + if (responseJson["success"] === false) { + toast(`Failed to ${action} the rule`); + } else { + toast(`Rule ${action}d successfully`); + callback(); + } + }) + ) + .catch((error) => { + console.log(`Error in ${action}ing the rule: `, error); + toast(`An error occurred while ${action}ing the rule`); + }); +}; + +const openEditBar = (file_id, setOpenCodeEditor, setFileData, globalUrl) => { + getFileContent(file_id, setFileData, globalUrl) + + setOpenCodeEditor(true); +}; + +const getFileContent = (file_id, setFileData, globalUrl) => { + setFileData(""); + fetch(globalUrl + "/api/v1/files/" + file_id + "/content", { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for file :O!"); + return ""; + } + return response.text(); + }) + .then((respdata) => { + if (respdata.length === 0) { + toast("Failed getting file. Is it deleted?"); + return; + } + return respdata + }) + .then((responseData) => { + + setFileData(responseData); + }) + .catch((error) => { + toast(error.toString()); + }); +}; +export default RuleCard; diff --git a/frontend/src/components/DiscordChat.jsx b/frontend/src/components/DiscordChat.jsx index e1d226b8..5f83c119 100644 --- a/frontend/src/components/DiscordChat.jsx +++ b/frontend/src/components/DiscordChat.jsx @@ -75,7 +75,12 @@ const DiscordChat = props => { fullWidth value={currentRefinement} onChange={(event) => refine(event.currentTarget.value)} - placeholder="Search Discord Chats" + onKeyDown={(event) => { + if(event.key === "Enter") { + event.preventDefault(); + } + }} + placeholder="Search Discord Chats..." style={{ backgroundColor: theme.palette.inputColor, borderRadius: borderRadius, margin: 10, width: "100%", }} InputProps={{ style: { @@ -143,7 +148,7 @@ const DiscordChat = props => { const CustomHits = connectHits(Hits); return ( -
+
diff --git a/frontend/src/components/DocsGrid.jsx b/frontend/src/components/DocsGrid.jsx index 06469251..7a32001c 100644 --- a/frontend/src/components/DocsGrid.jsx +++ b/frontend/src/components/DocsGrid.jsx @@ -124,6 +124,11 @@ const DocsGrid = props => { removeQuery("q") refine(event.currentTarget.value) }} + onKeyDown={(event) => { + if(event.key === "Enter") { + event.preventDefault(); + } + }} limit={5} /> {/*isSearchStalled ? 'My search is stalled' : ''*/} @@ -274,7 +279,7 @@ const DocsGrid = props => {
*/} -
+
diff --git a/frontend/src/components/EditWorkflow.jsx b/frontend/src/components/EditWorkflow.jsx index 9ac7fabe..a7694562 100644 --- a/frontend/src/components/EditWorkflow.jsx +++ b/frontend/src/components/EditWorkflow.jsx @@ -1,39 +1,41 @@ import React, { useEffect, useContext } from "react"; import theme from '../theme.jsx'; -import { isMobile } from "react-device-detect" +import { isMobile } from "react-device-detect" import { MuiChipsInput } from "mui-chips-input"; +import { toast } from "react-toastify" import UsecaseSearch from "../components/UsecaseSearch.jsx" import WorkflowGrid from "../components/WorkflowGrid.jsx" import dayjs from 'dayjs'; import WorkflowTemplatePopup from "./WorkflowTemplatePopup.jsx"; +import WorkflowValidationTimeline from "../components/WorkflowValidationTimeline.jsx" import { - Badge, - Avatar, - Grid, - InputLabel, - Select, - ListSubheader, - Paper, - Tooltip, - Divider, - Button, - TextField, - IconButton, - Menu, - MenuItem, - Link, - FormControlLabel, - Chip, - Switch, - Typography, - Zoom, - CircularProgress, - Drawer, - Dialog, - DialogTitle, - DialogActions, - DialogContent, + Badge, + Avatar, + Grid, + InputLabel, + Select, + ListSubheader, + Paper, + Tooltip, + Divider, + Button, + TextField, + IconButton, + Menu, + MenuItem, + Link, + FormControlLabel, + Chip, + Switch, + Typography, + Zoom, + CircularProgress, + Drawer, + Dialog, + DialogTitle, + DialogActions, + DialogContent, OutlinedInput, Checkbox, ListItemText, @@ -41,11 +43,12 @@ import { RadioGroup, FormControl, FormLabel, + Slider, } from "@mui/material"; -import { - DatePicker, +import { + DatePicker, LocalizationProvider, } from '@mui/x-date-pickers' @@ -53,93 +56,120 @@ import { AdapterDayjs } from '@mui/x-date-pickers/AdapterDayjs' import { useStyles } from '../views/AppCreator.jsx' import { - ExpandLess as ExpandLessIcon, - ExpandMore as ExpandMoreIcon, - Publish as PublishIcon, - OpenInNew as OpenInNewIcon, - Add as AddIcon, - Remove as RemoveIcon, + ExpandLess as ExpandLessIcon, + ExpandMore as ExpandMoreIcon, + Publish as PublishIcon, + OpenInNew as OpenInNewIcon, + Add as AddIcon, + Remove as RemoveIcon, + EditNote as EditNoteIcon, } from "@mui/icons-material"; const EditWorkflow = (props) => { - const { globalUrl, workflow, setWorkflow, modalOpen, setModalOpen, showUpload, usecases, setNewWorkflow, appFramework, isEditing, userdata, apps, } = props + const { globalUrl, workflow, setWorkflow, modalOpen, setModalOpen, showUpload, usecases, setNewWorkflow, appFramework, isEditing, userdata, apps, saveWorkflow, expanded, scrollTo, setRealtimeMarkdown, boxWidth, setBoxWidth, } = props - const [_, setUpdate] = React.useState(""); // Used for rendering, don't remove + const [_, setUpdate] = React.useState(""); // Used for rendering, don't remove - const [submitLoading, setSubmitLoading] = React.useState(false); - const [showMoreClicked, setShowMoreClicked] = React.useState(false); - const [innerWorkflow, setInnerWorkflow] = React.useState(workflow) + const [submitLoading, setSubmitLoading] = React.useState(false); + const [showMoreClicked, setShowMoreClicked] = React.useState(expanded === true ? true : false); - const [newWorkflowTags, setNewWorkflowTags] = React.useState(workflow.tags !== undefined && workflow.tags !== null ? JSON.parse(JSON.stringify(workflow.tags)) : []) - const [description, setDescription] = React.useState(workflow.description !== undefined ? workflow.description : "") + const [innerWorkflow, setInnerWorkflow] = React.useState(workflow) - const [selectedUsecases, setSelectedUsecases] = React.useState(workflow.usecase_ids !== undefined && workflow.usecase_ids !== null ? JSON.parse(JSON.stringify(workflow.usecase_ids)) : []); + const [newWorkflowTags, setNewWorkflowTags] = React.useState(workflow.tags !== undefined && workflow.tags !== null ? JSON.parse(JSON.stringify(workflow.tags)) : []) + const [description, setDescription] = React.useState(workflow.description !== undefined ? workflow.description : "") + + const [selectedUsecases, setSelectedUsecases] = React.useState(workflow.usecase_ids !== undefined && workflow.usecase_ids !== null ? JSON.parse(JSON.stringify(workflow.usecase_ids)) : []); const [foundWorkflowId, setFoundWorkflowId] = React.useState("") const [name, setName] = React.useState(workflow.name !== undefined ? workflow.name : "") - const [dueDate, setDueDate] = React.useState(workflow.due_date !== undefined && workflow.due_date !== null && workflow.due_date !== 0 ? dayjs(workflow.due_date*1000) : dayjs().subtract(1, 'day')) + const [workflowAsCode, setWorkflowAsCode] = React.useState(false) + const [dueDate, setDueDate] = React.useState(workflow.due_date !== undefined && workflow.due_date !== null && workflow.due_date !== 0 ? dayjs(workflow.due_date * 1000) : dayjs().subtract(1, 'day')) - console.log("WORKFLOW: ", workflow) - const [inputQuestions, setInputQuestions] = React.useState(workflow.input_questions !== undefined && workflow.input_questions !== null ? JSON.parse(JSON.stringify(workflow.input_questions)) : []) + const [inputQuestions, setInputQuestions] = React.useState(workflow.input_questions !== undefined && workflow.input_questions !== null ? JSON.parse(JSON.stringify(workflow.input_questions)) : []) + const [inputMarkdown, setInputMarkdown] = React.useState(workflow?.form_control?.input_markdown !== undefined && workflow?.form_control?.input_markdown !== null ? workflow?.form_control?.input_markdown : "") + const [scrollDone, setScrollDone] = React.useState(false) + const [selectedYieldActions, setSelectedYieldActions] = React.useState(workflow?.form_control?.output_yields !== undefined && workflow?.form_control?.output_yields !== null ? JSON.parse(JSON.stringify(workflow?.form_control?.output_yields)) : []) + const [formWidth, setFormWidth] = React.useState(boxWidth === undefined || boxWidth === null ? 500 : boxWidth) - const classes = useStyles(); + const classes = useStyles(); + + useEffect(() => { + if (setBoxWidth !== undefined && boxWidth !== formWidth) { + setBoxWidth(formWidth) + } + }, [formWidth]) + + if (scrollTo !== undefined && scrollTo !== null && scrollTo.length > 0 && scrollDone === false) { + setTimeout(() => { + const foundScroll = document.getElementById(scrollTo) + if (foundScroll !== null) { + // Smooth scroll + foundScroll.scrollIntoView({ behavior: "smooth" }) + } + + }, 200) + setScrollDone(true) + + } // Gets the generated workflow const getGeneratedWorkflow = (workflow_id) => { - fetch(globalUrl + "/api/v1/workflows/" + workflow_id, { - method: "GET", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - credentials: "include", - }) - .then((response) => { - if (response.status !== 200) { - console.log("Status not 200 when getting workflow"); - } - - return response.json(); + const url = `${globalUrl}/api/v1/workflows/${workflow_id}` + fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", }) - .then((responseJson) => { - if (responseJson.id === workflow_id) { - console.log("GOT WORKFLOW: ", responseJson) - if (name === "") { - innerWorkflow.name = responseJson.name - setName(responseJson.name) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 when getting workflow"); } - if (description === "") { - innerWorkflow.description = responseJson.description - setDescription(description) + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.id === workflow_id) { + console.log("GOT WORKFLOW: ", responseJson) + if (name === "") { + innerWorkflow.name = responseJson.name + setName(responseJson.name) + } + + if (description === "") { + innerWorkflow.description = responseJson.description + setDescription(description) + } + + if (newWorkflowTags === []) { + innerWorkflow.tags = responseJson.tags + setNewWorkflowTags(responseJson.tags) + } + + if (selectedUsecases === []) { + selectedUsecases = responseJson.usecase_ids + } + + innerWorkflow.id = responseJson.id + innerWorkflow.blogpost = responseJson.blogpost + innerWorkflow.actions = responseJson.actions + innerWorkflow.triggers = responseJson.triggers + innerWorkflow.branches = responseJson.branches + innerWorkflow.comments = responseJson.comments + innerWorkflow.workflow_variables = responseJson.workflow_variables + innerWorkflow.execution_variables = responseJson.execution_variables + + + setInnerWorkflow(innerWorkflow) + setUpdate(Math.random()) } - - if (newWorkflowTags === []) { - innerWorkflow.tags = responseJson.tags - setNewWorkflowTags(responseJson.tags) - } - - if (selectedUsecases === []) { - selectedUsecases = responseJson.usecase_ids - } - - innerWorkflow.id = responseJson.id - innerWorkflow.blogpost = responseJson.blogpost - innerWorkflow.actions = responseJson.actions - innerWorkflow.triggers = responseJson.triggers - innerWorkflow.branches = responseJson.branches - innerWorkflow.comments = responseJson.comments - innerWorkflow.workflow_variables = responseJson.workflow_variables - innerWorkflow.execution_variables = responseJson.execution_variables - - - setInnerWorkflow(innerWorkflow) - setUpdate(Math.random()) - } - }) - .catch((error) => { - //toast(error.toString()); - console.log("Get workflow error: ", error.toString()); - }) + }) + .catch((error) => { + //toast(error.toString()); + console.log("Get workflow error: ", error.toString()); + }) } if (foundWorkflowId.length > 0) { @@ -153,75 +183,92 @@ const EditWorkflow = (props) => { return null } - const newWorkflow = isEditing === true ? false : true - const priority = userdata === undefined || userdata === null ? null : userdata.priorities.find(prio => prio.type === "usecase" && prio.active === true) - var upload = ""; + const newWorkflow = isEditing === true ? false : true + const priority = userdata === undefined || userdata === null ? null : userdata.priorities.find(prio => prio.type === "usecase" && prio.active === true) + var upload = ""; var total_count = 0 return ( - { - setModalOpen(false); - }} - PaperProps={{ - style: { - color: "white", - minWidth: isMobile ? "90%" : 650, - maxWidth: isMobile ? "90%" : 650, - minHeight: 400, - paddingTop: 25, - paddingLeft: 50, - //minWidth: isMobile ? "90%" : newWorkflow === true ? 1000 : 550, - //maxWidth: isMobile ? "90%" : newWorkflow === true ? 1000 : 550, - }, - }} - > - -
-
-
- - {newWorkflow ? "New" : "Editing"} workflow - - {newWorkflow === true ? null : -
- - - - - + { + setModalOpen(false); + }} + PaperProps={{ + style: { + color: "white", + minWidth: isMobile ? "90%" : 650, + maxWidth: isMobile ? "90%" : 650, + minHeight: 400, + paddingTop: 25, + paddingLeft: 50, + //minWidth: isMobile ? "90%" : newWorkflow === true ? 1000 : 550, + //maxWidth: isMobile ? "90%" : newWorkflow === true ? 1000 : 550, + borderRadius: theme.palette.borderRadius, + backgroundColor: "black", + }, + }} + > + +
+
+
+ + {newWorkflow ? "New" : "Editing"} workflow + + + {newWorkflow === true ? null : +
+ + + + + + + +
+ } +
- } + + Workflows can be built from scratch, or from templates. Usecases can help you discover next steps, and you can search for them directly. Learn more + + + {/* +
+
- - Workflows can be built from scratch, or from templates. Usecases can help you discover next steps, and you can search for them directly. Learn more - - {showUpload === true ? -
- - - -
- : null} -
+ */} + + {showUpload === true ? +
+ + + +
+ : null} +
{/*newWorkflow === true ?
@@ -233,140 +280,132 @@ const EditWorkflow = (props) => {
: null*/}
- - -
- {/* - - */} - -
+ if (saveWorkflow !== undefined) { + saveWorkflow(innerWorkflow) - -
- { - setName(event.target.value) - }} - InputProps={{ - style: { - color: "white", - }, - }} - color="primary" - placeholder="Name" + if (setWorkflow !== undefined) { + setWorkflow(innerWorkflow) + } + } else if (setNewWorkflow !== undefined) { + setNewWorkflow( + innerWorkflow.name, + innerWorkflow.description, + innerWorkflow.tags, + innerWorkflow.default_return_value, + innerWorkflow, + newWorkflow, + innerWorkflow.usecase_ids, + innerWorkflow.blogpost, + innerWorkflow.status, + workflowAsCode + ) + setWorkflow({}) + } else { + setWorkflow(innerWorkflow) + console.log("editing workflow: ", innerWorkflow) + } + + setSubmitLoading(true) + + // If new workflow, don't close it + if (isEditing) { + setModalOpen(false) + } + }} + color="primary" + > + {submitLoading ? : "Save Changes"} + +
+ + +
+ { + setName(event.target.value) + }} + InputProps={{ + style: { + color: "white", + }, + }} + color="primary" + placeholder="Name" required - margin="dense" - defaultValue={innerWorkflow.name} + margin="dense" + defaultValue={innerWorkflow.name} label="Name" - autoFocus - fullWidth - /> -
- { - setDescription(event.target.value) - }} - InputProps={{ - style: { - color: "white", - }, - }} - maxRows={4} - color="primary" - defaultValue={innerWorkflow.description} - placeholder="Description" - multiline - label="Description" - margin="dense" - fullWidth - /> -
-
- {usecases !== null && usecases !== undefined && usecases.length > 0 ? - - Usecases - selected.join(', ')} @@ -374,15 +413,15 @@ const EditWorkflow = (props) => { console.log("Changed: ", event) }} > - - None - + + None + {usecases.map((usecase, index) => { //console.log(usecase) return ( {usecase.name} @@ -400,22 +439,22 @@ const EditWorkflow = (props) => { selectedUsecases.push(subcase.name) } - setUpdate(Math.random()); + setUpdate(Math.random()); setSelectedUsecases(selectedUsecases) }}> - - + + ) })} ) })} - - - : null} + + + : null} { fullWidth value={newWorkflowTags} onChange={(chip) => { - console.log("Chip: ", chip) - //newWorkflowTags.push(chip); setNewWorkflowTags(chip); }} + onBlur={(event) => { + if (event.target.value.length === 0) { + return + } + + if (newWorkflowTags.includes(event.target.value)) { + return + } + + newWorkflowTags.push(event.target.value) + setNewWorkflowTags(newWorkflowTags) + + setUpdate(Math.random()) + }} onAdd={(chip) => { - newWorkflowTags.push(chip); - setNewWorkflowTags(newWorkflowTags); + newWorkflowTags.push(chip) + setNewWorkflowTags(newWorkflowTags) }} onDelete={(chip, index) => { console.log("Deleting: ", chip, index) - newWorkflowTags.splice(index, 1); - setNewWorkflowTags(newWorkflowTags); - setUpdate(Math.random()); + newWorkflowTags.splice(index, 1) + setNewWorkflowTags(newWorkflowTags) + setUpdate(Math.random()) }} />
- {showMoreClicked === true ? - + {showMoreClicked === true ? +
+ { + setDescription(event.target.value) + }} + InputProps={{ + style: { + color: "white", + }, + }} + multiLine + rows={3} + color="primary" + defaultValue={innerWorkflow.description} + placeholder="Description" + multiline + label="Description" + margin="dense" + fullWidth + /> - - -
- +
+ Status - { - console.log("Data: ", e.target.value) - - //innerWorkflow.workflow_type = e.target.value - innerWorkflow.status = e.target.value - setInnerWorkflow(innerWorkflow) - }} - > - } label="Test" /> - } label="Production" /> + { + console.log("Data: ", e.target.value) - + //innerWorkflow.workflow_type = e.target.value + innerWorkflow.status = e.target.value + setInnerWorkflow(innerWorkflow) + }} + > + } label="Test" /> + } label="Production" /> + + - { @@ -486,25 +554,25 @@ const EditWorkflow = (props) => {
- + Type - { - console.log("Data: ", e.target.value) - - innerWorkflow.workflow_type = e.target.value - setInnerWorkflow(innerWorkflow) - }} - > - } label="Trigger" /> - } label="Subflow" /> - } label="Standalone" /> + { + console.log("Data: ", e.target.value) - + innerWorkflow.workflow_type = e.target.value + setInnerWorkflow(innerWorkflow) + }} + > + } label="Trigger" /> + } label="Subflow" /> + } label="Standalone" /> + + @@ -565,263 +633,589 @@ const EditWorkflow = (props) => { fullWidth /> - - MSSP Suborg Distribution (beta - contact support@shuffler.io) + + + + MSSP controls + + + + + MSSP Suborg Distribution (beta - contact support@shuffler.io for more info) {userdata !== undefined && userdata !== null && userdata.orgs !== undefined && userdata.orgs !== null && userdata.orgs.length > 0 ? userdata.orgs.filter(org => org.creator_org === userdata.active_org.id).length === 0 ? - - You can only distribute to suborgs from a parent org. - + userdata.active_org.creator_org === undefined || userdata.active_org.creator_org === null || userdata.active_org.creator_org === "" ? + + Your organization does not have any suborgs yet OR your user may not have access to available suborgs. Please make one or get access to suborgs by another admin, then try again. + + : + + {innerWorkflow.parentorg_workflow !== undefined && innerWorkflow.parentorg_workflow !== null && innerWorkflow.parentorg_workflow.length > 0 ? This workflow is distributed from your parent workflow (you may not have access). : null} +
+
+ You can only distribute to suborgs from a parent org. + + : + : - - : - - + + Create a sub-org to distribute workflows to suborgs. } - - - Input fields + {/**/} + + + + + Git Backup Repository - - Input fields are fields that will be used during the startup of the workflow. These will be formatted in JSON and is most commonly used from the workflow run page. + + Decide where this workflow is backed up in a Git repository. Will create logs and notifications if upload fails. The repository and branch must already have been initialized. Files will show up in the root folder in the format 'orgid/workflow status/workflow id.json' without images. Overrides your default backup repository. Credentials are encrypted. Creates notifications if it fails. + + + + + Workflow Backup Repository + { + //setUploadRepo(e.target.value); + innerWorkflow.backup_config.upload_repo = e.target.value + setInnerWorkflow(innerWorkflow) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + Branch + { + innerWorkflow.backup_config.upload_branch = e.target.value + setInnerWorkflow(innerWorkflow) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + + + Username + { + innerWorkflow.backup_config.upload_username = e.target.value + setInnerWorkflow(innerWorkflow) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + Git token/password + { + innerWorkflow.backup_config.upload_token = e.target.value + setInnerWorkflow(innerWorkflow) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + type="password" + /> + + + + + + + +
+ + Form Control + + + Form Control is used to control how the Form for the workflow is shown to users. You can add input fields, markdown, and more. This is the first step in the workflow, and is required for all workflows. + + + + Input fields + + + + + + + + + + +
+ + + Input fields are fields that will be used during the startup of the workflow. These will be formatted in JSON and is most commonly used from the Form page for this workflow. If chosen in the User Input node, these will be required fields. Use Semi-Colon ";" to create dropdown options. The first key will be the name shown, and subsequent keys will be the available values. {inputQuestions.map((data, index) => { - console.log("Inputfield: ", data) + var showListinfo = false + if (data.value !== undefined && data.value !== null && data.value.length > 0) { + if (data.value.includes(";")) { + showListinfo = true + } + } return ( -
+
{ - inputQuestions[index].name = e.target.value - setInputQuestions(inputQuestions) - setUpdate(Math.random()); - }} - InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, - style: { - color: "white", - minHeight: 50, - }, - }} + disabled={data.deleted === true} + style={{ + flex: 2, + marginTop: 0, + marginBottom: 0, + backgroundColor: theme.palette.inputColor, + marginRight: 5, + }} + fullWidth={true} + placeholder="Question" + id="standard-required" + margin="normal" + variant="outlined" + defaultValue={data.name} + onChange={(e) => { + inputQuestions[index].name = e.target.value + setInputQuestions(inputQuestions) + setUpdate(Math.random()); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + }} /> { - inputQuestions[index].value = e.target.value - setInputQuestions(inputQuestions) - setUpdate(Math.random()); - }} - InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, - style: { - color: "white", - minHeight: 50, - }, - }} + disabled={data.deleted === true} + style={{ + flex: 2, + marginTop: 0, + marginBottom: 0, + backgroundColor: theme.palette.inputColor, + marginRight: 5, + }} + fullWidth={true} + placeholder="$exec JSON key" + id="standard-required" + margin="normal" + variant="outlined" + helperText={showListinfo === true ? "Dropdown list" : null} + defaultValue={data.value} + onChange={(e) => { + // Replace multiple semicolon with one + e.target.value = e.target.value.replace(";;", ";") + + inputQuestions[index].value = e.target.value + setInputQuestions(inputQuestions) + setUpdate(Math.random()); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + }} /> - +
) })} - - : null} - - { - setShowMoreClicked(!showMoreClicked); - }} - > - {showMoreClicked ? : } - - -
+
+ + Input Markdown + + + Markdown will be shown on the Form page. The first image added will be used in your Form Toolbox list. Output for a Workflow is shown in Markdown, and is controlled by the LAST action that runs. Supports HTML. + + { + //console.log("KEY: ", e.key) + if (e.key === "Tab") { + e.preventDefault() + } + }} - + onChange={(e) => { + if (setRealtimeMarkdown !== undefined) { + setRealtimeMarkdown(e.target.value) + } - - {newWorkflow === true ? - - - Relevant Workflows - + setInputMarkdown(e.target.value) + workflow.form_control.input_markdown = e.target.value + setWorkflow(workflow) + setUpdate(Math.random()) + }} + /> +
- {priority === null || priority === undefined ? null : -
- + + Form Size + + + Control the width of the form. It will grow vertically as needed. + + { + setFormWidth(value) + }} + /> +
- srcapp={priority.description.split("&").length > 2 ? priority.description.split("&")[0] : ""} - img1={priority.description.split("&").length > 2 ? priority.description.split("&")[1] : ""} +
+ + Output Control ({selectedYieldActions.length === 0 ? "No Returns" : selectedYieldActions.length === 1 ? "Returning 1 node" : `Returning ${selectedYieldActions.length} nodes`}) + - dstapp={priority.description.split("&").length > 3 ? priority.description.split("&")[2] : ""} - img2={priority.description.split("&").length > 3 ? priority.description.split("&")[3] : ""} - title={priority.name} - description={priority.description.split("&").length > 4 ? priority.description.split("&")[4] : ""} + + When running this workflow, the output will be shown as a Markdown object by default, with JSON objects being rendered. By adding nodes below, they will be shown while the workflow is running as soon as they get a result. Failing/Skipped nodes are not shown. This makes it possible to track progress for more complex usecases. + - apps={apps} - /> -
- } - - - : null} + + + +
+
+ : null} + + {!isEditing ? <> +
+ } + label="Create workflow as code" + style={{ marginTop: '12px' }} + onChange={(e) => { + setWorkflowAsCode(e.target.checked) + workflow.workflow_as_code = e.target.checked + setWorkflow(workflow) + setInnerWorkflow(workflow) + }} + /> + +
+ : null} + + + + + +
+ + + + + {newWorkflow === true ? + + + Relevant Workflows + + + {priority === null || priority === undefined ? null : +
+ 2 ? priority.description.split("&")[0] : ""} + img1={priority.description.split("&").length > 2 ? priority.description.split("&")[1] : ""} + + dstapp={priority.description.split("&").length > 3 ? priority.description.split("&")[2] : ""} + img2={priority.description.split("&").length > 3 ? priority.description.split("&")[3] : ""} + title={priority.name} + description={priority.description.split("&").length > 4 ? priority.description.split("&")[4] : ""} + + apps={apps} + /> +
+ } + + + : null} + + {/*newWorkflow === true && name.length > 2 ?
{ onlyResults={true} />
- : null} - - + : null*/} + + ) } diff --git a/frontend/src/components/ExecutionPanel.jsx b/frontend/src/components/ExecutionPanel.jsx new file mode 100644 index 00000000..e3712bdf --- /dev/null +++ b/frontend/src/components/ExecutionPanel.jsx @@ -0,0 +1,546 @@ +import React, { useState, useEffect, useCallback } from 'react'; +import { Box, Typography, IconButton, CircularProgress, Tooltip } from '@mui/material'; +import { CheckCircle, Error, ArrowBack, Close, Cached as CachedIcon, Pause as PauseIcon } from '@mui/icons-material'; +import theme from '../theme.jsx'; +import ReactJson from "react-json-view-ssr"; +import { toast } from 'react-toastify'; +import { validateJson } from "../views/Workflows.jsx"; +// import HandleJsonCopy from "./ShuffleCodeEditor1"; + +const STATUS_CONFIG = { + EXECUTING: { + color: '#64B5F6', + icon: () => , + label: 'Executing' + }, + SUCCESS: { + color: '#4CAF50', + icon: () => , + label: 'Success' + }, + FINISHED: { + color: '#4CAF50', + icon: () => , + label: 'Finished' + }, + ABORTED: { + color: '#F44336', + icon: () => , + label: 'Aborted' + } +}; + +let to_be_copied = "" + +const handleReactJsonClipboard = (copy) => { + toast("Copied JSON path to clipboard, NOT Path") +}; + + +const HandleJsonCopy = (base, copy, base_node_name) => { + if (typeof copy.name === "string") { + copy.name = copy.name.replaceAll(" ", "_"); + } + + //lol + if (typeof base === 'object' || typeof base === 'dict') { + base = JSON.stringify(base) + } + + if (base_node_name === "execution_argument" || base_node_name === "Execution Argument") { + base_node_name = "exec" + } + + console.log("COPY: ", base_node_name, copy); + + //var newitem = JSON.parse(base); + var newitem = validateJson(base).result + to_be_copied = "$" + base_node_name.toLowerCase().replaceAll(" ", "_"); + for (let copykey in copy.namespace) { + if (copy.namespace[copykey].includes("Results for")) { + continue; + } + + if (newitem !== undefined && newitem !== null) { + newitem = newitem[copy.namespace[copykey]]; + if (!isNaN(copy.namespace[copykey])) { + to_be_copied += ".#"; + } else { + to_be_copied += "." + copy.namespace[copykey]; + } + } + } + + if (newitem !== undefined && newitem !== null) { + newitem = newitem[copy.name]; + if (!isNaN(copy.name)) { + to_be_copied += ".#"; + } else { + to_be_copied += "." + copy.name; + } + } + + to_be_copied.replaceAll(" ", "_"); + const elementName = "copy_element_shuffle"; + var copyText = document.getElementById(elementName); + if (copyText !== null && copyText !== undefined) { + console.log("NAVIGATOR: ", navigator); + const clipboard = navigator.clipboard; + if (clipboard === undefined) { + toast("Can only copy over HTTPS (port 3443)"); + return; + } + + navigator.clipboard.writeText(to_be_copied); + copyText.select(); + copyText.setSelectionRange(0, 99999); /* For mobile devices * + + /* Copy the text inside the text field */ + document.execCommand("copy"); + toast("Copied JSON path to clipboard.") + console.log("COPYING!"); + } else { + console.log("Couldn't find element ", elementName); + } +} + +const ExecuteWorkflow = async (executionData, globalUrl) => { + try { + const workflowData = executionData.workflow; + + // Execute workflow with original parameters + await fetch(`${globalUrl}/api/v1/workflows/${workflowData.id}/execute`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json' + }, + credentials: 'include', + body: workflowData + }).then(response => { + window.location.href = `/workflows/${workflowData.id}/code?execution_id=` + response.json().execution_id; + }); + + } catch (error) { + console.error('Error re-executing workflow:', error); + } +}; + +const ExecutionsList = ({ executions, onSelectExecution, activeExecutionId }) => { + return ( + + {executions.map((execution) => { + const status = STATUS_CONFIG[execution.status] || STATUS_CONFIG.ABORTED; + return ( + onSelectExecution(execution)} + sx={{ + display: 'flex', + alignItems: 'center', + cursor: 'pointer', + py: 1, + px: 2, + borderBottom: '1px solid #2A2A2A', + backgroundColor: activeExecutionId === execution.execution_id ? + 'rgba(255,255,255,0.05)' : 'transparent', + '&:hover': { + backgroundColor: 'rgba(255,255,255,0.05)' + } + }} + > + {status.icon()} + + + + {new Date(execution.started_at * 1000).toLocaleString()} + + + {status.label} + + + + ); + })} + + ); +}; + +const ExecutionDetail = ({ execution: initialExecution, onBack, globalUrl, onExecutionUpdate, selectedAction, executeWorkflow }) => { + const [execution, setExecution] = useState(initialExecution); + const [status, setStatus] = useState(STATUS_CONFIG[execution.status] || STATUS_CONFIG.EXECUTING); + const [validResult, setValidResult] = useState("{}") + + const abortExecution = async () => { + try { + await fetch(`${globalUrl}/api/v1/workflows/${execution.workflow.id}/executions/${execution.execution_id}/abort`, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }).then((response) => { + if (response.ok) { + const updatedExecution = { + ...execution, + status: "ABORTED", + }; + setExecution(updatedExecution); + onExecutionUpdate(updatedExecution); + } + }); + + } catch (error) { + console.log("Abort error:", error); + } + }; + + useEffect(() => { + setStatus(STATUS_CONFIG[execution.status] || STATUS_CONFIG.EXECUTING); + }, [execution]); + + const pollExecutionStatus = useCallback(async () => { + try { + const response = await fetch(`${globalUrl}/api/v1/streams/results`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + credentials: 'include', + body: JSON.stringify({ + execution_id: execution.execution_id, + authorization: execution.authorization, + }), + }); + + if (response.ok) { + const data = await response.json(); + const currentStatus = data.results?.[0]?.status || 'EXECUTING'; + + const updatedExecution = { + ...execution, + ...data, + status: currentStatus + }; + + setExecution(updatedExecution); + onExecutionUpdate(updatedExecution); + + } + } catch (error) { + console.error('Polling error:', error); + } + }, [execution, globalUrl, onExecutionUpdate]); + + useEffect(() => { + let pollTimeout; + if (execution.status === 'EXECUTING') { + pollTimeout = setTimeout(() => pollExecutionStatus(), 3000); + } + + if (execution?.results?.length === 1) { + setValidResult(JSON.parse(execution?.results[0]?.result || "{}")) + } + + return () => clearTimeout(pollTimeout); + }, [execution.status, pollExecutionStatus]); + + return ( + + + + + + + Execution Details + + {status.icon()} + + {status.label} + + + + {execution.status === "EXECUTING" && ( + + + + + + )} + + + { + ExecuteWorkflow( + execution, + globalUrl + ); + }} + sx={{ color: theme.palette.primary.main }} + > + + + + + + + + + + Started at + + + {new Date(execution.started_at * 1000).toLocaleString()} + + + + + + Execution ID + + + {execution.execution_id} + + + + + + + Result + + + 
+                {execution?.status === 'EXECUTING' ? (
+                  
+                    
+                    Executing...
+                  
+                ) : (
+                  execution?.results?.length === 1 ?
+                     {
+                        handleReactJsonClipboard(copy);
+                      }}
+                      collapsed={false}
+                      displayDataTypes={false}
+                      onSelect={(select) => {
+                        var basename = "exec"
+                        if (selectedAction !== undefined && selectedAction !== null && Object.keys(selectedAction).length !== 0) {
+                          basename = selectedAction.label.toLowerCase().replaceAll(" ", "_")
+                        }
+                        HandleJsonCopy(validResult, select, basename)
+                      }}
+                      name={"JSON autocompletion"}
+                    /> :
+                     { }}
+                      displayDataTypes={false}
+                      name={"JSON autocompletion"}
+                    />
+                )}
+
+ + + + + + ); +}; +const ExecutionPanel = ({ + workflow, + globalUrl, + onClose, + currentExecution, + mainAction +}) => { + const [executions, setExecutions] = useState([]); + const [selectedExecution, setSelectedExecution] = useState(null); + const [loading, setLoading] = useState(true); + + const handleExecutionUpdate = useCallback((updatedExecution) => { + setExecutions(prevExecutions => { + const updatedExecutions = [...prevExecutions]; + const index = updatedExecutions.findIndex( + e => e.execution_id === updatedExecution.execution_id + ); + if (index !== -1) { + updatedExecutions[index] = updatedExecution; + } + return updatedExecutions; + }); + }, []); + + const fetchExecutions = useCallback(async () => { + setLoading(true); + try { + const response = await fetch(`${globalUrl}/api/v2/workflows/${workflow.id}/executions`, { + credentials: 'include', + }); + if (response.ok) { + const data = await response.json(); + setExecutions(data.executions); + + const urlParams = new URLSearchParams(window.location.search); + const executionId = urlParams.get('execution_id'); + if (executionId) { + const execution = data.executions.find(e => e.execution_id === executionId); + if (execution) { + setSelectedExecution(execution); + } + } + } + } catch (error) { + console.error('Failed to fetch executions:', error); + } finally { + setLoading(false); + } + }, [workflow.id, globalUrl]); + + useEffect(() => { + fetchExecutions(); + }, [fetchExecutions]); + + useEffect(() => { + if (currentExecution?.execution_id) { + setExecutions(prev => { + const existingIndex = prev.findIndex(e => e.execution_id === currentExecution.execution_id); + if (existingIndex === -1) { + return [currentExecution, ...prev]; + } + const updated = [...prev]; + updated[existingIndex] = currentExecution; + return updated; + }); + setSelectedExecution(currentExecution); + } + }, [currentExecution]); + + return ( + + {loading && !executions.length ? ( + + + + ) : selectedExecution ? ( + { + setSelectedExecution(null); + const url = new URL(window.location); + url.searchParams.delete('execution_id'); + window.history.pushState({}, '', url); + fetchExecutions(); + }} + globalUrl={globalUrl} + selecteAction={mainAction} + onExecutionUpdate={handleExecutionUpdate} + /> + ) : ( + <> + + + Execution History + + + + + + + { + setSelectedExecution(execution); + const url = new URL(window.location); + url.searchParams.set('execution_id', execution.execution_id); + window.history.pushState({}, '', url); + }} + activeExecutionId={currentExecution?.execution_id} + /> + + + )} + + ); +}; + + +export default ExecutionPanel; diff --git a/frontend/src/components/ExploreWorkflow.jsx b/frontend/src/components/ExploreWorkflow.jsx index 69c44cd4..cac23192 100644 --- a/frontend/src/components/ExploreWorkflow.jsx +++ b/frontend/src/components/ExploreWorkflow.jsx @@ -227,7 +227,7 @@ const ExploreWorkflow = (props) => { -
+
{
-
+
{suggestedUsecases.length === 0 && usecasesSet ? diff --git a/frontend/src/components/Files.jsx b/frontend/src/components/Files.jsx index 692d2e62..0afaf0de 100644 --- a/frontend/src/components/Files.jsx +++ b/frontend/src/components/Files.jsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect } from "react"; +import React, { useState, useEffect, useContext, memo } from "react"; import { toast } from 'react-toastify'; import { @@ -21,6 +21,7 @@ import { DialogContent, DialogActions, Typography, + Skeleton, } from "@mui/material"; import { @@ -39,14 +40,16 @@ import { import Dropzone from "../components/Dropzone.jsx"; import ShuffleCodeEditor from "../components/ShuffleCodeEditor1.jsx"; import theme from "../theme.jsx"; +import { Context } from "../context/ContextApi.jsx"; -const Files = (props) => { +const Files = memo((props) => { const { globalUrl, userdata, serverside, selectedOrganization, isCloud,isSelectedFiles } = props; const [files, setFiles] = React.useState([]); - const [selectedNamespace, setSelectedNamespace] = React.useState("default"); + const [showLoader, setShowLoader] = useState(true) + const [selectedCategory, setSelectedCategory] = React.useState("default"); const [openFileId, setOpenFileId] = React.useState(false); - const [fileNamespaces, setFileNamespaces] = React.useState([]); + const [fileCategories, setFileCategories] = React.useState([]); const [fileContent, setFileContent] = React.useState(""); const [openEditor, setOpenEditor] = React.useState(false); const [renderTextBox, setRenderTextBox] = React.useState(false); @@ -57,18 +60,16 @@ const Files = (props) => { const [downloadUrl, setDownloadUrl] = React.useState("https://github.com/shuffle/standards") const [downloadBranch, setDownloadBranch] = React.useState("main"); const [downloadFolder, setDownloadFolder] = React.useState("translation_standards"); - + const [contentLoading, setContentLoading] = React.useState(false) //const alert = useAlert(); const allowedFileTypes = ["txt", "py", "yaml", "yml","json", "html", "js", "csv", "log", "eml", "msg", "md", "xml", "sh", "bat", "ps1", "psm1", "psd1", "ps1xml", "pssc", "psc1", "response"] var upload = ""; const handleKeyDown = (event) => { if (event.key === 'Enter') { - - console.log('do validate') - console.log("new namespace name->",event.target.value); - fileNamespaces.push(event.target.value); - setSelectedNamespace(event.target.value); + + fileCategories.push(event.target.value); + setSelectedCategory(event.target.value); setRenderTextBox(false); } @@ -107,12 +108,13 @@ const Files = (props) => { const getFiles = (namespace) => { var parsedurl = `${globalUrl}/api/v1/files` - if (namespace === undefined || namespace === "default") { + + if (namespace === undefined || namespace === null || namespace === "default") { } else if (namespace !== undefined && namespace !== null && namespace !== "") { parsedurl = `${globalUrl}/api/v1/files/namespaces/${namespace}?ids=true` - } else if (selectedNamespace !== undefined && selectedNamespace !== null && selectedNamespace !== "default" && selectedNamespace !== "") { - parsedurl = `${globalUrl}/api/v1/files/namespaces/${selectedNamespace}?ids=true` + } else if (selectedCategory !== undefined && selectedCategory !== null && selectedCategory !== "default" && selectedCategory !== "") { + parsedurl = `${globalUrl}/api/v1/files/namespaces/${selectedCategory}?ids=true` } fetch(parsedurl, { @@ -134,6 +136,7 @@ const Files = (props) => { .then((responseJson) => { if (responseJson.files !== undefined && responseJson.files !== null) { setFiles(responseJson.files); + setShowLoader(false) } else if (responseJson.list !== undefined && responseJson.list !== null) { // Set the "namespace" field in all items if (namespace !== undefined && namespace !== null) { @@ -145,13 +148,17 @@ const Files = (props) => { } setFiles(responseJson.list); + setShowLoader(false) } else { setFiles([]); + setShowLoader(false) } - if (responseJson.namespaces !== undefined && responseJson.namespaces !== null && (fileNamespaces.length === 0 || responseJson.namespaces.length > fileNamespaces.length)) { - setFileNamespaces(responseJson.namespaces); - } + if (namespace === undefined || namespace === null || namespace === "default") { + if (responseJson.namespaces !== undefined && responseJson.namespaces !== null && (fileCategories.length === 0 || responseJson.namespaces.length > fileCategories.length)) { + setFileCategories(responseJson.namespaces) + } + } }) .catch((error) => { toast(error.toString()); @@ -159,7 +166,21 @@ const Files = (props) => { }; useEffect(() => { - getFiles(selectedNamespace) + getFiles("default") + + setTimeout(() => { + var category = selectedCategory + if (window.location.search.includes("category=")) { + const urlParams = new URLSearchParams(window.location.search) + category = urlParams.get("category") + } + + if (category !== undefined && category !== null && category.length > 0 && category !== "default") { + setSelectedCategory(category) + } + + getFiles(category) + }, 1000) }, []); const importStandardsFromUrl = (url, folder) => { @@ -226,15 +247,27 @@ const Files = (props) => { const fileDownloadModal = loadFileModalOpen ? {}} PaperProps={{ - style: { - backgroundColor: theme.palette.surfaceColor, - color: "white", - minWidth: "800px", - minHeight: "320px", - }, - }} + sx: { + borderRadius: theme?.palette?.DialogStyle?.borderRadius, + border: theme?.palette?.DialogStyle?.border, + fontFamily: theme?.typography?.fontFamily, + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + zIndex: 1000, + minWidth: "800px", + minHeight: "320px", + overflow: "hidden", + '& .MuiDialogContent-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + }, + '& .MuiDialogTitle-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + }, + '& .MuiDialogActions-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + }, + } + }} >
@@ -346,7 +379,7 @@ const Files = (props) => { {/* */} @@ -676,45 +707,56 @@ const Files = (props) => { //setFile(fileObject) //const files = event.target.files[0] uploadFiles(event.target.files); + }} /> + {/*
*/} - - {fileNamespaces !== undefined && - fileNamespaces !== null && - fileNamespaces.length > 1 ? ( + {fileCategories !== undefined && + fileCategories !== null && + fileCategories.length > 1 ? ( - File Category ) : null} -
+
{renderTextBox ? - - - - : - - - } - + + + + : + + + + } {renderTextBox && { @@ -783,6 +826,7 @@ const Files = (props) => { isFileEditor = {true} key = {fileContent} //https://reactjs.org/docs/reconciliation.html#recursing-on-children runUpdateText = {runUpdateText} + contentLoading = {contentLoading} /> {isSelectedFiles?null: { backgroundColor: theme.palette.inputColor, }} />} - - - - - + + + {["Name", "Workflow", "Md5", "Status", "Filesize", "Actions"].map((header, index) => ( + - - - - - + /> + ))} - {files === undefined || files === null || files.length === 0 ? null : - files.map((file, index) => { - if (file.namespace === "") { - file.namespace = "default"; - } - - if (file.namespace !== selectedNamespace) { - return null; - } - - var bgColor = isSelectedFiles ? "#212121":"#27292d"; - if (index % 2 === 0) { - bgColor = isSelectedFiles ? "#1A1A1A":"#1f2023"; - } - - const filenamesplit = file.filename.split(".") - const iseditable = file.filesize < 2000000 && file.status === "active" && allowedFileTypes.includes(filenamesplit[filenamesplit.length-1]) - - return ( - - ( + + {Array(6) + .fill() + .map((_, colIndex) => ( + + + + ))} + + )): + files.length === 0 ? ( +
+ + No files found + +
+ ):( + files?.map((file, index) => { + if (file.namespace === "") { + file.namespace = "default"; + } + + if (file.namespace !== selectedCategory) { + return null; + } + + var bgColor = isSelectedFiles ? "#212121":"#27292d"; + if (index % 2 === 0) { + bgColor = isSelectedFiles ? "#1A1A1A":"#1f2023"; + } + + const filenamesplit = file.filename.split(".") + const iseditable = file.filesize < 2000000 && file.status === "active" && allowedFileTypes.includes(filenamesplit[filenamesplit.length-1]) + return ( + - - - - - : ( + > + {/* + + */} + + + + + : ( + + + + + + + + + + ) + } + style={{ + display: 'table-cell', + overflow: "hidden", + }} + /> + + {file.md5_sum} + + )} + primaryTypographyProps={{ + style:{ + display: 'table-cell', + marginLeft:isSelectedFiles? 15:null, + overflow: "hidden", + whiteSpace: 'nowrap', + textOverflow: 'ellipsis', + maxWidth: 200, + } + }} + /> + + + + + { + setOpenEditor(true) + setOpenFileId(file.id) + readFileData(file) + }} + > + edit icon + + + + {/* + + + { + // Open the file, without downloading it + window.open(`${globalUrl}/api/v1/files/${file.id}/content?type=text&authorization=${file.public_authorization}`, "_blank noreferrer noopener") + }} + > + + + + + */} + - { + downloadFile(file); }} - href={`/workflows/${file.workflow_id}`} - target="_blank" > - - - - + download icon + - ) - } - style={{ - minWidth: 100, - maxWidth: 100, - overflow: "hidden", - textAlign: isSelectedFiles?"center":null - }} - /> - - - - - - - { - setOpenEditor(true) - setOpenFileId(file.id) - readFileData(file) - }} - > - - - - - {/* - - - { - // Open the file, without downloading it - window.open(`${globalUrl}/api/v1/files/${file.id}/content?type=text&authorization=${file.public_authorization}`, "_blank noreferrer noopener") - }} - > - - - - - */} - - - { - downloadFile(file); - }} - > - - - - - - { - const elementName = "copy_element_shuffle"; - var copyText = - document.getElementById(elementName); - if ( - copyText !== null && - copyText !== undefined - ) { - const clipboard = navigator.clipboard; - if (clipboard === undefined) { - toast( - "Can only copy over HTTPS (port 3443)" - ); - return; - } - - navigator.clipboard.writeText(file.id); - copyText.select(); - copyText.setSelectionRange( - 0, - 99999 - ); /* For mobile devices */ - - /* Copy the text inside the text field */ - document.execCommand("copy"); - - toast(file.id + " copied to clipboard"); - } - }} + - - - - - { - deleteFile(file); + console.log("file is : ", file) + navigator.clipboard.writeText(file.id); + document.execCommand("copy"); + + toast(file.id + " copied to clipboard"); }} > - + copy icon - - - - style={{ - minWidth: 250, - maxWidth: 250, - // overflow: "hidden", - }} - /> - - ); - }) + + + + { + deleteFile(file) + }} + > + delete icon + + + + + style={{ + display: 'table-cell', + textAlign:'center' + // overflow: "hidden", + }} + /> + + ); + }) + ) } +
+
+
) -} +}) -export default Files; +export default memo(Files); + + +const DownloadFileIcon = memo(({ setLoadFileModalOpen, isSelectedFiles, }) => { + + const { leftSideBarOpenByClick } = useContext(Context) + + return( + + setLoadFileModalOpen(true)} + > + + + + ) +}) diff --git a/frontend/src/components/FixWorkflowValidationErrors.jsx b/frontend/src/components/FixWorkflowValidationErrors.jsx new file mode 100644 index 00000000..4781eebe --- /dev/null +++ b/frontend/src/components/FixWorkflowValidationErrors.jsx @@ -0,0 +1,687 @@ +import React, { useState, useEffect } from "react"; + +import { toast } from "react-toastify" +import theme from '../theme.jsx'; +import AuthenticationOauth2 from "../components/Oauth2Auth.jsx"; +import { validateJson, GetIconInfo } from "../views/Workflows.jsx"; + +import { + Tooltip, + Typography, + Button, + Divider, + + MenuItem, + Select, + Chip, + TextField, + CircularProgress, +} from "@mui/material" + +import { + CheckCircleOutline as CheckCircleOutlineIcon, + ErrorOutline as ErrorOutlineIcon, +} from "@mui/icons-material" + +import { + green, + red, +} from "../views/AngularWorkflow.jsx" + +const FixWorkflowValidationErrors = (props) => { + const { globalUrl, workflow, setWorkflow, setUpdateParent, } = props; + + const [appsLoading, setAppsLoading] = useState(false) + const [apps, setApps] = useState([]) + const [appAuth, setAppAuth] = useState([]) + const [_, setUpdate] = useState(0) + + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" || window.location.host === "migration.shuffler.io"; + + if (workflow === undefined || workflow === null) { + console.error("Workflow is undefined") + return null + } + + if (workflow.validation === undefined || workflow.validation === null) { + console.error("Workflow validation is undefined") + return null + } + + if (workflow.validation.valid === true) { + console.error("Workflow is valid - nothing to do for errors") + return null + } + + if (setWorkflow === undefined || setWorkflow === null) { + console.error("No setWorkflow") + return null + } + + const fetchApps = () => { + if (appsLoading === true) { + return + } + + setAppsLoading(true) + + const url = `${globalUrl}/api/v1/apps` + fetch(url,{ + method: "GET", + credentials: "include" + }) + .then(response => response.json()) + .then(data => { + setAppsLoading(false) + if (data.success === false) { + return + } + + setApps(data) + }) + .catch(error => { + setAppsLoading(false) + console.error("Error: ", error) + }) + } + + // Save the workflow as well + const saveWorkflow = (workflow) => { + if (workflow.id === undefined || workflow.id === null) { + toast("Workflow ID is missing during save. Please try again") + return + } + + const url = `${globalUrl}/api/v1/workflows/${workflow.id}` + fetch(url, { + method: "PUT", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + body: JSON.stringify(workflow), + }) + .then(response => response.json()) + .then(data => { + if (data.success === false) { + toast("Failed to save workflow") + return + } + }) + .catch(error => { + toast("Failed to save workflow: " + error.toString()) + }) + } + + const fetchAuthentication = (reset, updateAction, closeMenu, action_id) => { + if (appsLoading === true) { + return + } + + const url = `${globalUrl}/api/v1/apps/authentication` + fetch(url,{ + method: "GET", + credentials: "include" + }) + .then(response => response.json()) + .then(data => { + if (data.success === false) { + return + } + + const authlist = data.data + setAppAuth(authlist) + if (updateAction === true) { + console.log("Updating action: ", action_id) + + // Find the action in the workflow and set auth for it + var foundActionIndex = -1 + for (var i = 0; i < workflow.actions.length; i++) { + if (workflow.actions[i].id === action_id) { + foundActionIndex = i + break + } + } + + if (foundActionIndex === -1) { + console.error("Failed to find action in workflow") + return + } + + const appId = workflow.actions[foundActionIndex].app_id + var lastauth = -1 + for (var authKey in authlist) { + if (authlist[authKey].app_id !== appId) { + continue + } + + if (authlist[authKey].created > lastauth) { + lastauth = authlist[authKey].created + } else { + continue + } + + console.log("FOUND AUTH: ", authlist[authKey]) + workflow.actions[foundActionIndex].authentication_id = authlist[authKey].id + } + + + if (setWorkflow !== undefined) { + setWorkflow(workflow) + } + } + }) + .catch(error => { + console.error("Auth loading error: ", error) + }) + } + + if (apps !== undefined && apps !== null && apps.length === 0 && appsLoading === false) { + fetchApps() + fetchAuthentication() + } + + const setSelectedAction = (action) => { + if (workflow === undefined || workflow === null) { + return null + } + + if (workflow.actions === undefined || workflow.actions === null || workflow.actions.length === 0) { + return null + } + + if (setWorkflow === undefined || setWorkflow === null) { + return null + } + + for (var i = 0; i < workflow.actions.length; i++) { + if (workflow.actions[i].id === action.id) { + workflow.actions[i] = action + + // Update any action with the same app_id to have same auth + for (var j = 0; j < workflow.actions.length; j++) { + if (workflow.actions[j].app_id === action.app_id) { + workflow.actions[j].authentication_id = action.authentication_id + workflow.actions[j].selectedAuthentication = action.selectedAuthentication + } + } + + break + } + } + + setWorkflow(workflow) + } + + const ErrorItem = (props) => { + const { apps, error, index } = props + + const [validating, setValidating] = useState(false) + const [actionRunInfo, setActionRunInfo] = useState({}) + if (error === undefined || error === null) { + return null + } + + if (apps === undefined || apps === null || apps.length === 0) { + return null + } + + const validateApp = (app, action) => { + if (validating) { + return + } + + setValidating(true) + + // FIXME: Run execution: + // 1. Should set app authentication validation + if (isCloud) { + action.environment = "Cloud" + } else { + action.environment = "Shuffle" + } + + /* + setExecutionResult({ + valid: false, + result: baseResult, + }) + setExecuting(true); + */ + + setActionRunInfo({}) + const url = `${globalUrl}/api/v1/apps/${app.id}/run?validation=true` + fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(action), + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for stream results :O!"); + } + + return response.json(); + }) + .then((responseJson) => { + setValidating(false) + + setActionRunInfo(responseJson) + + //console.log("RESPONSE: ", responseJson) + if ( + responseJson.success === true && + responseJson.result !== null && + responseJson.result !== undefined && + responseJson.result.length > 0 + ) { + //toast(" + } + }) + .catch((error) => { + toast("Execution error: " + error.toString()); + setValidating(false) + }) + } + + var authReturn = null + var validationReturn = null + var foundApp = { + "name": "", + "id": "", + } + var foundAction = { + "name": "", + "label": "", + "id": "", + "app_id": "", + "app_name": "", + } + + var selectedImage = null + var resolveButton = null + if (error.app_id !== undefined && error.app_id !== null) { + for (var i = 0; i < apps.length; i++) { + if (apps[i].id === error.app_id) { + foundApp = apps[i] + break + } + } + + if (!foundApp) { + toast("Couldn't find relevant app. Is it activated?") + return "Failed to find app" + } + + selectedImage = + + } + + if (error.action_id !== undefined && error.action_id !== null && workflow.actions !== undefined && workflow.actions !== null && workflow.actions.length > 0) { + for (var i = 0; i < workflow.actions.length; i++) { + if (workflow.actions[i].id === error.action_id) { + foundAction = workflow.actions[i] + break + } + } + } + + const validationIcon = Object.getOwnPropertyNames(actionRunInfo).length === 0 ? null : + + {actionRunInfo.result} + + } + placement="bottom" + > + {actionRunInfo.validation.valid === true ? + + : + + } + + + const authenticationType = foundApp.authentication + if (error.type === "configuration" || error.type === "authentication") { + // FIXME: Check the error + if (appAuth === undefined || appAuth === null) { + return "Loading auth" + } + + var relevantAuthentication = [] + var foundAuth = {} + for (var key in appAuth) { + if (appAuth[key].app.id !== error.app_id) { + continue + } + + foundAuth = appAuth[key] + relevantAuthentication.push(appAuth[key]) + } + + if (foundAction.selectedAuthentication === undefined || foundAction.selectedAuthentication === null) { + foundAction.selectedAuthentication = {} + } + + console.log("FOUNDACTION: ", foundAction, foundAuth) + + var authFound = false + if (foundAuth.id !== undefined && foundAuth.id !== null && foundAuth.id.length > 0) { + var authGroups = [] + // Choose from a dropdown + authReturn = + } + + // FIXME: Validate the CURRENT authentication that has been chosen? + if (foundApp.authentication === undefined || foundApp.authentication === null) { + toast("Authentication error: No authentication found") + authReturn = "Failed to find auth" + } + + if (authReturn === null && foundApp.authentication.type === "oauth2" || foundApp.authentication.type === "oauth2-app") { + authReturn = + + } else if (authReturn === null) { + authReturn = "Other auth - Not implemented" + + } + + validationReturn = authReturn === null || foundAuth.id === undefined || foundAuth.id === null || foundAuth.id.length === 0 || !authFound ? null : + + + //resolveButton = !(Object.getOwnPropertyNames(actionRunInfo).length === 0 || actionRunInfo.validation.valid === true) ? null : + resolveButton = + + } + + + return ( +
+ {authReturn} +
+ {validationReturn} + + {resolveButton} +
+ ) + } + + console.log("Workflow validation: ", workflow.validation) + return ( +
+ {workflow.errors !== undefined && workflow.errors !== null ? +
+ General errors: {workflow.errors.length} + {workflow.errors.map((error, index) => { + return ( +
+ - {error} +
+ ) + })} +
+ : null} + + + + + {workflow.validation.errors !== undefined && workflow.validation.errors !== null ? +
+ Validation errors: {workflow.validation.errors.length} + {workflow.validation.errors.map((error, index) => { + return ( +
+ +
+ ) + })} +
+ : null} + + + Apps loaded: {apps.length} + +
+ ) +} + +export default FixWorkflowValidationErrors diff --git a/frontend/src/components/Header.jsx b/frontend/src/components/Header.jsx index 77faee2f..4d88d021 100644 --- a/frontend/src/components/Header.jsx +++ b/frontend/src/components/Header.jsx @@ -50,8 +50,6 @@ const hoverOutColor = "#e8eaf6" const Header = props => { const { globalUrl, setNotifications, notifications, isLoggedIn, removeCookie, homePage, userdata, serverside, } = props; - //const theme = useTheme(); - //const alert = useAlert() const [HomeHoverColor, setHomeHoverColor] = useState(hoverOutColor); @@ -829,7 +827,7 @@ const { globalUrl, setNotifications, notifications, isLoggedIn, removeCookie, ho }, }} style={{ - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, backgroundColor: theme.palette.surfaceColor, marginRight: 15, color: "white", @@ -904,7 +902,7 @@ const { globalUrl, setNotifications, notifications, isLoggedIn, removeCookie, ho
- {isCloud?{regiontag}:null} {image} {data.name} + {isCloud?{regiontag}:null} {image} {data.name}
@@ -935,7 +933,7 @@ const { globalUrl, setNotifications, notifications, isLoggedIn, removeCookie, ho null : -
= 0.9 ? "#f86a3e" : null, }} onClick={() => { +
= 0.9 ? "#f86a3e" : null, }} onClick={() => { console.log(userdata.appe_execution_usage/userdata.app_execution_limit) if (window.drift !== undefined) { window.drift.api.startInteraction({ interactionId: 326905 }) diff --git a/frontend/src/components/HealthPage.jsx b/frontend/src/components/HealthPage.jsx index 3ad6ad8b..fcadb776 100644 --- a/frontend/src/components/HealthPage.jsx +++ b/frontend/src/components/HealthPage.jsx @@ -11,14 +11,14 @@ import { import HealthBarChart from '../components/HealthBarChart.jsx'; const HealthPage = (props) => { - const { userdata } = props; + const { globalUrl, userdata } = props; const [healthData, setHealthData] = useState(null); const [selectedRange, setSelectedRange] = useState('30d'); const [filteredData, setFilteredData] = useState([]); const [averageUptime, setAverageUptime] = useState(0); const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; - const globalUrl = `https://shuffler.io` + //const globalUrl = `https://shuffler.io` console.log("HEALTHPAGE 1") diff --git a/frontend/src/components/LeftSideBar.jsx b/frontend/src/components/LeftSideBar.jsx new file mode 100644 index 00000000..934c14d9 --- /dev/null +++ b/frontend/src/components/LeftSideBar.jsx @@ -0,0 +1,1781 @@ +import React, { useEffect, useRef, useState, useContext, useCallback, useMemo } from "react"; +import { + ExpandLess as ExpandLessIcon, + ExpandMore as ExpandMoreIcon, + GridView as GridViewIcon, + ShieldOutlined as ShieldOutlinedIcon, + Add as AddIcon, + BorderColor, + Close as CloseIcon, + ConstructionOutlined, +} from "@mui/icons-material"; +import SearchBox from "./SearchData.jsx"; +import { + Button, + Typography, + Autocomplete, + TextField, + Popper, + Avatar, + Menu, + IconButton, + MenuItem, + Divider, + Box, + Dialog, + DialogContent, + Fade, + Portal, + Collapse, +} from "@mui/material"; +import theme from "../theme.jsx"; +import { + Settings as SettingsIcon +} from "@mui/icons-material"; +import RecentWorkflow from "../components/RecentWorkflow.jsx"; + +import { useNavigate } from "react-router"; + +import { Link } from "react-router-dom"; +import { + Business as BusinessIcon, + Notifications as NotificationsIcon, + HelpOutline as HelpOutlineIcon, + MeetingRoom as MeetingRoomIcon, + Lightbulb as LightbulbIcon, + Search as SearchIcon +} from "@mui/icons-material"; + +import { toast } from "react-toastify"; +import { Context } from "../context/ContextApi.jsx"; + +const ShuffleLogo = "/images/Shuffle_logo.png"; +const detectionIcon = "/icons/detection.svg"; +const documentationIcon = "/icons/documentation.svg"; +const ExpandMoreAndLessIcon = "/icons/expandMoreIcon.svg"; + +const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => { + + const navigate = useNavigate(); + const {setLeftSideBarOpenByClick, leftSideBarOpenByClick, setSearchBarModalOpen, searchBarModalOpen} = useContext(Context); + const [expandLeftNav, setExpandLeftNav] = useState(false); + const [activeOrgName, setActiveOrgName] = useState( + userdata?.active_org?.name || "Select Organziation" + ); + const [openAutocomplete, setOpenAutocomplete] = useState(false); + const [autocompleteValue, setAutocompleteValue] = useState(""); + const [openautomatetab, setOpenautomateTab] = useState(false); + const [openSecurityTab, setOpenSecurityTab] = useState(false); + const [selectedOrg, setSelectedOrg] = useState(activeOrgName); + const [anchorElAvatar, setAnchorElAvatar] = React.useState(null); + const [anchorEl, setAnchorEl] = React.useState(null); + const [recentworkflows, setRecentworkflows] = useState(null); + const [usersWorkFlows, setUsersWorkFlows] = useState([]); + const [currentOpenTab, setCurrentOpenTab] = useState(""); + const currentPath = window.location.pathname; + const [hoverOnAvatar, setHoverOnAvatar] = useState(false); + const [orgOptions, setOrgOptions] = useState( + userdata?.orgs?.map((org) => ({ + id: org.id, + name: org.name, + image: org.image, + region_url: org.region_url, + })) || [] + ); + const userOrgs = React.useMemo(() => { + return orgOptions.find((option) => option.name === selectedOrg); + }, [selectedOrg, orgOptions]); + +//With this code it is opening search bar on google chrome search bar as well which is not required +useEffect(() => { + const handleKeyDown = (event) => { + if ((event.ctrlKey || event.metaKey) && event.key === "k") { + event.preventDefault(); + setSearchBarModalOpen((prev)=> !prev); + } + }; + + window.addEventListener("keydown", handleKeyDown); + + return () => { + window.removeEventListener("keydown", handleKeyDown); + }; +}, [setSearchBarModalOpen]); + + const CustomPopper = (props) => { + return ( + + + + {props.children} + + + { + setSelectedOrg(userdata?.active_org?.name); + setOpenAutocomplete(false) + }} + > + Add suborg + + + + + ); + }; + + const iconRef = useRef(null); + + const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + + const parsedAvatar = + userdata?.avatar !== undefined && + userdata?.avatar !== null && + userdata?.avatar.length > 0 + ? userdata?.avatar + : ""; + + const autocompleteRef = useRef(null); + const hoverOutColor = "#e8eaf6"; + + const handleClose = () => { + setAnchorEl(null); + setAnchorElAvatar(null); + }; + + const hrefStyle = { + color: hoverOutColor, + textDecoration: "none", + textTransform: "none", + fontStyle: "normal", + width: "100%", + fontSize: "16px", + }; + + const isCloud = + serverside === true || typeof window === "undefined" + ? true + : window.location.host === "localhost:3002" || + window.location.host === "shuffler.io" || + window.location.host === "localhost:5002"; + + const UpdateTabStatus = useCallback(() => { + const lastTabOpenByUser = localStorage.getItem("lastTabOpenByUser"); + if ((lastTabOpenByUser === "automate" && currentPath.includes("/dashboards/automate")) || currentPath.includes("/dashboards/automate")) { + setOpenautomateTab(true); + setCurrentOpenTab("automate"); + setOpenSecurityTab(false); + } else if ((lastTabOpenByUser === "security" && currentPath.includes("/dashboards/security")) || currentPath.includes("/dashboards/security")) { + setOpenautomateTab(false); + setOpenSecurityTab(true); + setCurrentOpenTab("security"); + } else if ((lastTabOpenByUser === "usecases" && currentPath.includes("/usecases2")) || currentPath.includes("/usecases2")) { + setOpenautomateTab(true); + setOpenSecurityTab(false); + setCurrentOpenTab("usecases"); + } else if ((lastTabOpenByUser === "workflows" && currentPath === "/workflows") || currentPath === "/workflows") { + setOpenautomateTab(true); + setOpenSecurityTab(false); + setCurrentOpenTab("workflows"); + } else if ((lastTabOpenByUser === "apps" && currentPath.includes("/search")) || currentPath.includes("/search")) { + setOpenautomateTab(true); + setOpenSecurityTab(false); + setCurrentOpenTab("apps"); + } else if ((lastTabOpenByUser === "detection" && currentPath.includes("/detections")) || currentPath.includes("/detections")) { + setOpenautomateTab(false); + setOpenSecurityTab(true); + setCurrentOpenTab("detection"); + } else if ((lastTabOpenByUser === "response" && currentPath.includes("/response")) || currentPath.includes("/response")) { + setOpenautomateTab(false); + setOpenSecurityTab(true); + setCurrentOpenTab("response"); + } else if ((lastTabOpenByUser === "docs" || currentPath.includes("/docs")) || currentPath.includes("/docs")) { + setOpenautomateTab(false); + setOpenSecurityTab(false); + setCurrentOpenTab("docs"); + } else { + setOpenautomateTab(true); + setOpenSecurityTab(false); + setCurrentOpenTab(""); + } + },[currentPath]); + + useEffect(() => { + UpdateTabStatus(); + const expandLeftNav1 = localStorage.getItem("expandLeftNav"); + if (expandLeftNav1 === "false") { + setLeftSideBarOpenByClick(false); + setLeftSideBarOpenByClick(false); + } else { + setLeftSideBarOpenByClick(true); + setLeftSideBarOpenByClick(true); + setExpandLeftNav(true); + } + }, []); + + const getAvailableWorkflows = useCallback((amount) => { + + var url = `${globalUrl}/api/v1/workflows` + if (amount !== undefined && amount !== null) { + url += `?top=${amount}` + } + + fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + toast("Failed getting workflows. Are you logged in?"); + return + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson !== undefined) { + + var newarray = [] + for (var wfkey in responseJson) { + const wf = responseJson[wfkey] + if (wf.public === true || wf.hidden === true) { + continue + } + + newarray.push(wf) + } + + var actionnamelist = []; + var parsedactionlist = []; + for (var key in newarray) { + for (var actionkey in newarray[key].actions) { + const action = newarray[key].actions[actionkey]; + if (actionnamelist.includes(action.app_name)) { + continue; + } + + actionnamelist.push(action.app_name); + parsedactionlist.push(action); + } + } + + if (newarray.length > 0) { + try { + localStorage.setItem("workflows", JSON.stringify(newarray)) + } catch (e) { + console.log("Failed to set workflows in localstorage: ", e) + } + } + + setTimeout(() => { + setUsersWorkFlows(newarray); + if (newarray.length === 0) { + setUsersWorkFlows([]); + } + }, 250) + + } + }) + .catch((error) => { + toast(error.toString()); + }); + }, [usersWorkFlows]); + + useEffect(() => { + const fetchData = async () => { + if (recentworkflows === null || recentworkflows === undefined || recentworkflows?.length === 0) { + await delay(2000); + } + + const storagewf = localStorage.getItem("workflows"); + const storageWorkflows = JSON.parse(storagewf); + + if (storageWorkflows) { + setUsersWorkFlows(storageWorkflows); + } else { + getAvailableWorkflows(); + } + }; + + fetchData(); + }, []); + + useEffect(() => { + if (usersWorkFlows) { + const recentWorkflow = HandleGetUsersRecentWorkflows(usersWorkFlows); + setRecentworkflows(recentWorkflow); + } + }, [usersWorkFlows]); + + const removeCookie = (name, path = "/") => { + document.cookie = `${name}=; path=${path}; expires=Thu, 01 Jan 1970 00:00:00 GMT;`; + }; + + const handleClickLogout = () => { + // Logout API call + fetch(`${globalUrl}/api/v1/logout`, { + credentials: "include", + method: "POST", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => { + if (response.status !== 200) { + toast.error("Failed to logout. Please try again."); + return; + } + + // Remove cookies + removeCookie("session_token", { path: "/" }); + removeCookie("__session", { path: "/" }); + + // Clear localStorage + localStorage.clear(); + + // Redirect to home immediately + window.location.href = "/"; + }) + .catch((error) => { + console.error("Logout error:", error); + }); + }; + + const avatarMenu = ( + + { + setAnchorElAvatar(event.currentTarget); + }} + disableRipple + disableElevation + > + + + + + { + handleClose(); + }} + > + Organization + + + + { + handleClose(); + }} + > + Account + + + + + + { + handleClose(); + }} + > + Notifications ({ + notifications === undefined || notifications === null ? 0 : + notifications?.filter((notification) => notification.read === false).length + }) + + + + { + handleClose(); + }} + > + Use Cases + + + + + + + { + handleClose(); + }} + > + About + + + { + handleClickLogout(); + event.preventDefault(); + handleClose(); + }} + > +  Logout + + + + + Version: 2.0.0-beta + + + + ); + + const getWorkflowAppgroup = (data) => { + if (!data.actions) { + return []; + } + + let appsFound = []; + Object.keys(data.actions).forEach((key) => { + const parsedAction = data.actions[key]; + + if (!parsedAction.large_image) { + return; + } + + if ( + appsFound.findIndex((app) => app.app_name === parsedAction.app_name) < 0 + ) { + appsFound.push(parsedAction); + } + }); + + return appsFound; + }; + + const HandleGetUsersRecentWorkflows = useCallback(() => { + if (!usersWorkFlows) { + return []; + } + + let groupedWorkflows = []; + + usersWorkFlows.forEach((workflow) => { + const apps = getWorkflowAppgroup(workflow); + if (apps.length > 0) { + groupedWorkflows.push({ + name: workflow.name, + id: workflow.id, + apps: apps, + }); + } + }); + + return groupedWorkflows; + },[usersWorkFlows]); + + const handleClickChangeOrg = (orgId) => { + + toast.info("Changing active organization - please wait!"); + + const data = { + org_id: orgId, + }; + + localStorage.setItem("globalUrl", ""); + localStorage.setItem("getting_started_sidebar", "open"); + + fetch(`${globalUrl}/api/v1/orgs/${orgId}/change`, { + mode: "cors", + credentials: "include", + crossDomain: true, + method: "POST", + body: JSON.stringify(data), + withCredentials: true, + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + }) + .then(function (response) { + if (response.status !== 200) { + console.log("Error in response"); + } else { + localStorage.removeItem("apps"); + localStorage.removeItem("workflows"); + localStorage.removeItem("userinfo"); + localStorage.removeItem("lastTabOpenByUser"); + } + + return response.json(); + }) + .then(function (responseJson) { + if (responseJson.success === true) { + if ( + responseJson?.region_url !== undefined && + responseJson?.region_url !== null && + responseJson?.region_url.length > 0 + ) { + localStorage.setItem("globalUrl", responseJson.region_url); + } + + if (responseJson["reason"] === "SSO_REDIRECT") { + setTimeout(() => { + toast.info( + "Redirecting to SSO login page as SSO is required for this organization." + ); + window.location.href = responseJson["url"]; + return; + }, 2000); + } else { + setTimeout(() => { + window.location.reload(); + }, 2000); + } + + toast.success("Successfully changed active organization - refreshing!"); + } else { + if ( + responseJson.reason !== undefined && + responseJson.reason !== null && + responseJson.reason.length > 0 + ) { + toast(responseJson.reason); + } else { + toast( + "Failed changing org. Try again or contact support@shuffler.io if this persists." + ); + } + } + }) + .catch((error) => { + console.log("error changing: ", error); + }); +}; + + const getRegionTag = (region_url) => { + let regiontag = "UK"; + if ( + region_url !== undefined && + region_url !== null && + region_url.length > 0 + ) { + const regionsplit = region_url.split("."); + if (regionsplit.length > 2 && !regionsplit[0].includes("shuffler")) { + const namesplit = regionsplit[0].split("/"); + regiontag = namesplit[namesplit.length - 1]; + + if (regiontag === "california") { + regiontag = "US"; + } else if (regiontag === "frankfurt") { + regiontag = "EU"; + } else if (regiontag === "ca"){ + regiontag = "CA"; + } + } + } + + return regiontag; + }; + + useEffect(() => { + + if(activeOrgName !== userdata?.active_org?.name){ + setActiveOrgName(userdata?.active_org?.name || "Select Organization"); + } + }, [userdata]); + + const CheckOrgStates = useCallback(() => { + setOrgOptions( + userdata?.orgs?.map((org) => ({ + id: org.id, + name: org.name, + image: org.image, + region_url: getRegionTag(org.region_url), + })) || [] + ); + setActiveOrgName(userdata?.active_org?.name || "Select Organization"); + setSelectedOrg(userdata?.active_org?.name || "Select Organization"); + },[orgOptions, activeOrgName, selectedOrg]); + + useEffect(() => { + if (typeof userdata?.id === "string" && userdata?.id?.length > 0) { + CheckOrgStates(); + setAutocompleteValue(userdata?.active_org?.name || ""); + } + }, []); + + const ButtonStyle = { + width: expandLeftNav ? "100%" : 30, + justifyContent: expandLeftNav ? "flex-start" : "center", + height: 35, + color: openautomatetab ? "#F1F1F1" : "#C8C8C8", + textTransform: "none", + "& .MuiButton-root ": { + width: expandLeftNav ? "100%" : 30, + padding: 0, + }, + }; + + const modalView = ( + { + setSearchBarModalOpen(false); + }} + PaperProps={{ + style: { + color: "white", + minWidth: 750, + height: 785, + borderRadius: 16, + border: "1px solid var(--Container-Stroke, #494949)", + background: "var(--Container, #000000)", + boxShadow: "0px 16px 24px 8px rgba(0, 0, 0, 0.25)", + zIndex: 13000, + paddingTop: 20, + }, + }} + > + + + + + + + + ); + + const getRegionFlag = (region_url) => { + var region = "gb"; + const regionMapping = { + "US": "us", + "EU": "eu", + "CA": "ca", + "UK": "gb" + }; + + region = regionMapping[region_url] || "gb"; + + return `https://flagcdn.com/48x36/${region}.png`; + }; + + return ( +
+ {modalView} + + + Shuffle Logo + + + + + + {!leftSideBarOpenByClick && setExpandLeftNav(true);}} onMouseLeave={()=>{!leftSideBarOpenByClick && setExpandLeftNav(false);setOpenAutocomplete(false);}}> + + {expandLeftNav ? ( + + + ), + endAdornment: ( + + Ctrl/Cmd+K + + ), + disableUnderline: true, + }} + onClick={() => { + setSearchBarModalOpen(true); + }} + onChange={() => { + setSearchBarModalOpen(true); + }} + /> + + ):( + <> + + + + )} + + + + + + + + + { + setOpenautomateTab((prev) => !prev); + setOpenSecurityTab(false); + }} + style={{ + color: "#FFFFFF", + marginLeft: 0.625, + }} + onMouseOver={(event)=>{ + event.currentTarget.style.backgroundColor = "#2f2f2f"; + }} + onMouseOut={(event)=>{ + event.currentTarget.style.backgroundColor = "transparent"; + }} + > + {openautomatetab ? ( + + ) : ( + + )} + + + + + + + + + + + + + + + + + + + + + + { + setOpenSecurityTab((prev) => !prev); + setOpenautomateTab(false); + }} + style={{ + marginLeft: 0.625, + color: "#FFFFFF", + }} + onMouseOver={(event)=>{ + event.currentTarget.style.backgroundColor = "#2f2f2f"; + }} + onMouseOut={(event)=>{ + event.currentTarget.style.backgroundColor = "transparent"; + }} + > + {openSecurityTab ? ( + + ) : ( + + )} + + + + + + + + + + + + + + + + + + + + + + + + + + + + Recent Workflows + + + {recentworkflows?.slice(0, 2).map((workflow, index) => { + return ( + + ) + }) } + + + + + + orgOptions, [orgOptions])} + getOptionLabel={(option) => option.name} + PopperComponent={CustomPopper} + open={openAutocomplete} + onOpen={() => setOpenAutocomplete(true)} + renderOption={(props, option, { selected, index }) => ( +
  • { + e.currentTarget.style.backgroundColor = "#444444"; + }} + onMouseOut={(e) => { + e.currentTarget.style.backgroundColor = option.name === selectedOrg ? "#696969" : "transparent"; + }} + onClick={(e) => { + if (option.id !== userdata?.active_org?.id) { + setSelectedOrg(option.name); + handleClickChangeOrg(option.id); + } else { + setSelectedOrg(userdata?.active_org?.name); + } + setOpenAutocomplete(false); + }} + > + { + isCloud ? ( + + {option.region_url} + {option.region_url} + + ) : null + } + {option.name} + + {option.name} + +
    • + )} + onChange={(event, newValue) => { + if (newValue) { + if (userdata?.active_org?.id !== newValue.id) { + setSelectedOrg(newValue.name); + handleClickChangeOrg(newValue.id); + } else { + setSelectedOrg(userdata?.active_org?.name); + } + setOpenAutocomplete(false); + } + }} + onInputChange={(event, newInputValue) => { + setAutocompleteValue(newInputValue); + }} + filterOptions={(options, params) => { + return options.filter((option) => + option.name + .toLowerCase() + .includes(params.inputValue.toLowerCase()) + ); + }} + value={userOrgs} + renderInput={(params) => ( + + {expandLeftNav ? ( + + ) : ( + + + setOpenAutocomplete((prev) => !prev)} + style={{ + cursor: "pointer", + fontSize: "24px", + opacity: expandLeftNav ? 0 : 1, + transition: "opacity 0.3s ease", + }} + /> + + )} + + )} + /> +     + {event.currentTarget.style.backgroundColor = "#2f2f2f";setHoverOnAvatar(true)}} + onMouseLeave ={(event)=>{event.currentTarget.style.backgroundColor = "transparent";setHoverOnAvatar(false)}} + > + {expandLeftNav ? ( + <> + + + ) : ( + <> + + + )} + +     +     +
    + ); +}; + +export default LeftSideBar; diff --git a/frontend/src/components/LicencePopup.jsx b/frontend/src/components/LicencePopup.jsx index 1c12d7fd..883f156d 100644 --- a/frontend/src/components/LicencePopup.jsx +++ b/frontend/src/components/LicencePopup.jsx @@ -2,7 +2,6 @@ import React, { useState, useEffect } from "react"; import ReactGA from 'react-ga4'; import theme from "../theme.jsx"; -import { useTheme } from "@mui/styles"; import countries from "../components/Countries.jsx"; import { Box, @@ -88,7 +87,7 @@ const LicencePopup = (props) => { const paperStyle = { padding: 20, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, height: "100%", } @@ -194,7 +193,7 @@ const LicencePopup = (props) => { return (
    @@ -320,7 +319,7 @@ const LicencePopup = (props) => { margin: "auto", width: 100, backgroundColor: "white", - // borderRadius: theme.palette.borderRadius, + // borderRadius: theme.palette?.borderRadius, }} /> : null} @@ -390,7 +389,7 @@ const LicencePopup = (props) => { value={feature.split("Worker License: ")[1]} style={{ // backgroundColor: theme.palette.inputColor, - // borderRadius: theme.palette.borderRadius, + // borderRadius: theme.palette?.borderRadius, }} id={fieldId} onClick={() => { }} @@ -515,16 +514,16 @@ const LicencePopup = (props) => { } useEffect(() => { - console.log("New variant: ", shuffleVariant) + console.log("New variant: ", shuffleVariant) - if (shuffleVariant === 1) { - setCalculatedCost("$600") - setSelectedValue(8) - } else { - setCalculatedCost("$540") - setSelectedValue(300) - } - }, [shuffleVariant]) + if (shuffleVariant === 1) { + setCalculatedCost("$960") + setSelectedValue(8) + } else { + setCalculatedCost("$960") + setSelectedValue(300) + } + }, [shuffleVariant]) if (typeof window === 'undefined' || window.location === undefined) { return null @@ -680,7 +679,7 @@ const LicencePopup = (props) => { color: "white", } - + console.log("Priceitem: ", shuffleVariant) const isLoggedInHandler = () => { if (calculatedCost === payasyougo) { handlePayasyougo(props.userdata) @@ -690,7 +689,7 @@ const LicencePopup = (props) => { const priceItem = window.location.origin === "https://shuffler.io" ? shuffleVariant === 0 ? "app_executions" : "cores" : - shuffleVariant === 0 ? "price_1MROFrDzMUgUjxHShcSxgHO1" : "price_1NXjQqDzMUgUjxHSg690R4FP" + shuffleVariant === 0 ? "price_1PZPSSEJjT17t98NLJoTMYja" : "price_1PZPQuEJjT17t98N3yORUtd9" const successUrl = `${window.location.origin}/admin?admin_tab=billing&payment=success` const failUrl = `${window.location.origin}/pricing?admin_tab=billing&payment=failure` @@ -824,7 +823,7 @@ const LicencePopup = (props) => { {errorMessage.length > 0 ? Error: {errorMessage} : null}
    diff --git a/frontend/src/components/MFASetUP.jsx b/frontend/src/components/MFASetUP.jsx new file mode 100644 index 00000000..e7196ee2 --- /dev/null +++ b/frontend/src/components/MFASetUP.jsx @@ -0,0 +1,197 @@ +import React, { useEffect, useState } from "react"; +import { Paper, Typography, Box, CircularProgress, TextField, Button } from "@mui/material"; +import { toast } from "react-toastify"; + +const MFASetup = ({ isLoaded, globalUrl, setCookie }) => { + const [image2FA, setImage2FA] = useState(""); + const [secret2FA, setSecret2FA] = useState(""); + const [mfaCode, setMfaCode] = useState(""); + const [code, setCode] = useState(null); + + useEffect(() => { + handleGet2FACode(); + }, []); + + useEffect(() => { + if (isLoaded) { + const code = window.location.pathname.split("/")[2]; + setMfaCode(code); + } + }, [isLoaded]); + + const handleGet2FACode = () => { + if (mfaCode === "") { + return; + } + + fetch(`${globalUrl}/api/v1/users/${mfaCode}/get2fa`, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status === 404) { + toast("User not found. Redirecting to login page in 3 seconds..."); + setTimeout(() => { + window.location.pathname = "/login"; + return; + }, 3000); + } + if (response.status !== 200) { + console.log("Status not 200 for apps :O!"); + } + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success === true) { + setImage2FA(responseJson.reason); + setSecret2FA(responseJson.extra); + } + }) + .catch((error) => { + toast(error.toString()); + }); + }; + + useEffect(() => { + if (mfaCode) { + handleGet2FACode(); + } + }, [mfaCode]); + + const handleVerify2FA = (mfaCode, code, changeMFAActive) => { + const data = { + code: code, + changeMFAActive: changeMFAActive, + }; + + toast("Verifying 2fa code. Please wait..."); + + fetch(`${globalUrl}/api/v1/users/${mfaCode}/set2fa`, { + mode: "cors", + method: "POST", + body: JSON.stringify(data), + credentials: "include", + crossDomain: true, + withCredentials: true, + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + }) + .then((response) => { + if (response.status === 500) { + toast("Wrong code sent. Please try again."); + return; + } + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success === true) { + toast.success("Successfully setup 2fa. Redirecting in 3 seconds..."); + for (var key in responseJson["cookies"]) { + setCookie(responseJson["cookies"][key].key, responseJson["cookies"][key].value, { path: "/" }); + } + + const tmpView = new URLSearchParams(window.location.search).get("view"); + if (tmpView !== undefined && tmpView !== null) { + var newUrl = `/${tmpView}`; + if (tmpView.startsWith("/")) { + newUrl = `${tmpView}`; + } + window.location.pathname = newUrl; + return; + } + + if (responseJson.tutorials !== undefined && responseJson.tutorials !== null) { + const welcome = responseJson.tutorials.find((element) => element.name === "welcome"); + if (welcome === undefined || welcome === null) { + setTimeout(() => { + window.location.pathname = "/welcome"; + }, 3000); + } + } + + setTimeout(() => { + window.location.pathname = "/workflows"; + }, 3000); + } else { + toast("Failed to setup 2fa. Please try again."); + } + }) + .catch((error) => { + console.error("Error:", error); + }); + }; + + return ( +
    + + + Multi-Factor Authentication Setup + +
    + +
    + + Enter the code from your authenticator app below. + + setCode(e.target.value)} + onKeyPress={(e) => { + if (e.key === "Enter" && code !== null && code !== "" && code.length === 6) { + handleVerify2FA(mfaCode, code, true); + } + }} + /> + + +     +
    + ); +}; + +const QRCodeSection = ({ secret2FA, image2FA }) => { + return ( +
    + {secret2FA && image2FA ? ( +
    + + Scan the image below with the two-factor authentication app on your phone. If you can’t use a QR code, use the code {secret2FA} instead. + + 2FA QR code +
    + ) : ( + + )} +
    + ); +}; + +export default MFASetup; diff --git a/frontend/src/components/NewHeader.jsx b/frontend/src/components/NewHeader.jsx index 0af0c9bc..abdbcafd 100644 --- a/frontend/src/components/NewHeader.jsx +++ b/frontend/src/components/NewHeader.jsx @@ -1,4 +1,4 @@ -import React, { useState } from "react"; +import React, { useState, useEffect } from "react"; import { toast } from "react-toastify"; import theme from "../theme.jsx"; import { BrowserView, MobileView } from "react-device-detect"; @@ -25,9 +25,10 @@ import { Divider, LinearProgress, AppBar, - Dialog, - DialogTitle, + Dialog, + DialogTitle, } from "@mui/material"; +import { makeStyles } from "@mui/styles"; import { Close as CloseIcon, @@ -45,16 +46,82 @@ import { Analytics as AnalyticsIcon, Lightbulb as LightbulbIcon, ExpandMore as ExpandMoreIcon, + KeyboardArrowDown as KeyboardArrowDownIcon } from "@mui/icons-material"; +import zIndex from "@mui/material/styles/zIndex.js"; -const hoverColor = "#f85a3e"; -const hoverOutColor = "#e8eaf6"; +const useStyles = makeStyles((theme) => ({ + menuButton: { + textTransform: "none !important", + fontStyle: "normal", + color: "#333", + textAlign: "center", + fontSize: "16px !important", + fontWeight: "500 !important", + display: "flex", + alignItems: "center", + '&:hover': { + backgroundColor: "transparent", + }, + }, + dropdownMenu: { + borderRadius: "12px !important", + zIndex: 10, + "& .MuiPaper-root": { + border: "1px solid #f85a3e", + boxShadow: "0 4px 6px rgba(0, 0, 0, 0.1)", + borderRadius: "12px !important", + top: -10, + overflow: "visible", + background: "#1A1A1A", + "&::before": { + content: '""', + display: "block", + position: "absolute", + top: -10, + left: "82%", + borderLeft: "10px solid transparent", + borderRight: "10px solid transparent", + borderBottom: "10px solid #f85a3e", + }, + }, + }, + dropdownMenuItem: { + fontSize: "16px", + fontWeight: 400, + color: "#fff", + background: "#1A1A1A", + borderRadius: 16, // Ensure the border radius matches the container + transition: "background-color 0.3s, color 0.3s", + '&:hover': { + color: "#1A73E8", + background: "#3c3c3c", + }, + }, + menuList: { + display: "flex", + flexDirection: "row", + alignItems: "center", + padding: 0, + margin: 0, + listStyle: "none", + textTransform: "none", + }, + cssStcg3yMenuList: { + borderRadius: "12px !important", + }, + divider: { + width: "80%", + border: "0.5px solid #494949", + backgroundColor: "#fff", + alignItems: "center", + marginLeft: 17 + }, +})); const Header = (props) => { const { globalUrl, - setNotifications, - notifications, isLoaded, isLoggedIn, removeCookie, @@ -63,26 +130,64 @@ const Header = (props) => { isMobile, serverside, billingInfo, - } = props; - const [HomeHoverColor, setHomeHoverColor] = useState(hoverOutColor); - const [SoarHoverColor, setSoarHoverColor] = useState(hoverOutColor); - const [LoginHoverColor, setLoginHoverColor] = useState(hoverOutColor); - const [DocsHoverColor, setDocsHoverColor] = useState(hoverOutColor); - const [HelpHoverColor, setHelpHoverColor] = useState(hoverOutColor); + notifications, + } = props; const [isHeader, setIsHeader] = React.useState(false); const [modalOpen, setModalOpen] = useState(false); + const [tooltipOpen, setTooltipOpen] = useState(false); const [anchorEl, setAnchorEl] = React.useState(null); const [anchorElAvatar, setAnchorElAvatar] = React.useState(null); const [subAnchorEl, setSubAnchorEl] = React.useState(null); const [upgradeHovered, setUpgradeHovered] = React.useState(false); - const [showTopbar, setShowTopbar] = useState(false) - const stripeKey = typeof window === 'undefined' || window.location === undefined ? "" : window.location.origin === "https://shuffler.io" ? "pk_live_XAxwE2Fp9DEbEcNYw4UKmyby00vIlIPPRp" : "pk_test_EdxgKfqmQGXY5JLjdBqtuhCw00BHbiKJDB" + const [showTopbar, setShowTopbar] = useState(false) // Set to true to show top bar + const stripeKey = typeof window === 'undefined' || window.location === undefined ? "" : window.location.origin === "https://shuffler.io" ? "pk_live_51PXYYMEJjT17t98N20qEqItyt1fLQjrnn41lPeG2PjnSlZHTDNKHuisAbW00s4KAn86nGuqB9uSVU4ds8MutbnMU00DPXpZ8ZD" : "pk_test_51PXYYMEJjT17t98NbDkojZ3DRvsFUQBs35LGMx3i436BXwEBVFKB9nCvHt0Q3M4MG3dz4mHheuWvfoYvpaL3GmsG00k1Rb2ksO" let navigate = useNavigate(); + const classes = useStyles(); const handleClick = (event) => { setAnchorEl(event.currentTarget); }; + const handleMenuOpen = (event) => { + setAnchorEl(event.currentTarget); + }; + + const handleMenuClose = () => { + setAnchorEl(null); + }; + + const handleMenuItemClick = (path) => { + navigate(path); + handleMenuClose(); + }; + + const handleTooltipClose = () => { + setTooltipOpen(false); + }; + + const handleTooltipOpen = () => { + setTooltipOpen(true); + }; + + const topbar_var = "topbar_closed5" + useEffect(() => { + // Manually setShowTopbar(true) to show topbar by default + const topbar = localStorage.getItem(topbar_var) + if (topbar === "true") { + setShowTopbar(false) + } + }, []) + + const hoverColor = "#f85a3e"; + const hoverOutColor = "#e8eaf6"; + + const handleHover = (event) => { + event.target.style.color = hoverColor; + }; + + const handleHoverOut = (event) => { + event.target.style.color = hoverOutColor; + }; const handleClose = () => { setAnchorEl(null); @@ -94,6 +199,10 @@ const Header = (props) => { const hrefStyle = { color: hoverOutColor, textDecoration: "none", + textTransform: "none", + fontStyle: "normal", + width: "100%", + fontSize: "16px", }; const menuText = { @@ -117,71 +226,6 @@ const Header = (props) => { ? window.location.pathname : ""; - const clearNotifications = () => { - // Don't really care about the logout - - toast("Clearing notifications") - fetch(`${globalUrl}/api/v1/notifications/clear`, { - credentials: "include", - method: "GET", - headers: { - "Content-Type": "application/json", - }, - }) - .then(function (response) { - if (response.status !== 200) { - console.log("Error in response"); - } - - return response.json(); - }) - .then(function (responseJson) { - if (responseJson.success === true) { - setNotifications([]); - handleClose(); - } else { - toast("Failed dismissing notifications. Please try again later."); - } - }) - .catch((error) => { - console.log("error in notification dismissal: ", error); - //removeCookie("session_token", {path: "/"}) - }); - }; - - const dismissNotification = (alert_id) => { - // Don't really care about the logout - fetch(`${globalUrl}/api/v1/notifications/${alert_id}/markasread`, { - credentials: "include", - method: "GET", - headers: { - "Content-Type": "application/json", - }, - }) - .then(function (response) { - if (response.status !== 200) { - console.log("Error in response"); - } - - return response.json(); - }) - .then(function (responseJson) { - if (responseJson.success === true) { - const newNotifications = notifications.filter( - (data) => data.id !== alert_id - ); - console.log("NEW NOTIFICATIONS: ", newNotifications); - setNotifications(newNotifications); - } else { - toast("Failed dismissing notification. Please try again later."); - } - }) - .catch((error) => { - console.log("error in notification dismissal: ", error); - //removeCookie("session_token", {path: "/"}) - }); - }; - // DEBUG HERE const handleClickLogout = () => { console.log("SHOULD LOG OUT"); @@ -220,247 +264,10 @@ const Header = (props) => { }); }; - // Rofl this is weird - const handleDocsHover = () => { - setDocsHoverColor(hoverColor); - }; - - const handleDocsHoverOut = () => { - setDocsHoverColor(hoverOutColor); - }; - - const handleHomeHover = () => { - setHomeHoverColor(hoverColor); - }; - - const handleHelpHover = () => { - setHelpHoverColor(hoverColor); - }; - - const handleHelpHoverOut = () => { - setHelpHoverColor(hoverOutColor); - }; - - const handleSoarHover = () => { - setSoarHoverColor(hoverColor); - }; - - const handleSoarHoverOut = () => { - setSoarHoverColor(hoverOutColor); - }; - - const handleHomeHoverOut = () => { - setHomeHoverColor(hoverOutColor); - }; - - const handleLoginHover = () => { - setLoginHoverColor(hoverColor); - }; - - const handleLoginHoverOut = () => { - setLoginHoverColor(hoverOutColor); - }; - const notificationWidth = 335 const imagesize = 22; const boxColor = "#86c142"; - const NotificationItem = (props) => { - const { data } = props - - var image = ""; - var orgName = ""; - var orgId = ""; - - if (userdata.orgs !== undefined) { - const foundOrg = userdata.orgs.find((org) => org.id === data["org_id"]); - if (foundOrg !== undefined && foundOrg !== null) { - //position: "absolute", bottom: 5, right: -5, - const imageStyle = { - width: imagesize, - height: imagesize, - pointerEvents: "none", - marginLeft: - data.creator_org !== undefined && data.creator_org.length > 0 - ? 20 - : 0, - borderRadius: 10, - border: - foundOrg.id === userdata.active_org.id - ? `3px solid ${boxColor}` - : null, - cursor: "pointer", - marginRight: 10, - } - - image = - foundOrg.image === "" ? ( - {foundOrg.name} - ) : ( - {foundOrg.name} { }} - /> - ); - - orgName = foundOrg.name; - orgId = foundOrg.id; - } - } - - return ( - - {data.reference_url !== undefined && data.reference_url !== null && data.reference_url.length > 0 ? - - - {data.title} ({data.amount}) - - - : - - {data.title} - - } - - {data.image !== undefined && data.image !== null && data.image.length > 0 ? - {data.title} - : - null - } - - {data.description} - -
    - {data.read === false ? ( - - ) : null} - -
    { }} - > - {image} -
    -     -
    -     - ); - }; - - const notificationMenu = ( - - { - setAnchorEl(event.currentTarget); - }} - > - {/* n.read === false).length} color="primary">*/} - - - { - handleClose(); - }} - > - -
    - - Notifications ({notifications.filter((data) => !data.read).length}) - - - {notifications.length > 1 ? ( - - ) : null} - - -
    - - Notifications generated made by Shuffle to help you discover issues or - improvements. - Learn more - -     - {notifications.map((data, index) => { - if (data.read) { - return null - } - - return ; - })} -     -     - ); - const handleClickChangeOrg = (orgId) => { // Don't really care about the logout //name: org.name, @@ -487,39 +294,52 @@ const Header = (props) => { if (response.status !== 200) { console.log("Error in response"); } else { - localStorage.removeItem("apps") - localStorage.removeItem("workflows") - localStorage.removeItem("userinfo") + localStorage.removeItem("apps"); + localStorage.removeItem("workflows"); + localStorage.removeItem("userinfo"); } return response.json(); }) .then(function (responseJson) { - console.log("In here?") + console.log("In here?"); if (responseJson.success === true) { - if (responseJson.region_url !== undefined && responseJson.region_url !== null && responseJson.region_url.length > 0) { + if ( + responseJson.region_url !== undefined && + responseJson.region_url !== null && + responseJson.region_url.length > 0 + ) { console.log("Region Change: ", responseJson.region_url); localStorage.setItem("globalUrl", responseJson.region_url); //globalUrl = responseJson.region_url } + if (responseJson["reason"] === "SSO_REDIRECT") { - setTimeout(() => { - toast.info("Redirecting to SSO login page as SSO is required for this organization.") - window.location.href = responseJson["url"] - return - }, 2000) + toast.info("Redirecting to SSO login page as SSO is required for this organization.") + setTimeout(() => { + toast.info( + "Redirecting to SSO login page as SSO is required for this organization." + ); + window.location.href = responseJson["url"]; + return; + }, 2000); + } else { + toast("Successfully changed active organization - refreshing!"); + setTimeout(() => { + window.location.reload(); + }, 2000); } - - setTimeout(() => { - window.location.reload() - }, 2000); - - toast("Successfully changed active organization - refreshing!"); } else { - if (responseJson.reason !== undefined && responseJson.reason !== null && responseJson.reason.length > 0) { + if ( + responseJson.reason !== undefined && + responseJson.reason !== null && + responseJson.reason.length > 0 + ) { toast(responseJson.reason); } else { - toast("Failed changing org. Try again or contact support@shuffler.io if this persists."); + toast( + "Failed changing org. Try again or contact support@shuffler.io if this persists." + ); } } }) @@ -537,7 +357,7 @@ const Header = (props) => { rel="noopener noreferrer" target="_blank" > - + { aria-haspopup="true" onClick={(event) => { }} > - Discord Community Join + + {/*#f865f2*/} @@ -611,18 +428,31 @@ const Header = (props) => { - + + + { + handleClose(); + }} + > + Notifications ({ + notifications === undefined || notifications === null ? 0 : + notifications?.filter((notification) => notification.read === false).length + }) + + + { handleClose(); }} > - Notifications + Use Cases - {/*notificationMenu*/} + { @@ -632,36 +462,6 @@ const Header = (props) => { About - {/* - - { - handleClose(); - }} - > - Get Started - - - */} - - { - handleClose(); - }} - > - Use Cases - - - - { - handleClose(); - }} - > - Creator page - - - { @@ -675,7 +475,7 @@ const Header = (props) => { - Version: 1.4.0 + Version: 1.4.5 @@ -685,67 +485,68 @@ const Header = (props) => { textAlign: "center", marginTop: "auto", marginBottom: "auto", - marginRight: 10, + // marginRight: 10, }; - const modalView = - { - setModalOpen(false); - }} - PaperProps={{ - style: { - color: "white", - minWidth: 850, - minHeight: 370, - padding: 20, - backgroundColor: "rgba(0, 0, 0, 1)", - borderRadius: theme.palette.borderRadius, - }, - }} - > - - - Upgrade your plan - - { - if (isCloud) { + const modalView = + { + setModalOpen(false); + }} + PaperProps={{ + style: { + color: "white", + minWidth: 850, + minHeight: 370, + padding: 20, + backgroundColor: "rgba(0, 0, 0, 1)", + borderRadius: theme.palette?.borderRadius, + }, + }} + > + + + Upgrade your plan + + { + if (isCloud) { ReactGA.event({ category: "header", action: "close_Upgread_popup", label: "", - })}; - setModalOpen(false); - }} - style={{ - marginLeft: "auto", - position: "absolute", - top: 20, - right: 20, - }} - > - - - -
    - + + + +
    + -
    -
    + userdata={userdata} + stripeKey={stripeKey} + setModalOpen={setModalOpen} + {...props} + /> +
    + // Handle top bar or something const defaultTop = -2 @@ -787,13 +588,13 @@ const Header = (props) => { - + - + - {isCloud ? ( - - - - - - ) : null} - + + + {isCloud && ( + handleMenuItemClick('/pricing')}> + + Pricing + + + )} +
    + handleMenuItemClick('/professional-services')}> + + Professional Services + + +
    + handleMenuItemClick('/training')}> + + Training Courses + + +
    + handleMenuItemClick('/partners')}> + + Partner Program + + +
    +     + + + + {/* - +     */}     { margin: "auto", }} > -
    +
    {
    @@ -979,10 +796,9 @@ const Header = (props) => {
    {
    {
    {
    + {/* + + +
    + + Pricing & Services + +
    + +     + */}
    @@ -1069,7 +902,6 @@ const Header = (props) => { }} > {avatarMenu} - {/*notificationMenu*/} {supportMenu} {logoCheck} @@ -1094,7 +926,7 @@ const Header = (props) => { }, }} style={{ - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, backgroundColor: theme.palette.surfaceColor, marginRight: 15, color: "white", @@ -1222,7 +1054,7 @@ const Header = (props) => { { title={""} placement="left" > -
    +
    Add suborgs @@ -1280,14 +1112,17 @@ const Header = (props) => { marginRight: 7, marginTop: 0, }} + title={upgradeHovered ? "Upgrade License" : ""} + open={tooltipOpen} + onClose={handleTooltipClose} > @@ -1315,7 +1150,11 @@ const Header = (props) => { userdata.app_execution_usage === undefined || userdata.app_execution_usage < 1000 ? null : ( + App Runs used: {userdata.app_execution_usage} / {userdata.app_execution_limit}. When the limit is reached, you can still use Shuffle normally, but your Workflow triggers will stop workflows from starting. Reach out to support@shuffler.io to extend this limit. Customer workflows are NOT stopped this way. + + } >
    { padding: 8, textAlign: "center", cursor: "pointer", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, marginTop: 5, backgroundColor: theme.palette.surfaceColor, minWidth: 60, @@ -1392,9 +1231,9 @@ const Header = (props) => {
    @@ -1416,9 +1255,9 @@ const Header = (props) => {
    About
    @@ -1460,9 +1299,7 @@ const Header = (props) => {
    @@ -1507,10 +1344,10 @@ const Header = (props) => { >
    Logout
    @@ -1536,50 +1373,12 @@ const Header = (props) => { const topbarHeight = showTopbar ? 40 : 0 const topbar = !isCloud || !showTopbar ? null : - curpath === "/" || curpath.includes("/docs") || curpath === "/pricing" || curpath === "/contact" || curpath === "/search" || curpath === "/usecases" || curpath === "/training" ? + curpath === "/" || curpath.includes("/docs") || curpath === "/pricing" || curpath === "/contact" || curpath === "/search" || curpath === "/usecases" || curpath === "/usecases2" || curpath === "/training" || curpath === "/professional-services" ?
    - + {/* Shuffle 1.4.0 is out! Read more about  */} - Shuffle now offers  - {/* - { - ReactGA.event({ - category: "landingpage", - action: "click_header_features", - label: "", - }) - - //if (window.drift !== undefined) { - // window.drift.api.startInteraction({ interactionId: 341911 }) - //} else { - // console.log("Couldn't find drift in window.drift and not .drift-open-chat with querySelector: ", window.drift) - //} - }} style={{ cursor: "pointer", textDecoration: "none", color: "rgba(255,255,255,0.8)" }}> - Features - - - ,  - - { - ReactGA.event({ - category: "landingpage", - action: "click_header_pricing", - label: "", - }) - - navigate("/pricing") - - //if (window.drift !== undefined) { - // window.drift.api.startInteraction({ interactionId: 341911 }) - //} else { - // console.log("Couldn't find drift in window.drift and not .drift-open-chat with querySelector: ", window.drift) - //} - }} style={{ cursor: "pointer", textDecoration: "none", color: "rgba(255,255,255,0.8)" }}> - Pricing - - -  and  */} + New  { ReactGA.event({ @@ -1590,17 +1389,18 @@ const Header = (props) => { navigate("/training") - //if (window.drift !== undefined) { - // window.drift.api.startInteraction({ interactionId: 341911 }) - //} else { - // console.log("Couldn't find drift in window.drift and not .drift-open-chat with querySelector: ", window.drift) - //} - }} style={{ cursor: "pointer", textDecoration: "none", color: "rgba(255,255,255,0.8)" }}> - Public Training! + }} style={{ cursor: "pointer", textDecoration: "none", fontWeight: 600, color: "rgba(255,255,255,0.9)" }}> + Public Training +  Dates Released! - { setShowTopbar(false) }}> + { + setShowTopbar(false) + + // Set storage that it's clicked + localStorage.setItem(topbar_var, "true") + }}>
    diff --git a/frontend/src/components/Newsletter.jsx b/frontend/src/components/Newsletter.jsx index fa3b0fa9..402f687c 100644 --- a/frontend/src/components/Newsletter.jsx +++ b/frontend/src/components/Newsletter.jsx @@ -1,7 +1,7 @@ import React, {useState} from 'react'; -import { useTheme } from '@mui/styles'; import {isMobile} from "react-device-detect"; import ReactGA from 'react-ga4'; +import theme from '../theme.jsx'; import { TextField, @@ -12,7 +12,6 @@ import { const Newsletter = (props) => { const { globalUrl, } = props; - const theme = useTheme(); const [email, setEmail] = useState(""); const [msg, setMsg] = useState(""); const [buttonActive, setButtonActive] = useState(true); diff --git a/frontend/src/components/Oauth2Auth.jsx b/frontend/src/components/Oauth2Auth.jsx index 1ebf81dd..fb4319e6 100755 --- a/frontend/src/components/Oauth2Auth.jsx +++ b/frontend/src/components/Oauth2Auth.jsx @@ -96,7 +96,7 @@ const AuthenticationOauth2 = (props) => { autoAuth, authButtonOnly, isLoggedIn, - + org_id, setFinalized, } = props; @@ -148,16 +148,14 @@ const AuthenticationOauth2 = (props) => { navigate(`/login?view=${window.location.pathname}&message=Log in to authenticate this app`) } - console.log("Should automatically click the auto-auth button?: ", autoAuth) if (autoAuth === true && selectedApp !== undefined) { startOauth2Request() } }, []) - if (selectedApp.authentication === undefined) { - return null; - } - + if (selectedApp.authentication === undefined) { + return null; + } const startOauth2Request = (admin_consent) => { // Admin consent also means to add refresh tokens @@ -167,7 +165,7 @@ const AuthenticationOauth2 = (props) => { //console.log("APP: ", selectedApp) if (selectedApp.name.toLowerCase() == "outlook_graph" || selectedApp.name.toLowerCase() == "outlook_office365") { handleOauth2Request( - "efe4c3fe-84a1-4821-a84f-23a6cfe8e72d", + "fd55c175-aa30-4fa6-b303-09a29fb3f750", "", "https://graph.microsoft.com", ["Mail.ReadWrite", "Mail.Send", "offline_access"], @@ -299,12 +297,11 @@ const AuthenticationOauth2 = (props) => { const handleOauth2Request = (client_id, client_secret, oauth_url, scopes, admin_consent, prompt, skipScopeReplace) => { - console.log("SKIP SCOPE: ", skipScopeReplace) if (skipScopeReplace === false || skipScopeReplace === undefined) { console.log("Selected scopes: ", selectedScopes) if (selectedScopes !== undefined && selectedScopes !== null && selectedScopes.length > 0) { - toast("Using your scopes instead of the default ones") + //toast("Using your scopes instead of the default ones") scopes = selectedScopes } } @@ -453,6 +450,8 @@ const AuthenticationOauth2 = (props) => { if (orgId !== undefined && orgId !== null && orgId.length > 0) { console.log("Adding org_id from user side") state += `%26org_id%3d${orgId}`; + }else{ + state += `%26org_id%3d${org_id}` } if (oauth_url !== undefined && oauth_url !== null && oauth_url.length > 0) { @@ -461,42 +460,56 @@ const AuthenticationOauth2 = (props) => { } - if ( - authenticationType.refresh_uri !== undefined && - authenticationType.refresh_uri !== null && - authenticationType.refresh_uri.length > 0 - ) { - state += `%26refresh_uri%3d${authenticationType.refresh_uri}`; + if (authenticationType.refresh_uri !== undefined && authenticationType.refresh_uri !== null && authenticationType.refresh_uri.length > 0) { + state += `%26refresh_uri%3d${authenticationType.refresh_uri}` } else { - state += `%26refresh_uri%3d${authentication_url}`; + state += `%26refresh_uri%3d${authentication_url}` } - // No prompt forcing - //var url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=login&scope=${resources}&state=${state}&access_type=offline`; + if (workflow?.org_id !== undefined && workflow?.org_id !== null && workflow?.org_id.length > 0) { + state += `%26org_id%3d${workflow.org_id}` + } + + // FIXME: Should this be =consent? var defaultPrompt = "login" if (prompt !== undefined && prompt !== null && prompt.length > 0) { - defaultPrompt = prompt - } + defaultPrompt = prompt + } - var url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=${defaultPrompt}&scope=${resources}&state=${state}&access_type=offline`; + var url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=${defaultPrompt}&scope=${resources}&state=${state}&access_type=offline`; + if (admin_consent === true) { + console.log("Running Oauth2 WITH admin consent") + //url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=consent&scope=${resources}&state=${state}&access_type=offline`; + url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=admin_consent&scope=${resources}&state=${state}&access_type=offline`; + } + + if (url !== undefined && url !== null && url.length > 0) { + if (url.toLowerCase().includes("{tenant")) { + // Check location of {tenant, then find the next } and replace with 'common'. Make sure next } is AFTER {tenant + try { + const tenantIndex = url.toLowerCase().indexOf("{tenant") + const substring = url.substring(tenantIndex) + const nextBracket = substring.indexOf("}") + const newUrl = url.substring(0, tenantIndex) + "common" + url.substring(tenantIndex + nextBracket + 1) + url = newUrl + } catch (e) { + console.log("Failed to replace {tenant} with common: ", e) + } - if (admin_consent === true) { - console.log("Running Oauth2 WITH admin consent") - //url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=consent&scope=${resources}&state=${state}&access_type=offline`; - url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=admin_consent&scope=${resources}&state=${state}&access_type=offline`; } + } - // Force new consent + /* + console.log("OAUTH2 URL: ", url) + return + */ + + + // Force new consent //const url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&scope=${resources}&prompt=consent&state=${state}&access_type=offline`; - // Admin consent + // Admin consent //const url = `https://accounts.zoho.com/oauth/v2/auth?response_type=code&client_id=${client_id}&scope=AaaServer.profile.Read&redirect_uri=${redirectUri}&prompt=consent` - - // &resource=https%3A%2F%2Fgraph.microsoft.com& - - // FIXME: Awful, but works for prototyping - // How can we get a callback properly realtime? - // How can we properly try-catch without breaks on error? try { var newwin = window.open(url, "", "width=582,height=700"); //console.log(newwin) @@ -511,7 +524,12 @@ const AuthenticationOauth2 = (props) => { //alert('"Secure Payment" window closed!'); if (getAppAuthentication !== undefined) { - getAppAuthentication(true, true, true); + // This should be orgId, not action Id as to load auth properly + if (workflow !== undefined && workflow !== null && workflow.org_id !== undefined && workflow.org_id !== null && workflow.org_id.length > 0) { + getAppAuthentication(true, true, true, workflow.org_id) + } else { + getAppAuthentication(true, true, true) + } } toast("Authentication successful!") @@ -525,7 +543,7 @@ const AuthenticationOauth2 = (props) => { setFinalized(true) } } else { - console.log("Not closed") + //console.log("Not closed") } }, 1000); //do { @@ -678,7 +696,7 @@ const AuthenticationOauth2 = (props) => { justifyContent: "flex-start", backgroundColor: "#ffffff", color: "#2f2f2f", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, minWidth: 300, maxWidth: 300, maxHeight: 50, @@ -703,7 +721,7 @@ const AuthenticationOauth2 = (props) => { {selectedAction.app_name} @@ -726,7 +744,7 @@ const AuthenticationOauth2 = (props) => { - Oauth2 requires a client ID and secret to authenticate, defined in the remote system. {authenticationType.type === "oauth2-app" ? null : Your redirect URL is {window.location.origin}/set_authentication - } + Oauth2 requires a client ID and secret to authenticate, defined in the remote system. Your redirect URL is {window.location.origin}/set_authentication -  {
    - {isCloud && registeredApps.includes(selectedApp.name.toLowerCase()) ? + {isCloud && registeredApps?.includes(selectedApp?.name?.replaceAll(" ", "_").toLowerCase()) ? {autoAuthButton} @@ -785,7 +803,7 @@ const AuthenticationOauth2 = (props) => { : null} {/* { setOauthUrl(data.value); } - const defaultValue = data.name === "url" && authenticationType.token_uri !== undefined && authenticationType.token_uri !== null && authenticationType.token_uri.length > 0 && (authenticationType.authorizationUrl === undefined || authenticationType.authorizationUrl === null || authenticationType.authorizationUrl.length === 0) && authenticationType.type === "oauth2-app" ? authenticationType.token_uri : data.value === undefined || data.value === null ? "" : data.value + const isNormalOauth = authenticationType.redirect_uri !== undefined && authenticationType.redirect_uri !== null && authenticationType.redirect_uri.length > 0 + const defaultValue = !isNormalOauth && data.name === "url" && authenticationType.token_uri !== undefined && authenticationType.token_uri !== null && authenticationType.token_uri.length > 0 && (authenticationType.authorizationUrl === undefined || authenticationType.authorizationUrl === null || authenticationType.authorizationUrl.length === 0) && authenticationType.type === "oauth2-app" ? authenticationType.token_uri : data.value === undefined || data.value === null ? "" : data.value - const fieldname = data.name === "url" && authenticationType.grant_type !== undefined && authenticationType.grant_type !== null && authenticationType.grant_type.length > 0 && authenticationType.type === "oauth2-app" ? "Token URL" : data.name + const fieldname = !isNormalOauth && data.name === "url" && authenticationType.grant_type !== undefined && authenticationType.grant_type !== null && authenticationType.grant_type.length > 0 && authenticationType.type === "oauth2-app" ? "Token URL" : data.name return ( -
    +
    {fieldname} @@ -875,7 +894,7 @@ const AuthenticationOauth2 = (props) => { { style={{ marginTop: 20, backgroundColor: theme.palette.inputColor, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} InputProps={{ style: { @@ -924,7 +943,7 @@ const AuthenticationOauth2 = (props) => { { { { color: "white", padding: 5, minWidth: 300, - maxWidth: 300, }} onChange={(e, value) => { //handleScopeChange(e) @@ -1044,7 +1062,7 @@ const AuthenticationOauth2 = (props) => { style={{ marginBottom: 40, marginTop: 20, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} disabled={ clientSecret.length === 0 || clientId.length === 0 || buttonClicked || (allscopes.length !== 0 && selectedScopes.length === 0) @@ -1075,7 +1093,7 @@ const AuthenticationOauth2 = (props) => { + + +
    */} +
    + + + OpenID connect + + + + + Client ID + 0 + } + id="outlined-with-placeholder" + margin="normal" + variant="outlined" + placeholder="The OpenID client ID from the identity provider" + value={openidClientId} + onChange={(e) => { + setOpenidClientId(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + Client Secret (optional) + 0 + } + id="outlined-with-placeholder" + margin="normal" + variant="outlined" + placeholder="The OpenID client secret - DONT use this if dealing with implicit auth / PKCE" + value={openidClientSecret} + onChange={(e) => { + setOpenidClientSecret(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + + + Authorization URL + { + setOpenidAuthorization(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + Token URL + { + setOpenidToken(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + {/* } */} + {/*isCloud ? null : */} + + + SAML SSO (v1.1) + + + + + SSO Entrypoint (IdP) + 0 + } + id="outlined-with-placeholder" + margin="normal" + variant="outlined" + placeholder="The entrypoint URL from your provider" + value={ssoEntrypoint} + onChange={(e) => { + setSsoEntrypoint(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + SSO Certificate (X509) + { + setSsoCertificate(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + {isCloud ? ( + + IdP URL for Shuffle: https://shuffler.io/api/v1/login_sso + + ) : null} + + {isCloud ? null : ( + + + App Download URL + { + setAppDownloadUrl(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + )} + {isCloud ? null : ( + + + App Download Branch + { + setAppDownloadBranch(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + )} + {isCloud ? null : ( + + + Workflow Download URL + { + setWorkflowDownloadUrl(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + )} + {isCloud ? null : ( + + + Workflow Download Branch + { + setWorkflowDownloadBranch(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + )} + +
    + {orgSaveButton} +
    + {/* {expanded ? @@ -903,9 +1206,9 @@ const OrgHeaderexpanded = (props) => { } */} - -
    - ) -} + +
    + ); +}; export default OrgHeaderexpanded; diff --git a/frontend/src/components/PaperComponent.jsx b/frontend/src/components/PaperComponent.jsx index d1e2dc22..2b8f31f9 100644 --- a/frontend/src/components/PaperComponent.jsx +++ b/frontend/src/components/PaperComponent.jsx @@ -2,7 +2,7 @@ import React, {useState, useEffect, useLayoutEffect} from 'react'; import Draggable from "react-draggable"; import { - Paper + Paper } from "@mui/material"; const PaperComponent = (props) => { @@ -11,7 +11,9 @@ const PaperComponent = (props) => { handle="#draggable-dialog-title" cancel={'[class*="MuiDialogContent-root"]'} > - + ) } diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 29297082..5749edfa 100755 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -1,14 +1,15 @@ -import React, { useState, useEffect, useLayoutEffect } from "react"; +import React, { useState, useEffect, useLayoutEffect, useMemo } from "react"; import { toast } from 'react-toastify'; import { makeStyles, createStyles } from "@mui/styles"; import theme from '../theme.jsx'; - +import { useNavigate, Link, useParams } from "react-router-dom"; import { validateJson, GetIconInfo } from "../views/Workflows.jsx"; import { GetParsedPaths } from "../views/Apps.jsx"; import { sortByKey } from "../views/AngularWorkflow.jsx"; import { NestedMenuItem } from "mui-nested-menu"; import { parsedDatatypeImages } from "../components/AppFramework.jsx"; +import { green, yellow, red } from "../views/AngularWorkflow.jsx" //import { useAlert import { @@ -46,11 +47,13 @@ import { CircularProgress, Switch, Collapse, - Autocomplete + Autocomplete, + Box } from "@mui/material"; import { HelpOutline as HelpOutlineIcon, + OpenInFull as OpenInFullIcon, Description as DescriptionIcon, GetApp as GetAppIcon, Search as SearchIcon, @@ -88,12 +91,10 @@ import { Circle as CircleIcon, SquareFoot as SquareFootIcon, Storage as StorageIcon, + Check as CheckIcon, } from '@mui/icons-material'; -const useStyles = makeStyles({ - notchedOutline: { - borderColor: "#f85a3e !important", - }, +export const useStyles = makeStyles({ root: { "& .MuiAutocomplete-listbox": { border: "2px solid grey", @@ -166,22 +167,36 @@ const ParsedAction = (props) => { setExpansionModalOpen, listCache, - + setActiveDialog, + authGroups, apps, setEditorData, setcodedata, setAiQueryModalOpen, } = props; + let navigate = useNavigate(); const classes = useStyles(); - const [hideBody, setHideBody] = React.useState(true); - const [activateHidingBodyButton, setActivateHidingBodyButton] = React.useState(false); + const [hideBody, setHideBody] = React.useState(false) + const [activateHidingBodyButton, setActivateHidingBodyButton] = React.useState(false) + const [appActionName, setAppActionName] = React.useState(selectedAction?.label); + const [delay, setDelay] = React.useState(selectedAction?.execution_delay || 0); + const [prevActionName, setPrevActionName] = React.useState(selectedAction?.label); const [fieldCount, setFieldCount] = React.useState(0); const [hiddenDescription, setHiddenDescription] = React.useState(true); - + const [hiddenParameters, setHiddenParameters] = React.useState(true); const [autoCompleting, setAutocompleting] = React.useState(false); - + const [selectedActionParameters, setSelectedActionParameters] = React.useState(selectedAction?.parameters || []); + const [selectedVariableParameter, setSelectedVariableParameter] = React.useState(""); + const [paramUpdate, setParamUpdate] = React.useState(""); + const [actionlist, setActionlist] = React.useState([]); + const [jsonList, setJsonList] = React.useState([]); + const [showDropdown, setShowDropdown] = React.useState(false); + const [showDropdownNumber, setShowDropdownNumber] = React.useState(0); + const [showAutocomplete, setShowAutocomplete] = React.useState(false); + const [menuPosition, setMenuPosition] = useState(null); + const [uiBox, setUiBox] = useState(null); const isIntegration = selectedAction.app_id === "integration" useEffect(() => { @@ -190,27 +205,141 @@ const ParsedAction = (props) => { } }, [expansionModalOpen]) + useEffect(() => { - if (selectedAction.parameters === null || selectedAction.parameters === undefined) { - return - } + // Changes the order of params to show in order: + // auth, required, optional + var changed = false + if (selectedActionParameters === undefined || selectedActionParameters === null || selectedActionParameters.length === 0) { + return + } - const paramcheck = selectedAction.parameters.find(param => param.name === "body") - if (paramcheck === undefined || paramcheck === null) { - return - } - - // This was just opposite.. - if (paramcheck.id === "TOGGLED"){ - setHideBody(true) - } else { - setHideBody(false) - - if (paramcheck.id === "UNTOGGLED") { - setActivateHidingBodyButton(false) + // Fixing required fields with a shitty structure :) + if (selectedApp !== undefined && selectedApp !== null && selectedApp.generated === true && selectedAction !== undefined && selectedAction !== null && selectedAction.name !== undefined && selectedAction.name !== null && selectedApp.actions !== undefined && selectedApp.actions !== null && selectedApp.actions.length > 0 && (selectedAction.required_body_fields === undefined || selectedAction.required_body_fields === null || selectedAction.required_body_fields.length === 0)) { + // Check for required fields + for (var actionkey in selectedApp.actions) { + var action = selectedApp.actions[actionkey] + if (action.name === selectedAction.name) { + selectedAction.required_body_fields = action.required_body_fields + break + } } - } + } + // Check if missing parameters? + var auth = [] + var required = [] + var optional = [] + + var bodyfield = [] + var special_optional = [] + var generated_optional = [] + + var keyorder = [] + for (let paramkey in selectedActionParameters) { + var param = selectedActionParameters[paramkey] + keyorder.push(param.name) + + if (param.configuration) { + auth.push(param) + continue + } + + if (param?.value?.toLowerCase().includes("secret. replace")) { + param.value = "" + } + + + if (selectedApp?.generated === true && param?.name === "body") { + param.required = true + bodyfield.push(param) + continue + } + + if (param.required === false && param.name.startsWith("${") && param.name.endsWith("}")) { + // Check if it's a required param + param.autocompleted = false + if (selectedAction.required_body_fields !== undefined && selectedAction.required_body_fields !== null && selectedAction.required_body_fields.length > 0) { + if (selectedAction.required_body_fields.includes(param.name)) { + param.required = true + } + } + } + + if (param.required) { + required.push(param) + continue + } + + if (param.name === "headers" || param.name === "queries") { + special_optional.push(param) + continue + } + + if (hideBody && param?.description.includes("Generated")) { + continue + } + + if (param.field_active === true) { + generated_optional.push(param) + continue + } + + optional.push(param) + } + + // Sort order: auth > body(used for simple/advanced) > required > optional + // Optional field order: + // 1. headers & queries + // 2. other fields + // 3. generated fields & all else + + + const newparams = auth + .concat(bodyfield) + .concat(required) + .concat(special_optional) + .concat(generated_optional) + .concat(optional) + + var newkeyorder = [] + for (let paramkey in newparams) { + //console.log("Param: ", newparams[paramkey]) + + newkeyorder.push(newparams[paramkey].name) + } + + if (keyorder.join(",") !== newkeyorder.join(",")) { + //toast("KEYORDER CHANGED!") + + setSelectedActionParameters(newparams) + selectedAction.parameters = newparams + setSelectedAction(selectedAction) + } + }, [selectedActionParameters]) + + useEffect(() => { + const shouldHide = localStorage.getItem("hideBody") + if (shouldHide !== null) { + const ishiding = shouldHide !== "true" + if (ishiding !== hideBody) { + setHideBody(ishiding) + } + } + + if (selectedActionEnvironment === undefined || selectedActionEnvironment === null || Object.keys(selectedActionEnvironment).length === 0) { + + if (environments !== undefined && environments !== null && environments.length > 0) { + if (selectedAction.environment !== undefined && selectedAction.environment !== null) { + + const foundenv = environments.find(env => env.id === selectedAction.environment || selectedAction.environment === env.Name) + + if (foundenv !== undefined && foundenv !== null) { + setSelectedActionEnvironment(foundenv) + } + } + } + } }, []) const keywords = [ @@ -280,7 +409,6 @@ const ParsedAction = (props) => { (action) => action.name.toLowerCase() === selectedAction.name.toLowerCase() ); - console.log("FOUNDACTION: ", foundAction); if (foundAction !== null && foundAction !== undefined) { var foundparams = []; for (let [paramkey,paramkeyval] in Object.entries(foundAction.parameters)) { @@ -371,251 +499,259 @@ const ParsedAction = (props) => { //setStartNode(selectedAction.id) }; - const AppActionArguments = (props) => { - const [selectedActionParameters, setSelectedActionParameters] = React.useState([]); - const [selectedVariableParameter, setSelectedVariableParameter] = React.useState(""); - const [actionlist, setActionlist] = React.useState([]); - const [jsonList, setJsonList] = React.useState([]); - const [showDropdown, setShowDropdown] = React.useState(false); - const [showDropdownNumber, setShowDropdownNumber] = React.useState(0); - const [showAutocomplete, setShowAutocomplete] = React.useState(false); - const [menuPosition, setMenuPosition] = useState(null); useEffect(() => { - if (selectedActionParameters !== undefined && selectedActionParameters !== null && selectedActionParameters.length === 0 - ) { - if (selectedAction.parameters !== undefined && selectedAction.parameters !== null && selectedAction.parameters.length > 0) { - setSelectedActionParameters(selectedAction.parameters); - } - } + // Only set app action name if it has changed + if (selectedAction.label !== appActionName) { + setAppActionName(selectedAction.label); - if ((selectedVariableParameter === null || selectedVariableParameter === undefined) && workflow.workflow_variables !== null && workflow.workflow_variables.length > 0) { - - // FIXME - this is the bad thing - setSelectedVariableParameter(workflow.workflow_variables[0].name); - } - - if (actionlist.length === 0) { - // FIXME: Have previous execution values in here - if (workflowExecutions.length > 0) { - for (let [key,keyval] in Object.entries(workflowExecutions)) { - if ( - workflowExecutions[key].execution_argument === undefined || - workflowExecutions[key].execution_argument === null || - workflowExecutions[key].execution_argument.length === 0 - ) { - continue; - } - - const valid = validateJson(workflowExecutions[key].execution_argument) - if (valid.valid) { - actionlist.push({ - type: "Execution Argument", - name: "Execution Argument", - value: "$exec", - highlight: "exec", - autocomplete: "exec", - example: valid.result, - }) - break - } - } - - } - - if (actionlist.length === 0) { - actionlist.push({ - type: "Execution Argument", - name: "Execution Argument", - value: "$exec", - highlight: "exec", - autocomplete: "exec", - example: "", - }) - } - - /* - actionlist.push({ - type: "Shuffle DB", - name: "Shuffle DB", - value: "$shuffle_cache", - highlight: "shuffle_cache", - autocomplete: "shuffle_cache", - example: { - "what": "", - "unique gmail ids new": "", - }, - }) - */ - - var cachekey = { - type: "Shuffle DB", - name: "Shuffle DB", - value: "$shuffle_cache", - highlight: "shuffle_cache", - autocomplete: "shuffle_cache", - example: "", - } - - if (listCache !== undefined && listCache !== null && listCache.keys !== undefined && listCache.keys !== null && listCache.keys.length > 0) { - cachekey.example = {} - - for (var i in listCache.keys) { - const item = listCache.keys[i] - if (item.key === undefined || item.key === null || item.key.length === 0) { - continue - } - - var itemvalue = item.value === undefined || item.value === null ? "" : item.value - try{ - if (itemvalue.length > 10000) { - itemvalue = "" - } - - } catch (e) { - itemvalue = "" - } - - var itemkey = item.key.split(" ").join("_") - cachekey.example[itemkey] = { - "value": itemvalue, + const shouldHide = localStorage.getItem("hideBody") + if (shouldHide !== null) { + const ishiding = shouldHide !== "true" + if (ishiding !== hideBody) { + setHideBody(ishiding) } } - } else { } - actionlist.push(cachekey) + if(selectedAction.label !== prevActionName){ + setPrevActionName(selectedAction.label) + } + + // Only set delay if it has changed + const newDelay = selectedAction?.execution_delay || 0; + if (newDelay !== delay) { + setDelay(newDelay); + } + + // Only set selected action parameters if they have changed + if (selectedAction?.parameters?.length > 0 && selectedAction.label !== appActionName) { + //console.log("PARAMS CHANGED DURING APPCHANGE: ", selectedAction.parameters) + setSelectedActionParameters(selectedAction.parameters); + } + + // Only set selected variable parameter if it is null or undefined + if (!selectedVariableParameter && workflow.workflow_variables?.length > 0) { + setSelectedVariableParameter(workflow.workflow_variables[0].name); + } + },[selectedAction,selectedApp,setNewSelectedAction,workflow, workflowExecutions, getParents]) - if (workflow.workflow_variables !== null && workflow.workflow_variables !== undefined && workflow.workflow_variables.length > 0) { - for (let [key,keyval] in Object.entries(workflow.workflow_variables)) { - const item = workflow.workflow_variables[key]; - actionlist.push({ - type: "workflow_variable", - name: item.name, - value: item.value, - id: item.id, - autocomplete: `${item.name.split(" ").join("_")}`, - example: item.value, - }); - } + useEffect(() => { + const newActionList = []; + const parentActionList = []; + + // Process workflowExecutions + if (workflowExecutions.length > 0) { + for (let execution of workflowExecutions) { + const execArg = execution.execution_argument; + if (execArg && execArg.length > 0) { + const valid = validateJson(execArg); + if (valid.valid) { + newActionList.push({ + type: "Execution Argument", + name: "Execution Argument", + value: "$exec", + highlight: "exec", + autocomplete: "exec", + example: valid.result, + }) + + break + } + } + } } - if (workflow.execution_variables !== null && workflow.execution_variables !== undefined && workflow.execution_variables.length > 0) { - for (let [key,keyval] in Object.entries(workflow.execution_variables)) { - const item = workflow.execution_variables[key] + // Add default Execution Argument if none were added + if (newActionList.length === 0) { + newActionList.push({ + type: "Execution Argument", + name: "Execution Argument", + value: "$exec", + highlight: "exec", + autocomplete: "exec", + example: "", + }) + } - var exampleoutput = "" - for (let execkey in workflowExecutions) { - const exec = workflowExecutions[execkey] - if (exec["execution_variables"] === undefined || exec["execution_variables"] === null) { - continue - } + // Look for cachekey + if (newActionList.find((item) => item.type === "Shuffle DB") === undefined) { + let cacheKey = { + type: "Shuffle DB", + name: "Shuffle DB", + value: "$shuffle_cache", + highlight: "shuffle_cache", + autocomplete: "shuffle_cache", + example: "", + }; - const foundExec = exec.execution_variables.find((exvar) => exvar.name === item.name) - if (!foundExec) { - continue - } - - if (foundExec.value !== undefined && foundExec.value !== null && foundExec.value.length > 0) { - exampleoutput = foundExec.value - break - } - } - - actionlist.push({ - type: "execution_variable", - name: item.name, - value: item.value, - id: item.id, - autocomplete: `${item.name.split(" ").join("_")}`, - example: exampleoutput, - }); - } - } - - // Loops parent nodes' old results to fix autocomplete - if (getParents !== undefined) { - var parents = getParents(selectedAction) - - if (parents.length > 1) { - var labels = [] - //for (let [parentkey, parentkeyval] in Object.entries(parents)) { - for (let parentkey in parents) { - const parentNode = parents[parentkey] - if (parentNode.label === "Execution Argument") { - continue - } - - //if (labels.includes(item.label)) { - // continue - //} - - labels.push(parentNode.label) - - var exampledata = parentNode.example === undefined || parentNode.example === null ? "" : parentNode.example - // Find previous execution and their variables - //exampledata === "" && - if (workflowExecutions.length > 0) { - // Look for the ID - const found = false; - for (let wfkey in workflowExecutions) { - if (workflowExecutions[wfkey].results === undefined || workflowExecutions[wfkey].results === null) { - - continue; - } - - var foundResult = workflowExecutions[wfkey].results.find((result) => result.action.id === parentNode.id) - - if (foundResult === undefined || foundResult === null) { - continue - } - - if (foundResult.result !== undefined && foundResult.result !== null) { - foundResult = foundResult.result - } - - const valid = validateJson(foundResult) - if (valid.valid) { - if (valid.result.success === false) { - //console.log("Skipping success false autocomplete") - } else { - - // FIXME: Have a merge system to allow to use kind of any key from that node in the last 10-20 execs - //if (exampledata.length > 0) { - // exampledata = valid.result - //} else { - // exampledata = valid.result - //} - - exampledata = valid.result - break + if (listCache?.keys?.length > 0) { + cacheKey.example = {}; + for (let item of listCache.keys) { + if (item.key) { + let itemValue = item.value ?? ""; + if (itemValue.length > 10000) { + itemValue = ""; } - } else { - exampledata = foundResult + cacheKey.example[item.key.split(" ").join("_")] = { value: itemValue }; } - } - } + } + } - // 1. Take - const itemlabelComplete = parentNode.label === null || parentNode.label === undefined ? "" : parentNode.label.split(" ").join("_"); - - const actionvalue = { - type: "action", - id: parentNode.id, - name: parentNode.label, - autocomplete: itemlabelComplete, - example: exampledata, - } - - actionlist.push(actionvalue) - } - } - - setActionlist(actionlist); + newActionList.push(cacheKey); } - } - }); + // Process workflow variables + if (workflow.workflow_variables?.length > 0) { + for (let variable of workflow.workflow_variables) { + newActionList.push({ + type: "workflow_variable", + name: variable.name, + value: variable.value, + id: variable.id, + autocomplete: variable.name.split(" ").join("_"), + example: variable.value, + }); + } + } + + // Process execution variables + if (workflow.execution_variables?.length > 0) { + for (let variable of workflow.execution_variables) { + let exampleOutput = ""; + for (let exec of workflowExecutions) { + const foundExec = exec.execution_variables?.find(exvar => exvar.name === variable.name); + if (foundExec?.value) { + exampleOutput = foundExec.value; + break; + } + } + newActionList.push({ + type: "execution_variable", + name: variable.name, + value: variable.value, + id: variable.id, + autocomplete: variable.name.split(" ").join("_"), + example: exampleOutput, + }); + } + } + + // Process parent actions if getParents is provided + if (getParents) { + const parents = getParents(selectedAction); + if (parents.length > 1) { + const labels = []; + for (let parentNode of parents) { + if (parentNode.label !== "Execution Argument" && !labels.includes(parentNode.label)) { + labels.push(parentNode.label); + let exampleData = parentNode.example ?? ""; + if (!exampleData && workflowExecutions.length > 0) { + for (let exec of workflowExecutions) { + const foundResult = exec.results?.find(result => result.action.id === parentNode.id); + if (foundResult) { + const valid = validateJson(foundResult.result); + if (valid.valid && valid.result.success !== false) { + exampleData = valid.result; + break; + } + } + } + } + + if (parentNode.label === undefined) { + parentNode.label = "" + } + + newActionList.push({ + type: "action", + id: parentNode.id, + name: parentNode.label, + autocomplete: parentNode.label.split(" ").join("_"), + example: exampleData, + }); + + parentActionList.push({ + type: "action", + id: parentNode.id, + name: parentNode.label, + autocomplete: parentNode.label.split(" ").join("_"), + example: exampleData, + }); + + + } + } + } + } + + let newParameters = selectedAction?.parameters?.map((param) => { + let paramvalue = param.value === undefined || param.value === null ? "" : param.value; + let errorVars = []; + + if(paramvalue.includes("$")){ + let actions = workflow.actions?.map((action) => { + return "$"+action.label?.toLowerCase(); + }) + + if(newActionList?.length > 0){ + let appParentActions = parentActionList?.map(action => "$" + action.name.toLowerCase()); + let notPresentAction = actions?.filter((action) => !appParentActions?.includes(action)) + notPresentAction?.forEach((action) => { + action = action.replace(" ", "_"); + if(paramvalue.includes(action)){ + errorVars.push(action); + // paramvalue = paramvalue.replace(action, "") + // paramvalue = paramvalue.replace(/^\s*[\r\n]/gm, ""); + } + }) + } + } + + let message = ""; + if(errorVars.length > 0){ + if(errorVars.length === 1){ + message = errorVars[0] + " is not accessible in this action."; + }else{ + message = errorVars.join(", ") + " are not accessible in this action."; + } + } + + if (param?.configuration) { + let regex = /(^|[^\\])\$/; + if (regex.test(paramvalue)) { + if(message.length > 0){ + message += "\nUse \"\\$\" instead of \"$\" if you want to escape $ (1)"; + }else{ + message = "Use \"\\$\" instead of \"$\" if you want to escape $ (2)"; + } + } + } + return {...param, value: paramvalue, error: message} + }); + setSelectedActionParameters(newParameters); + setActionlist(newActionList); + }, [workflow.execution_variables, paramUpdate, workflow.workflow_variables, workflowExecutions, workflow, selectedAction, listCache, getParents,setNewSelectedAction]); + + useEffect(() => { + selectedNameChange(appActionName) + + if (actionDelayChange !== undefined) { + actionDelayChange(delay) + } + },[appActionName,delay]) + + const handleParamChange = (event, count,data) => { + const newParams = [...selectedActionParameters]; + newParams.map((param) => { + if (param.name === data.name) { + param.value = event.target.value; + } + }) + setSelectedActionParameters(newParams); + setParamUpdate(event.target.value); + changeActionParameter(event, count, data) + } const calculateHelpertext = (input_data) => { var helperText = "" @@ -682,7 +818,7 @@ const ParsedAction = (props) => { } const changeActionParameter = (event, count, data, viewForceUpdate) => { - //console.log("Action change: ", selectedAction, data) + //console.log("Action change: ", selectedAction, data) if (data.name.startsWith("${") && data.name.endsWith("}")) { // PARAM FIX - Gonna use the ID field, even though it's a hack const paramcheck = selectedAction.parameters.find((param) => param.name === "body"); @@ -692,9 +828,9 @@ const ParsedAction = (props) => { var toReplace = event.target.value.trim() - if (!toReplace.startsWith("{") && !toReplace.startsWith("[")) { - toReplace = toReplace.replaceAll('\\"', '"').replaceAll('"', '\\"') - } + if (!toReplace.startsWith("{") && !toReplace.startsWith("[")) { + toReplace = toReplace.replaceAll('\\"', '"').replaceAll('"', '\\"') + } console.log("REPLACE WITH: ", toReplace); if ( @@ -708,7 +844,6 @@ const ParsedAction = (props) => { }, ]; - console.log("IN IF: ", paramcheck); } else { const subparamindex = paramcheck["value_replace"].findIndex( (param) => param.key === data.name @@ -721,25 +856,27 @@ const ParsedAction = (props) => { } else { paramcheck["value_replace"][subparamindex]["value"] = toReplace; } - - console.log("IN ELSE: ", paramcheck); } - //console.log("PARAM: ", paramcheck) - //if (paramcheck.id === undefined) { - // console.log("Normal paramcheck") - //} else { - // selectedActionParameters[count]["value_replace"] = paramcheck - // selectedAction.parameters[count]["value_replace"] = paramcheck - //} + + if (selectedActionParameters[count].value_replace === undefined) { + selectedActionParameters[count].value_replace = paramcheck + } + + if (selectedAction?.parameters[count] !== undefined && selectedAction?.parameters[count].value_replace === undefined) { + selectedAction.parameters[count].value_replace = paramcheck + } if (paramcheck["value_replace"] === undefined) { - selectedActionParameters[count]["value_replace"] = paramcheck; - selectedAction.parameters[count]["value_replace"] = paramcheck; + selectedActionParameters[count]["value_replace"] = paramcheck + + if (selectedAction?.parameters[count] !== undefined) { + selectedAction.parameters[count]["value_replace"] = paramcheck + } } else { - selectedActionParameters[count]["value_replace"] = - paramcheck["value_replace"]; - selectedAction.parameters[count]["value_replace"] = - paramcheck["value_replace"]; + selectedActionParameters[count]["value_replace"] = paramcheck["value_replace"]; + if (selectedAction?.parameters[count] !== undefined) { + selectedAction.parameters[count]["value_replace"] = paramcheck["value_replace"]; + } } setSelectedAction(selectedAction); //setUpdate(Math.random()) @@ -840,41 +977,58 @@ const ParsedAction = (props) => { } } - //console.log("CHANGING ACTION COUNT !") - selectedActionParameters[count].autocompleted = false - selectedAction.parameters[count].autocompleted = false - selectedActionParameters[count].value = event.target.value; - selectedAction.parameters[count].value = event.target.value; + selectedActionParameters[count].autocompleted = false + selectedAction.parameters[count].autocompleted = false + selectedActionParameters[count].value = event.target.value; + selectedAction.parameters[count].value = event.target.value; - var forceUpdate = false - if (isCloud && (selectedAction.app_name === "Shuffle Tools" || selectedAction.app_name === "email") && (selectedAction.name === "send_email_shuffle" || selectedAction.name === "send_sms_shuffle") && data.name === "apikey") { - console.log("APIKEY - this shouldn't show up!") - } + var forceUpdate = false + if (isCloud && (selectedAction.app_name === "Shuffle Tools" || selectedAction.app_name === "email") && (selectedAction.name === "send_email_shuffle" || selectedAction.name === "send_sms_shuffle") && data.name === "apikey") { + console.log("APIKEY - this shouldn't show up!") + } - if (selectedAction.app_name === "Shuffle Tools" && selectedAction.name === "filter_list" && data.name === "input_list") { - //console.log("FILTER LIST!: ", event, count, data) - const parsedvalue = event.target.value - if (parsedvalue.includes("#")) { - const splitparsed = parsedvalue.split(".#.") - //console.log("Cant contain #: ", splitparsed) - if (splitparsed.length > 1) { - data.value = splitparsed[0] + if (selectedAction.app_name === "Shuffle Tools" && selectedAction.name === "filter_list" && data.name === "input_list") { + //console.log("FILTER LIST!: ", event, count, data) + const parsedvalue = event.target.value + if (parsedvalue.includes(".#")) { + const splitparsed = parsedvalue.split(".#.") + //console.log("Cant contain #: ", splitparsed) + if (splitparsed.length > 1) { + data.value = splitparsed[0] - selectedActionParameters[count].value = splitparsed[0] - selectedAction.parameters[count].value = splitparsed[0] + selectedActionParameters[count].value = splitparsed[0] + selectedAction.parameters[count].value = splitparsed[0] - selectedActionParameters[1].value = splitparsed[1] - selectedAction.parameters[1].value = splitparsed[1] - forceUpdate = true - } + selectedActionParameters[1].value = splitparsed[1] + selectedAction.parameters[1].value = splitparsed[1] + } else { + // Remove .# and after + const splitparsed = parsedvalue.split(".#") + data.value = splitparsed[0] + selectedActionParameters[0].value = splitparsed[0] + selectedAction.parameters[0].value = splitparsed[0] + + selectedActionParameters[1].value = "" + selectedAction.parameters[1].value = "" + + toast.warn("No value found in the list. Please select an item in the list to filter based on.") } - } - setSelectedAction(selectedAction); - if (forceUpdate || viewForceUpdate === true) { - setUpdate(Math.random()) + forceUpdate = true + selectedActionParameters[0].autocompleted = true + selectedAction.parameters[0].autocompleted = true + selectedActionParameters[1].autocompleted = true + selectedAction.parameters[1].autocompleted = true } - //setUpdate(event.target.value) + } + + setSelectedAction(selectedAction) + if (forceUpdate || viewForceUpdate === true) { + setUpdate(Math.random()) + } + + //console.log("END OF THIS THING") + //setUpdate(event.target.value) }; @@ -904,8 +1058,6 @@ const ParsedAction = (props) => { } else { paramcheck["value_replace"][subparamindex]["value"] = toReplace } - - console.log("IN ELSE: ", paramcheck) } //console.log("PARAM: ", paramcheck) //if (paramcheck.id === undefined) { @@ -941,6 +1093,7 @@ const ParsedAction = (props) => { } } + // bad detection mechanism probably if (event.target.value[event.target.value.length-1] === "." && actionlist.length > 0) { console.log("GET THE LAST ARGUMENT FOR NODE!") @@ -1014,10 +1167,12 @@ const ParsedAction = (props) => { } } + setTimeout(() => { selectedActionParameters[count].autocompleted = false selectedAction.parameters[count].autocompleted = false selectedActionParameters[count].value = data selectedAction.parameters[count].value = data + }, 100); setSelectedAction(selectedAction) //setUpdate(Math.random()) //setUpdate(event.target.value) @@ -1111,7 +1266,7 @@ const ParsedAction = (props) => { var helperText = "" if (name.includes("url")) { if (value.includes("localhost") || value.includes("127.0.0.1")) { - helperText = "Can't use localhost. Please change to your external IP." + helperText = "Can't use localhost in Shuffle. Please change to server's IP." } } @@ -1123,10 +1278,77 @@ const ParsedAction = (props) => { return helperText } + const errorHelperText = (name, value, error) => { + return ( +
    + {error} +
    + ); + } + + + const analyzeFields = () => { + + if (selectedAction === undefined || selectedAction === null) { + return null + } + + if (selectedActionParameters === undefined || selectedActionParameters === null || selectedActionParameters.length === 0) { + return null + } + + // Only shuffle tools for now + if (selectedAction.app_name !== "Shuffle Tools") { + return null + } + + // Custom rules + if (selectedAction.name === "set_cache_value") { + var actionKey = "" + var actionValue = "" + for (let [key,keyval] in Object.entries(selectedActionParameters)) { + const param = selectedActionParameters[key] + if (param.name === "key") { + actionKey = param.value + } + + if (param.name === "value") { + actionValue = param.value + } + } + + if (actionKey === "" || actionValue === "") { + return null + } + + if (!actionKey.includes(".#") && actionValue.includes(".#")) { + return When the key ({actionKey}) is static, but the value is a list ({actionValue}), it will overwrite the list. You may be looking for the {}} style={{cursor: "pointer", color: "#FF8544", }}>Check Cache Contains action instead. + } + } + + return null + + } + + const suggestionInfo = () => { + const suggestionText = analyzeFields() + if (suggestionText === undefined || suggestionText === null) { + return null + } + + if (selectedAction.errors === undefined || selectedAction.errors === null || selectedAction.errors.length === 0) { + selectedAction.errors = ["Suggestion: " + suggestionText] + } + + return + + Tip: {suggestionText} + + + } // FIXME: Issue #40 - selectedActionParameters not reset - if (Object.getOwnPropertyNames(selectedAction).length > 0 && selectedActionParameters.length > 0) { - + if (Object.getOwnPropertyNames(selectedAction)?.length > 0 && selectedActionParameters?.length > 0) { var wrapperapp = { "id": "", "name": "noapp", @@ -1150,15 +1372,1186 @@ const ParsedAction = (props) => { } } - var authWritten = false; + var authWritten = false; var noAppSelected = false - const paramIndex = selectedAction.parameters.findIndex((param) => param.name === "app_name") - if (paramIndex === -1 || selectedAction.parameters[paramIndex].value === "" || selectedAction.parameters[paramIndex].value === "noapp") { - // Check the actual value and if it's the same - noAppSelected = true + if (selectedAction.parameters !== undefined && selectedAction.parameters !== null && selectedAction.parameters.length > 0) { + var paramIndex = selectedAction.parameters.findIndex((param) => param.name === "app_name") + if (paramIndex === -1 || selectedAction.parameters[paramIndex].value === "" || selectedAction.parameters[paramIndex].value === "noapp") { + // Check the actual value and if it's the same + noAppSelected = true + } } - return ( -
    + } + + + const ActionSelectOption = (actionprops) => { + const { option, newActionname, newActiondescription, useIcon, extraDescription, } = actionprops; + const [hover, setHover] = React.useState(false); + + return ( + +
    setHover(true)} onMouseLeave={() => setHover(false)} + onClick={(event) => { + // event.preventDefault() + //setSelectedAction(actionprops) + //setShowActionList(false) + //setUpdate(Math.random()) + // + if (option !== undefined && option !== null) { + setNewSelectedAction({ + target: { + value: option.name + } + }); + } + + document.activeElement.blur(); + + const disabledUiBox = localStorage.getItem("disabled_ui_box") + if (disabledUiBox === "true") { + } else { + setHiddenDescription(false) + } + }} + > +
    + + {useIcon} + + {newActionname} +
    + {extraDescription.length > 0 ? + + {extraDescription} + + : null} +
    +     + ) + } + + const sortByCategoryLabel = (a, b) => { + const aHasCategoryLabel = a.category_label !== undefined && a.category_label !== null && a.category_label.length > 0 + const bHasCategoryLabel = b.category_label !== undefined && b.category_label !== null && b.category_label.length > 0 + + // Sort by existence and length of "category_label" + if (aHasCategoryLabel && !bHasCategoryLabel) { + return -1 + } else if (!aHasCategoryLabel && bHasCategoryLabel) { + return 1 + } else { + return 0 + } + } + + // Function to deduplicate based on the "name" field + const deduplicateByName = (array) => { + const uniqueNames = {}; + return array.filter(item => { + if (!item.hasOwnProperty('name') || !item.name.length) { + return true + } + if (!uniqueNames[item.name]) { + uniqueNames[item.name] = true + return true + } + return false + }) + } + + // Gets the most important actions first + const renderedActionOptions = deduplicateByName(( + selectedApp.actions === undefined || selectedApp.actions === null ? [] : + selectedApp.actions.filter((a) => + a.category_label !== undefined && a.category_label !== null && a.category_label.length > 0).concat(sortByKey(selectedApp.actions, "label")) + ).sort(sortByCategoryLabel)) + + + const selectedAppIcon = selectedAction.large_image + return ( +
    + + {hideExtraTypes === true ? null : ( + +
    +
    +
    { + //window.open("/apps/${selectedAction.app_id}", "_blank") + }} + > + + + +

    + {( + selectedAction?.app_name?.charAt(0).toUpperCase() + + selectedAction?.app_name?.substring(1) + )?.replaceAll("_", " ")} +

    +
    +
    + { + if (workflowExecutions.length > 0) { + // Look for the ID + var found = false; + for (let [key,keyval] in Object.entries(workflowExecutions)) { + if (workflowExecutions[key].results === undefined || workflowExecutions[key].results === null) { + continue; + } + + var foundResult = workflowExecutions[key].results.find( + (result) => result.action.id === selectedAction.id + ) + + if (foundResult === undefined || foundResult === null) { + continue + } + + const oldstartnode = cy.getElementById(selectedAction.id); + if (oldstartnode !== undefined && oldstartnode !== null) { + const foundname = oldstartnode.data("label") + if (foundname !== undefined && foundname !== null) { + foundResult.action.label = foundname + } + } + + setSelectedResult(foundResult); + if (setCodeModalOpen !== undefined) { + setCodeModalOpen(true) + + found = true + } + + break + } + + if (!found) { + toast("No result for this action yet. Please run the workflow first.") + } + } + }} + > + + + + + { + setAuthenticationModalOpen(true) + }} + > + + + + + + { + if (setAiQueryModalOpen !== undefined) { + setAiQueryModalOpen(true) + } else { + aiSubmit("Fill based on previous values", undefined, undefined, selectedAction) + } + + setAutocompleting(true) + setTimeout(() => { + setAutocompleting(false) + }, 3000) + }} + > + + {autoCompleting ? + + : + + } + + +
    +
    +
    + {selectedApp.versions !== null && + selectedApp.versions !== undefined && + selectedApp.versions.length > 1 ? ( + + ) : null} +
    +
    +
    +
    + Name + { + let newValue = event.target.value + newValue = newValue.replaceAll(" ", "_") + setAppActionName(newValue) + } + } + onBlur={(e) => { + // Copy the name value + const name = e.target.value + const parsedBaseLabel = "$"+prevActionName.toLowerCase().replaceAll(" ", "_") + const newname = "$"+name.toLowerCase().replaceAll(" ", "_") + + // Check if it's the same as the current name in use + //if (name === selectedAction.label) { + // console.log("Returning from name thing") + // return + //} + + // Change in actions, triggers & conditions + // Highlight the changes somehow with a glow? + if (workflow.branches !== undefined && workflow.branches !== null) { + for (let [key,keyval] in Object.entries(workflow.branches)) { + if (workflow.branches[key].conditions !== undefined && workflow.branches[key].conditions !== null) { + for (let [subkey,subkeyval] in Object.entries(workflow.branches[key].conditions)) { + const condition = workflow.branches[key].conditions[subkey] + const sourceparam = condition.source + const destinationparam = condition.destination + + // Should have a smarter way of discovering node names + // Finding index(es) and replacing at the location + if (sourceparam.value.includes("$")) { + try { + var cnt = -1 + var previous = 0 + while (true) { + cnt += 1 + // Need to make sure e.g. changing the first here doesn't change the 2nd + // $change_me + // $change_me_2 + + const foundindex = sourceparam.value.toLowerCase().indexOf(parsedBaseLabel, previous) + if (foundindex === previous && foundindex !== 0) { + break + } + + if (foundindex >= 0) { + previous = foundindex+newname.length + // Need to add diff of length to word + + // Check location: + // If it's a-zA-Z_ then don't replace + if (sourceparam.value.length > foundindex+parsedBaseLabel.length) { + const regex = /[a-zA-Z0-9_]/g; + const match = sourceparam.value[foundindex+parsedBaseLabel.length].match(regex); + if (match !== null) { + continue + } + } + + console.log("Old found: ", workflow.branches[key].conditions[subkey].source.value) + const extralength = newname.length-parsedBaseLabel.length + sourceparam.value = sourceparam.value.substring(0, foundindex) + newname + sourceparam.value.substring(foundindex-extralength+newname.length, sourceparam.value.length) + + console.log("New: ", workflow.branches[key].conditions[subkey].source.value) + } else { + break + } + + // Break no matter what after 5 replaces. May need to increase + if (cnt >= 5) { + break + } + + } + } catch (e) { + console.log("Failed value replacement based on index: ", e) + } + } + + if (destinationparam.value.includes("$")) { + try { + var cnt = -1 + var previous = 0 + while (true) { + cnt += 1 + // Need to make sure e.g. changing the first here doesn't change the 2nd + // $change_me + // $change_me_2 + + const foundindex = destinationparam.value.toLowerCase().indexOf(parsedBaseLabel, previous) + if (foundindex === previous && foundindex !== 0) { + break + } + + if (foundindex >= 0) { + previous = foundindex+newname.length + // Need to add diff of length to word + + // Check location: + // If it's a-zA-Z_ then don't replace + if (destinationparam.value.length > foundindex+parsedBaseLabel.length) { + const regex = /[a-zA-Z0-9_]/g; + const match = destinationparam.value[foundindex+parsedBaseLabel.length].match(regex); + if (match !== null) { + continue + } + } + + console.log("Old found: ", workflow.branches[key].conditions[subkey].destination.value) + const extralength = newname.length-parsedBaseLabel.length + destinationparam.value = destinationparam.value.substring(0, foundindex) + newname + destinationparam.value.substring(foundindex-extralength+newname.length, destinationparam.value.length) + + console.log("New: ", workflow.branches[key].conditions[subkey].destination.value) + } else { + break + } + + // Break no matter what after 5 replaces. May need to increase + if (cnt >= 5) { + break + } + + } + } catch (e) { + console.log("Failed value replacement based on index: ", e) + } + } + } + } + } + } + + for (let [key,keyval] in Object.entries(workflow.actions)) { + if (workflow.actions[key].id === selectedAction.id) { + continue + } + + const params = workflow.actions[key].parameters + if (params === null || params === undefined) { + continue + } + + for (let [subkey, subkeyval] in Object.entries(params)) { + const param = workflow.actions[key].parameters[subkey]; + if (!param.value.includes("$")) { + continue + } + + // Should have a smarter way of discovering node names + // Do regex? + // Finding index(es) and replacing at the location + // + + try { + var cnt = -1 + var previous = 0 + while (true) { + cnt += 1 + // Need to make sure e.g. changing the first here doesn't change the 2nd + // $change_me + // $change_me_2 + const foundindex = param.value.toLowerCase().indexOf(parsedBaseLabel, previous) + if (foundindex === previous && foundindex !== 0) { + break + } + + if (foundindex >= 0) { + previous = foundindex+newname.length + // Need to add diff of length to word + + // Check location: + // If it's a-zA-Z_ then don't replace + if (param.value.length > foundindex+parsedBaseLabel.length) { + const regex = /[a-zA-Z0-9_]/g; + const match = param.value[foundindex+parsedBaseLabel.length].match(regex); + if (match !== null) { + continue + } + } + + const extralength = newname.length-parsedBaseLabel.length + param.value = param.value.substring(0, foundindex) + newname + param.value.substring(foundindex-extralength+newname.length, param.value.length) + + } else { + break + } + + // Break no matter what after 5 replaces. May need to increase + if (cnt >= 5) { + break + } + + } + } catch (e) { + console.log("Failed value replacement based on index: ", e) + } + } + } + + setWorkflow(workflow); + setUpdate(Math.random()); + setPrevActionName(name) + }} + /> +
    + {/*!isCloud ? null :*/} +
    + + + Delay + { + setDelay(event.target.value) + }} + /> + + +
    + {/**/} +
    +     + )} + + {selectedApp.name !== undefined && + selectedAction.authentication !== null && + selectedAction.authentication !== undefined && + selectedAction.authentication.length === 0 && + requiresAuthentication ? ( +
    + + + + + +
    + ) : null} + + {selectedAction.authentication !== undefined && + selectedAction.authentication !== null && + selectedAction.authentication.length > 0 ? ( +
    + Authentication +
    + + + + { + setAuthenticationModalOpen(true); + }} + > + + + +
    +
    + ) : null} + + {selectedAction.authentication_id === "authgroups" && (authGroups === undefined || authGroups === null || authGroups.length === 0) ? +     + + Create your first Authentication group + + + : null} + + + {/*showEnvironment !== undefined && showEnvironment && environments.length > 1 && !isIntegration ? ( +
    + Environment + + +
    + ) : null*/} + + {workflow.execution_variables !== undefined && workflow.execution_variables !== null && workflow.execution_variables.length > 0 ? ( +
    + Execution variable (optional) + +
    + ) : null} + + +
    + {/*hideExtraTypes ? null : +
    + Actions +
    + */} + + {setNewSelectedAction !== undefined ? ( + { + // FIXME: Sorting + // Most popular + // Is categorized + // Uncategorized + return option.category_label !== undefined && option.category_label !== null && option.category_label.length > 0 ? "Most used" : "All Actions"; + }} + renderGroup={(params) => { + + return ( +
  • + {params.group} + {params.children} +
    • + ) + }} + options={renderedActionOptions} + ListboxProps={{ + style: { + backgroundColor: theme.palette.surfaceColor, + color: "white", + }, + }} + filterOptions={(options, { inputValue }) => { + const lowercaseValue = inputValue.toLowerCase() + options = options.filter(x => x.name.replaceAll("_", " ").toLowerCase().includes(lowercaseValue) || x.description.toLowerCase().includes(lowercaseValue)) + + return options + }} + getOptionLabel={(option) => { + if (option === undefined || option === null || option.name === undefined || option.name === null ) { + return null; + } + + const newname = ( + option.name.charAt(0).toUpperCase() + option.name.substring(1) + ).replaceAll("_", " "); + + return newname; + }} + fullWidth + sx={{ + '& .MuiOutlinedInput-root': { + height: 40, // Adjust the input height + }, + '& .MuiAutocomplete-input': { + padding: '8px', // Adjust the text padding + }, + }} + + style={{ + backgroundColor: theme.palette.backgroundColor, + height: 35, + borderRadius: theme.palette?.borderRadius, + }} + onChange={(event, newValue) => { + // Workaround with event lol + if (newValue !== undefined && newValue !== null) { + setNewSelectedAction({ + target: { + value: newValue.name + } + }); + } + }} + renderOption={(props, option, state) => { + var newActionname = option.name; + if (option.label !== undefined && option.label !== null && option.label.length > 0) { + newActionname = option.label; + } + + var newActiondescription = option.description; + //console.log("DESC: ", newActiondescription) + if (option.description === undefined || option.description === null) { + newActiondescription = "Description: No description defined for this action" + } else { + newActiondescription = "Description: "+newActiondescription + } + + const iconInfo = GetIconInfo({ name: option.name }); + const useIcon = iconInfo.originalIcon; + + if (newActionname === undefined || newActionname === null) { + newActionname = "No name" + option.name = "No name" + option.label = "No name" + } + + newActionname = (newActionname.charAt(0).toUpperCase() + newActionname.substring(1)).replaceAll("_", " "); + + var method = "" + var extraDescription = "" + if (option.name.includes("get_")) { + method = "GET" + } else if (option.name.includes("post_")) { + method = "POST" + } else if (option.name.includes("put_")) { + method = "PUT" + } else if (option.name.includes("patch_")) { + method = "PATCH" + } else if (option.name.includes("delete_")) { + method = "DELETE" + } else if (option.name.includes("options_")) { + method = "OPTIONS" + } else if (option.name.includes("connect_")) { + method = "CONNECT" + } + + // FIXME: Should it require a base URL? + if (method.length > 0 && option.description !== undefined && option.description !== null && option.description.includes("http")) { + var extraUrl = "" + const descSplit = option.description.split("\n") + // Last line of descSplit + if (descSplit.length > 0) { + extraUrl = descSplit[descSplit.length-1] + } + + if (extraUrl.length > 0) { + if (extraUrl.includes(" ")) { + extraUrl = extraUrl.split(" ")[0] + } + + if (extraUrl.includes("#")) { + extraUrl = extraUrl.split("#")[0] + } + + extraDescription = `${method} ${extraUrl}` + } else { + //console.log("No url found. Check again :)") + } + } + + return ( + + ); + }} + renderInput={(params) => { + if (params.inputProps?.value) { + const prefixes = ["Post", "Put", "Patch"]; + for (let prefix of prefixes) { + if (params.inputProps.value.startsWith(prefix)) { + let newValue = params.inputProps.value.replace(prefix + " ", ""); + if (newValue.length > 1) { + newValue = newValue.charAt(0).toUpperCase() + newValue.substring(1); + } + // Set the new value without mutating inputProps + params = { ...params, inputProps: { ...params.inputProps, value: newValue } }; + break; + } + } + // Check if it starts with "Get List" and method is "Get" + if (params.inputProps.value.startsWith("Get List")) { + console.log("Get List"); + } + } + + + const actionDescription = null + + return ( + + + + ); + }} + /> + ) : null} + +
    { + selectedActionParameters !== undefined && selectedActionParameters !== null && Object.getOwnPropertyNames(selectedAction).length > 0 && selectedActionParameters.length > 0 ? +
    {isIntegration ? apps !== undefined && apps !== null && apps.length > 0 ?
    @@ -1296,20 +2689,30 @@ const ParsedAction = (props) => { title={"Click to learn more about this action"} placement="top" > +
    + {/* + */} } + {selectedAction.template === true && selectedAction.matching_actions !== undefined && selectedAction.matching_actions !== null && selectedAction.matching_actions.length > 0 ?
    @@ -1346,8 +2749,8 @@ const ParsedAction = (props) => { fullWidth style={{ backgroundColor: theme.palette.inputColor, - height: 50, - borderRadius: theme.palette.borderRadius, + height: 35, + borderRadius: theme.palette?.borderRadius, }} onChange={(event, newValue) => { console.log("SELECT: ", event, newValue) @@ -1409,7 +2812,7 @@ const ParsedAction = (props) => { { />
    : null} - {selectedAction.description !== undefined && selectedAction.description !== null && selectedAction.description.length > 0 && hiddenDescription === false ? ( + {selectedAction.description !== undefined && selectedAction.description !== null && selectedAction.description.length > 0 && hiddenParameters === false ? (
    {
    ) : null} - {selectedActionParameters.map((data, count) => { + {suggestionInfo()} + {selectedActionParameters?.map((data, count) => { if (data.variant === "") { data.variant = "STATIC_VALUE"; } @@ -1450,20 +2854,31 @@ const ParsedAction = (props) => { return null } - // selectedAction.selectedAuthentication = e.target.value - // selectedAction.authentication_id = e.target.value.id - if ( - !selectedAction.auth_not_required && - selectedAction.selectedAuthentication !== undefined && - selectedAction.selectedAuthentication.fields !== undefined && - selectedAction.selectedAuthentication.fields[data.name] !== - undefined - ) { + if (data.value === "authgroup controlled") { + if (data?.name === "url" && authenticationType?.type === "oauth2-app") { + } else { + return null + } + } + + if (selectedAction.parameters === undefined || selectedAction.parameters === null || selectedAction.parameters.length !== selectedActionParameters.length) { + + //selectedAction.parameters = selectedActionParameters + //console.log("PARAM BUG - length change(?): ", selectedAction) + } + + //!selectedAction.auth_not_required && + if (selectedAction.selectedAuthentication !== undefined && selectedAction.selectedAuthentication.fields !== undefined && selectedAction.selectedAuthentication.fields[data.name] !== undefined) { + // This sets the placeholder in the frontend. (Replaced in backend) - selectedActionParameters[count].value = - selectedAction.selectedAuthentication.fields[data.name]; - selectedAction.parameters[count].value = - selectedAction.selectedAuthentication.fields[data.name]; + if (selectedActionParameters[count] !== undefined) { + selectedActionParameters[count].value = selectedAction.selectedAuthentication.fields[data.name] + } + + if (selectedAction.parameters[count] !== undefined) { + selectedAction.parameters[count].value = selectedAction.selectedAuthentication.fields[data.name] + } + setSelectedAction(selectedAction); //setUpdate(Math.random()) @@ -1485,6 +2900,7 @@ const ParsedAction = (props) => { ); } + // Added autofill to make this ALOT simpler if (isCloud && (selectedAction.app_name === "Shuffle Tools" || selectedAction.app_name === "email") && (selectedAction.name === "send_email_shuffle" || selectedAction.name === "send_sms_shuffle") && data.name === "apikey") { if (selectedActionParameters[count].length === 0) { @@ -1512,13 +2928,9 @@ const ParsedAction = (props) => { data.value = data.value.join(",") } - if ( - data.value !== undefined && - data.value !== null && - data.value.startsWith("{") && - data.value.endsWith("}") - ) { - multiline = true; + if (data.value !== undefined && data.value !== null && + data.value.startsWith("{") && data.value.endsWith("}")) { + multiline = true } var placeholder = "Value"; @@ -1526,9 +2938,9 @@ const ParsedAction = (props) => { placeholder = data.example; - if (data.name === "url" && data.value !== undefined && data.value !== null && data.value.length === 0) { - data.value = data.example; - } + // if (data.name === "url") { + // data.value = data.example; + // } // In case of data.example if (data.value === undefined || data.value === null) { data.value = "" @@ -1536,7 +2948,7 @@ const ParsedAction = (props) => { if (data.value.length === 0) { if (data.name.toLowerCase() === "headers") { - console.log("Should show headers field instead with + and -!") + //console.log("Should show headers field instead with + and -!") // Check if file ID exists // @@ -1601,6 +3013,7 @@ const ParsedAction = (props) => { var rows = "3"; var openApiHelperText = "This is an OpenAPI specific field"; + if (selectedApp.generated && data.name === "headers") { //console.log("HEADER: ", data) //if (data.value.length === 0) { @@ -1608,194 +3021,300 @@ const ParsedAction = (props) => { //setSelectedActionParameters(selectedActionParameters) } - var hideBodyButton = ""; - const hideBodyButtonValue = ( -
    - - { - var tag = "TOGGLED" - if (hideBody) { - tag = "UNTOGGLED" - } - - setHideBody(!hideBody) - for (let paramkey in Object.entries(selectedActionParameters)) { - var currentItem = selectedActionParameters[paramkey]; - if (currentItem.name === "ssl_verify") { - - } - - if (currentItem.name === "body") { - currentItem.id = tag - } - - if (currentItem.description === openApiFieldDesc) { - currentItem.field_active = !hideBody - } - } - - - // Scroll to hide_body_button - setTimeout(() => { - var element = document.getElementById("hide_body_button") - if (element !== undefined && element !== null) { - // Keep the button a little below the top - element.scrollIntoView({ - behavior: "smooth", - block: "center", - }) - } - }, 100) - - - }} - name="requires_unique" - /> - } - label={hideBody ? "Show Body" : "Hide Body"} - /> - -
    - ) - - if (selectedApp.generated && data.name === "body") { - const regex = /\${(\w+)}/g; - const found = placeholder.match(regex); - - // setActivateHidingBodyButton(false) - // - hideBodyButton = hideBodyButtonValue; - if (found === null || !hideBody) { - if (found === null) { - setActivateHidingBodyButton(true); - } else { - //console.log("In found: ", found, hideBody) - } - } else { - - rows = "1"; - disabled = true; - openApiHelperText = "OpenAPI spec: fill the following fields."; - - var changed = false; - var tempArray = [] - for (let specKey in found) { - const tmpitem = found[specKey]; - var skip = false; - - for (let innerkey in selectedActionParameters) { - if (selectedActionParameters[innerkey].name === tmpitem) { - skip = true; - break; - } - } - - if (skip) { - //console.log("SKIPPING ", tmpitem) - continue; - } - - changed = true; - var isRequired = false - // Check if original field name is in the selectedAction.required_body_fields - if (selectedAction.required_body_fields !== undefined && selectedAction.required_body_fields !== null) { - for (let innerkey in selectedAction.required_body_fields) { - if (selectedAction.required_body_fields[innerkey] === tmpitem) { - isRequired = true - break - } - } - } - - tempArray.push({ - action_field: "", - configuration: false, - description: openApiFieldDesc, - example: "", - id: "", - multiline: true, - name: tmpitem, - options: null, - required: isRequired, - schema: { type: "string" }, - skip_multicheck: false, - tags: null, - value: "", - variant: "STATIC_VALUE", - field_active: true, - - autocompleted: true, - }); - } - - console.log("TEMP ARRAY: ", tempArray) - var required = selectedActionParameters.filter(item => item.required === true) - var notRequired = selectedActionParameters.filter(item => item.required === false) - - if (tempArray.length > 0) { - // Sort tempArray based on tempArray.required - tempArray.sort((a, b) => (a.required < b.required) ? 1 : -1) - // Add all items to the selectedActionParameters array - for (let innerkey in tempArray) { - tempArray[innerkey].id = "ADDED" - - if (tempArray[innerkey].required === true) { - required.push(tempArray[innerkey]) - } else { - notRequired.push(tempArray[innerkey]) - } + var hideBodyButtonValue = ( +
    + + { + // Set localstorage + localStorage.setItem("hideBody", "true") + + setHideBody(false) + const updatedParameters = selectedActionParameters.map((param) => { + if (param.name === "body") { + return { + ...param, + id: "UNTOGGLED", + } + } - if (changed) { - // Sort selectedActionParameters based on selectedActionParameters.required - //selectedActionParameters.sort((a, b) => (a.required < b.required) ? 1 : -1) - // Find the "headers" and "queries" field names and put them on the first indexes anyway - var newArray = required.concat(notRequired) - + if (param.description === openApiFieldDesc) { + // Check required fields here + if (selectedAction.required_body_fields !== undefined && selectedAction.required_body_fields !== null && selectedAction.required_body_fields.length > 0) { + // Look for the field name in the required_body_fields + if (selectedAction.required_body_fields.includes(param.name)) { + param.required = true + } else { + param.required = false + } + } - setSelectedActionParameters(newArray) + return { ...param, field_active: true } + } + + return param + }) + + setSelectedActionParameters(updatedParameters) + }} + /> + { + localStorage.setItem("hideBody", "false") + setHideBody(true) + // Make sure the body field is shown + const updatedParameters = selectedActionParameters.map((param) => { + if (param.name === "body") { + return { + ...param, + id: "TOGGLED", + } + } + + if (param.description === openApiFieldDesc) { + return { ...param, field_active: false } + } + + return param + }) + + setSelectedActionParameters(updatedParameters) + }} + /> + + {/* + + + + + + + + + */} +
    + ) + + var showButtonField = false + if (selectedApp.generated === true && data.name === "body") { + const regex = /\${(\w+)}/g; + const found = placeholder.match(regex); + + var newhidebody = hideBody + showButtonField = true + if (found === undefined || found === null || found.length === 0) { + newhidebody = false + hideBodyButtonValue = null + + if (hideBody === false) { + setHideBody(true) + } + } + + if (newhidebody === true) { + //toast("BODYBUTTON TRUE") + } else { + + rows = "1"; + disabled = true; + openApiHelperText = "OpenAPI spec: fill the following fields."; + + var changed = false; + var tempArray = [] + for (let specKey in found) { + const tmpitem = found[specKey]; + var skip = false; + + for (let innerkey in selectedActionParameters) { + if (selectedActionParameters[innerkey].name === tmpitem) { + skip = true; + break; + } + } + + if (skip) { + //console.log("SKIPPING ", tmpitem) + continue; + } + + changed = true; + var isRequired = false + // Check if original field name is in the selectedAction.required_body_fields + if (selectedAction.required_body_fields !== undefined && selectedAction.required_body_fields !== null) { + for (let innerkey in selectedAction.required_body_fields) { + if (selectedAction.required_body_fields[innerkey] === tmpitem) { + isRequired = true + break + } + } + } + + tempArray.push({ + action_field: "", + configuration: false, + description: openApiFieldDesc, + example: "", + id: "", + multiline: false, + name: tmpitem, + options: null, + required: isRequired, + schema: { type: "string" }, + skip_multicheck: false, + tags: null, + value: "", + variant: "STATIC_VALUE", + field_active: true, + + autocompleted: false, + }); } + + var required = selectedActionParameters.filter(item => item.required === true) + var notRequired = selectedActionParameters.filter(item => item.required === false) - return hideBodyButton; - } - } + if (tempArray.length > 0) { + // Sort tempArray based on tempArray.required + tempArray.sort((a, b) => (a.required < b.required) ? 1 : -1) + // Add all items to the selectedActionParameters array + for (let innerkey in tempArray) { + tempArray[innerkey].id = "ADDED" - if (activateHidingBodyButton === true) { - hideBodyButton = ""; - } + if (tempArray[innerkey].required === true) { + required.push(tempArray[innerkey]) + } else { + notRequired.push(tempArray[innerkey]) + } + } + } + + if (changed) { + // Sort selectedActionParameters based on selectedActionParameters.required + // Find the "headers" and "queries" field names and put them on the first indexes anyway + var newArray = required.concat(notRequired) + + + setSelectedActionParameters(newArray) + } + + } + } const clickedFieldId = "rightside_field_" + count; var baseHelperText = "" - if (data !== undefined && data !== null && data.value !== undefined && data.value !== null && data.value.length > 0) { - baseHelperText = calculateHelpertext(data.value) - } - + if (data !== undefined && data !== null && data.value !== undefined && data.value !== null && data.value.length > 0) { + baseHelperText = calculateHelpertext(data.value) + } var tmpitem = data.name.valueOf(); if (data.name.startsWith("${") && data.name.endsWith("}")) { @@ -1814,57 +3333,130 @@ const ParsedAction = (props) => { tmpitem = "Username" } else if (tmpitem === "Password basic") { tmpitem = "Password" + } + + // No longer multiline for new fields + //multiline = data.name.startsWith("${") && data.name.endsWith("}") ? true : multiline + + if (data.name === "body") { + //console.log("BODY: ", data) + if (hideBody === false) { + return hideBodyButtonValue + } + + rows = "4" + multiline = true + disabled = false + } + + const description = data.description === undefined ? "" : data?.description; + + const tooltipDescription = ( + + + + {tmpitem.charAt(0).toUpperCase() + tmpitem.slice(1)} + + { + setUiBox("closed") + }} + > + + + + + + + Required: {data.required === true || data.configuration === true ? "True" : "False"} + + + Description: {description} + + + Ex. : {data?.example?.length > 0 ? data.example : "No example available"} + + { + data?.configuration === true ? + ( + + Auth: Use "\$" instead of "$" + + ) : null } - multiline = data.name.startsWith("${") && data.name.endsWith("}") ? true : multiline - +
    { + e.preventDefault() + e.stopPropagation() + + localStorage.setItem("disabled_ui_box", "true") + setUiBox("closed") + }}> + + Don't show again + +
    +     +     + ); + + var datafield = ( + - - - { - event.preventDefault() - setFieldCount(count) - setExpansionModalOpen(true) - - //setcodedata(data.value) - var parsedvalue = data.value - if (parsedvalue === undefined || parsedvalue === null) { - parsedvalue = "" - } - - setEditorData({ - "name": data.name, - "value": parsedvalue, - "field_number": count, - "actionlist": actionlist, - "field_id": clickedFieldId, - }) - }} - /> - - + + { event.preventDefault() @@ -1884,7 +3476,7 @@ const ParsedAction = (props) => { ), }} - multiline={data.name.startsWith("${") && data.name.endsWith("}") ? true : multiline} + multiline={multiline} onClick={() => { /* setExpansionModalOpen(false); @@ -1896,18 +3488,22 @@ const ParsedAction = (props) => { setScrollConfig(scrollConfig) } }} - id={clickedFieldId} rows={data.name.startsWith("${") && data.name.endsWith("}") ? 2 : rows} color="primary" - defaultValue={data.value} - //value={data.value} + // defaultValue={data.value} + value={ + data?.value + } + error={ + data?.error?.length > 0 ? true : false + } + helperText={data?.error?.length > 0 ? errorHelperText(data?.name,data?.value,data?.error) : returnHelperText(data.name, data.value)} //options={{ // theme: 'gruvbox-dark', // keyMap: 'sublime', // mode: 'python', //}} //height={multiline ? 50 : 150} - type={ placeholder.includes("***") || (data.configuration && @@ -1920,16 +3516,35 @@ const ParsedAction = (props) => { placeholder={placeholder} onChange={(event) => { //changeActionParameterCodemirror(event, count, data) - changeActionParameter(event, count, data); + // changeActionParameter(event, count, data); + handleParamChange(event, count, data) }} - helperText={returnHelperText(data.name, data.value)} + onFocus={(event) => { + // Get local storage key "disabled_ui_box" and check if it's true + const disabledUiBox = localStorage.getItem("disabled_ui_box") + if (disabledUiBox === "true") { + } else { + //setUiBox(event.target.id) + } + }} onBlur={(event) => { baseHelperText = calculateHelpertext(event.target.value) if (setLastSaved !== undefined) { setLastSaved(false) } + + // Check if we clicked the tooltip or not + const tooltipid = "rightside_field_tooltip" + count + const foundElement = document.getElementById(tooltipid) + if (foundElement !== null && foundElement !== undefined) { + console.log("FOUND: ", foundElement) + } else { + //console.log("TOOLTIP -> NOT FOUND") + //setUiBox("closed") + } }} /> + ); // Finds headers from a string to be used for autocompletion @@ -2115,7 +3730,7 @@ const ParsedAction = (props) => { selectedActionParameters[count].value += "\n" selectedAction.parameters[count].value += "\n" - setSelectedActionParameters(selectedActionParameters) + setSelectedActionParameters(selectedActionParameters) setSelectedAction(selectedAction) setUpdate(Math.random()) }}> @@ -2124,7 +3739,6 @@ const ParsedAction = (props) => {
    } - //console.log("FIELD VALUE: ", data.value) //const regexp = new RegExp("\W+\.", "g") //let match //while ((match = regexp.exec(data.value)) !== null) { @@ -2140,15 +3754,13 @@ const ParsedAction = (props) => { // } //} - // Basic helpertext - if (files !== undefined && files !== null && data.name.toLowerCase() === "file_category") { - //selectedActionParameters[count].options.length > 0 - console.log("FileS: ", files) - if (files.namespaces !== undefined && files.namespaces !== null && files.namespaces.length > 0) { - data.options = files.namespaces - } - } + //selectedActionParameters[count].options.length > 0 + console.log("FileS: ", files) + if (files.namespaces !== undefined && files.namespaces !== null && files.namespaces.length > 0) { + data.options = files.namespaces + } + } //const keywords = ["len", "lower", "upper", "trim", "split", "length", "number", "parse", "join"] if ( @@ -2160,7 +3772,7 @@ const ParsedAction = (props) => { { ), }} - helperText={returnHelperText(data.name, data.value)} + helperText={returnHelperText(data.name, data.value)} fullWidth multiline={multiline} rows={"3"} @@ -2196,26 +3808,11 @@ const ParsedAction = (props) => { }} onBlur={(event) => {}} /> - ); - //const fileId = "6daabec1-892b-469c-b603-c902e47223a9" - //datafield = `SHOW FILES FROM OTHER NODES? Filename: ${selectedActionParameters[count].value}` - /* - if (selectedActionParameters[count].value != fileId) { - changeActionParameter(fileId, count, data) - setUpdate(Math.random()) - - } - */ + ) } else if ( - (data.options !== undefined && - data.options !== null && - data.options.length > 0) - || - (selectedActionParameters[count].options !== undefined && - selectedActionParameters[count].options !== null && - selectedActionParameters[count].options.length > 0) - ) { - const parsedoptions = data.options !== undefined && data.options !== null && data.options.length > 0 ? data.options : selectedActionParameters[count].options + (data.options !== undefined && data.options !== null && data.options.length > 0) || + (selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0)) { + const parsedoptions = data.options !== undefined && data.options !== null && data.options.length > 0 ? data.options : selectedActionParameters[count].options if (selectedActionParameters[count].value === "") { // && selectedActionParameters[count].required) { @@ -2259,7 +3856,7 @@ const ParsedAction = (props) => { backgroundColor: theme.palette.surfaceColor, color: "white", height: "50px", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, }} > {parsedoptions.map( @@ -2289,14 +3886,14 @@ const ParsedAction = (props) => { ); } else if (data.variant === "STATIC_VALUE") { - staticcolor = "#f85a3e"; + staticcolor = "#FF8544"; } if (data.field_active === false) { - return null; + console.log("Field not active: ", data?.name) + return null } - // Shows nested list of nodes > their JSON lists const ActionlistWrapper = (props) => { const handleMenuClose = () => { @@ -2315,7 +3912,6 @@ const ParsedAction = (props) => { }; const handleItemClick = (values) => { - console.log("In normal itemclick") if (values === undefined ||values === null ||values.length === 0) { return; } @@ -2338,31 +3934,28 @@ const ParsedAction = (props) => { // Handles the fields under OpenAPI body to be parsed. if (data.name.startsWith("${") && data.name.endsWith("}")) { - console.log("INSIDE VALUE REPLACE: ", data.name, toComplete); - // PARAM FIX - Gonna use the ID field, even though it's a hack const paramcheck = selectedAction.parameters.find( (param) => param.name === "body" - ); + ) + if (paramcheck !== undefined) { - if ( - paramcheck["value_replace"] === undefined || - paramcheck["value_replace"] === null - ) { + if (paramcheck["value_replace"] === undefined || paramcheck["value_replace"] === null) { paramcheck["value_replace"] = [ { key: data.name, value: toComplete, }, - ]; + ] } else { - const subparamindex = paramcheck[ - "value_replace" - ].findIndex((param) => param.key === data.name); + const subparamindex = paramcheck["value_replace"] + .findIndex((param) => param.key === data.name); + if (subparamindex === -1) { paramcheck["value_replace"].push({ key: data.name, value: toComplete, - }); + }) + } else { paramcheck["value_replace"][subparamindex]["value"] += toComplete; @@ -2370,7 +3963,9 @@ const ParsedAction = (props) => { } selectedActionParameters[count]["value_replace"] = paramcheck; - selectedAction.parameters[count]["value_replace"] = paramcheck; + + selectedAction.parameters = selectedActionParameters + //selectedAction.parameters[count]["value_replace"] = paramcheck; setSelectedAction(selectedAction); setUpdate(Math.random()); @@ -2387,7 +3982,7 @@ const ParsedAction = (props) => { //selectedAction.parameters[count].value = selectedActionParameters[count].value; //setSelectedAction(selectedAction); //setUpdate(Math.random()); - + setShowDropdown(false); setMenuPosition(null); }; @@ -2427,7 +4022,7 @@ const ParsedAction = (props) => { ); if (exec_text_field !== null) { if (inside) { - exec_text_field.style.border = "2px solid #f85a3e"; + exec_text_field.style.border = "2px solid #FF8544"; } else { exec_text_field.style.border = ""; } @@ -2613,7 +4208,8 @@ const ParsedAction = (props) => { newname = newname.slice(0, newname.length-5) } - selectedActionParameters[count].value += `{{ $${innerdata.name}.${newname} | size }}` + //selectedActionParameters[count].value += `{{ $${innerdata.name}.${newname} | size }}` + selectedActionParameters[count].value += `$${innerdata.name}.${newname}` selectedAction.parameters[count].value = selectedActionParameters[count].value; setSelectedAction(selectedAction); setUpdate(Math.random()); @@ -2660,40 +4256,17 @@ const ParsedAction = (props) => { })} ); - }; - - const description = - data.description === undefined ? "" : data.description; - const tooltipDescription = ( - - - - Required:{" "} - {data.required === true || data.configuration === true - ? "True" - : "False"} - - - - Example: {data.example} - - - - Description: {description} - - - ); - - //var itemColor = "#f85a3e" - //if (!data.required) { - // itemColor = "#ffeb3b" - //} - { - /*
    */ } + + const buttonTitle = `Authenticate API ${selectedApp.name.replaceAll("_", " ")}` + const hasAutocomplete = data?.autocompleted === true + if (data.variant === undefined || data.variant === null) { + data.variant = "STATIC_VALUE" + } - const buttonTitle = `Authenticate ${selectedApp.name.replaceAll("_", " ")}` - const hasAutocomplete = data.autocompleted === true return (
    - {hideBodyButton} + {showButtonField === true ? hideBodyButtonValue : null}
    @@ -2739,7 +4312,7 @@ const ParsedAction = (props) => { > @@ -2751,67 +4324,40 @@ const ParsedAction = (props) => { flex: "10", marginTop: "auto", marginBottom: "auto", + color: "#C5C5C5", }} > - - {tmpitem} - + {tmpitem} {selectedActionParameters[count].required || selectedActionParameters[count].configuration ? "*" : ""}
    - {/*selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 ? null : -
    - -
    { - e.preventDefault() - changeActionParameterVariant("STATIC_VALUE", count) - }}> - -
    -     -  |  - -
    { - e.preventDefault() - changeActionParameterVariant("ACTION_RESULT", count) - }}> - -
    -     -  |  - -
    { - e.preventDefault() - changeActionParameterVariant("WORKFLOW_VARIABLE", count) - }}> - -
    -     -
    - */} - {/*(selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 && selectedActionParameters[count].required === true && selectedActionParameters[count].unique_toggled !== undefined) || hideExtraTypes ? null : -
    - -
    {}}> - { - //console.log("CHECKED!: ", selectedActionParameters[count]) - selectedActionParameters[count].unique_toggled = !selectedActionParameters[count].unique_toggled - selectedAction.parameters[count].unique_toggled = selectedActionParameters[count].unique_toggled - setSelectedActionParameters(selectedActionParameters) - setSelectedAction(selectedAction) - setUpdate(Math.random()) - }} - name="requires_unique" - /> -
    -     -
    - */} + + + { + event.preventDefault() + setFieldCount(count) + setExpansionModalOpen(true) + setActiveDialog("codeeditor") + //setcodedata(data.value) + var parsedvalue = data.value + if (parsedvalue === undefined || parsedvalue === null) { + parsedvalue = "" + } + + setEditorData({ + "name": data.name, + "value": parsedvalue, + "field_number": count, + "actionlist": actionlist, + "field_id": clickedFieldId, + + "example": selectedActionParameters[count].example, + }) + }} + /> + +
    {datafield} {/*shufflecode*/} @@ -2827,9 +4373,9 @@ const ParsedAction = (props) => { Autocomplete { - const newversion = selectedApp.versions.find( - (tmpApp) => tmpApp.version == event.target.value - ) - - if (newversion !== undefined && newversion !== null) { - getApp(newversion.id, true) - } - - // Change in all actions in the workflow at the same time and add a toast.success() about it - for (var actionkey in workflow.actions) { - const action = workflow.actions[actionkey] - if (action.app_name === selectedAction.app_name) { - workflow.actions[actionkey].app_version = event.target.value - } - } - - toast.success("Changed version of all nodes to "+event.target.value) - }} - style={{ - marginTop: 10, - backgroundColor: theme.palette.surfaceColor, - backgroundColor: theme.palette.inputColor, - color: "white", - height: 35, - marginleft: 10, - borderRadius: theme.palette.borderRadius, - }} - SelectDisplayProps={{ - style: { - }, - }} - > - {selectedApp.versions.map((data, index) => { - return ( - - {data.version} - - ); - })} - - ) : null} -
    -
    -
    -
    - Name - { - // Copy the name value - const name = e.target.value - const parsedBaseLabel = "$"+baselabel.toLowerCase().replaceAll(" ", "_") - const newname = "$"+name.toLowerCase().replaceAll(" ", "_") - - // Check if it's the same as the current name in use - //if (name === selectedAction.label) { - // console.log("Returning from name thing") - // return - //} - - // Change in actions, triggers & conditions - // Highlight the changes somehow with a glow? - if (workflow.branches !== undefined && workflow.branches !== null) { - for (let [key,keyval] in Object.entries(workflow.branches)) { - if (workflow.branches[key].conditions !== undefined && workflow.branches[key].conditions !== null) { - for (let [subkey,subkeyval] in Object.entries(workflow.branches[key].conditions)) { - const condition = workflow.branches[key].conditions[subkey] - const sourceparam = condition.source - const destinationparam = condition.destination - - // Should have a smarter way of discovering node names - // Finding index(es) and replacing at the location - if (sourceparam.value.includes("$")) { - try { - var cnt = -1 - var previous = 0 - while (true) { - cnt += 1 - // Need to make sure e.g. changing the first here doesn't change the 2nd - // $change_me - // $change_me_2 - - const foundindex = sourceparam.value.toLowerCase().indexOf(parsedBaseLabel, previous) - if (foundindex === previous && foundindex !== 0) { - break - } - - if (foundindex >= 0) { - previous = foundindex+newname.length - // Need to add diff of length to word - - // Check location: - // If it's a-zA-Z_ then don't replace - if (sourceparam.value.length > foundindex+parsedBaseLabel.length) { - const regex = /[a-zA-Z0-9_]/g; - const match = sourceparam.value[foundindex+parsedBaseLabel.length].match(regex); - if (match !== null) { - continue - } - } - - console.log("Old found: ", workflow.branches[key].conditions[subkey].source.value) - const extralength = newname.length-parsedBaseLabel.length - sourceparam.value = sourceparam.value.substring(0, foundindex) + newname + sourceparam.value.substring(foundindex-extralength+newname.length, sourceparam.value.length) - - console.log("New: ", workflow.branches[key].conditions[subkey].source.value) - } else { - break - } - - // Break no matter what after 5 replaces. May need to increase - if (cnt >= 5) { - break - } - - } - } catch (e) { - console.log("Failed value replacement based on index: ", e) - } - } - - if (destinationparam.value.includes("$")) { - try { - var cnt = -1 - var previous = 0 - while (true) { - cnt += 1 - // Need to make sure e.g. changing the first here doesn't change the 2nd - // $change_me - // $change_me_2 - - const foundindex = destinationparam.value.toLowerCase().indexOf(parsedBaseLabel, previous) - if (foundindex === previous && foundindex !== 0) { - break - } - - if (foundindex >= 0) { - previous = foundindex+newname.length - // Need to add diff of length to word - - // Check location: - // If it's a-zA-Z_ then don't replace - if (destinationparam.value.length > foundindex+parsedBaseLabel.length) { - const regex = /[a-zA-Z0-9_]/g; - const match = destinationparam.value[foundindex+parsedBaseLabel.length].match(regex); - if (match !== null) { - continue - } - } - - console.log("Old found: ", workflow.branches[key].conditions[subkey].destination.value) - const extralength = newname.length-parsedBaseLabel.length - destinationparam.value = destinationparam.value.substring(0, foundindex) + newname + destinationparam.value.substring(foundindex-extralength+newname.length, destinationparam.value.length) - - console.log("New: ", workflow.branches[key].conditions[subkey].destination.value) - } else { - break - } - - // Break no matter what after 5 replaces. May need to increase - if (cnt >= 5) { - break - } - - } - } catch (e) { - console.log("Failed value replacement based on index: ", e) - } - } - } - } - } - } - - for (let [key,keyval] in Object.entries(workflow.actions)) { - if (workflow.actions[key].id === selectedAction.id) { - continue - } - - const params = workflow.actions[key].parameters - console.log(params) - if (params === null || params === undefined) { - continue - } - - for (let [subkey, subkeyval] in Object.entries(params)) { - const param = workflow.actions[key].parameters[subkey]; - if (!param.value.includes("$")) { - continue - } - - // Should have a smarter way of discovering node names - // Do regex? - // Finding index(es) and replacing at the location - // - - try { - var cnt = -1 - var previous = 0 - while (true) { - cnt += 1 - // Need to make sure e.g. changing the first here doesn't change the 2nd - // $change_me - // $change_me_2 - - const foundindex = param.value.toLowerCase().indexOf(parsedBaseLabel, previous) - if (foundindex === previous && foundindex !== 0) { - break - } - - if (foundindex >= 0) { - previous = foundindex+newname.length - // Need to add diff of length to word - - // Check location: - // If it's a-zA-Z_ then don't replace - if (param.value.length > foundindex+parsedBaseLabel.length) { - const regex = /[a-zA-Z0-9_]/g; - const match = param.value[foundindex+parsedBaseLabel.length].match(regex); - if (match !== null) { - continue - } - } - - console.log("Old found: ", workflow.actions[key].parameters[subkey].value) - const extralength = newname.length-parsedBaseLabel.length - param.value = param.value.substring(0, foundindex) + newname + param.value.substring(foundindex-extralength+newname.length, param.value.length) - - console.log("New: ", workflow.actions[key].parameters[subkey].value) - } else { - break - } - - // Break no matter what after 5 replaces. May need to increase - if (cnt >= 5) { - break - } - - } - } catch (e) { - console.log("Failed value replacement based on index: ", e) - } - } - } - - setWorkflow(workflow); - setUpdate(Math.random()); - baselabel = name - }} - /> -
    - {/*!isCloud ? null :*/} -
    - - - Delay - { - if (actionDelayChange !== undefined) { - actionDelayChange(event) - } - }} - /> - - -
    - {/**/} -
    - - )} - {selectedApp.name !== undefined && - selectedAction.authentication !== null && - selectedAction.authentication !== undefined && - selectedAction.authentication.length === 0 && - requiresAuthentication ? ( -
    - - - - - -
    - ) : null} - - {selectedAction.authentication !== undefined && - selectedAction.authentication !== null && - selectedAction.authentication.length > 0 ? ( -
    - Authentication -
    - - - {/* - - - curaction.authentication = authenticationOptions - if (curaction.selectedAuthentication === null || curaction.selectedAuthentication === undefined || curaction.selectedAuthentication.length === "") - */} - - { - setAuthenticationModalOpen(true); - }} - > - - - -
    -
    - ) : null} - - {showEnvironment !== undefined && showEnvironment && environments.length > 1 && !isIntegration ? ( -
    - Environment - -
    - ) : null} - - {workflow.execution_variables !== undefined && - workflow.execution_variables !== null && - workflow.execution_variables.length > 0 ? ( -
    - Execution variable (optional) - -
    - ) : null} - - -
    - {/*hideExtraTypes ? null : -
    - Actions -
    - */} - - {setNewSelectedAction !== undefined ? ( - { - // Most popular - // Is categorized - // Uncategorized - return option.category_label !== undefined && option.category_label !== null && option.category_label.length > 0 ? "Most used" : "All Actions"; - }} - renderGroup={(params) => { - - return ( -
  • - {params.group} - {params.children} -
    • - ) - }} - options={renderedActionOptions} - ListboxProps={{ - style: { - backgroundColor: theme.palette.surfaceColor, - color: "white", - }, - }} - filterOptions={(options, { inputValue }) => { - //console.log("Option contains?: ", inputValue, options) - const lowercaseValue = inputValue.toLowerCase() - options = options.filter(x => x.name.replaceAll("_", " ").toLowerCase().includes(lowercaseValue) || x.description.toLowerCase().includes(lowercaseValue)) - - return options - }} - getOptionLabel={(option) => { - if (option === undefined || option === null || option.name === undefined || option.name === null ) { - return null; - } - - const newname = ( - option.name.charAt(0).toUpperCase() + option.name.substring(1) - ).replaceAll("_", " "); - - return newname; - }} - fullWidth - style={{ - backgroundColor: theme.palette.inputColor, - height: 50, - borderRadius: theme.palette.borderRadius, - }} - onChange={(event, newValue) => { - // Workaround with event lol - if (newValue !== undefined && newValue !== null) { - setNewSelectedAction({ - target: { - value: newValue.name - } - }); - } - }} - renderOption={(props, data, state) => { - var newActionname = data.name; - if (data.label !== undefined && data.label !== null && data.label.length > 0) { - newActionname = data.label; - } - - var newActiondescription = data.description; - //console.log("DESC: ", newActiondescription) - if (data.description === undefined || data.description === null) { - newActiondescription = "Description: No description defined for this action" - } else { - newActiondescription = "Description: "+newActiondescription - } - - const iconInfo = GetIconInfo({ name: data.name }); - const useIcon = iconInfo.originalIcon; - - if (newActionname === undefined || newActionname === null) { - newActionname = "No name" - data.name = "No name" - data.label = "No name" - } - - newActionname = (newActionname.charAt(0).toUpperCase() + newActionname.substring(1)).replaceAll("_", " "); - - var method = "" - var extraDescription = "" - if (data.name.includes("get_")) { - method = "GET" - } else if (data.name.includes("post_")) { - method = "POST" - } else if (data.name.includes("put_")) { - method = "PUT" - } else if (data.name.includes("patch_")) { - method = "PATCH" - } else if (data.name.includes("delete_")) { - method = "DELETE" - } else if (data.name.includes("options_")) { - method = "OPTIONS" - } else if (data.name.includes("connect_")) { - method = "CONNECT" - } - - // FIXME: Should it require a base URL? - if (method.length > 0 && data.description !== undefined && data.description !== null && data.description.includes("http")) { - var extraUrl = "" - const descSplit = data.description.split("\n") - // Last line of descSplit - if (descSplit.length > 0) { - extraUrl = descSplit[descSplit.length-1] - } - - //for (let [line,lineval] in Object.entries(descSplit)) { - // if (descSplit[line].includes("http") && descSplit[line].includes("://")) { - // const urlsplit = descSplit[line].split("/") - // try { - // extraUrl = "/"+urlsplit.slice(3, urlsplit.length).join("/") - // } catch (e) { - // //console.log("Failed - running with -1") - // extraUrl = "/"+urlsplit.slice(3, urlsplit.length-1).join("/") - // } - - - // //console.log("NO BASEURL TOO!! Why missing last one in certain scenarios (sevco)?", extraUrl, urlsplit, descSplit[line]) - // //break - // } - //} - - if (extraUrl.length > 0) { - if (extraUrl.includes(" ")) { - extraUrl = extraUrl.split(" ")[0] - } - - if (extraUrl.includes("#")) { - extraUrl = extraUrl.split("#")[0] - } - - extraDescription = `${method} ${extraUrl}` - } else { - //console.log("No url found. Check again :)") - } - } - - return ( - - ); - }} - renderInput={(params) => { - if (params.inputProps !== undefined && params.inputProps !== null && params.inputProps.value !== undefined && params.inputProps.value !== null) { - const prefixes = ["Post", "Put", "Patch"] - for (let [key,keyval] in Object.entries(prefixes)) { - if (params.inputProps.value.startsWith(prefixes[key])) { - params.inputProps.value = params.inputProps.value.replace(prefixes[key]+" ", "", -1) - if (params.inputProps.value.length > 1) { - params.inputProps.value = params.inputProps.value.charAt(0).toUpperCase()+params.inputProps.value.substring(1) - } - break - } - } - - // Check if it starts with "Get List" and method is "Get" - if (params.inputProps.value.startsWith("Get List")) { - console.log("Get List") - } - } - - return ( - - ); - }} - /> - ) : null} - - {/*setNewSelectedAction !== undefined ? - - : null*/} - -
    - +
    diff --git a/frontend/src/components/Priorities.jsx b/frontend/src/components/Priorities.jsx index 234269ae..29078a6a 100644 --- a/frontend/src/components/Priorities.jsx +++ b/frontend/src/components/Priorities.jsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect } from "react"; +import React, { useState, useEffect, useContext, memo } from "react"; import { toast } from "react-toastify"; import theme from "../theme.jsx"; @@ -13,22 +13,26 @@ import { Card, Chip, Switch, + Skeleton, } from "@mui/material"; +import { Context } from "../context/ContextApi.jsx"; import { useNavigate, Link } from "react-router-dom"; import Priority from "../components/Priority.jsx"; +import { constrainMatrix } from "reaviz"; //import { useAlert -const Priorities = (props) => { +const Priorities = memo((props) => { const { globalUrl, userdata,clickedFromOrgTab, serverside, billingInfo, stripeKey, checkLogin, setAdminTab, setCurTab, notifications, setNotifications, } = props; + const [showDismissed, setShowDismissed] = React.useState(false); const [showRead, setShowRead] = React.useState(false); const [appFramework, setAppFramework] = React.useState({}); - const [selectedWorkflow, setSelectedWorkflow] = React.useState("NO HIGHLIGHT"); const [selectedExecutionId, setSelectedExecutionId] = React.useState("NO HIGHLIGHT"); + const [highlightKMS, setHighlightKMS] = React.useState(false) + let navigate = useNavigate(); - useEffect(() => { getFramework() @@ -36,6 +40,12 @@ const Priorities = (props) => { const urlParams = new URLSearchParams(window.location.search) const workflow = urlParams.get("workflow") const execution_id = urlParams.get("execution_id") + const kms = urlParams.get("kms") + + if (kms !== null && kms !== undefined && kms.length > 0 && kms === "true") { + toast.info("KMS-related notifications are highlighted.") + setHighlightKMS(true) + } if (execution_id !== null) { setSelectedExecutionId(execution_id) @@ -209,176 +219,16 @@ const Priorities = (props) => { const notificationWidth = "100%" const imagesize = 22 const boxColor = "#86c142" - const NotificationItem = (props) => { - const {data} = props - - var image = ""; - var orgName = ""; - var orgId = ""; - - - const highlighted = selectedExecutionId === "" && selectedWorkflow === "" ? false : data.reference_url === undefined || data.reference_url === null || data.reference_url.length === 0 ? false : data.reference_url.includes(selectedExecutionId) || data.reference_url.includes(selectedWorkflow) - - if (userdata.orgs !== undefined) { - const foundOrg = userdata.orgs.find((org) => org.id === data["org_id"]); - if (foundOrg !== undefined && foundOrg !== null) { - //position: "absolute", bottom: 5, right: -5, - const imageStyle = { - width: imagesize, - height: imagesize, - pointerEvents: "none", - marginLeft: - data.creator_org !== undefined && data.creator_org.length > 0 - ? 20 - : 0, - borderRadius: 10, - border: - foundOrg.id === userdata.active_org.id - ? `3px solid ${boxColor}` - : null, - cursor: "pointer", - marginRight: 10, - }; - - image = - foundOrg.image === "" ? ( - {foundOrg.name} - ) : ( - {foundOrg.name} {}} - /> - ); - - orgName = foundOrg.name; - orgId = foundOrg.id; - } - } - - return ( - -
    - {data.amount === 1 && data.read === false ? - - : null} - {data.ignored === true ? - - : null} - {data.read === false ? - - : - - } - - {data.title} - -
    - - {data.image !== undefined && data.image !== null && data.image.length > 0 ? - {data.title} - : - null - } - - {data.description} - -
    - - - {data.read === false ? ( - - ) : null} - - - - - - - First seen: {new Date(data.created_at * 1000).toISOString().slice(0, 19)} - - - Last seen: {new Date(data.updated_at * 1000).toISOString().slice(0, 19)} - - - Times seen: {data.amount} - -
    -     - ); - } return ( - ) -} +}) export default Priorities; + + +const NotificationItem = memo((props) => { + const {data, selectedExecutionId, selectedWorkflow, highlightKMS, userdata, imagesize, boxColor, clickedFromOrgTab, notificationWidth, dismissNotification} = props + + var image = ""; + var orgName = ""; + var orgId = ""; + + + var highlighted = selectedExecutionId === "" && selectedWorkflow === "" ? false : data.reference_url === undefined || data.reference_url === null || data.reference_url.length === 0 ? false : data.reference_url.includes(selectedExecutionId) || data.reference_url.includes(selectedWorkflow) + + if (!highlighted && highlightKMS) { + if (data.title !== undefined && data.title !== null && data.title.toLowerCase().includes("kms")) { + highlighted = true + } else if (data.description !== undefined && data.description !== null && data.description.toLowerCase().includes("kms")) { + highlighted = true + } + + } + + if (userdata.orgs !== undefined) { + const foundOrg = userdata.orgs.find((org) => org.id === data["org_id"]); + if (foundOrg !== undefined && foundOrg !== null) { + //position: "absolute", bottom: 5, right: -5, + const imageStyle = { + width: imagesize, + height: imagesize, + pointerEvents: "none", + marginLeft: + data.creator_org !== undefined && data.creator_org.length > 0 + ? 20 + : 0, + borderRadius: 10, + border: + foundOrg.id === userdata.active_org.id + ? `3px solid ${boxColor}` + : null, + cursor: "pointer", + marginRight: 10, + }; + + image = + foundOrg.image === "" ? ( + {foundOrg.name} + ) : ( + {foundOrg.name} {}} + /> + ); + + orgName = foundOrg.name; + orgId = foundOrg.id; + } + } + + return ( + +
    + {data.amount === 1 && data.read === false ? + + : null} + {data.ignored === true ? + + : null} + {data.read === false ? + + : + + } + + {data.title} + +
    + + {data.image !== undefined && data.image !== null && data.image.length > 0 ? + {data.title} + : + null + } + + {data.description} + +
    + + + {data.read === false ? ( + + ) : null} + + + + + + + First seen: {new Date(data.created_at * 1000).toISOString().slice(0, 19)} + + + + Last seen: {new Date(data.updated_at * 1000).toISOString().slice(0, 19)} + + + + Times seen: {data.amount} + +
    +     + ); +}) + + +const NotificationComponent = memo(({notifications, showRead, selectedExecutionId, selectedWorkflow, highlightKMS, userdata, imagesize, boxColor, clickedFromOrgTab, notificationWidth, dismissNotification}) => { + + return( +
    + {notifications === null || notifications === undefined || notifications?.length === 0 ? ( + null + ) : +
    + {notifications?.map((notification, index) => { + if (showRead === false && notification.read === true) { + return null + } + + return ( + + ) + })} +
    + } +
    + ) +}) + +// const PaddingWrapper = memo(({children, clickedFromOrgTab}) => { + +// const { leftSideBarOpenByClick } = useContext(Context) + +// return( +//
    +// {children} +//
    +// ) +// }) + +// const Wrapper = memo(({children, clickedFromOrgTab}) => { + +// return( +// +// {children} +// +// ) +// }) diff --git a/frontend/src/components/Priority.jsx b/frontend/src/components/Priority.jsx index 67d56af7..4d37142f 100644 --- a/frontend/src/components/Priority.jsx +++ b/frontend/src/components/Priority.jsx @@ -24,9 +24,16 @@ import { const Priority = (props) => { const { globalUrl, clickedFromOrgTab,userdata, serverside, priority, checkLogin, setAdminTab, setCurTab, appFramework, } = props; - const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + const isCloud = (window.location.host === "localhost:3002" || window.location.host === "shuffler.io") ? true : (process.env.IS_SSR === "true"); let navigate = useNavigate(); + if (window.location.pathname === "/workflows") { + const hidePriorities = localStorage.getItem("hidePriorities", "true") + if (hidePriorities === "true") { + return null + } + } + var realignedSrc = false var realignedDst = false let newdescription = priority.description @@ -114,7 +121,7 @@ const Priority = (props) => { const srcSize = realignedSrc ? 35 : 30 const dstSize = realignedDst ? 35 : 30 return ( -
    +
    {priority.type === "usecase" || priority.type == "apps" ? : null} @@ -124,7 +131,7 @@ const Priority = (props) => { {priority.type === "usecase" && priority.description.includes("&") ? - {priority.name} + {priority.name} {newdescription.split("&")[0]} @@ -132,7 +139,7 @@ const Priority = (props) => { {newdescription.split("&").length > 3 ? - {priority.name+"2"} + {priority.name+"2"} {newdescription.split("&")[2]} @@ -176,6 +183,20 @@ const Priority = (props) => { diff --git a/frontend/src/components/RecentWorkflow.jsx b/frontend/src/components/RecentWorkflow.jsx new file mode 100644 index 00000000..f61492e2 --- /dev/null +++ b/frontend/src/components/RecentWorkflow.jsx @@ -0,0 +1,155 @@ +import React from "react" + +import { Link } from "react-router-dom"; +import { + Avatar, + Box, + Button, + Typography, + Tooltip, +} from "@mui/material" +import { useNavigate } from "react-router"; +import theme from "../theme.jsx"; + +import { + Lock as LockIcon, +} from '@mui/icons-material'; + +// onclickHandler = function override from parent onclick +const RecentWorkflow = ({ workflow, onclickHandler, leftNavOpen, currentWorkflowId, }) => { + + const navigate = useNavigate(); + + const [hovered, setHovered] = React.useState(false) + if (workflow === undefined || workflow === null) { + console.log("No workflow") + return null + } + + /* + * Note for @Lalit: + * + * When you want to make a list of something that is complex, + * make a component. This way, you can easily manage + * the logic, and we can actually reuse it. This component is used + * multiple places, so do make sure to not break it randomly. + */ + + const expandLeftNav = leftNavOpen === true || leftNavOpen === undefined ? true : false + + // Check if workflow.input_markdown has an image in it + // If it does, show it as the main thing + var relevantImageUrl = "" + if (workflow?.form_control?.input_markdown !== undefined && workflow?.form_control?.input_markdown !== null && workflow?.form_control?.input_markdown !== "") { + // Look for tag or ![alt](src) markdown + // html > markdown + const imgTag = workflow?.form_control?.input_markdown.match(/]+>/g) + + if (imgTag !== null) { + const src = imgTag[0].match(/src="([^"]+)"/) + if (src !== null) { + relevantImageUrl = src[1] + } + } else { + const markdownTag = workflow?.form_control?.input_markdown.match(/!\[.*\]\(.*\)/g) + + if (markdownTag !== null) { + const src = markdownTag[0].match(/\(([^)]+)\)/) + if (src !== null) { + relevantImageUrl = src[1] + } + } + } + } + + return ( +
    setHovered(true)} + onMouseLeave={() => setHovered(false)} + > + + + +
    + ) +} + +export default RecentWorkflow diff --git a/frontend/src/components/RuntimeDebugger.jsx b/frontend/src/components/RuntimeDebugger.jsx index 9a91eb3e..83345bbc 100644 --- a/frontend/src/components/RuntimeDebugger.jsx +++ b/frontend/src/components/RuntimeDebugger.jsx @@ -35,6 +35,7 @@ import { PlayArrow as PlayArrowIcon, Insights as InsightsIcon, Replay as ReplayIcon, + EditNote as EditNoteIcon, } from '@mui/icons-material'; import { DataGrid, GridColDef, GridValueGetterParams } from '@mui/x-data-grid' @@ -76,6 +77,10 @@ const RuntimeDebugger = (props) => { {"id": "", "name": "All Workflows",} ]) + if (document != undefined) { + document.title = "Workflow Run Debugger" + } + // Shitty workflow search on purpose :) const handleWorkflowUsageCount = (workflows) => { if (workflows === undefined || workflows === null || workflows.length === 0) { @@ -219,7 +224,7 @@ const RuntimeDebugger = (props) => { } - const getAvailableWorkflows = () => { + const getAvailableWorkflows = (workflowId) => { fetch(globalUrl + "/api/v1/workflows", { method: "GET", headers: { @@ -240,6 +245,15 @@ const RuntimeDebugger = (props) => { var foundWorkflows = [{"id": "", "name": "All Workflows",}] foundWorkflows.push(...responseJson) setWorkflows(foundWorkflows) + + if (workflowId !== undefined && workflowId !== null && workflowId !== "" && workflowId.length === 36) { + for (var key in responseJson) { + if (responseJson[key].id === workflowId) { + setWorkflow(responseJson[key]) + break + } + } + } } }) .catch((error) => { @@ -248,7 +262,6 @@ const RuntimeDebugger = (props) => { } useEffect(() => { - getAvailableWorkflows() // Find workflow_id in url query const urlParams = new URLSearchParams(window.location.search); @@ -265,6 +278,8 @@ const RuntimeDebugger = (props) => { } } + getAvailableWorkflows(workflowId) + const foundStatus = urlParams.get('status'); if (foundStatus !== undefined && foundStatus !== null && foundStatus !== "") { setStatus(foundStatus) @@ -314,15 +329,17 @@ const RuntimeDebugger = (props) => { var source = params.row.execution_source if (source === "schedule") { - foundSource = schedule + foundSource = schedule } else if (source === "webhook") { - foundSource = webhook + foundSource = webhook } else if (source === "subflow" || source.length === 36) { - foundSource = subflow + foundSource = subflow source = "subflow" } else if (source === "rerun" || source.length === 36) { foundSource = source = "rerun of a previous run" + } else if (source === "form") { + foundSource = } else { source = "manual" } @@ -888,13 +905,13 @@ const RuntimeDebugger = (props) => { {userdata.support === true ? : null}
    @@ -942,11 +959,8 @@ const RuntimeDebugger = (props) => { }, }} getOptionLabel={(option) => { - if ( - option === undefined || - option === null || - option.name === undefined || - option.name === null + if (option === undefined || option === null || + option.name === undefined || option.name === null ) { return "No Workflow Selected"; } @@ -961,7 +975,7 @@ const RuntimeDebugger = (props) => { style={{ backgroundColor: theme.palette.inputColor, height: 50, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, marginTop: 5, marginLeft: 5, }} @@ -995,13 +1009,13 @@ const RuntimeDebugger = (props) => { {data.image !== undefined && data.image !== null && data.image.length > 0 ? - {data.name} + {data.name} : null} Choose {data.name} - } placement="bottom"> + }> { { - const { serverside, globalUrl, userdata, setModalOpen, modalOpen } = props + const { serverside, globalUrl, userdata } = props let navigate = useNavigate(); + const { searchBarModalOpen, setSearchBarModalOpen } = useContext(Context); const borderRadius = 3 const node = useRef() const [searchOpen, setSearchOpen] = useState(false) @@ -56,8 +59,8 @@ const SearchData = props => { const [value, setValue] = useState(""); const handleLinkClick = () => { - if (modalOpen) { - setModalOpen(false); // Assuming setModalOpen is defined correctly + if (searchBarModalOpen) { + setSearchBarModalOpen(false); // Assuming setModalOpen is defined correctly } else { console.log("Condition not met, staying on the same page"); } @@ -71,7 +74,7 @@ const SearchData = props => { // return null //} - const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + const isCloud = (window.location.host === "localhost:3002" || window.location.host === "shuffler.io") ? true : (process.env.IS_SSR === "true"); // if (window.location.pathname === "/docs" || window.location.pathname === "/apps" || window.location.pathname === "/usecases" ) { // setModalOpen(false) // } @@ -88,14 +91,14 @@ const SearchData = props => { const textFieldRef = useRef(null); const keyPressHandler = (e) => { - if (e.which === 13) { + if (e.key === "Enter") { + e.preventDefault(); // navigate(`/search?q=${currentRefinement}`, { state: value, replace: true }); // setModalOpen(false); const trimmedValue = inputValue.trim(); if (trimmedValue !== '') { - e.preventDefault(); navigate(`/search?q=${trimmedValue}`, { state: trimmedValue, replace: true }); - setModalOpen(false); + setSearchBarModalOpen(false); } } }; @@ -249,7 +252,7 @@ const SearchData = props => { { //console.log("CLICK") setSearchOpen(true) - setModalOpen(false) + setSearchBarModalOpen(false) aa('init', { appId: searchClient.appId, apiKey: searchClient.transporter.queryParameters["x-algolia-api-key"] @@ -498,7 +501,7 @@ const SearchData = props => { return ( { setSearchOpen(true) - setModalOpen(false) + setSearchBarModalOpen(false) aa('init', { appId: searchClient.appId, apiKey: searchClient.transporter.queryParameters["x-algolia-api-key"] @@ -547,8 +550,6 @@ const SearchData = props => { e.preventDefault() e.stopPropagation() - console.log("OBJECT CHANGE: ", hit.objectID) - // This does nothing rofl if (userdata.active_apps === undefined || userdata.active_apps === null) { activateApp(hit.name, hit.objectID, "activate") @@ -702,7 +703,7 @@ const SearchData = props => { console.log("CLICK") setSearchOpen(true) - setModalOpen(false) + setSearchBarModalOpen(false) }}> { setMouseHoverIndex(index) @@ -873,10 +874,11 @@ const SearchData = props => { - + + + ) : null diff --git a/frontend/src/components/Searchfield.jsx b/frontend/src/components/Searchfield.jsx index dc5f9179..ec257cc6 100644 --- a/frontend/src/components/Searchfield.jsx +++ b/frontend/src/components/Searchfield.jsx @@ -1,9 +1,11 @@ -import React, { useState, useEffect, useRef } from 'react'; +import React, { useState, useEffect, useRef, useContext } from 'react'; import theme from '../theme.jsx'; import { useNavigate, Link, useParams } from "react-router-dom"; import SearchBox from "../components/SearchData.jsx"; +import { Context } from '../context/ContextApi.jsx'; + import { Chip, IconButton, @@ -45,20 +47,22 @@ const chipStyle = { const SearchField = props => { const { serverside, userdata, isMobile, isLoaded, globalUrl, isHeader, isLoggedIn, small, rounded } = props + const {searchBarModalOpen, setSearchBarModalOpen} = useContext(Context); + let navigate = useNavigate(); const borderRadius = 3 const node = useRef() const [searchOpen, setSearchOpen] = useState(false) - const [modalOpen, setModalOpen] = React.useState(false); + // const [modalOpen, setModalOpen] = React.useState(false); const [oldPath, setOldPath] = useState("") const [value, setValue] = useState(""); useEffect(() => { Mousetrap.bind(['command+k', 'ctrl+k'], () => { - setModalOpen(true); + setSearchBarModalOpen(true); return false; // Prevent the default action }); Mousetrap.bind(['esc'], () => { - setModalOpen(false); + setSearchBarModalOpen(false); return false; // Prevent the default action }); @@ -72,9 +76,9 @@ const SearchField = props => { // console.log("key:", dataValue.key), //console.log("value:",dataValue.value), { - setModalOpen(false); + setSearchBarModalOpen(false); }} PaperProps={{ style: { @@ -92,12 +96,12 @@ const SearchField = props => { {isHeader ?
    Search for Docs, Apps, Workflows and more
    : null} - + @@ -124,10 +128,10 @@ const SearchField = props => { ); return ( -
    +
    {modalView} { color="primary" placeholder="Search Apps, Workflows, Docs..." onClick={(event) => { - setModalOpen(true) + setSearchBarModalOpen(true) }} limit={5} /> diff --git a/frontend/src/components/ShuffleCodeEditor1.jsx b/frontend/src/components/ShuffleCodeEditor1.jsx index a23de3e5..875637a1 100644 --- a/frontend/src/components/ShuffleCodeEditor1.jsx +++ b/frontend/src/components/ShuffleCodeEditor1.jsx @@ -40,11 +40,12 @@ import { Close as CloseIcon, DragIndicator as DragIndicatorIcon, + RestartAlt as RestartAltIcon, } from '@mui/icons-material'; import { validateJson } from "../views/Workflows.jsx"; -import ReactJson from "react-json-view"; +import ReactJson from "react-json-view-ssr"; import PaperComponent from "../components/PaperComponent.jsx"; import { padding, textAlign } from '@mui/system'; @@ -56,6 +57,7 @@ import { tags as t } from '@lezer/highlight'; import AceEditor from "react-ace"; import ace from "ace-builds"; import 'ace-builds/src-noconflict/mode-python'; +import 'ace-builds/src-noconflict/mode-json'; //import 'ace-builds/src-noconflict/theme-twilight'; //import 'ace-builds/src-noconflict/theme-solarized_dark'; import 'ace-builds/src-noconflict/theme-gruvbox'; @@ -105,12 +107,16 @@ const CodeEditor = (props) => { selectedAction , workflowExecutions, getParents, - + activeDialog, + setActiveDialog, fieldname, + contentLoading, + editorData, + + setAiQueryModalOpen, + fullScreenMode } = props - - const [localcodedata, setlocalcodedata] = React.useState(codedata === undefined || codedata === null || codedata.length === 0 ? "" : codedata); //const { setContainer } = useCodeMirror({ @@ -163,6 +169,11 @@ const CodeEditor = (props) => { setMenuPosition(null); } + useEffect(() => { + highlight_variables(localcodedata) + expectedOutput(localcodedata) + }, [localcodedata]) + let navigate = useNavigate(); useEffect(() => { @@ -627,8 +638,8 @@ const CodeEditor = (props) => { newMarkers.push({ startRow: i, startCol: startCh, - endRow: i+1, - endCol: endCh+1, + endRow: i, + endCol: endCh, className: correctVariable ? "good-marker" : "bad-marker", type: "text", }) @@ -811,11 +822,7 @@ const CodeEditor = (props) => { } const handleItemClick = (values) => { - if ( - values === undefined || - values === null || - values.length === 0 - ) { + if (values === undefined || values === null || values.length === 0) { return; } @@ -830,7 +837,11 @@ const CodeEditor = (props) => { toComplete += values[key].autocomplete; } - setlocalcodedata(localcodedata+toComplete) + + handleClick({ + "value": toComplete + }) + //setlocalcodedata(localcodedata+toComplete) setMenuPosition(null) } @@ -839,10 +850,37 @@ const CodeEditor = (props) => { return } - if (!item.value.includes("{%") && !item.value.includes("{{")) { - setlocalcodedata(localcodedata+" | "+item.value+" }}") - } else { - setlocalcodedata(localcodedata+item.value) + // Injects it in the right spot instead of random + var edited = false + if (currentCharacter !== undefined && currentCharacter !== null && currentCharacter !== -1 && currentLine !== undefined && currentLine !== null && currentLine !== -1) { + // Input at the right spot + var codedatasplit = localcodedata.split('\n') + if (codedatasplit.length > currentLine) { + var currentLineData = codedatasplit[currentLine] + + // Remove newlines from item.value + if (item.value.includes("% python %")) { + item.value = item.value.replaceAll("\n", ";") + item.value = item.value.replaceAll("python %};", "python %}") + } else { + item.value = item.value.replaceAll("\n", "") + } + + currentLineData = currentLineData.slice(0, currentCharacter) + item.value + currentLineData.slice(currentCharacter) + codedatasplit[currentLine] = currentLineData + + setlocalcodedata(codedatasplit.join('\n')) + + edited = true + } + } + + if (edited === false) { + if (!item.value.includes("{%") && !item.value.includes("{{")) { + setlocalcodedata(localcodedata+" | "+item.value+" }}") + } else { + setlocalcodedata(localcodedata+item.value) + } } setAnchorEl(null) @@ -858,8 +896,8 @@ const CodeEditor = (props) => { // Shuffle Tools 1.2.0 (in most cases?) const appid = toolsAppId !== undefined && toolsAppId !== null && toolsAppId.length > 0 ? toolsAppId : "3e2bdf9d5069fe3f4746c29d68785a6a" - const actionname = selectedAction.name === "execute_python" && !inputdata.replaceAll(" ", "").includes("{%python%}") ? "execute_python" : "repeat_back_to_me" - const params = actionname === "execute_python" ? [{"name": "code", "value":inputdata}] : [{"name":"call", "value": inputdata}] + const actionname = selectedAction.name === "execute_python" && !inputdata.replaceAll(" ", "").includes("{%python%}") ? "execute_python" : selectedAction.name === "execute_bash" ? "execute_bash" : "repeat_back_to_me" + const params = actionname === "execute_python" ? [{"name": "code", "value":inputdata}] : actionname === "execute_bash" ? [{"name": "code", "value":inputdata}, {"name": "shuffle_input", "value": "", }] : [{"name":"call", "value": inputdata}] const actiondata = {"description":"Repeats the call parameter","id":"","name":actionname,"label":"","node_type":"","environment":"","sharing":false,"private_id":"","public_id":"","app_id": appid,"tags":null,"authentication":[],"tested":false,"parameters": params, "execution_variable":{"description":"","id":"","name":"","value":""},"returns":{"description":"","example":"","id":"","schema":{"type":"string"}},"authentication_id":"","example":"","auth_not_required":false,"source_workflow":"","run_magic_output":false,"run_magic_input":false,"execution_delay":0,"app_name":"Shuffle Tools","app_version":"1.2.0","selectedAuthentication":{}} @@ -946,23 +984,77 @@ const CodeEditor = (props) => { // Define a custom completer for the Ace Editor - const customVariables = availableVariables const customCompleter = { getCompletions: function(editor, session, pos, prefix, callback) { - callback(null, customVariables.map((variable) => ({ - caption: variable, - value: variable, - meta: 'custom', - }))); + console.log("CUSTOM COMPLETER: ", prefix) + + callback(null, availableVariables.map((variable) => { + console.log("CUSTOM VAR: ", variable) + + return ({ + caption: variable, + value: variable, + meta: 'custom', + }) + })) } } + if (fullScreenMode) { + return ( + { + // setlocalcodedata(value) + // expectedOutput(value) + // highlight_variables(value,editor) + setlocalcodedata(value) + setcodedata(value) + }} + name="python-editor" + fontSize={14} + width="100%" + height="100%" + showPrintMargin={false} + showGutter={true} + markers={markers} + highlightActiveLine={false} + + enableBasicAutocompletion={true} + completers={[customCompleter]} + + style={{ + wordBreak: "break-word", + marginTop: 0, + paddingBottom: 10, + overflowY: "auto", + whiteSpace: "pre-wrap", + wordWrap: "break-word", + backgroundColor: "rgba(40,40,40,1)", + zIndex: activeDialog === "codeeditor" ? 1200 : 1100, + }} + + setOptions={{ + enableBasicAutocompletion: true, + enableLiveAutocompletion: true, + enableSnippets: true, + showLineNumbers: true, + tabSize: 4, + fontFamily: "'JetBrains Mono', Consolas, monospace", + useSoftTabs: true + }} + /> + ) + } + return ( { @@ -977,8 +1069,14 @@ const CodeEditor = (props) => { }} PaperComponent={PaperComponent} PaperProps={{ + onClick: () => { + if (setActiveDialog !== undefined) { + setActiveDialog("codeeditor") + } + }, style: { - zIndex: 12501, + // zIndex: 12501, + pointerEvents: "auto", color: "white", minWidth: isMobile ? "100%" : isFileEditor ? 650 : "80%", maxWidth: isMobile ? "100%" : isFileEditor ? 650 : 1100, @@ -986,9 +1084,22 @@ const CodeEditor = (props) => { maxHeight: isMobile ? "100%" : 700, border: theme.palette.defaultBorder, padding: isMobile ? "25px 10px 25px 10px" : 25, + zoom: 0.8, + backgroundColor: "black", }, }} > + + {contentLoading === true ? + + + + : null} + { */} { isFileEditor ? null :
    - {selectedAction.name === "execute_python" ? + {selectedAction?.name === "execute_python" ? Run Python Code + : + selectedAction.name === "execute_bash" ? + + Run Bash Code + :
    @@ -1618,6 +1762,7 @@ const CodeEditor = (props) => { overflow: "auto", minWidth: 450, maxWidth: "100%", + zIndex: activeDialog === "codeeditor" ? 1200 : 1100, }} collapsed={false} enableClipboard={(copy) => { @@ -1645,11 +1790,12 @@ const CodeEditor = (props) => { padding: 10, marginTop: -2, border: `2px solid ${theme.palette.inputColor}`, - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, maxHeight: 450, minHeight: 450, overflow: "auto", wordWrap: "anywhere", + zIndex: activeDialog === "codeeditor" ? 1200 : 1100, }} > {expOutput} diff --git a/frontend/src/components/SuggestedWorkflows.jsx b/frontend/src/components/SuggestedWorkflows.jsx index 1f01529a..7780b1a9 100644 --- a/frontend/src/components/SuggestedWorkflows.jsx +++ b/frontend/src/components/SuggestedWorkflows.jsx @@ -102,7 +102,7 @@ const SuggestedWorkflows = (props) => { placement="top" style={{ zIndex: 10011 }} > -
    { +
    { setHovering(true) }} onMouseOut={() => { setHovering(false) @@ -140,7 +140,7 @@ const SuggestedWorkflows = (props) => { // return ( - + 0 && usecaseSearchType.length > 0} onClose={() => { @@ -193,7 +193,7 @@ const SuggestedWorkflows = (props) => { apps={apps} /> -
    +
    Suggested Workflows ({finishedUsecases.length}/{usecaseSuggestions.length}) diff --git a/frontend/src/components/UsecaseSearch.jsx b/frontend/src/components/UsecaseSearch.jsx index 474fcce8..2460876d 100644 --- a/frontend/src/components/UsecaseSearch.jsx +++ b/frontend/src/components/UsecaseSearch.jsx @@ -349,7 +349,7 @@ const UsecaseSearch = (props) => { const [selectedAction, setSelectedAction] = React.useState({}); const [firstRequest, setFirstRequest] = React.useState(true); - const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + const isCloud = (window.location.host === "localhost:3002" || window.location.host === "shuffler.io") ? true : (process.env.IS_SSR === "true"); //const alert = useAlert() useEffect(() => { @@ -594,7 +594,7 @@ const UsecaseSearch = (props) => { width: 30, height: 30, border: "2px solid rgba(255,255,255,0.6)", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, maxHeight: 30, maxWidth: 30, overflow: "hidden", @@ -616,7 +616,7 @@ const UsecaseSearch = (props) => { width: 30, height: 30, border: "2px solid rgba(255,255,255,0.6)", - borderRadius: theme.palette.borderRadius, + borderRadius: theme.palette?.borderRadius, maxWidth: 30, maxHeight: 30, overflow: "hidden", @@ -1266,9 +1266,11 @@ const UsecaseSearch = (props) => { .then((responseJson) => { if (responseJson.success === false) { var msgString = "Failed to activate the app" + if (responseJson.reason !== undefined) { msgString += ": " + responseJson.reason } + toast(msgString) } else { //toast("App activated for your organization! Refresh the page to use the app.") @@ -1388,7 +1390,7 @@ const UsecaseSearch = (props) => { {startText} -
    +
    {selectionOpen === true ? { // {defaultSearch}: {allusecases[usecaseIndex].name} //console.log("UseCase: ", usecases) return ( -
    +
    {configureWorkflowModal} {authenticationModal} {showTitle !== false && defaultSearch !== undefined ? diff --git a/frontend/src/components/WelcomeForm2.jsx b/frontend/src/components/WelcomeForm2.jsx index 3c9200e0..3e11cfc2 100644 --- a/frontend/src/components/WelcomeForm2.jsx +++ b/frontend/src/components/WelcomeForm2.jsx @@ -64,6 +64,7 @@ const WelcomeForm = (props) => { discoveryWrapper, setDiscoveryWrapper, appFramework, + setAppFramework, getFramework, activeStep, setActiveStep, @@ -160,7 +161,7 @@ const WelcomeForm = (props) => { const [clickdiff, setclickdiff] = useState(0); const [mouseHoverIndex, setMouseHoverIndex] = useState(-1) - const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + const isCloud = (window.location.host === "localhost:3002" || window.location.host === "shuffler.io") ? true : (process.env.IS_SSR === "true"); //const alert = useAlert(); let navigate = useNavigate(); @@ -435,20 +436,6 @@ const WelcomeForm = (props) => { setSkipped(newSkipped); }; - const handleBack = () => { - setActiveStep((prevActiveStep) => prevActiveStep - 1); - - if (activeStep === 2) { - setDiscoveryWrapper({}); - - if (getFramework !== undefined) { - getFramework(); - } - navigate("/welcome?tab=2"); - } else if (activeStep === 1) { - navigate("/welcome?tab=1"); - } - }; const handleReset = () => { setActiveStep(0); @@ -645,6 +632,7 @@ const WelcomeForm = (props) => { globalUrl={globalUrl} userdata={userdata} appFramework={appFramework} + setAppFramework={setAppFramework} setActiveStep={setActiveStep} defaultSearch={defaultSearch} setDefaultSearch={setDefaultSearch} @@ -660,7 +648,7 @@ const WelcomeForm = (props) => {
    -
    +
    { const { maxRows, showName, showSuggestion, isMobile, globalUrl, parsedXs, alternativeView, onlyResults, inputsearch } = props - const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + const isCloud = (window.location.host === "localhost:3002" || window.location.host === "shuffler.io") ? true : (process.env.IS_SSR === "true"); const rowHandler = maxRows === undefined || maxRows === null ? 50 : maxRows const xs = parsedXs === undefined || parsedXs === null ? isMobile ? 6 : 4 : parsedXs //const [apps, setApps] = React.useState([]); @@ -204,6 +204,9 @@ const AppGrid = props => { style={{backgroundColor: theme.palette.inputColor, borderRadius: borderRadius, margin: 10, width: "100%",}} InputProps={{ style:{ + color: "white", + fontSize: "1em", + height: 50, }, startAdornment: ( @@ -221,6 +224,11 @@ const AppGrid = props => { removeQuery("q") refine(event.currentTarget.value) }} + onKeyDown={(event) => { + if(event.key === "Enter") { + event.preventDefault(); + } + }} limit={5} /> : null} @@ -233,6 +241,8 @@ const AppGrid = props => { flexWrap: "wrap", alignContent: "space-between", marginTop: 5, + padding: "0px 180px", + width:"auto" } var workflowDelay = -50 diff --git a/frontend/src/components/WorkflowTemplatePopup.jsx b/frontend/src/components/WorkflowTemplatePopup.jsx index 13a6735f..74459d59 100644 --- a/frontend/src/components/WorkflowTemplatePopup.jsx +++ b/frontend/src/components/WorkflowTemplatePopup.jsx @@ -47,7 +47,7 @@ const WorkflowTemplatePopup = (props) => { const [requestSent, setRequestSent] = React.useState(false) - const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + const isCloud = (window.location.host === "localhost:3002" || window.location.host === "shuffler.io") ? true : (process.env.IS_SSR === "true"); let navigate = useNavigate(); useEffect(() => { if (modalOpen !== true) { @@ -585,7 +585,7 @@ const WorkflowTemplatePopup = (props) => { {/*errorMessage === "" && configurationFinished === true && workflow.id !== undefined && workflowLoading === false ? { // Open in new tab window.open("/workflows/" + workflow.id, "_blank") diff --git a/frontend/src/components/WorkflowTemplatePopup2.jsx b/frontend/src/components/WorkflowTemplatePopup2.jsx new file mode 100644 index 00000000..2e666637 --- /dev/null +++ b/frontend/src/components/WorkflowTemplatePopup2.jsx @@ -0,0 +1,901 @@ +import React, { useState, useEffect } from "react"; + +import { toast } from "react-toastify" +import theme from '../theme.jsx'; +import { useNavigate, Link, useParams } from "react-router-dom"; +import AppSearchButtons from "../components/AppSearchButtons.jsx"; +import { isMobile } from "react-device-detect"; +import RenderCytoscape from "../components/RenderCytoscape.jsx"; +import { + Button, + Typography, + Dialog, + DialogTitle, + DialogContent, + DialogActions, + Drawer, + CircularProgress, + Fade, + IconButton, + Tooltip, +} from "@mui/material"; + +import { + Check as CheckIcon, + TrendingFlat as TrendingFlatIcon, + Close as CloseIcon, + East as EastIcon, + Interests as InterestsIcon, +} from '@mui/icons-material'; + +import { + green, + yellow, + red, + grey, +} from "../views/AngularWorkflow.jsx" + +import WorkflowTemplatePopup2 from "./WorkflowTemplatePopup.jsx"; +import ConfigureWorkflow from "../components/ConfigureWorkflow.jsx"; +import WorkflowValidationTimeline from "../components/WorkflowValidationTimeline.jsx"; +import FixWorkflowValidationErrors from "../components/FixWorkflowValidationErrors.jsx"; + +const WorkflowTemplatePopup = (props) => { + const { + userdata, appFramework, globalUrl, img1, srcapp, img2, dstapp, title, description, visualOnly, apps, isLoggedIn, isHomePage, getAppFramework, showTryit, shownColor, workflowBuilt, usecaseDetails, + + isModalOpenDefault, + setIsClicked, + inputWorkflowId, + } = props; + + const [isActive, setIsActive] = useState(workflowBuilt === true); + const [isHovered, setIsHovered] = useState(false); + const [modalOpen, setModalOpen] = useState(isModalOpenDefault === true ? true : false) + const [errorMessage, setErrorMessage] = useState(""); + const [workflowLoading, setWorkflowLoading] = useState(false) + const [showLoginButton, setShowLoginButton] = useState(false); + const [appAuthentication, setAppAuthentication] = React.useState(undefined); + const [missingSource, setMissingSource] = React.useState(undefined) + const [missingDestination, setMissingDestination] = React.useState(undefined); + const [configurationFinished, setConfigurationFinished] = React.useState(false) + const [appSetupDone, setAppSetupDone] = React.useState(false) + + const [requestSent, setRequestSent] = React.useState(false) + const [showTryitOut, setShowTryitout] = React.useState(showTryit === true ? true : false) + + const [loadingWorkflow, setLoadingWorkflow] = React.useState(false) + const [workflow, setWorkflow] = useState({}); + const [_, setUpdate] = useState(0) + + const fetchWorkflow = (id) => { + if (id === undefined || id === null || id === "") { + return + } + + if (loadingWorkflow === true) { + return + } + + setLoadingWorkflow(true) + const url = `${globalUrl}/api/v1/workflows/${id}` + fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + setLoadingWorkflow(false) + if (response.status !== 200) { + console.log("Status not 200 for framework!"); + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success === false) { + console.log("Error in workflow loading for ID ", id) + } else { + setWorkflow(responseJson) + } + }) + .catch((error) => { + console.log("err in framework: ", error.toString()); + setLoadingWorkflow(false) + }) + + + } + + if (inputWorkflowId !== undefined && inputWorkflowId !== null && inputWorkflowId !== "" && workflow.id !== inputWorkflowId) { + fetchWorkflow(inputWorkflowId) + } + + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + let navigate = useNavigate(); + useEffect(() => { + if (modalOpen !== true) { + if (workflowLoading === true) { + setWorkflowLoading(false) + } + + //console.log("Modal is not open, so we are not doing anything.") + return + } + + if (workflowLoading !== true) { + //console.log("Workflow loading is false, so we can try to get the workflow.") + return + } + + console.log("DEBUG: Skipped direct generation without Try it for now.") + + /* + if (!srcapp.includes(":default") && !dstapp.includes(":default")) { + if (appSetupDone === false && setAppSetupDone !== undefined) { + setAppSetupDone(true) + } + + getGeneratedWorkflow() + } + + if (missingSource !== undefined && missingDestination !== undefined) { + if (appSetupDone === false && setAppSetupDone !== undefined) { + setAppSetupDone(true) + } + } + + if (getAppFramework !== undefined) { + setTimeout(() => { + getAppFramework() + }, 500) + } + */ + }, [modalOpen, missingSource, missingDestination]) + + useEffect(() => { + //console.log("IN USEEFFECT FOR CONFIG: ", configurationFinished) + if (configurationFinished === true && workflow.id !== undefined && workflow.id !== null && workflow.id !== "") { + //toast.success("Generation Successful") + + /* + setTimeout(() => { + navigate("/workflows/" + workflow.id) + }, 2000) + */ + } + }, [configurationFinished, workflow]) + + const imageSize = 32 + const defaultBorder = "1px solid rgba(255,255,255,0.6)" + const imagestyleWrapper = { + height: imageSize, + width: imageSize, + borderRadius: imageSize, + border: isHomePage ? null : defaultBorder, + overflow: "hidden", + display: "flex", + + backgroundColor: theme.palette.inputColor, + } + + const imagestyleWrapperDefault = { + height: imageSize, + width: imageSize, + borderRadius: imageSize, + border: isHomePage ? null : defaultBorder, + overflow: "hidden", + display: "flex", + + backgroundColor: theme.palette.inputColor, + } + + const imagestyle = { + height: imageSize, + width: imageSize, + borderRadius: imageSize, + //border: isHomePage ? null : defaultBorder, + overflow: "hidden", + + backgroundColor: theme.palette.inputColor, + } + + const imagestyleDefault = { + display: "block", + marginLeft: 9, + marginTop: 9, + height: imageSize, + width: "auto", + + backgroundColor: theme.palette.inputColor, + } + + if (modalOpen === false && (title === undefined || title === null || title === "")) { + if (setIsClicked !== undefined) { + setIsClicked(false) + } + + console.log("No title for workflow template popup!"); + return null + } + + + const loadAppAuth = () => { + // Check if it exists, and has keys + // + if (userdata === undefined || userdata === null || Object.keys(userdata).length === 0) { + setErrorMessage("You need to be logged in to try the pre-built Workflow Templates.") + setShowLoginButton(true) + + // Send the user to the login screen after 3 seconds + setTimeout(() => { + // Make it cancel if the state modalOpen changes + if (modalOpen === false) { + return + } + + navigate("/login?view=" + window.location.pathname + window.location.search) + }, 4500) + + return + } + + + fetch(`${globalUrl}/api/v1/apps/authentication`, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for setting app auth :O!"); + } + + return response.json(); + }) + .then((responseJson) => { + if (!responseJson.success) { + toast("Failed to get app auth: " + responseJson.reason); + return + } + + var newauth = []; + for (let authkey in responseJson.data) { + if (responseJson.data[authkey].defined === false) { + continue; + } + + newauth.push(responseJson.data[authkey]); + } + + setAppAuthentication(newauth); + }) + .catch((error) => { + //toast(error.toString()); + console.log("New auth error: ", error.toString()); + }); + } + + // Can create and set workflows + const reloadWorkflow = (workflow_id) => { + + const new_url = `${globalUrl}/api/v1/workflows/${workflow_id}` + return fetch(new_url, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for workflows :O!"); + return; + } + //setSubmitLoading(false); + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success === false) { + if (responseJson.reason !== undefined) { + toast("Error setting workflow: ", responseJson.reason) + } else { + toast("Error setting workflow.") + } + + return + } else if (responseJson.id !== undefined && responseJson.id !== null && responseJson.id !== "") { + setWorkflow(responseJson) + } + + return responseJson; + }) + .catch((error) => { + toast("Failed reloading configured workflow: ", error.toString()); + }); + }; + + // Can create and set workflows + const saveWorkflow = (workflowdata) => { + + const new_url = `${globalUrl}/api/v1/workflows?set_auth=true` + return fetch(new_url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(workflowdata), + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for workflows :O!"); + return; + } + //setSubmitLoading(false); + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success === false) { + if (responseJson.reason !== undefined) { + toast("Error setting workflow: ", responseJson.reason) + } else { + toast("Error setting workflow.") + } + + return + } + + // In case it got a new id, this is to make sure it loads with the correct config + if (responseJson.id !== undefined && responseJson.id !== null && responseJson.id !== "") { + reloadWorkflow(responseJson.id) + } + + return responseJson; + }) + .catch((error) => { + toast("Failed generating workflow: ", error.toString()); + }); + }; + + + const getGeneratedWorkflow = () => { + // POST + // https://shuffler.io/api/v1/workflows/merge + // destination: {app_id: "b9c2feaf99b6309dabaeaa8518c61d3d", app_name: "Servicenow_API", app_version: "",…} + // id: "" + // middle:[] + // name: "Email analysis" + // source:{app_id: "accdaaf2eeba6a6ed43b2efc0112032d", app_name + if (requestSent === true) { + return + } + + console.log("SRCAPP: ", srcapp, "DSTAPP: ", dstapp) + if (srcapp === undefined || srcapp === null) { + srcapp = "" + } + + if ((srcapp !== undefined && srcapp !== null && srcapp.includes(":default")) || (dstapp !== undefined && dstapp !== null && dstapp.includes(":default"))) { + toast("You need to select both a source and destination app before generating this workflow.") + + if (srcapp !== undefined && srcapp !== null && srcapp.includes(":default")) { + setMissingSource({ + "type": srcapp.split(":")[0], + }) + } + + if (dstapp !== undefined && dstapp !== null && dstapp.includes(":default")) { + setMissingDestination({ + "type": dstapp.split(":")[0], + }) + } + + return + } + + setWorkflowLoading(true) + + const newsrcapp = srcapp + const newdstapp = dstapp + + const mergedata = { + name: title, + id: "", + source: { + app_name: newsrcapp, + }, + middle: [], + destination: { + app_name: newdstapp, + }, + } + + setRequestSent(true) + const url = isCloud ? `${globalUrl}/api/v1/workflows/merge` : `https://shuffler.io/api/v1/workflows/merge` + fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + body: JSON.stringify(mergedata), + }) + .then((response) => { + if (response.status !== 200) { + //console.log("Status not 200 for framework!"); + setRequestSent(false) + } + + setWorkflowLoading(false) + return response.json(); + }) + .then((responseJson) => { + if (responseJson.id !== undefined && responseJson.id !== null && responseJson.id !== "" && responseJson.name !== undefined && responseJson.name !== null && responseJson.name !== "") { + console.log("Success in workflow template (prebuilt): ", responseJson); + setWorkflow(responseJson) + + // Sets it in the database properly + saveWorkflow(responseJson) + return + } + + if (responseJson.success === false) { + //console.log("Error in workflow template: ", responseJson.error); + setRequestSent(false) + + const defaultMessage = "Error: Failed to generate workflow the workflow - the Shuffle team has been notified. Contact support@shuffler.io if you want manual help building this usecase until the AI system is handled." + if (responseJson.reason !== undefined && responseJson.reason !== null && responseJson.reason !== "") { + setErrorMessage(defaultMessage + "\n\n" + responseJson.reason) + } else { + setErrorMessage(defaultMessage) + } + + setIsActive(true) + //setTimeout(() => { + // setModalOpen(false) + //}, 5000) + } else { + console.log("Success in workflow template: ", responseJson); + setIsActive(true) + if (responseJson.workflow_id === "") { + console.log("Failed to build workflow for these tools. Closing in 3 seconds.") + return + } + + fetchWorkflow(responseJson.workflow_id) + } + }) + .catch((error) => { + console.log("err in framework: ", error.toString()); + setRequestSent(false) + setWorkflowLoading(false) + }) + } + + if (modalOpen === true && !srcapp?.includes(":default") && !dstapp?.includes(":default")) { + if (appSetupDone === false && setAppSetupDone !== undefined) { + setAppSetupDone(true) + } + + // No autoruns anymore without clicking "Try it" + if (workflow.id === undefined && workflowLoading === false && errorMessage === "") { + //getGeneratedWorkflow() + } + } + + const isFinished = () => { + // Look for configuration fields being done in the current modal + // 1. Start by finding the modal + const template = document.getElementById("workflow-template") + if (template === null || template == undefined) { + return true + } + + // Find item in template with id app-config + const appconfig = template.getElementsByClassName("app-config") + if (appconfig === null || appconfig == undefined) { + return true + } + + return false + } + + const ModalView = () => { + if (modalOpen === false) { + return null + } + + const divHeight = 500 + const divWidth = 500 + + return ( + { + setModalOpen(false); + + if (setIsClicked !== undefined) { + setIsClicked(false) + } + }} + PaperProps={{ + style: { + backgroundColor: "black", + color: "white", + minWidth: isHomePage ? null : isMobile ? 300 : 850, + maxWidth: isHomePage ? null : isMobile ? 300 : 850, + paddingTop: isMobile ? null : 75, + itemAlign: "center", + }, + }} + > + { + setModalOpen(false); + }} + > + + + + + Configure Workflow + + + {title === undefined || title === null || title === "" ? null : + + + Selected Workflow: + +
    + + +
    +     + } + +
    + {/* Fix the timeline when errors are fixed.. how? */} + + + +
    + + {workflowLoading === true ? +
    + Generating the Workflow... + + +
    + : +
    + {usecaseDetails === undefined ? null : + + {usecaseDetails?.description} + + } + + {errorMessage !== "" ? errorMessage : ""} + + {showLoginButton ? + + + Sign up + + + + : + !showTryitOut && !isActive ? + + : null} +
    + } + + {!isLoggedIn ? null : +
    + {(appSetupDone === false && missingSource !== undefined || missingDestination !== undefined) ? + + {"Find relevant Apps for this Usecase"} + + : null} + + {(missingSource !== undefined) ? +
    + +
    + : null} + + {(missingDestination !== undefined) ? +
    + +
    + : null} +
    + } + + + +     +     + ) + } + + if (isModalOpenDefault === true) { + return + } + + var parsedTitle = title !== undefined && title !== null ? title : "" + const maxlength = 50 + if (title !== undefined && title !== null && title.length > maxlength) { + parsedTitle = title.substring(0, maxlength) + "..." + } + + parsedTitle = parsedTitle.replaceAll("_", " ") + + const parsedDescription = description !== undefined && description !== null ? description.replaceAll("_", " ") : "" + + const boxHeight = 104 + const highlightColor = shownColor !== undefined && shownColor !== null && shownColor !== "" ? shownColor : "#f85a3e" + + var hasInterest = false + if (userdata.interests !== undefined && userdata.interests !== null && userdata.interests.length > 0) { + const comparisonTitle = title === undefined || title === null ? "" : title.trim().toLowerCase().replaceAll(" ", "_") + for (var interestkey in userdata.interests) { + if (userdata.interests[interestkey].name === undefined || userdata.interests[interestkey].name === null || userdata.interests[interestkey].name === "") { + continue + } + + if (modalOpen) { + console.log("COMPARE: ", userdata.interests[interestkey].name.trim().toLowerCase().replaceAll(" ", "_"), comparisonTitle) + } + + if (userdata.interests[interestkey].name.trim().toLowerCase().replaceAll(" ", "_") === comparisonTitle) { + if (modalOpen) { + console.log("FOUND: ", comparisonTitle) + } + + hasInterest = true + break + } + } + } + + const borderStyle = isHomePage ? null : isHovered && isActive ? errorMessage !== "" ? "1px solid red" : `2px solid ${theme.palette.green}` : isHovered ? `1px solid ${highlightColor}` : "1px solid rgba(33, 33, 33, 1)" + + return ( +
    + + +
    { + setIsHovered(true) + + setShowTryitout(true) + }} + onMouseLeave={() => { + setIsHovered(false) + + if (showTryit !== true) { + setShowTryitout(false) + } + }} + onClick={() => { + if (visualOnly === true) { + console.log("Not showing more than visuals.") + return + } + + if (!isLoggedIn) { + loadAppAuth() + setModalOpen(true) + } else if (isLoggedIn && errorMessage !== "") { + toast.error("Already failed to generate a workflow for this usecase. Please try again later or contact support@shuffler.io.") + + setModalOpen(true) + } else if (isActive) { + // toast.success("Workflow already generated. Please try another workflow template!") + + // FIXME: Remove these? + loadAppAuth() + setModalOpen(true) + //getGeneratedWorkflow() + } else { + setModalOpen(true) + //setWorkflowLoading(false) + } + }} + > + +
    + {shownColor !== undefined && shownColor !== null && shownColor !== "" ? +
    + : null} + +
    +
    +
    + {img1 !== undefined && img1 !== "" && srcapp !== undefined && srcapp !== "" ? + +
    + +
    +     + : +
    + } +
    + + + {img2 !== undefined && img2 !== "" && dstapp !== undefined && dstapp !== "" ? + +
    +
    + +
    +
    +     + : +
    + } + +
    +
    + + {parsedTitle} + +
    +
    +
    + +
    + {isActive === true && errorMessage === "" ? + + + + : ""} + + {!isActive && hasInterest === true ? + + + + : null} +
    + + + {showTryitOut && !isActive ? + + + + : null} +
    + + +
    + ) +} + +export default WorkflowTemplatePopup diff --git a/frontend/src/components/WorkflowValidationTimeline.jsx b/frontend/src/components/WorkflowValidationTimeline.jsx new file mode 100644 index 00000000..5ef7ba70 --- /dev/null +++ b/frontend/src/components/WorkflowValidationTimeline.jsx @@ -0,0 +1,694 @@ +import React, { useState, } from "react"; +import { makeStyles, createStyles } from "@mui/styles"; +import { toast } from "react-toastify" + +import { + Tooltip, + Chip, + Typography, + IconButton, + + Avatar, + AvatarGroup, +} from "@mui/material" + +import { + ErrorOutline as ErrorOutlineIcon, +} from "@mui/icons-material" + +import { + green, + yellow, + red, + grey, +} from "../views/AngularWorkflow.jsx" + +import WorkflowTemplatePopup2 from "../components/WorkflowTemplatePopup2.jsx" +import { validateJson, GetIconInfo } from "../views/Workflows.jsx"; +import theme from "../theme.jsx"; +const itemHeight = 24 + +export const getParentNodes = (workflow, action) => { + if (action === undefined || action === null) { + return [] + } + + if (workflow.actions === undefined || workflow.actions === null) { + workflow.actions = [] + } + + if (workflow.triggers === undefined || workflow.triggers === null) { + workflow.triggers = [] + } + + if (workflow.branches === undefined || workflow.branches === null) { + workflow.branches = [] + } + + var allkeys = [action.id]; + var handled = []; + var results = []; + + // maxiter = max amount of parent nodes to loop + // also handles breaks if there are issues + var iterations = 0; + var maxiter = 10; + while (true) { + for (let parentkey in allkeys) { + if (allkeys[parentkey] === undefined) { + continue + } + + var currentnode = workflow.actions.find((element) => element.id === allkeys[parentkey]) + if (currentnode === undefined) { + currentnode = workflow.triggers.find((element) => element.id === allkeys[parentkey]) + + if (currentnode === undefined) { + //console.log("Could not find parent node for: ", allkeys[parentkey]) + continue + } + } + + if (handled.includes(currentnode.id)) { + continue + } else { + handled.push(currentnode.id); + results.push(currentnode); + } + + // Get the name / label here too? + if (currentnode.length === 0) { + continue; + } + + // FIXME: This part is only handling first level, + // but needs to recurse + var incomingEdges = [] + for (var branchkey in workflow.branches) { + const branch = workflow.branches[branchkey] + if (branch.destination_id !== currentnode.id) { + continue + } + + // Go up in the levels + const parents = getParentNodes(workflow, { + id: branch.source_id, + }) + if (parents.length > 0) { + incomingEdges = incomingEdges.concat(parents) + } + + incomingEdges.push(branch) + } + + for (let i = 0; i < incomingEdges.length; i++) { + var tmp = incomingEdges[i]; + if (tmp.decorator === true) { + continue + } + + if (!allkeys.includes(tmp.source_id)) { + allkeys.push(tmp.source_id) + } + } + } + + if (results.length === allkeys.length || iterations === maxiter) { + break + } + + iterations += 1 + } + + // Remove on the end as we don't want to remove everything + results = results.filter((data) => data.id !== action.id) + results = results.filter((data) => data.type === "ACTION" || data.app_name === "Shuffle Workflow" || data.app_name === "User Input" || data.app_name === "shuffle-subflow") + results.push({ label: "Execution Argument", type: "INTERNAL" }) + + return results +} + +const WorkflowValidationTimeline = (props) => { + const { globalUrl, userdata, workflow, originalWorkflow, apps, getParents, execution, showHoverColor, } = props + + const [hovering, setHovering] = useState(false) + const [decidedColor, setDecidedColor] = useState(grey) + const [isClicked, setIsClicked] = useState(false) + + const showMiddle = false + if (workflow === undefined || workflow === null) { + return null + } + + if (workflow.validation === undefined || workflow.validation === null) { + return null + } + + if (workflow.actions === undefined || workflow.actions === null) { + workflow.actions = [] + } + + if (workflow.triggers === undefined || workflow.triggers === null) { + workflow.triggers = [] + } + + if (workflow.branches === undefined || workflow.branches === null) { + workflow.branches = [] + } + + var results = [] + if (execution !== undefined) { + results = execution.results + } + + // 1. Find startnode + // 2. Map childnodes from it + var startnodeId = workflow.start + + if (execution !== undefined && execution !== null) { + startnodeId = execution.start + } + + // Find parent of startnodeId and if it's a webhook + var relevantactions = [] + for (var key in workflow.branches) { + const branch = workflow.branches[key] + if (branch.destination_id !== startnodeId) { + continue + } + + for (var triggerkey in workflow.triggers) { + const trigger = workflow.triggers[triggerkey] + if (trigger.trigger_type !== "WEBHOOK") { + continue + } + + if (trigger.id === branch.source_id) { + trigger.order = -1 + relevantactions.push(trigger) + break + } + } + } + + for (var key in workflow.actions) { + const action = workflow.actions[key] + if (action.id === startnodeId) { + action.order = 0 + relevantactions.push(action) + continue + } + + var parents = [] + if (getParents !== undefined) { + parents = getParents(action) + } else { + parents = getParentNodes(workflow, action) + } + + if (action.app_name === "Integration Framework" || action.app_name === "Integration") { + for (var paramkey in action.parameters) { + const param = action.parameters[paramkey] + if (param.name === "app_name") { + action.app_name = param.value.charAt(0).toUpperCase() + param.value.slice(1) + } + } + } + + //const parents = getParentNodes(workflow, action) + //console.log("PARENTS", key, parents) + if (parents !== undefined && parents !== null && parents.length > 0) { + const parentfound = parents.find((element) => element.id === startnodeId) + if (parentfound !== undefined) { + + // FIXME: add order here based on how many steps away from the startnode + // This just has the parent count + action.order = parents.length + + relevantactions.push(action) + } + } + } + + if (getParents === undefined) { + var newactions = [] + for (var key in workflow.triggers) { + const trigger = workflow.triggers[key] + if (trigger.trigger_type !== "SUBFLOW" && trigger.trigger_type !== "USERINPUT") { + continue + } + + for (var branchkey in workflow.branches) { + const branch = workflow.branches[branchkey] + + // Checking for OUTBOUND branches from it. + // This will mean it's NOT the last node and is easy to visualize + if (branch.source_id !== trigger.id) { + continue + } + + trigger.order = 2 + } + + + // Just in case (: + if (workflow.actions.find((element) => element.id === trigger.id) === undefined) { + newactions.push(trigger) + //workflow.actions.push(trigger) + } + } + + relevantactions.push(...newactions) + } + + if (relevantactions.length <= 1) { + return null + } + + // Sort according to how many parents a node has. MAY be wrong~ + relevantactions.sort((a, b) => { + if (a.order === undefined) { + return 1 + } + + if (b.order === undefined) { + return -1 + } + + return a.order - b.order + }) + + // FIXME: Add other relevant items as well from subflows (?) + var nodecolor = grey + var branchcolor = grey + var skipped = false + + var previousTools = false + var scheduleNotStarted = false + + if (workflow.validation !== undefined && workflow.validation !== null && workflow.validation.validation_ran === false) { + console.log("Validation didn't run. Why?") + return null + } + + if (workflow.validation !== undefined && workflow.validation !== null && workflow.validation.errors !== undefined && workflow.validation.errors !== null && workflow.validation.errors.length > 0) { + var newErrors = [] + for (var key in workflow.validation.errors) { + const error = workflow.validation.errors[key] + if (error.type === "SCHEDULE") { + scheduleNotStarted = true + continue + } + + newErrors.push(error) + } + + workflow.validation.errors = newErrors + } + + // Use this variable to control visualization + //const showMiddle = false + // border: workflow.validation.valid ? `2px solid ${green}` : "1px solid rgba(255,255,255,0.4)", + var middleError = "" + var startBranchColor = "" + var middleBranchColor = "" + + + const showHoverForClick = showHoverColor === true ? true : false + return ( +
    { + if (isClicked === false) { + setHovering(true) + } + }} + onMouseLeave={() => { + if (isClicked === false) { + setHovering(false) + } + }} + onClick={() => { + if (showHoverForClick === true) { + setIsClicked(true) + } + }} + > + + {isClicked === false ? null : + + } + +
    + + {scheduleNotStarted === true ? + null + : null} + + {relevantactions.map((action, index) => { + action.result = {} + if (results !== undefined) { + const foundResult = results.find((element) => element.action.id === action.id) + if (foundResult !== undefined) { + action.result = foundResult + + action.status = foundResult.status + } + } + + const lastitem = index === relevantactions.length - 1 + if (!lastitem) { + if (action.app_name === "Shuffle Tools") { + if (action.status === "SUCCESS") { + branchcolor = red + + // Check action.result for the actual status + const validate = validateJson(action.result.result) + if (validate.valid) { + if (validate.result.success === true) { + nodecolor = green + branchcolor = green + } else { + nodecolor = grey + branchcolor = grey + } + } + + + } else if (action.status === "SKIPPED") { + branchcolor = grey + } else { + // FIXME: How do we handle this? + if (action.status === undefined) { + nodecolor = grey + branchcolor = grey + } else { + nodecolor = red + branchcolor = red + } + } + + previousTools = true + + if (startnodeId !== action.id) { + //return null + } + } else { + if (action.status === "SUCCESS") { + nodecolor = green + } else if (action.status === "SKIPPED") { + nodecolor = grey + } else { + if (action.status === undefined) { + nodecolor = green + } else { + nodecolor = red + } + } + } + } else { + nodecolor = grey + } + + if (action.status === "SKIPPED") { + skipped = true + } + + var image = "" + if (action.large_image !== undefined && action.large_image !== null && action.large_image !== "") { + image = action.large_image + } else { + if (originalWorkflow !== undefined) { + for (var key in originalWorkflow.actions) { + if (originalWorkflow.actions[key].id === action.id) { + image = originalWorkflow.actions[key].large_image + break + } + } + + if (image === "") { + for (var key in originalWorkflow.triggers) { + if (originalWorkflow.triggers[key].id === action.id) { + image = originalWorkflow.triggers[key].large_image + break + } + } + + } + } + } + + var founderror = "" + //console.log("WORKFLOW VALIDATION: ", workflow.validation) + if (workflow.validation !== undefined && workflow.validation !== null && workflow.validation.errors !== undefined && workflow.validation.errors !== null) { + const foundError = workflow.validation.errors.find((element) => element.action_id === action.id) + if (foundError !== undefined) { + founderror = foundError.error + nodecolor = red + branchcolor = red + } else { + //console.log("NO ERROR: ", action.id) + } + } + + var appgroup = [] + if (action.app_name === "shuffle-subflow") { + if (action.status === "SUCCESS") { + nodecolor = green + branchcolor = green + } + + if (workflow.validation.subflow_apps !== undefined && workflow.validation.subflow_apps !== null && workflow.validation.subflow_apps.length > 0) { + nodecolor = red + branchcolor = red + + for (var subflowkey in workflow.validation.subflow_apps) { + const subflowApp = workflow.validation.subflow_apps[subflowkey] + founderror += "- " + subflowApp.error+"\n" + + if (subflowApp.error === action.id) { + appgroup.push(subflowApp) + } + } + } + } + + if (!showMiddle && relevantactions.length > 2 && index > 0 && index === relevantactions.length - 2) { + if (founderror.length > 0) { + middleError += founderror+"\n" + + middleBranchColor = branchcolor + } + + if (index === relevantactions.length-2 && relevantactions.length > 2) { + + const selectedIcon = middleError.length > 0 ? + + {middleError} + + }> + + + + + : null + + return selectedIcon + } else { + return null + } + } + + // Returns for anything non-middle + if (relevantactions.length > 2 && index >= 1 && index < relevantactions.length - 2) { + if (founderror.length > 0) { + middleError += founderror+"\n" + } + + return null + } + + if (skipped && !lastitem) { + nodecolor = grey + branchcolor = grey + } + + if (previousTools) { + previousTools = false + } else { + branchcolor = nodecolor + } + + if (action.trigger_type === "WEBHOOK") { + nodecolor = green + branchcolor = green + } + + + + var flex = index !== 0 && index !== relevantactions.length - 1 ? 1 : 3 + if (nodecolor === green) { + branchcolor = green + } else if (nodecolor === yellow) { + branchcolor = yellow + } else if (nodecolor === red) { + branchcolor = red + } + + if (index === 0) { + startBranchColor = branchcolor + } else if (index !== 0 && index !== relevantactions.length - 1) { + // FIXME: This doesn't work yet + middleBranchColor = branchcolor + } + + if (lastitem) { + if (middleError.length === 0) { + branchcolor = startBranchColor + } else { + //branchcolor = middleBranchColor + } + + if (founderror === "") { + nodecolor = green + } + } + + // FIXME: This could mean the workflow hasn't ran yet + if (workflow.validation.valid === false && (workflow.validation.errors === undefined || workflow.validation.errors === null || workflow.validation.errors.length == 0) && (workflow.validation.subflow_apps === undefined || workflow.validation.subflow_apps === null || workflow.validation.subflow_apps.length == 0)) { + nodecolor = grey + branchcolor = grey + } + + const branchTooltip = branchcolor === yellow ? "Check nodes for errors" : "" + const appname = action.app_name.replaceAll('_', ' ').slice(0, 16) + + const chipBackground = nodecolor === green ? "rgba(2,203,112, 0.2)" : nodecolor === grey ? "#494949": "rgba(245,52,52,0.8)" + const chipColor = nodecolor === green ? "#02cb70" : nodecolor === grey ? "#CDCDCD" : "white" + + //const ballcolor = lastitem ? nodecolor : branchcolor + const ballcolor = branchcolor + const ballsize = 8 + const topMargin = 20 + + const chipStyle = { + height: 40, + minWidth: 125, + maxWidth: 125, + borderRadius: 50, + color: chipColor, + backgroundColor: chipBackground, + + //border: `2px solid ${chipBackground}`, + //backgroundColor: "rgba(0,0,0,0.0)", + } + + if (image === "") { + console.log("MISSING IMAGE: ", appname, image, action) + } + + if (decidedColor === grey && nodecolor === green) { + setDecidedColor(red) + } + + if (decidedColor !== red && nodecolor === red) { + setDecidedColor(red) + } + + return ( +
    + {lastitem ? + +
    +
    +
    +
    + + : null} + + {appgroup.length > 0 ? + + {appgroup.map((subflowApp, subflowIndex) => { + var appimage = "" + if (apps !== undefined && apps !== null && apps.length > 0) { + for (var key in apps) { + const app = apps[key] + if (app.name === subflowApp.app_name) { + appimage = apps[key].large_image + break + } + } + } + + return ( + + + + ) + })} + + : + + {founderror.length > 0 ? founderror : `App: ${appname} - Action: ${action.label}`} + + } placement="top"> + + + {image !== "" ? + + : null} + + } /> + + + } + + {lastitem ? null : + +
    +
    +
    +
    + + } +
    + ) + })} +
    + +
    + ) +} + +export default WorkflowValidationTimeline diff --git a/frontend/src/context/ContextApi.jsx b/frontend/src/context/ContextApi.jsx new file mode 100644 index 00000000..c9b02e34 --- /dev/null +++ b/frontend/src/context/ContextApi.jsx @@ -0,0 +1,37 @@ +import { createContext, useState, useEffect } from 'react'; +export const Context = createContext(); + +export const AppContext =(props) => { + + // Left side bar global states + const [searchBarModalOpen, setSearchBarModalOpen] = useState(false); + const [leftSideBarOpenByClick, setLeftSideBarOpenByClick] = useState(false); + const [windowWidth, setWindowWidth] = useState(window.innerWidth); + + + //Calculate window width + useEffect(() => { + const handleResize = () => { + setWindowWidth(window?.innerWidth); + }; + + window.addEventListener('resize', handleResize); + + return () => { + window.removeEventListener('resize', handleResize); + }; + }, []); + + + return ( + + {props.children} + + ) +} diff --git a/frontend/src/defaultCytoscapeStyle.jsx b/frontend/src/defaultCytoscapeStyle.jsx index d36c8e8d..d61f5809 100644 --- a/frontend/src/defaultCytoscapeStyle.jsx +++ b/frontend/src/defaultCytoscapeStyle.jsx @@ -13,6 +13,30 @@ const data = [ return elementname }, "text-valign": "center", + "text-margin-x": function(element) { + // Attempt at bottom-positioning + // Required text-valign: bottom + // FIXME: Disabled for now. + return "15px" + + + + const name = element.data("label") + console.log("Name: ", name) + if (name === null || name === undefined || name == "" || document=== undefined || document === null) { + return "0px" + } + + const canvas = document.createElement('canvas'); + const context = canvas.getContext('2d') + + context.font = '18px Segoe UI, Tahoma, Geneva, Verdana, sans-serif, sans-serif' + + const textWidth = context.measureText(name).width + return textWidth + "px" + //return -1*(textWidth) + "px" + }, + "font-family": "Segoe UI, Tahoma, Geneva, Verdana, sans-serif, sans-serif", "font-weight": "lighter", "font-size": "18px", @@ -23,7 +47,6 @@ const data = [ padding: "10px", margin: "5px", "border-width": "1px", - "text-margin-x": "10px", "z-index": 5001, }, }, @@ -35,7 +58,7 @@ const data = [ "curve-style": "unbundled-bezier", label: "data(label)", "text-margin-y": "-15px", - width: "5px", + width: "3px", color: "white", "line-fill": "linear-gradient", "line-gradient-stop-positions": ["0.0", "100"], @@ -119,10 +142,10 @@ const data = [ }, }, { - selector: `node[app_name="Shuffle Tools"]`, + selector: `node[app_name="Shuffle Tools"], node[app_name="email"], node[app_name="http"]`, css: { - width: "30px", - height: "30px", + width: "35px", + height: "35px", "z-index": 5000, "font-size": "0px", "background-width": "75%", @@ -258,7 +281,9 @@ const data = [ { selector: "node[?isStartNode]", css: { - shape: "ellipse", + shape: function(element) { + return "ellipse" + }, "border-color": "#80deea", width: "80px", height: "80px", @@ -270,8 +295,8 @@ const data = [ { selector: "node[!is_valid]", css: { - "border-color": "red", - "border-width": "10px", + "border-color": "#f53434", + "border-width": "5px", }, }, { @@ -357,7 +382,7 @@ const data = [ css: { "background-color": "#f85a3e", "border-color": "#f85a3e", - "border-width": "12px", + "border-width": "7px", "transition-property": "border-width", "transition-duration": "0.25s", label: "data(label)", @@ -462,13 +487,66 @@ const data = [ "font-size": "0px", }, }, - { - selector: "node:selected", - css: { - "border-color": "#f86a3e", - "border-width": "7px", - }, - }, + { + selector: "node:selected", + css: { + "border-color": "#f86a3e", + "border-width": "7px", + }, + }, + { + selector: `node[buttonType="condition-drag"]`, + css: { + "width": "5px", + "height": "5px", + "background-color": "#f85a3e", + }, + }, + { + selector: `node[name="switch"]`, + css: { + label: function(element) { + // Load from the actual element + var nodeheight = 400 + var conditions = [{ + "name": "Condition 1", + "check": "X equals Y", + }, + { + "name": "Condition 2", + "check": "X2 equals Y2", + }, + { + "name": "Condition 3", + "check": "X3 equals Y3", + }] + + conditions.push({ + "name": "Else", + "check": "If all else fails", + }) + + const newlines = nodeheight / conditions.length + console.log("Newlines: ", newlines) + + const label = conditions.map((condition) => { + return `${condition.name}\n\n\n` + }).join("\n") + + return label + }, + color: "white", + "border-color": "#f85a3e", + "background-color": "#1f1f1f", + "font-size": "19px", + "text-margin-x": "-110px", + "text-wrap": "wrap", + shape: "roundrectangle", + width: "100", + height: "300", + + }, + }, ]; //{ diff --git a/frontend/src/index.css b/frontend/src/index.css index 9b1fc96c..3ab58ef1 100755 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -18,11 +18,9 @@ code { margin-right: -13px; } -.toc:hover { - background-color: gray; - color: white; +table th, table td { + border: 1px solid; } - /* .cm-string{ z-index: -1; } diff --git a/frontend/src/theme.jsx b/frontend/src/theme.jsx index 6e66e28c..68934746 100644 --- a/frontend/src/theme.jsx +++ b/frontend/src/theme.jsx @@ -1,28 +1,30 @@ import React from "react"; import { createTheme, adaptV4Theme } from "@mui/material/styles"; -//const theme = createTheme({ const theme = createTheme(adaptV4Theme({ palette: { theme: "dark", - main: "#F86743", + main: "#FF8544", primary: { - main: "#F86743", + main: "#FF8544", contrastText: "#ffffff", }, secondary: { - main: "#e8eaf6", + main: "rgba(255,255,255,0.7)", contrastText: "#000000", }, text: { secondary: "rgba(255,255,255,0.7)", }, type: "dark", - inputColor: "rgba(39,41,45,1)", //inputColor: "#383B40", + + inputColor: "rgba(39,41,45,1)", surfaceColor: "#27292d", - platformColor: "#1c1c1d", + //platformColor: "#1c1c1d", + platformColor: "#212121", backgroundColor: "#1a1a1a", + green: "#5cc879", borderRadius: 10, defaultBorder: "1px solid rgba(255,255,255,0.3)", @@ -44,10 +46,20 @@ const theme = createTheme(adaptV4Theme({ overflowX: "auto", }, textFieldStyle: { - backgroundColor: "#383B40", + backgroundColor: "#212121", borderRadius: 5, + height: 40, + }, + DialogStyle: { + backgroundColor: "#212121", + borderRadius: 2, + boxShadow: "0px 0px 10px 0px rgba(0,0,0,0.75)", + border: "1px solid #494949", }, innerTextfieldStyle: { + height: 40, + fontSize: 16, + backgroundColor: "#212121", // Removed since upgrading to mui 18 //color: "white", //minHeight: 50, diff --git a/frontend/src/views/404.jsx b/frontend/src/views/404.jsx new file mode 100644 index 00000000..0a08f081 --- /dev/null +++ b/frontend/src/views/404.jsx @@ -0,0 +1,138 @@ +import React from 'react'; +import { useNavigate } from 'react-router-dom'; +import { Button, Typography } from '@mui/material'; +import theme from "../theme.jsx"; + +const NotFound = () => { + const navigate = useNavigate(); + + const buttonStyle = { + borderRadius: 25, + height: 50, + fontSize: 18, + width: "100%", + marginBottom: "10px", + }; + + const primaryButtonStyle = { + ...buttonStyle, + background: "linear-gradient(89.83deg, #FF8444 0.13%, #F2643B 99.84%)", + color: "white", + '&:hover': { + background: "linear-gradient(89.83deg, #FF8444 0.13%, #F2643B 99.84%)", + opacity: 0.9, + } + }; + + const secondaryButtonStyle = { + ...buttonStyle, + background: "#383B40", + border: "1px solid #494949", + color: "white", + '&:hover': { + background: "#434649", + } + }; + + return ( +
    +
    + Shuffle Logo + + + 404 + + + + Page Not Found + + + + Our code doggo couldn't find the page you were looking for. + + + -
    + + { + e.preventDefault(); + setVideoViewOpen(false) + }} + > + + + + + +
    const loadedCheck = isLoaded && isLoggedIn && workflowDone ? (
    - {/* + {/* */} - 1366 ? 1366 : isMobile ? "100%" : 1200, - margin: "auto", - padding: 20, - }} - onDrop={uploadFile} - > - - + {/*modalView*/} {deleteModal} {exportVerifyModal} {publishModal} {workflowDownloadModalOpen} - {!drawerOpen ?
    - - { - setDrawerOpen(true) - localStorage.setItem(sidebarKey, "open"); - }}> - - - -
    : null} - {isMobile ? null : gettingStartedDrawer} - {videoView} + {/*!drawerOpen ? +
    + + { + setDrawerOpen(true) + localStorage.setItem(sidebarKey, "open"); + }}> + + + +
    : null*/} + {isMobile ? null : gettingStartedDrawer} + {videoView} - {modalOpen === true ? - - : null} - {/*
    + workflows={workflows} + apps={apps} + setWorkflows={setWorkflows} + /> + : null} + {/*
    Need assistance? Ask our support team (it's free!).
    */} -
    +
    ) : (
    { + + return { + datagrid: { + border: 0, + "& .MuiDataGrid-columnsContainer": { + backgroundColor: + theme?.palette?.type === "light" ? "#fafafa" : theme?.palette?.inputColor, + }, + "& .MuiDataGrid-iconSeparator": { + display: "none", + }, + "& .MuiDataGrid-colCell, .MuiDataGrid-cell": { + borderRight: `1px solid ${theme?.palette?.type === "light" ? "white" : "#303030" + }`, + }, + "& .MuiDataGrid-columnsContainer, .MuiDataGrid-cell": { + borderBottom: `1px solid ${theme?.palette?.type === "light" ? "#f0f0f0" : "#303030" + }`, + }, + "& .MuiDataGrid-cell": { + color: + theme?.palette?.type === "light" ? "white" : "rgba(255,255,255,0.65)", + }, + "& .MuiPaginationItem-root, .MuiTablePagination-actions, .MuiTablePagination-caption": + { + borderRadius: 0, + color: "white", + }, + }, + } +}) + + + + + +// Takes an action in Shuffle and +// Returns information about the icon, the color etc to be used +// This can be used for actions of all types +export const GetIconInfo = (action) => { + // Finds the icon based on the action. Should be verbs. + const iconList = [ + { key: "cases", values: ["cases"] }, + { key: "cache_add", values: ["set_cache"] }, + { key: "cache_get", values: ["get_cache"] }, + { key: "filter", values: ["filter"] }, + { key: "merge", values: ["join", "merge", "route", "router"] }, + { + key: "search", + values: ["search", "find", "locate", "index", "analyze", "anal", "match", "check cache", "check", "verify", "validate"], + }, + { key: "list", values: ["list", "head", "options"] }, + { + key: "download", + values: [ + "capture", + "get", + "download", + "return", + "hello_world", + "curl", + "request", + "export", + "preview", + ], + }, + { key: "add", values: ["add", "accept",] }, + { key: "delete", values: ["delete", "remove", "clear", "clean", "dismiss",] }, + { + key: "send", + values: [ + "send", + "dispatch", + "mail", + "forward", + "post", + "submit", + "mark", + "set", + "release", + ], + }, + { + key: "repeat", + values: ["repeat", "retry", "pause", "skip", "copy", "replicat", "demo",], + }, + { key: "execute", values: ["execute", "run", "play", "raise"] }, + { key: "extract", values: ["extract", "unpack", "decompress", "open"] }, + { key: "inflate", values: ["inflate", "pack", "compress"] }, + { + key: "edit", + values: [ + "modify", + "update", + "create", + "edit", + "put", + "patch", + "change", + "replace", + "conver", + "map", + "format", + "escape", + "describe", + ], + }, + { + key: "compare", + values: ["compare", "convert", "to", "filter", "translate", "parse"], + }, + { key: "close", values: ["close", "stop", "cancel", "block"] }, + { key: "communication", values: ["communication", "comms", "email", "mail",] }, + ]; + + var selectedKey = "" + if (action.app_name == "Integration Framework") { + selectedKey = "magic" + } else if (action.name === undefined || action.name === null) { + } else { + const actionname = action.name.toLowerCase() + for (var key in iconList) { + //console.log(iconList[key], actionname) + const found = iconList[key].values.find((value) => + actionname.includes(value) + ) + if (found !== null && found !== undefined) { + selectedKey = iconList[key].key + break + } + } + } + + // Some of these are manually parsed or created instead of material ui + //M8 0C3.58 0 0 1.79 0 4C0 6.21 3.58 8 8 8C12.42 8 16 6.21 16 4C16 1.79 12.42 0 8 0ZM0 6V9C0 11.21 3.58 13 8 13C12.42 13 16 11.21 16 9V6C16 8.21 12.42 10 8 10C3.58 10 0 8.21 0 6ZM0 11V14C0 16.21 3.58 18 8 18C9.41 18 10.79 17.81 12 17.46V14.46C10.79 14.81 9.41 15 8 15C3.58 15 0 13.21 0 11ZM17 11V14H14V16H17V19H19V16H22V14H19V11 + //https://www.figma.com/file/uCfnMs5w6wnLx6ehPHEV74/Figma-Material-Design-System-v3_0?node-id=834%3A21 + //COLORS: https://www.pinterest.co.uk/pin/326299935499972946/ + const defaultColor = "#f76b1c"; + const defaultGradient = ["#fad961", "#f76b1c"]; + const parsedIcons = { + magic: { + icon: "M7.5 5.6 10 7 8.6 4.5 10 2 7.5 3.4 5 2l1.4 2.5L5 7zm12 9.8L17 14l1.4 2.5L17 19l2.5-1.4L22 19l-1.4-2.5L22 14zM22 2l-2.5 1.4L17 2l1.4 2.5L17 7l2.5-1.4L22 7l-1.4-2.5zm-7.63 5.29a.9959.9959 0 0 0-1.41 0L1.29 18.96c-.39.39-.39 1.02 0 1.41l2.34 2.34c.39.39 1.02.39 1.41 0L16.7 11.05c.39-.39.39-1.02 0-1.41zm-1.03 5.49-2.12-2.12 2.44-2.44 2.12 2.12z", + iconColor: "white", + iconBackgroundColor: "red", + originalIcon: "", + fillGradient: ["#FF0000", "#FF7F00", "#FFFF00", "#00FF00", "#0000FF", "#4B0082", "#8A2BE2"], + }, + communication: { + icon: "M9.89516 7.71433H8.60945V5.1429H9.89516V7.71433ZM9.89516 10.2858H8.60945V9.00004H9.89516V10.2858ZM14.3952 2.57147H4.10944C3.76845 2.57147 3.44143 2.70693 3.20031 2.94805C2.95919 3.18917 2.82373 3.51619 2.82373 3.85719V15.4286L5.39516 12.8572H14.3952C14.7362 12.8572 15.0632 12.7217 15.3043 12.4806C15.5454 12.2395 15.6809 11.9125 15.6809 11.5715V3.85719C15.6809 3.14361 15.1023 2.57147 14.3952 2.57147Z", + iconColor: "white", + iconBackgroundColor: "#8acc3f", + originalIcon: "", + fillGradient: ["#8acc3f", "#459622"], + }, + cases: { + icon: "M15.6408 8.39233H18.0922V10.0287H15.6408V8.39233ZM0.115234 8.39233H2.56663V10.0287H0.115234V8.39233ZM9.92083 0.21051V2.66506H8.28656V0.21051H9.92083ZM3.31839 2.25596L5.05889 4.00687L3.89856 5.16051L2.15807 3.42596L3.31839 2.25596ZM13.1485 3.99869L14.8808 2.25596L16.0493 3.42596L14.3088 5.16051L13.1485 3.99869ZM9.10369 4.30142C10.404 4.30142 11.651 4.81863 12.5705 5.73926C13.4899 6.65989 14.0065 7.90854 14.0065 9.21051C14.0065 11.0269 13.0178 12.6141 11.5551 13.4651V14.9378C11.5551 15.1548 11.469 15.3629 11.3158 15.5163C11.1625 15.6698 10.9547 15.756 10.738 15.756H7.46943C7.25271 15.756 7.04487 15.6698 6.89163 15.5163C6.73839 15.3629 6.6523 15.1548 6.6523 14.9378V13.4651C5.18963 12.6141 4.2009 11.0269 4.2009 9.21051C4.2009 7.90854 4.71744 6.65989 5.63689 5.73926C6.55635 4.81863 7.80339 4.30142 9.10369 4.30142ZM10.738 16.5741V17.3923C10.738 17.6093 10.6519 17.8174 10.4986 17.9709C10.3454 18.1243 10.1375 18.2105 9.92083 18.2105H8.28656C8.06984 18.2105 7.862 18.1243 7.70876 17.9709C7.55552 17.8174 7.46943 17.6093 7.46943 17.3923V16.5741H10.738ZM8.28656 14.1196H9.92083V12.3769C11.3345 12.0169 12.3722 10.7323 12.3722 9.21051C12.3722 8.34253 12.0279 7.5101 11.4149 6.89634C10.8019 6.28259 9.97056 5.93778 9.10369 5.93778C8.23683 5.93778 7.40546 6.28259 6.79249 6.89634C6.17953 7.5101 5.83516 8.34253 5.83516 9.21051C5.83516 10.7323 6.87292 12.0169 8.28656 12.3769V14.1196Z", + iconColor: "white", + iconBackgroundColor: "#8acc3f", + originalIcon: "", + fillGradient: ["#8acc3f", "#459622"], + }, + cache_add: { + icon: "M11 3C6.58 3 3 4.79 3 7C3 9.21 6.58 11 11 11C15.42 11 19 9.21 19 7C19 4.79 15.42 3 11 3ZM3 9V12C3 14.21 6.58 16 11 16C15.42 16 19 14.21 19 12V9C19 11.21 15.42 13 11 13C6.58 13 3 11.21 3 9ZM3 14V17C3 19.21 6.58 21 11 21C12.41 21 13.79 20.81 15 20.46V17.46C13.79 17.81 12.41 18 11 18C6.58 18 3 16.21 3 14ZM20 14V17H17V19H20V22H22V19H25V17H22V14", + iconColor: "white", + iconBackgroundColor: "#8acc3f", + originalIcon: "", + fillGradient: ["#8acc3f", "#459622"], + }, + cache_get: { + icon: "M12 2C7.58 2 4 3.79 4 6C4 8.06 7.13 9.74 11.15 9.96C12.45 8.7 14.19 8 16 8C16.8 8 17.59 8.14 18.34 8.41C19.37 7.74 20 6.91 20 6C20 3.79 16.42 2 12 2ZM4 8V11C4 12.68 6.08 14.11 9 14.71C9.06 13.7 9.32 12.72 9.77 11.82C6.44 11.34 4 9.82 4 8ZM15.93 9.94C14.75 9.95 13.53 10.4 12.46 11.46C8.21 15.71 13.71 22.5 18.75 19.17L23.29 23.71L24.71 22.29L20.17 17.75C22.66 13.97 19.47 9.93 15.93 9.94ZM15.9 12C17.47 11.95 19 13.16 19 15C19 15.7956 18.6839 16.5587 18.1213 17.1213C17.5587 17.6839 16.7956 18 16 18C13.33 18 12 14.77 13.88 12.88C14.47 12.29 15.19 12 15.9 12ZM4 13V16C4 18.05 7.09 19.72 11.06 19.95C10.17 19.07 9.54 17.95 9.22 16.74C6.18 16.17 4 14.72 4 13Z", + iconColor: "white", + iconBackgroundColor: "#8acc3f", + originalIcon: "", + fillGradient: ["#8acc3f", "#459622"], + }, + repeat: { + icon: "M19 8l-4 4h3c0 3.31-2.69 6-6 6-1.01 0-1.97-.25-2.8-.7l-1.46 1.46C8.97 19.54 10.43 20 12 20c4.42 0 8-3.58 8-8h3l-4-4zM6 12c0-3.31 2.69-6 6-6 1.01 0 1.97.25 2.8.7l1.46-1.46C15.03 4.46 13.57 4 12 4c-4.42 0-8 3.58-8 8H1l4 4 4-4H6z", + iconColor: "white", + iconBackgroundColor: defaultColor, + originalIcon: , + }, + add: { + icon: "M19 13h-6v6h-2v-6H5v-2h6V5h2v6h6v2z", + iconColor: "white", + iconBackgroundColor: defaultColor, + originalIcon: , + }, + edit: { + icon: "M3 17.25V21h3.75L17.81 9.94l-3.75-3.75L3 17.25zM20.71 7.04c.39-.39.39-1.02 0-1.41l-2.34-2.34a.9959.9959 0 00-1.41 0l-1.83 1.83 3.75 3.75 1.83-1.83z", + iconColor: "white", + iconBackgroundColor: defaultColor, + originalIcon: , + }, + filter: { + icon: "M4.25 5.61C6.27 8.2 10 13 10 13v6c0 .55.45 1 1 1h2c.55 0 1-.45 1-1v-6s3.72-4.8 5.74-7.39c.51-.66.04-1.61-.79-1.61H5.04c-.83 0-1.3.95-.79 1.61z", + iconColor: "white", + iconBackgroundColor: "#f5515f", + originalIcon: "", + fillGradient: ["#f5515f", "#a1051d"], + }, + merge: { + icon: "M17 20.41 18.41 19 15 15.59 13.59 17 17 20.41zM7.5 8H11v5.59L5.59 19 7 20.41l6-6V8h3.5L12 3.5 7.5 8z", + iconColor: "white", + iconBackgroundColor: "#f5515f", + originalIcon: "", + fillGradient: ["#f5515f", "#a1051d"], + }, + compare: { + icon: "M10 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h5v2h2V1h-2v2zm0 15H5l5-6v6zm9-15h-5v2h5v13l-5-6v9h5c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2z", + iconColor: "white", + iconBackgroundColor: defaultColor, + originalIcon: , + }, + extract: { + icon: "M3 3h18v2H3z", + iconColor: "white", + iconBackgroundColor: defaultColor, + originalIcon: , + }, + inflate: { + icon: "M6 19h12v2H6z", + iconColor: "white", + iconBackgroundColor: defaultColor, + originalIcon: , + }, + list: { + icon: "M3 9h14V7H3v2zm0 4h14v-2H3v2zm0 4h14v-2H3v2zm16 0h2v-2h-2v2zm0-10v2h2V7h-2zm0 6h2v-2h-2v2z", + iconColor: "white", + iconBackgroundColor: defaultColor, + originalIcon: , + }, + execute: { + icon: "M8 5v14l11-7z", + iconColor: "white", + iconBackgroundColor: defaultColor, + originalIcon: , + }, + delete: { + icon: "M6 19c0 1.1.9 2 2 2h8c1.1 0 2-.9 2-2V7H6v12zM19 4h-3.5l-1-1h-5l-1 1H5v2h14V4z", + iconColor: "white", + iconBackgroundColor: "#03030e", + originalIcon: , + fillGradient: ["#03030e", "#205d66"], + }, + close: { + icon: "M19 6.41L17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z", + iconColor: "white", + iconBackgroundColor: "#03030e", + originalIcon: , + fillGradient: ["#03030e", "#205d66"], + }, + send: { + icon: "M2.01 21L23 12 2.01 3 2 10l15 2-15 2z", + iconColor: "white", + iconBackgroundColor: "#0373da", + originalIcon: , + fillGradient: ["#0bc8bf", "#0373da"], + }, + download: { + icon: "M19 9h-4V3H9v6H5l7 7 7-7zM5 18v2h14v-2H5z", + iconColor: "white", + iconBackgroundColor: "#0373da", + originalIcon: , + fillGradient: ["#0bc8bf", "#0373da"], + }, + search: { + icon: "M15.5 14h-.79l-.28-.27C15.41 12.59 16 11.11 16 9.5 16 5.91 13.09 3 9.5 3S3 5.91 3 9.5 5.91 16 9.5 16c1.61 0 3.09-.59 4.23-1.57l.27.28v.79l5 4.99L20.49 19l-4.99-5zm-6 0C7.01 14 5 11.99 5 9.5S7.01 5 9.5 5 14 7.01 14 9.5 11.99 14 9.5 14z", + iconColor: "white", + iconBackgroundColor: "green", + originalIcon: , + }, + }; + + var selectedItem = parsedIcons[selectedKey]; + if (selectedItem === undefined || selectedItem === null) { + return { + icon: "", + iconColor: "", + iconBackground: "black", + originalIcon: "", + }; + } + + if (selectedItem.fillGradient === undefined) { + selectedItem.fillGradient = defaultGradient; + selectedItem.iconBackgroundColor = defaultColor; + } + + if (selectedItem.icon === "" || selectedItem.icon === undefined) { + console.log( + `MISSING PATH FOR ${selectedKey} (find in scope): `, + selectedItem.originalIcon.type.type + ); + } + + if ( + (selectedItem.originalIcon === undefined || + selectedItem.originalIcon === "") && + selectedItem.icon !== "" && + selectedItem.icon !== undefined + ) { + const svg_pin = ( + + + + ); + selectedItem.originalIcon = svg_pin; + } + + return selectedItem; +}; + +const chipStyle = { + backgroundColor: "#2F2F2F", + marginRight: 5, + paddingLeft: 5, + paddingRight: 5, + height: 35, + cursor: "pointer", + borderColor: "#2F2F2F", + color: "#C8C8C8", + fontSize: "14px", + fontFamily: theme?.typography?.fontFamily, + borderRadius: "17.5px" +}; + +export const collapseField = (field) => { + if (field === undefined || field === null) { + return true + } + + if (field.name === "headers" || field.name === "cookies") { + return true + } + + if (field.type === "array") { + return true + } + + // If more than 10 keys in object, collapse + if (field.type === "object") { + if (Object.keys(field.src).length > 7) { + return true + } + } + + return false +} + +export const validateJson = (showResult) => { + if (showResult === undefined || showResult === null) { + return { + valid: false, + result: "", + } + } + + if (typeof showResult === 'string') { + showResult = showResult.split(" False").join(" false") + showResult = showResult.split(" True").join(" true") + + showResult.replaceAll("False,", "false,") + showResult.replaceAll("True,", "true,") + } + + if (typeof showResult === "object" || typeof showResult === "array") { + return { + valid: true, + result: showResult, + } + } + + if (showResult[0] === "\"") { + return { + valid: false, + result: showResult, + } + } + + var jsonvalid = true + try { + if (!showResult.includes("{") && !showResult.includes("[")) { + jsonvalid = false + + return { + valid: jsonvalid, + result: showResult, + }; + } + } catch (e) { + + try { + showResult = showResult.split("'").join('"'); + if (!showResult.includes("{") && !showResult.includes("[")) { + jsonvalid = false; + } + } catch (e) { + + jsonvalid = false; + } + } + + var result = showResult; + try { + result = jsonvalid ? JSON.parse(showResult, { "storeAsString": true }) : showResult; + } catch (e) { + ////console.log("Failed parsing JSON even though its valid: ", e) + jsonvalid = false; + } + + if (jsonvalid === false) { + + if (typeof showResult === 'string') { + showResult = showResult.trim() + } + + try { + var newstr = showResult.replaceAll("'", '"') + + // Basic workarounds for issues with Python Dicts -> JSON + if (newstr.includes(": None")) { + newstr = newstr.replaceAll(": None", ': null') + } + + if (newstr.includes("[\"{") && newstr.includes("}\"]")) { + newstr = newstr.replaceAll("[\"{", '[{') + newstr = newstr.replaceAll("}\"]", '}]') + } + + if (newstr.includes("{\"[") && newstr.includes("]\"}")) { + newstr = newstr.replaceAll("{\"[", '[{') + newstr = newstr.replaceAll("]\"}", '}]') + } + + result = JSON.parse(newstr) + jsonvalid = true + } catch (e) { + + //console.log("Failed parsing JSON even though its valid (2): ", e) + jsonvalid = false + } + } + + if (jsonvalid && typeof result === "number") { + jsonvalid = false + } + + // This is where we start recursing + if (jsonvalid) { + // Check fields if they can be parsed too + try { + for (const [key, value] of Object.entries(result)) { + if (typeof value === "string" && (value.startsWith("{") || value.startsWith("["))) { + //console.log("CHECKING STRING: ", value) + + const inside_result = validateJson(value) + if (inside_result.valid) { + //console.log("INSIDE RESULT: ", inside_result.result) + + if (typeof inside_result.result === "string") { + const newres = JSON.parse(inside_result.result) + + result[key] = newres + } else { + result[key] = inside_result.result + } + } + } else { + + // Usually only reaches here if raw array > dict > value + if (typeof showResult !== "array") { + for (const [subkey, subvalue] of Object.entries(value)) { + if (typeof subvalue === "string" && (subvalue.startsWith("{") || subvalue.startsWith("["))) { + const inside_result = validateJson(subvalue) + if (inside_result.valid) { + if (typeof inside_result.result === "string") { + const newres = JSON.parse(inside_result.result) + result[key][subkey] = newres + } else { + result[key][subkey] = inside_result.result + } + } + } + + } + } + } + } + } catch (e) { + //console.log("Failed parsing inside json subvalues: ", e) + } + } + + return { + valid: jsonvalid, + result: result, + }; +}; + +//Custom hook for handling styling of the dropzone +const useDropzoneStyles = () => { + const { leftSideBarOpenByClick } = useContext(Context); + + return { + paddingTop: 70, + // minHeight: 1000, + backgroundColor: "#1A1A1A", + fontFamily: theme?.typography?.fontFamily, + // maxWidth: window.innerWidth > 1366 ? 1366 : isMobile ? "100%" : 1200, + paddingLeft: leftSideBarOpenByClick ? 200 : 0, + transition: "padding-left 0.3s ease", + }; +}; + +//Wrapper for the dropzone component +const DropzoneWrapper = memo(({ onDrop, WorkflowView }) => { + const dropzoneStyles = useDropzoneStyles(); + return ( + + + + ); +}); + + + +const Workflows2 = (props) => { + const { globalUrl, isLoggedIn, isLoaded, userdata, checkLogin } = props; + const { leftSideBarOpenByClick } = useContext(Context); + const location = useLocation(); + const navigate = useNavigate(); + const [currTab, setCurrTab] = useState(0); + const [searchQuery, setSearchQuery] = useState(""); + const [selectedCategory, setSelectedCategory] = useState([]); + const [selectedLabel, setSelectedLabel] = useState([]); + const [mouseHoverIndex, setMouseHoverIndex] = useState(-1); + const [isLoadingWorkflow, setIsLoadingWorkflow] = useState(false); + const [isLoadingPublicWorkflow, setIsLoadingPublicWorkflow] = useState(false); + const [view, setView] = useState("grid"); + const classes = useStyles(theme) + const imgSize = 60; + + const referenceUrl = globalUrl + "/api/v1/hooks/"; + + var upload = ""; + + const [workflows, setWorkflows] = React.useState([]); + const [_, setUpdate] = React.useState(""); // Used for rendering, don't remove + const [selectedUsecases, setSelectedUsecases] = React.useState([]); + const [filteredWorkflows, setFilteredWorkflows] = React.useState([]); + const [selectedWorkflow, setSelectedWorkflow] = React.useState({}); + const [workflowDone, setWorkflowDone] = React.useState(false); + const [selectedWorkflowId, setSelectedWorkflowId] = React.useState(""); + + const [field1, setField1] = React.useState(""); + const [field2, setField2] = React.useState(""); + const [downloadUrl, setDownloadUrl] = React.useState("https://github.com/shuffle/workflows") + const [downloadBranch, setDownloadBranch] = React.useState("master"); + const [loadWorkflowsModalOpen, setLoadWorkflowsModalOpen] = + React.useState(false); + const [exportModalOpen, setExportModalOpen] = React.useState(false); + const [exportData, setExportData] = React.useState(""); + const [dialogModalOpen, setDialogModalOpen] = React.useState(false); + const [modalOpen, setModalOpen] = React.useState(false); + const [isEditing, setIsEditing] = React.useState(true); + const [newWorkflowName, setNewWorkflowName] = React.useState(""); + const [newWorkflowDescription, setNewWorkflowDescription] = + React.useState(""); + const [newWorkflowTags, setNewWorkflowTags] = React.useState([]); + + const [defaultReturnValue, setDefaultReturnValue] = React.useState(""); + const [blogpost, setBlogpost] = React.useState(""); + const [status, setStatus] = React.useState("test"); + + const [deleteModalOpen, setDeleteModalOpen] = React.useState(false); + const [publishModalOpen, setPublishModalOpen] = React.useState(false); + const [editingWorkflow, setEditingWorkflow] = React.useState({}); + const [isDropzone, setIsDropzone] = React.useState(false); + const [filters, setFilters] = React.useState([]); + const [submitLoading, setSubmitLoading] = React.useState(false); + const [actionImageList, setActionImageList] = React.useState([{ "large_image": "" }]) + + const [firstLoad, setFirstLoad] = React.useState(true); + const [showMoreClicked, setShowMoreClicked] = React.useState(false); + const [usecases, setUsecases] = React.useState([]); + const [allUsecases, setAllUsecases] = React.useState({ + "success": false, + }); + const [appFramework, setAppFramework] = React.useState({}); + const [drawerOpen, setDrawerOpen] = React.useState(false) + const [videoViewOpen, setVideoViewOpen] = React.useState(false) + const [gettingStartedItems, setGettingStartedItems] = React.useState([]) + const [selectedWorkflowIndexes, setSelectedWorkflowIndexes] = React.useState([]) + const [highlightIds, setHighlightIds] = React.useState([]) + + const [apps, setApps] = React.useState([]); + + document.title = "Shuffle - Workflows"; + const handleTabChange = (event, newValue) => { + // Set loading when switching to public workflows tab + if (newValue === 2) { + setIsLoadingPublicWorkflow(true); + // Simulate loading time for the Algolia search results + setTimeout(() => { + setIsLoadingPublicWorkflow(false); + }, 2000); + } + + setCurrTab(newValue); + }; + + const handleCreateWorkflow = () => { + setModalOpen(true) + setIsEditing(false) + setNewWorkflowName("") + setNewWorkflowDescription("") + setDefaultReturnValue("") + setEditingWorkflow({}) + setNewWorkflowTags([]) + setSelectedUsecases([]) + + }; + + + const drawerWidth = drawerOpen ? 325 : 0 + + const sidebarKey = "getting_started_sidebar" + if (isLoggedIn === true && gettingStartedItems.length === 0 && (userdata.tutorials !== undefined && userdata.tutorials !== null && userdata.tutorials.length > 0) && workflowDone === true) { + + const activeFiltered = userdata.tutorials.filter((item) => item.active === true) + if (activeFiltered.length > 0) { + var newfiltered = [] + for (var key in activeFiltered) { + if (activeFiltered[key].name === "Discover Usecases") { + if (workflows.length > 1) { + activeFiltered[key].done = true + activeFiltered[key].description = `${workflows.length} workflows created` + } + } + + newfiltered.push(activeFiltered[key]) + } + setGettingStartedItems(activeFiltered) + + /* + const sidebar = localStorage.getItem(sidebarKey) + if (sidebar === null || sidebar === undefined) { + console.log("No sidebar defined") + + localStorage.setItem(sidebarKey, "open"); + setDrawerOpen(true) + } else { + if (sidebar === "open") { + setDrawerOpen(true) + } else { + setDrawerOpen(false) + } + } + */ + } + + } + + const isCloud = + window.location.host === "localhost:3002" || + window.location.host === "shuffler.io"; + + const findWorkflow = (filters) => { + console.log("Using filters: ", filters) + if (filters.length === 0) { + setFilteredWorkflows(workflows); + handleKeysetting(allUsecases, workflows) + return; + } + + var newWorkflows = []; + for (var workflowKey in workflows) { + const curWorkflow = workflows[workflowKey]; + + var found = [false]; + if (curWorkflow.tags === undefined || curWorkflow.tags === null) { + found = filters.map((filter) => + curWorkflow.name.toLowerCase().includes(filter) + ); + } + + if (curWorkflow.tags !== undefined && curWorkflow.tags !== null && curWorkflow.tags.length > 0) { + // Make them all lowercase + curWorkflow.tags = curWorkflow.tags.map((tag) => tag.toLowerCase()) + } + + + if (found.every((v) => v !== true)) { + found = filters.map((filter) => { + if (filter === undefined || filter === null) { + return false; + } + + const newfilter = filter.toLowerCase(); + + if (curWorkflow.name.toLowerCase().includes(filter.toLowerCase())) { + return true; + } else if (curWorkflow.tags !== undefined && curWorkflow.tags !== null && curWorkflow.tags.includes(filter.toLowerCase())) { + return true; + } else if (curWorkflow.owner === filter) { + return true; + } else if (curWorkflow.org_id === filter) { + return true; + } else if (curWorkflow.usecase_ids !== undefined && curWorkflow.usecase_ids !== null && curWorkflow.usecase_ids.length > 0) { + // Check if the usecase is the right category + for (var key in usecases) { + if (usecases[key].name.toLowerCase() !== newfilter) { + continue + } + + for (var subkey in usecases[key].list) { + if (curWorkflow.usecase_ids.includes(usecases[key].list[subkey].name)) { + return true + } + } + } + } else if ( + curWorkflow.actions !== null && + curWorkflow.actions !== undefined + ) { + for (var key in curWorkflow.actions) { + const action = curWorkflow.actions[key]; + + if ( + action.app_name.toLowerCase() === newfilter || + action.app_name.toLowerCase().includes(newfilter) + ) { + return true; + } + } + } + + return false; + }); + } + + if (found.every((v) => v === true)) { + newWorkflows.push(curWorkflow); + continue; + } + } + + console.log("Changing workflow filter, and finding new usecase mappings!") + if (newWorkflows.length !== workflows.length) { + handleKeysetting(allUsecases, newWorkflows) + + setFilteredWorkflows(newWorkflows); + } + }; + + const getApps = () => { + try { + const appstorage = localStorage.getItem("apps") + const privateapps = JSON.parse(appstorage) + setApps(privateapps) + } catch (e) { + //console.log("Failed to get apps from localstorage: ", e) + } + + fetch(`${globalUrl}/api/v1/apps`, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for apps :O!"); + } + + return response.json(); + }) + .then((responseJson) => { + setApps(responseJson); + }) + .catch((error) => { + console.log("App loading error: " + error.toString()); + }); + } + + const addFilter = (data) => { + if (data === null || data === undefined) { + console.log("No filter data") + return; + } + + if (data.includes("<") && data.includes(">")) { + console.log("Filter includes < or >") + return; + } + + if (filters.includes(data) || filters.includes(data.toLowerCase())) { + console.log("Filter already has the data") + return; + } + + filters.push(data.toLowerCase()); + setFilters(filters); + + findWorkflow(filters); + }; + + const removeFilter = (index) => { + var newfilters = filters; + + if (index < 0) { + console.log("Can't handle index (remove): ", index); + return; + } + + newfilters.splice(index, 1); + + if (newfilters.length === 0) { + newfilters = []; + setFilters(newfilters); + } else { + setFilters(newfilters); + } + + findWorkflow(newfilters); + }; + + const exportVerifyModal = exportModalOpen ? ( + { + setExportModalOpen(false); + setSelectedWorkflow({}); + }} + PaperProps={{ + style: { + backgroundColor: theme.palette.surfaceColor, + color: "white", + minWidth: 500, + padding: 30, + }, + }} + > + +
    + Want to auto-sanitize this workflow before exporting? +
    +     + + + This will make potentially sensitive fields such as username, + password, url etc. empty + + + + +     + ) : null; + + const publishModal = publishModalOpen ? ( + { + setPublishModalOpen(false); + setSelectedWorkflow({}); + }} + PaperProps={{ + style: { + backgroundColor: theme.palette.surfaceColor, + color: "white", + minWidth: 500, + padding: 50, + }, + }} + > + +
    + Are you sure you want to PUBLISH this workflow? +
    +     + +
    + + Before publishing, we will sanitize all inputs, remove references to + you, randomize ID's and remove your authentication. + + + The published workflow is yours, and you can always change your public workflows after they are released. + +
    + + +     +     + ) : null; + + const deleteModal = deleteModalOpen ? ( + { + setDeleteModalOpen(false); + setSelectedWorkflowId(""); + }} + PaperProps={{ + style: { + backgroundColor: theme.palette.surfaceColor, + color: "white", + minWidth: 500, + padding: 50, + }, + }} + > + +
    + Are you sure you want to delete {selectedWorkflowId.length > 0 ? filteredWorkflows.find((w) => w.id === selectedWorkflowId)?.name : `${selectedWorkflowIndexes.length} workflow${selectedWorkflowIndexes.length === 1 ? '' : 's'}`}?
    + + Other workflows relying on {selectedWorkflowIndexes.length > 0 ? "them" : "it"} one will stop working +
    + + + + + +
    + ) : null; + + const uploadFile = (e) => { + const isDropzone = + e.dataTransfer === undefined ? false : e.dataTransfer.files.length > 0; + const files = isDropzone ? e.dataTransfer.files : e.target.files; + + const reader = new FileReader(); + toast("Starting upload. Please wait while we validate the workflows"); + + try { + reader.addEventListener("load", (e) => { + var data = e.target.result; + setIsDropzone(false); + try { + data = JSON.parse(reader.result); + } catch (e) { + toast("Invalid JSON: " + e); + return; + } + + // Initialize the workflow itself + setNewWorkflow( + data.name, + data.description, + data.tags, + data.default_return_value, + {}, + false, + [], + "", + data.status, + ) + .then((response) => { + if (response !== undefined) { + // SET THE FULL THING + data.id = response.id; + + // Actually create it + setNewWorkflow( + data.name, + data.description, + data.tags, + data.default_return_value, + data, + false, + [], + "", + data.status + ).then((response) => { + if (response !== undefined) { + toast(`Successfully imported ${data.name}`); + } + }); + } + }) + .catch((error) => { + toast("Import error: " + error.toString()); + }); + }); + } catch (e) { + console.log("Error in dropzone: ", e); + } + + reader.readAsText(files[0]); + }; + + useEffect(() => { + if (isDropzone) { + setIsDropzone(false); + } + + }, [isDropzone]); + + + const getFramework = () => { + fetch(globalUrl + "/api/v1/apps/frameworkConfiguration", { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for framework!"); + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success === false) { + setAppFramework({}) + if (responseJson.reason !== undefined) { + //toast("Failed loading: " + responseJson.reason) + } else { + //toast("Failed to load framework for your org.") + } + } else { + setAppFramework(responseJson) + } + }) + .catch((error) => { + console.log("err in framework: ", error.toString()); + }) + } + + + + const getAvailableWorkflows = (amount) => { + var storageWorkflows = [] + try { + const storagewf = localStorage.getItem("workflows") + storageWorkflows = JSON.parse(storagewf) + if (storageWorkflows === null || storageWorkflows === undefined || storageWorkflows.length === 0) { + storageWorkflows = [] + } else { + setWorkflows(storageWorkflows) + setFilteredWorkflows(storageWorkflows) + fetchUsecases(storageWorkflows) + setWorkflowDone(true) + } + } catch (e) { + //console.log("Failed to get workflows from localstorage: ", e) + } + + var url = `${globalUrl}/api/v1/workflows` + if (amount !== undefined && amount !== null) { + url += `?top=${amount}` + } + + fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for workflows :O!: ", response.status); + + //if (isCloud) { + // navigate("/search?tab=workflows") + //} + + toast("Failed getting workflows. Are you logged in?"); + return + } + + return response.json(); + }) + .then((responseJson) => { + console.log("Response : /api/v1/workflows", responseJson) + if (responseJson !== undefined) { + var newarray = [] + for (var wfkey in responseJson) { + const wf = responseJson[wfkey] + if (wf.public === true || wf.hidden === true) { + continue + } + + newarray.push(wf) + } + + var setProdFilter = false + + var actionnamelist = []; + var parsedactionlist = []; + for (var key in newarray) { + const workflow = newarray[key] + //if (workflow.status === "production") { + // setProdFilter = true + //} + + for (var actionkey in newarray[key].actions) { + const action = newarray[key].actions[actionkey]; + //console.log("Action: ", action) + if (actionnamelist.includes(action.app_name)) { + continue; + } + + actionnamelist.push(action.app_name); + parsedactionlist.push(action); + } + } + + try { + localStorage.setItem("workflows", JSON.stringify(newarray)) + } catch (e) { + console.log("Failed to set workflows in localstorage: ", e) + } + + // Ensures the zooming happens only once per load + setTimeout(() => { + fetchUsecases(newarray) + + setActionImageList(parsedactionlist); + if (setProdFilter === true) { + const newWorkflows = newarray.filter(workflow => workflow.status === "production") + if (newWorkflows !== undefined && newWorkflows !== null) { + setFilteredWorkflows(newWorkflows); + } else { + setFilteredWorkflows(newarray); + } + + setFilters(["status:production"]); + } else { + setFilteredWorkflows(newarray) + } + + setFirstLoad(false) + }, 250) + + /* + setTimeout(() => { + var timeout = 0 + for (var key in newarray) { + const wf = newarray[key] + if (wf.actions === undefined || wf.actions === null || wf.actions.length === 0) { + setTimeout(() => { + sideloadWorkflow(wf.id, false) + }, timeout) + + timeout += 1000 + } + + } + }, 1000) + */ + + } else { + if (isLoggedIn) { + toast("An error occurred while loading workflows"); + } + + return; + } + }) + .catch((error) => { + toast(error.toString()); + }); + } + + const findMatches = (category, workflows) => { + category.matches = [] + for (var subcategorykey in category.list) { + var subcategory = category.list[subcategorykey] + subcategory.matches = [] + + for (var workflowkey in workflows) { + const workflow = workflows[workflowkey] + + if (workflow.usecase_ids !== undefined && workflow.usecase_ids !== null) { + for (var usecasekey in workflow.usecase_ids) { + + if (workflow.usecase_ids[usecasekey].toLowerCase() === subcategory.name.toLowerCase()) { + //console.log("Got match: ", workflow.usecase_ids[usecasekey]) + + category.matches.push({ + "workflow": workflow.id, + "category": subcategory.name, + }) + subcategory.matches.push(workflow.id) + break + } + } + } + + if (subcategory.matches.length > 0) { + break + } + } + } + + return category + } + + const handleKeysetting = (categorydata, workflows) => { + if (workflows !== undefined && workflows !== null) { + var newcategories = [] + for (var key in categorydata) { + var category = categorydata[key] + // Check if category is bool + if (typeof category === "boolean") { + continue + } + + category = findMatches(category, workflows) + newcategories.push(category) + } + + setUsecases(newcategories) + } else { + setUsecases(categorydata) + } + } + + const fetchUsecases = (workflows) => { + fetch(globalUrl + "/api/v1/workflows/usecases", { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for usecases"); + } + + return response.json(); + }) + .then((responseJson) => { + setWorkflows(workflows); + setWorkflowDone(true); + + if (responseJson.success !== false) { + setAllUsecases(responseJson); + handleKeysetting(responseJson, workflows) + } + }) + .catch((error) => { + //toast("ERROR: " + error.toString()); + console.log("ERROR: " + error.toString()); + setWorkflows(workflows); + setWorkflowDone(true); + }); + }; + + // eslint-disable-next-line react-hooks/exhaustive-deps + useEffect(() => { + if (workflows.length <= 0) { + const tmpView = localStorage.getItem("view"); + if (tmpView !== undefined && tmpView !== null) { + setView(tmpView); + } + + const urlSearchParams = new URLSearchParams(window.location.search); + const params = Object.fromEntries(urlSearchParams.entries()); + const foundTab = params["top"]; + if (foundTab !== null && foundTab !== undefined) { + // Check if it's a number + if (isNaN(foundTab)) { + getAvailableWorkflows() + } else { + getAvailableWorkflows(foundTab) + } + } else { + getAvailableWorkflows() + } + + getApps() + getFramework() + } + }, []) + + const viewStyle = { + color: "#ffffff", + width: "100%", + display: "flex", + minWidth: isMobile ? "100%" : 1024, + maxWidth: isMobile ? "100%" : 1024, + margin: drawerWidth === 0 ? "auto" : `auto ${drawerWidth + 100} auto auto`, + paddingBottom: 200, + }; + + const emptyWorkflowStyle = { + paddingTop: "200px", + width: 1024, + margin: "auto", + }; + + const boxStyle = { + padding: "20px 20px 20px 20px", + width: "100%", + height: "250px", + color: "white", + backgroundColor: theme.palette.surfaceColor, + display: "flex", + flexDirection: "column", + }; + + //flexDirection: !isMobile ? "column" : "row", + const paperAppContainer = { + //display: "flex", + //flexWrap: "wrap", + //alignContent: "space-between", + }; + + const paperAppStyle = { + minHeight: 146, + maxHeight: 146, + overflow: "hidden", + width: "100%", + color: "white", + padding: "12px 12px 0px 15px", + display: "flex", + fontFamily: theme?.typography?.fontFamily, + boxSizing: "border-box", + position: "relative", + borderRadius: "8px", + // backgroundColor: "#212121", + }; + + const gridContainer = { + height: "auto", + color: "white", + margin: "10px", + backgroundColor: "#212121", + position: "relative", + }; + + const workflowActionStyle = { + display: "flex", + width: 160, + height: 44, + justifyContent: "space-between", + fontFamily: theme?.typography?.fontFamily, + }; + + const exportAllWorkflows = (allWorkflows) => { + for (var i = 0; i < allWorkflows.length; i++) { + const wf = allWorkflows[i] + + if (wf === undefined || wf.id === undefined) { + continue + } + + console.log("Exporting workflow: ", wf) + setTimeout(() => { + exportWorkflow(JSON.parse(JSON.stringify(wf)), false) + }, i * 100); + } + + toast(`Exporting and keeping original for all ${allWorkflows.length} workflows`); + } + + const deduplicateIds = (data, skip_sanitize) => { + if (data.triggers !== null && data.triggers !== undefined) { + for (var key in data.triggers) { + const trigger = data.triggers[key]; + if (skip_sanitize !== true) { + if (trigger.app_name === "Shuffle Workflow") { + if (trigger.parameters !== null && trigger.parameters !== undefined) { + if (trigger.parameters.length > 2) { + trigger.parameters[2].value = ""; + } + } + } + } + + if (trigger.status === "running") { + trigger.status = "stopped"; + } + + const newId = uuidv4(); + if (trigger.trigger_type === "WEBHOOK") { + if ( + trigger.parameters !== undefined && + trigger.parameters !== null && + trigger.parameters.length === 2 + ) { + trigger.parameters[0].value = + referenceUrl + "webhook_" + trigger.id; + trigger.parameters[1].value = "webhook_" + trigger.id; + } else if ( + trigger.parameters !== undefined && + trigger.parameters !== null && + trigger.parameters.length === 3 + ) { + trigger.parameters[0].value = + referenceUrl + "webhook_" + trigger.id; + trigger.parameters[1].value = "webhook_" + trigger.id; + // FIXME: Add auth here? + } else { + toast("Something is wrong with the webhook in the copy"); + } + } + + for (var branchkey in data.branches) { + const branch = data.branches[branchkey]; + if (branch.source_id === trigger.id) { + branch.source_id = newId; + } + + if (branch.destination_id === trigger.id) { + branch.destination_id = newId; + } + } + + trigger.environment = isCloud ? "cloud" : "Shuffle"; + trigger.id = newId; + } + } + + if (data.actions !== null && data.actions !== undefined && skip_sanitize !== true) { + for (key in data.actions) { + data.actions[key].authentication_id = ""; + + for (var subkey in data.actions[key].parameters) { + const param = data.actions[key].parameters[subkey]; + + // Removed October 10th, 2022 as key usually isn't + // containing anything secret, but rather necessary configurations. + // param.name.includes("key") || + // + if ( + param.name.includes("user") || + param.name.includes("pass") || + param.name.includes("api") || + param.name.includes("auth") || + param.name.includes("secret") || + param.name.includes("domain") || + param.name.includes("url") || + param.name.includes("mail") + ) { + // FIXME: This may be a vuln if api-keys are generated that start with $ + if (param.value.startsWith("$")) { + console.log("Skipping field, as it's referencing a variable"); + } else { + param.value = ""; + param.is_valid = false; + } + } + } + + const newId = uuidv4(); + for (branchkey in data.branches) { + const branch = data.branches[branchkey]; + if (branch.source_id === data.actions[key].id) { + branch.source_id = newId; + } + + if (branch.destination_id === data.actions[key].id) { + branch.destination_id = newId; + } + } + + if (data.actions[key].id === data.start) { + data.start = newId; + } + + data.actions[key].environment = ""; + data.actions[key].id = newId; + } + } + + if (data.workflow_variables !== null && data.workflow_variables !== undefined && skip_sanitize !== true) { + for (key in data.workflow_variables) { + const param = data.workflow_variables[key]; + //param.name.includes("key") || + + if ( + param.name.includes("user") || + param.name.includes("pass") || + param.name.includes("api") || + param.name.includes("auth") || + param.name.includes("secret") || + param.name.includes("email") + ) { + param.value = ""; + param.is_valid = false; + } + } + } + + return data; + }; + + const sanitizeWorkflow = (data) => { + data = JSON.parse(JSON.stringify(data)); + console.log("Sanitize start: ", data); + data = deduplicateIds(data); + + console.log("Sanitize end: ", data); + + return data; + }; + + const exportWorkflow = (data, sanitize) => { + try { + data = JSON.parse(JSON.stringify(data)); + } catch (e) { + console.log("Failed to parse JSON: ", e); + } + + let exportFileDefaultName = data.name + ".json"; + + data["owner"] = ""; + data["org"] = []; + data["org_id"] = ""; + data["execution_org"] = {}; + + // These are backwards.. True = saved before. Very confuse. + data["previously_saved"] = false; + data["first_save"] = false; + + if (sanitize === true) { + data = sanitizeWorkflow(data); + + if (data.subflows !== null && data.subflows !== undefined) { + toast( + "Not exporting with subflows when sanitizing. Please manually export them." + ) + + data.subflows = [] + } + + // for (var key in data.subflows) { + // if (data.sublof + // } + //} + } + + // Add correct ID's for triggers + // Add mag + + data.status = "test" + let dataStr = JSON.stringify(data); + let dataUri = + "data:application/json;charset=utf-8," + encodeURIComponent(dataStr); + let linkElement = document.createElement("a"); + linkElement.setAttribute("href", dataUri); + linkElement.setAttribute("download", exportFileDefaultName); + linkElement.click(); + }; + + const publishWorkflow = (data) => { + data = JSON.parse(JSON.stringify(data)); + data = sanitizeWorkflow(data); + toast("Sanitizing and publishing " + data.name); + + // This ALWAYS talks to Shuffle cloud + fetch(globalUrl + "/api/v1/workflows/" + data.id + "/publish", { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(data), + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for workflow publish :O!"); + } else { + if (isCloud) { + toast("Successfully published workflow"); + } else { + toast( + "Successfully published workflow to https://shuffler.io" + ); + } + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.reason !== undefined) { + toast("Failed publishing: ", responseJson.reason); + } + + getAvailableWorkflows(); + }) + .catch((error) => { + toast("Failed publishing: is the workflow valid? Remember to save the workflow first.") + console.log(error.toString()); + }); + }; + + const duplicateWorkflow = (data) => { + //data = JSON.parse(JSON.stringify(data)); + toast("Copying workflow '" + data.name + "'. The new workflow will load in and be highlighted."); + //data.id = ""; + //data.name = data.name + "_copy"; + //data = deduplicateIds(data, true); + + const duplicateData = { + name: data.name + "_copy", + } + + fetch(`${globalUrl}/api/v1/workflows/${data.id}/duplicate`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(duplicateData), + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for workflows :O!"); + return; + } + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success === false) { + if (responseJson.reason !== undefined) { + toast("Failed copying workflow: " + responseJson.reason) + } else { + toast("Failed copying workflow") + } + + return + } + + if (responseJson.id !== undefined) { + setHighlightIds([responseJson.id]) + } + setTimeout(() => { + getAvailableWorkflows(); + }, 1000); + }) + .catch((error) => { + toast(error.toString()); + }) + } + + const setEditing = (data) => { + ReactDOM.unstable_batchedUpdates(() => { + setIsEditing(true) + setModalOpen(true); + setNewWorkflowName(data.name); + setNewWorkflowDescription(data.description); + setDefaultReturnValue(data.default_return_value); + if (data.tags !== undefined && data.tags !== null) { + setNewWorkflowTags(JSON.parse(JSON.stringify(data.tags))); + } + + if (data.usecase_ids !== undefined && data.usecase_ids !== null && data.usecase_ids.length > 0) { + setSelectedUsecases(data.usecase_ids) + } + + setEditingWorkflow(JSON.parse(JSON.stringify(data))) + }) + } + + const exportSingleWorkflow = (data, setOpen) => { + setExportModalOpen(true) + + if (data.triggers !== null && data.triggers !== undefined) { + var newSubflows = []; + for (var key in data.triggers) { + const trigger = data.triggers[key]; + + if ( + trigger.parameters !== null && + trigger.parameters !== undefined + ) { + for (var subkey in trigger.parameters) { + const param = trigger.parameters[subkey]; + if ( + param.name === "workflow" && + param.value !== data.id && + !newSubflows.includes(param.value) + ) { + newSubflows.push(param.value); + } + } + } + } + + var parsedworkflows = []; + for (var key in newSubflows) { + const foundWorkflow = workflows.find( + (workflow) => workflow.id === newSubflows[key] + ); + if (foundWorkflow !== undefined && foundWorkflow !== null) { + parsedworkflows.push(foundWorkflow); + } + } + + if (parsedworkflows.length > 0) { + console.log( + "Appending subflows during export: ", + parsedworkflows.length + ); + data.subflows = parsedworkflows; + } + } + + setExportData(data) + setOpen(false) + } + + const sideloadWorkflow = (id, action, setOpen) => { + + const storagewf = localStorage.getItem("workflows") + const storageWorkflows = JSON.parse(storagewf) + if (storageWorkflows === null || storageWorkflows === undefined || storageWorkflows.length === 0) { + } else { + for (var i = 0; i < storageWorkflows.length; i++) { + if (storageWorkflows[i].id === id) { + if (storageWorkflows[i].image !== "" && storageWorkflows[i].image !== undefined && storageWorkflows[i].image !== null) { + + if (action === undefined || action === null || action === "") { + console.log("RETURNING") + return + } + } + } + } + } + + fetch(globalUrl + "/api/v1/workflows/" + id, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + } + + return response.json() + }) + .then((responseJson) => { + if (responseJson.success !== false && responseJson.id !== undefined) { + if (action === "edit") { + setEditing(responseJson) + } else if (action === "publish") { + setPublishModalOpen(true) + setSelectedWorkflow(responseJson) + } else if (action === "export") { + exportSingleWorkflow(responseJson, setOpen) + } + + } + + for (var i = 0; i < storageWorkflows.length; i++) { + if (storageWorkflows[i].id === id) { + storageWorkflows[i] = responseJson + localStorage.setItem("workflows", JSON.stringify(storageWorkflows)) + break + } + } + + //setWorkflows(storageWorkflows) + setFilteredWorkflows(storageWorkflows) + //setUpdate(Math.random()) + }) + .catch((error) => { + console.log(error.toString()) + }) + } + + const deleteWorkflow = (id, bulk) => { + fetch(globalUrl + "/api/v1/workflows/" + id, { + method: "DELETE", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for setting workflows :O!"); + toast("Failed deleting workflow. Do you have access?"); + } else { + if (bulk !== true) { + toast(`Deleted workflow ${id}. Child Workflows in Suborgs were also removed.`) + } + } + + return response.json(); + }) + .then(() => { + if (bulk !== true) { + setTimeout(() => { + getAvailableWorkflows(); + }, 1000); + } + }) + .catch((error) => { + toast(error.toString()); + }) + } + + const handleChipClick = (e) => { + addFilter(e.target.innerHTML); + }; + + const hasWorkflows = workflows === undefined || workflows === null || workflows.length === 0 + const NewWorkflowPaper = () => { + const [hover, setHover] = React.useState(false); + + const innerColor = "rgba(255,255,255,0.3)" + + const setupPaperStyle = { + minHeight: paperAppStyle.minHeight, + maxWidth: "100%", + minWidth: paperAppStyle.width, + color: innerColor, + padding: paperAppStyle.padding, + display: "flex", + boxSizing: "border-box", + position: "relative", + border: hasWorkflows ? `2px solid #f85a3e` : `2px solid ${innerColor}`, + cursor: "pointer", + backgroundColor: hover ? "rgba(39,41,45,0.5)" : "rgba(39,41,45,1)", + borderRadius: paperAppStyle.borderRadius, + } + + return ( + + { + setModalOpen(true) + setIsEditing(false) + }} + onMouseOver={() => { + setHover(true); + }} + onMouseOut={() => { + setHover(false); + }} + > + + + + + New Workflow + + + + + + ); + }; + + const getWorkflowAppgroup = (data) => { + if (currTab !== 2) { + if (data.actions === undefined || data.actions === null) { + return [] + } + + var appsFound = [] + for (var key in data.actions) { + const parsedAction = data.actions[key] + if (parsedAction.large_image === undefined || parsedAction.large_image === null || parsedAction.large_image === "") { + continue + } + + if (parsedAction.app_name === "Shuffle Tools" || parsedAction.app_id === "bc78f35c6c6351b07a09b7aed5d29652") { + continue + } + + if (appsFound.findIndex(data => data.app_name === parsedAction.app_name) < 0) { + appsFound.push(parsedAction) + } + } + } else { + if (data.action_references === undefined || data.action_references === null) { + return [] + } + + var appsFound = [] + for (var key in data.action_references) { + const parsedAction = data.action_references[key] + if (parsedAction.image_url === undefined || parsedAction.image_url === null || parsedAction.image_url === "") { + continue + } + + // if (parsedAction.name === "Shuffle Tools" || parsedAction.id === "bc78f35c6c6351b07a09b7aed5d29652") { + // continue + // } + + if (appsFound.findIndex(data => data.name === parsedAction.name) < 0) { + appsFound.push(parsedAction) + } + } + } + + return appsFound + } + + const WorkflowSkeleton = () => { + return ( + +
    + +
    + + + +
    +
    +     + ); + }; + + // Replace the loading sections in the main component with this + const LoadingWorkflowGrid = () => { + return ( +
    + {[...Array(7)].map((_, index) => ( + + ))} +
    + ); + }; + + const WorkflowPaper = (props) => { + const { data, type = "org" } = props; + const [open, setOpen] = React.useState(false); + const [anchorEl, setAnchorEl] = React.useState(null); + + var boxColor = "#FECC00"; + if (data.is_valid) { + boxColor = "#86c142"; + } + + if (!data.previously_saved) { + boxColor = "#f86a3e"; + } + + const menuClick = (event) => { + setOpen(!open); + setAnchorEl(event.currentTarget); + } + + + var parsedName = data.name; + if ( + parsedName !== undefined && + parsedName !== null && + parsedName.length > 20 + ) { + parsedName = parsedName.slice(0, 21) + ".."; + } + + const actions = data.actions !== null ? data.actions.length : 0; + const appGroup = getWorkflowAppgroup(data) + const [triggers, subflows] = getWorkflowMeta(data) + + const hasSuborgs = data.suborg_distribution !== undefined && data.suborg_distribution !== null && data.suborg_distribution.length > 0 + const isDistributed = (data.parentorg_workflow !== undefined && data.parentorg_workflow !== null && data.parentorg_workflow.length > 0) //|| (data.org_id !== userdata.active_org.id && data.org_id !== undefined && data.org_id !== null && data.org_id.length > 0) + + const workflowMenuButtons = ( + { + setOpen(false); + setAnchorEl(null); + }} + > + {isDistributed ? + { + navigate(`/workflows/${data.id}`) + }} + > + + Explore Workflow + + : null} + { + event.stopPropagation() + if (data.actions !== undefined && data.actions !== null && data.actions.length > 0 && data.image !== "") { + setEditing(data) + + } else { + //toast("Need to side-load workflow to be edited properly") + sideloadWorkflow(data.id, "edit") + + toast.info("Loading full workflow for editing. Please wait...") + } + }} + key={"change"} + > + + {"Edit details"} + + + { + window.open(`/forms/${data.id}`, "_blank") + }} + key={"explore forms"} + > + + {"Create Form"} + + + + + { + sideloadWorkflow(data.id, "publish") + + toast.info("Loading full workflow for publishing. Please wait...") + }} + key={"publish"} + > + + {"Publish Workflow"} + + + { + sideloadWorkflow(data.id, "export", setOpen) + + toast.info("Loading full workflow to be exported. Please wait...") + }} + key={"export"} + > + + {"Export Workflow"} + + + + + { + duplicateWorkflow(data) + setOpen(false) + }} + key={"duplicate"} + > + + {"Duplicate Workflow"} + + + { + setDeleteModalOpen(true); + setSelectedWorkflowId(data.id); + setOpen(false); + }} + key={"delete"} + > + + {"Delete Workflow"} + + + ); + + var image = ""; + + var orgName = ""; + var orgId = ""; + if (userdata.orgs !== undefined) { + const foundOrg = userdata.orgs.find((org) => org.id === data["org_id"]); + if (foundOrg !== undefined && foundOrg !== null) { + //position: "absolute", bottom: 5, right: -5, + const imageStyle = { + width: imagesize, + height: imagesize, + pointerEvents: "none", + marginLeft: + data.creator_org !== undefined && data.creator_org.length > 0 + ? 20 + : 0, + borderRadius: 10, + border: + foundOrg.id === userdata.active_org.id + ? `3px solid ${boxColor}` + : null, + cursor: "pointer", + marginRight: 10, + }; + + + image = + foundOrg.image === "" || foundOrg.image === null || foundOrg.image === undefined ? ( + {foundOrg.name} + ) : ( + {foundOrg.name} { }} + /> + ); + + orgName = foundOrg.name; + orgId = foundOrg.id; + } + } + + var selectedCategory = "" + if (data.usecase_ids !== undefined && data.usecase_ids !== null && data.usecase_ids.length > 0 && usecases !== null && usecases !== undefined && usecases.length > 0) { + const oldcolor = boxColor.valueOf() + + // Find the first usecase and use that ones' ID + for (var key in usecases) { + var category = usecases[key] + category.matches = [] + + for (var subcategorykey in category.list) { + var subcategory = category.list[subcategorykey] + subcategory.matches = [] + + for (var usecasekey in data.usecase_ids) { + if (data.usecase_ids[usecasekey].toLowerCase() === subcategory.name.toLowerCase()) { + boxColor = category.color + break + } + } + + if (boxColor !== oldcolor) { + break + } + } + + if (boxColor !== oldcolor) { + selectedCategory = category.name + break + } + } + } + + if (type === "public" && currTab === 2) { + + const imageStyle = { + width: 24, + height: 24, + marginRight: 10, + border: "1px solid rgba(255,255,255,0.3)", + } + + image = data.creator_info !== undefined && data.creator_info !== null && data.creator_info.image !== undefined && data.creator_info.image !== null && data.creator_info.image.length > 0 ? : + const creatorname = data.creator_info !== undefined && data.creator_info !== null && data.creator_info.username !== undefined && data.creator_info.username !== null && data.creator_info.username.length > 0 ? data.creator_info.username : "Shuffle" + if ((data.objectID === undefined || data.objectID === null) && data.id !== undefined && data.id !== null) { + data.objectID = data.id + } + + //console.log("IMG: ", data) + var parsedUrl = `/workflows/${data.objectID}` + if (data.__queryID !== undefined && data.__queryID !== null) { + parsedUrl += `?queryID=${data.__queryID}` + } + } + + return ( +
    + + {selectedCategory !== "" ? + +
    { + addFilter(selectedCategory) + }} + /> + + : null} + + + +
    { + navigate("/admin") + }} + > + {image} +
    +     + { + /* + if (data.image === undefined || data.image === null || data.image === "" && !loadingWorkflows.includes(data.id)) { + sideloadWorkflow(data.id, false) + loadingWorkflows.push(data.id) + } + */ + }} + title={ +
    + {(data?.image !== undefined || data?.image_url !== undefined) ? ( +
    + {data?.name} +
    + ) : null} + + + Edit: {data.name} + + + {(isDistributed || hasSuborgs) && ( +
    + + This is a parentorg-controlled workflow + +
    + )} +
    + } placement="right"> + + + + {parsedName} + + +     +     + + {appGroup.length > 0 ? +
    + + {currTab !== 2 && appGroup.map((data, index) => { + return ( +
    { + addFilter(data.app_name); + }} + > + + + +
    + ) + })} + { + currTab === 2 && appGroup.map((data, index) => { + return ( +
    { + addFilter(data.app_name); + }} + > + + + +
    + ) + })} +     +
    + : + + + + + {actions} + + + + } + + + + + {triggers} + + + + + { + if (subflows === 0) { + toast("No subflows for " + data.name); + return; + } + + var newWorkflows = [data]; + for (var key in data.triggers) { + const trigger = data.triggers[key]; + if (trigger.app_name !== "Shuffle Workflow") { + continue; + } + + if ( + trigger.parameters !== undefined && + trigger.parameters !== null && + trigger.parameters.length > 0 && + trigger.parameters[0].name === "workflow" + ) { + const newWorkflow = workflows.find( + (item) => item.id === trigger.parameters[0].value + ); + if (newWorkflow !== null && newWorkflow !== undefined) { + newWorkflows.push(newWorkflow); + continue; + } + } + } + + setFilters(["Subflows of " + data.name]); + setFilteredWorkflows(newWorkflows); + }} + > + + + + + {subflows} + + + +     + + {data.tags !== undefined && data.tags !== null + ? data.tags.map((tag, index) => { + if (index >= 3) { + return null; + } + + return ( + + ); + }) + : null} + + {data.actions !== undefined && data.actions !== null && type !== "public" ? ( +
    + + + + {workflowMenuButtons} +
    + ) : null} + {(data.sharing !== undefined && data.sharing !== null && data.sharing === "form") || (data?.form_control?.input_markdown !== undefined && data?.form_control?.input_markdown !== null && data?.form_control?.input_markdown !== "") && type !== "public" ? + +
    + { + navigate(`/forms/${data.id}`) + }} + style={{ padding: "0px", color: "#979797" }} + > + + + {workflowMenuButtons} +
    +     + : null} +     + +
    + ) + } + + // Can create and set workflows + const setNewWorkflow = ( + name, + description, + tags, + defaultReturnValue, + editingWorkflow, + redirect, + currentUsecases, + inputblogpost, + inputstatus, + ) => { + + var method = "POST"; + var extraData = ""; + var workflowdata = {}; + + if (editingWorkflow.id !== undefined) { + console.log("Building original workflow"); + method = "PUT"; + extraData = "/" + editingWorkflow.id + "?skip_save=true"; + workflowdata = editingWorkflow; + + console.log("REMOVING OWNER"); + workflowdata["owner"] = ""; + workflowdata["org"] = []; + workflowdata["org_id"] = ""; + workflowdata["execution_org"] = {}; + workflowdata["previously_saved"] = false; + // FIXME: Loop triggers and turn them off? + } + + workflowdata["name"] = name; + workflowdata["description"] = description; + if (tags !== undefined) { + workflowdata["tags"] = tags; + } + + workflowdata["blogpost"] = inputblogpost + workflowdata["status"] = inputstatus + + if (defaultReturnValue !== undefined) { + workflowdata["default_return_value"] = defaultReturnValue; + } + + if (currentUsecases !== undefined && currentUsecases !== null) { + workflowdata["usecase_ids"] = currentUsecases + //workflows[0].category = ["detect"] + //workflows[0].usecase_ids = ["Correlate tickets"] + } + + const new_url = `${globalUrl}/api/v1/workflows${extraData}` + return fetch(new_url, { + method: method, + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(workflowdata), + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for workflows :O!"); + return; + } + setSubmitLoading(false); + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success === false) { + if (responseJson.reason !== undefined) { + toast("Error setting workflow: ", responseJson.reason) + } else { + toast("Error setting workflow.") + } + + return + } + + if (redirect) { + //window.location.pathname = "/workflows/" + responseJson["id"]; + navigate("/workflows/" + responseJson["id"]) + //setModalOpen(false); + } else if (!redirect) { + // Update :) + setTimeout(() => { + getAvailableWorkflows(); + }, 4000); + setSubmitLoading(false) + setModalOpen(false); + } else { + //toast("Successfully changed basic info for workflow"); + setModalOpen(false); + } + + return responseJson; + }) + .catch((error) => { + toast(error.toString()); + setSubmitLoading(false) + setModalOpen(false); + setSubmitLoading(false); + }); + }; + + const importFiles = (event) => { + console.log("Importing!"); + setSubmitLoading(true) + + if (event.target.files.length > 0) { + console.log("Files: !", event.target.files.length); + for (var key in event.target.files) { + const file = event.target.files[key]; + if (file.type !== "application/json") { + if (file.type !== undefined) { + toast("File has to contain valid json"); + setSubmitLoading(false) + } + + continue; + } + + const reader = new FileReader(); + var workflowids = [] + + // Waits for the read + reader.addEventListener("load", (event) => { + var data = reader.result; + try { + data = JSON.parse(reader.result); + } catch (e) { + toast("Invalid JSON: " + e); + setSubmitLoading(false) + return; + } + + console.log("File being loaded: ", data.name); + + // Initialize the workflow itself + setNewWorkflow( + data.name, + data.description, + data.tags, + data.default_return_value, + {}, + false, + [], + "", + data.status, + ) + .then((response) => { + if (response !== undefined) { + // SET THE FULL THING + data.id = response.id; + data.first_save = false; + data.previously_saved = false; + data.is_valid = false; + data.org_id = userdata.active_org.id + data.org = [] + data.execution_org = {} + + workflowids.push(data.id) + + // Actually create it + setNewWorkflow( + data.name, + data.description, + data.tags, + data.default_return_value, + data, + false, + [], + "", + data.status, + ).then((response) => { + if (response !== undefined) { + toast("Successfully imported " + data.name); + } + }); + } + }) + .catch((error) => { + toast("Import error: " + error.toString()); + }); + }); + + // Actually reads + reader.readAsText(file); + } + } + + setLoadWorkflowsModalOpen(false); + + if (workflowids.length > 0) { + setHighlightIds(workflowids) + } + }; + + const getWorkflowMeta = (data) => { + let triggers = 0; + let subflows = 0; + if ( + data.triggers !== undefined && + data.triggers !== null && + data.triggers.length > 0 + ) { + triggers = data.triggers.length; + for (let key in data.triggers) { + if (data.triggers[key].app_name === "Shuffle Workflow") { + subflows += 1; + } + } + } + + return [triggers, subflows]; + }; + + const WorkflowListView = () => { + let workflowData = ""; + if (workflows.length > 0) { + const columns = [ + { + field: "image", + headerName: "Logo", + width: 50, + sortable: false, + renderCell: (params) => { + const data = params.row.record; + + var boxColor = "#FECC00"; + if (data.is_valid) { + boxColor = "#86c142"; + } + + if (!data.previously_saved) { + boxColor = "#f85a3e"; + } + + var image = ""; + if (userdata.orgs !== undefined) { + const foundOrg = userdata.orgs.find( + (org) => org.id === data["org_id"] + ); + if (foundOrg !== undefined && foundOrg !== null) { + //position: "absolute", bottom: 5, right: -5, + const imageStyle = { + width: imagesize + 7, + height: imagesize + 7, + pointerEvents: "none", + marginLeft: + data.creator_org !== undefined && + data.creator_org.length > 0 + ? 20 + : 0, + borderRadius: 10, + border: + foundOrg.id === userdata.active_org.id + ? `3px solid ${boxColor}` + : null, + cursor: "pointer", + marginTop: 5, + }; + + // + image = + foundOrg.image === "" ? ( + {foundOrg.name} + ) : ( + {foundOrg.name} { + //setFilteredWorkflows(newWorkflows) + }} + /> + ); + } + } + + return
    {image}
    ; + }, + }, + { + field: "title", + headerName: "Title", + width: 330, + renderCell: (params) => { + const data = params.row.record; + + + return ( + + + {data.name} + + + ); + }, + }, + + { + field: "options", + headerName: "Options", + width: 200, + sortable: false, + disableClickEventBubbling: true, + renderCell: (params) => { + const data = params.row.record; + const actions = data.actions !== null ? data.actions.length : 0; + let [triggers, subflows] = getWorkflowMeta(data); + const appGroup = getWorkflowAppgroup(data) + + return ( + +
    + {appGroup.length > 0 ? +
    + + {appGroup.map((data, index) => { + return ( +
    { + addFilter(data.app_name); + }} + > + + + +
    + ) + })} +     +
    + : + + + + + {actions} + + + + } + + + + + {triggers} + + + + + { + if (subflows === 0) { + toast("No subflows for " + data.name); + return; + } + + var newWorkflows = [data]; + for (var key in data.triggers) { + const trigger = data.triggers[key]; + if (trigger.app_name !== "Shuffle Workflow") { + continue; + } + + if ( + trigger.parameters !== undefined && + trigger.parameters !== null && + trigger.parameters.length > 0 && + trigger.parameters[0].name === "workflow" + ) { + const newWorkflow = workflows.find( + (item) => item.id === trigger.parameters[0].value + ); + if ( + newWorkflow !== null && + newWorkflow !== undefined + ) { + newWorkflows.push(newWorkflow); + continue; + } + } + } + + setFilters(["Subflows of " + data.name]); + setFilteredWorkflows(newWorkflows); + }} + > + + + + + {subflows} + + + +
    +     + ); + }, + }, + { + field: "tags", + headerName: "Tags", + maxHeight: 15, + width: 300, + sortable: false, + disableClickEventBubbling: true, + renderCell: (params) => { + const data = params.row.record; + return ( + + {data.tags !== undefined + ? data.tags.map((tag, index) => { + if (index >= 3) { + return null; + } + + return ( + + ); + }) + : null} + + ); + }, + }, + { + field: "", + headerName: "", + maxHeight: 15, + width: 100, + sortable: false, + disableClickEventBubbling: true, + renderCell: (params) => { }, + }, + ]; + + let rows = []; + rows = filteredWorkflows.map((data, index) => { + let obj = { + id: index + 1, + title: data.name, + record: data, + }; + + return obj; + }) + + workflowData = ( + { + setSelectedWorkflowIndexes(newSelection) + }} + selectionModel={selectedWorkflowIndexes} + components={{ + Toolbar: GridToolbar, + }} + /> + ); + } + return ( +
    + + { + setModalOpen(true) + setIsEditing(false) + }} + > + + + + {filteredWorkflows.length === 0 ? null : + { + setDeleteModalOpen(true) + }} + > + + + + + } + {workflowData} +
    + ) + }; + + var total_count = 0 + const modalView = dialogModalOpen ? ( + { + setDialogModalOpen(false); + }} + PaperProps={{ + style: { + backgroundColor: theme.palette.surfaceColor, + color: "white", + minWidth: isMobile ? "90%" : "800px", + maxWidth: isMobile ? "90%" : "800px", + borderRadius: 8, + }, + }} + > + +
    + + {editingWorkflow.id !== undefined ? "Edit Workflow" : "Create New Workflow"} + +
    + + upload.click()} + style={{ backgroundColor: "rgba(255,255,255,0.08)" }} + > + + + + setDialogModalOpen(false)} + style={{ backgroundColor: "rgba(255,255,255,0.08)" }} + > + + +
    +
    +     + + + setNewWorkflowName(event.target.value)} + InputProps={{ + style: { + color: "white", + backgroundColor: theme.palette.inputColor, + borderRadius: 8, + }, + }} + color="primary" + label="Workflow Name" + required + margin="dense" + defaultValue={newWorkflowName} + autoFocus + fullWidth + variant="outlined" + /> + setNewWorkflowDescription(event.target.value)} + InputProps={{ + style: { + color: "white", + backgroundColor: theme.palette.inputColor, + borderRadius: 8, + }, + }} + color="primary" + label="Description" + defaultValue={newWorkflowDescription} + multiline + rows={3} + margin="dense" + fullWidth + variant="outlined" + style={{ marginTop: 16 }} + /> +
    + { + // Directly set the new array instead of mutating + setNewWorkflowTags(chips); + }} + onBlur={(event) => { + if (event.target.value.length === 0) { + return + } + + if (newWorkflowTags.includes(event.target.value)) { + return + } + + // Create new array instead of pushing + setNewWorkflowTags([...newWorkflowTags, event.target.value]); + }} + onAdd={(chip) => { + // Create new array instead of pushing + setNewWorkflowTags([...newWorkflowTags, chip]); + }} + onDelete={(chip, index) => { + // Filter instead of splice + setNewWorkflowTags(newWorkflowTags.filter((_, i) => i !== index)); + }} + variant="outlined" + /> + {usecases !== null && usecases !== undefined && usecases.length > 0 ? + + Usecases + + + : null} +
    + + +
    + { + if (event.target.value.toLowerCase() === "test") { + setStatus("test") + } else if (event.target.value.toLowerCase() === "production") { + setStatus("production") + } + }} + InputProps={{ + style: { + color: "white", + backgroundColor: theme.palette.inputColor, + borderRadius: 8, + }, + }} + color="primary" + label="Status" + defaultValue={status} + helperText="Can be test or production" + fullWidth + variant="outlined" + /> + setBlogpost(event.target.value)} + InputProps={{ + style: { + color: "white", + backgroundColor: theme.palette.inputColor, + borderRadius: 8, + }, + }} + color="primary" + label="Reference" + defaultValue={blogpost} + helperText="Blogpost or documentation reference" + fullWidth + variant="outlined" + /> + setDefaultReturnValue(event.target.value)} + InputProps={{ + style: { + color: "white", + backgroundColor: theme.palette.inputColor, + borderRadius: 8, + }, + }} + color="primary" + label="Default Return Value" + defaultValue={defaultReturnValue} + helperText="Used for Subflows if the subflow fails" + multiline + rows={3} + fullWidth + variant="outlined" + /> +
    +     + + +     + + + + +     +     + ) : null; + + const viewSize = { + workflowView: 4, + executionsView: 3, + executionResults: 4, + }; + + const workflowViewStyle = { + flex: viewSize.workflowView, + marginLeft: 10, + marginRight: 10, + }; + + if (viewSize.workflowView === 0) { + workflowViewStyle.display = "none"; + } + + const workflowButtons = ( + + + + + {view === "list" && ( + + + + )} + {view === "grid" && ( + + + + )} + + + + (upload = ref)} + onChange={importFiles} + /> + {workflows.length > 0 ? ( + + + + ) : null} + {isCloud ? null : ( + + + + )} + + ); + + // const tourOptions = { + // defaultStepOptions: { + // classes: "shadow-md bg-purple-dark", + // scrollTo: true + // }, + // useModalOverlay: true, + // tourName: workflows, + // exitOnEsc: true, + // } + + // //classes: "custom-class-name-1 custom-class-name-2", + // const newSteps = [ + // { + // id: "intro", + // scrollTo: true, + // beforeShowPromise: function() { + // return new Promise(function(resolve) { + // setTimeout(function() { + // window.scrollTo(0, 0); + // resolve(); + // }, 500); + // }); + // }, + // buttons: [ + // { + // classes: "shepherd-button-primary", + // style: { + // backgroundColor: "red", + // color: "white", + // }, + // text: "Next", + // type: "next" + // } + // ], + // highlightClass: "highlight", + // showCancelLink: true, + // text: [ + // "React-Shepherd is a JavaScript library for guiding users through your React app." + // ], + // when: { + // show: () => { + // console.log("show step 1"); + // }, + // hide: () => { + // console.log("hide step 1"); + // } + // } + // }, + // { + // id: "second", + // attachTo: { + // element: "second-step", + // on: "top" + // }, + // text: [ + // "Yuk eksplorasi hasil Tes Minat Bakat-mu dan rekomendasi Jurusan dan Karier." + // ], + // buttons: [ + // { + // classes: "btn btn-info", + // text: "Kembali", + // type: "back" + // }, + // { + // classes: "btn btn-success", + // text: "Saya Mengerti", + // type: "cancel" + // } + // ], + // when: { + // show: () => { + // console.log("show stepp"); + // }, + // hide: () => { + // console.log("complete step"); + // } + // }, + // showCancelLink: false, + // scrollTo: true, + // modalOverlayOpeningPadding: 4, + // useModalOverlay: false, + // canClickTarget: false + // } + // ] + + // function TourButton() { + // const tour = useContext(ShepherdTourContext); + + // return ( + // + // ); + // } + + useEffect(() => { + if (userdata !== undefined && userdata !== null && currTab !== 2) { + setIsLoadingWorkflow(true); + var filteredWorkflows = [] + if (currTab === 0) { + filteredWorkflows = workflows.filter(workflow => workflow?.org_id === userdata?.active_org?.id) + } + else if (currTab === 1) { + filteredWorkflows = workflows.filter(workflow => workflow?.org_id === userdata?.active_org?.id && workflow?.owner === userdata?.id) + } + setFilteredWorkflows(filteredWorkflows) + setTimeout(() => { + setIsLoadingWorkflow(false); + }, 500); + + } + }, [currTab, workflows, userdata]) + + useEffect(() => { + console.log("SearchQuery: ", searchQuery) + }, [searchQuery]) + + const Hits = ({ hits }) => { + var counted = 0 + console.log("Public workflows", hits) + + return ( + + isLoadingPublicWorkflow ? + ( + + ) : ( + +
    + {hits.map((data, index) => { + return + })} +
    + ) + + ) + } + + const CustomHits = connectHits(Hits) + + + const handleCategoryChange = (e) => { + setSelectedCategory(e.target.value) + } + + + const iconButtonStyle = { + color: 'white', + backgroundColor: '#212121', + borderRadius: '4px', + padding: "12px 16px", + cursor: 'pointer', + minWidth: '40px', + height: 'auto', + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + + } + + const WorkflowView = memo(() => { + if (workflows.length === 0) { + } + + var workflowDelay = -150 + var appDelay = -75 + + const foundPriority = userdata === undefined || userdata === null || userdata.priorities === undefined || userdata.priorities === null ? null : userdata.priorities.find(prio => prio.type === "usecase" && prio.active === true) + return ( + <> +
    + + Workflows + + +
    + + + + + +
    + + +
    + + + + // + // + // ), + onKeyDown: (e) => { + // Prevent default behavior for Enter and Backspace + if (e.key === 'Enter' || e.key === 'Backspace') { + e.preventDefault(); + e.stopPropagation(); + e.target.focus(); + } + }, + }} + clearInputOnBlur={false} + sx={{ + // Container styling + '& .MuiOutlinedInput-root': { + height: "fit-content", + borderRadius: '4px', + backgroundColor: '#212121', + '& fieldset': { + borderColor: 'rgba(255, 255, 255, 0.23)', + }, + '&:hover fieldset': { + borderColor: 'rgba(255, 255, 255, 0.4)', + }, + }, + + // Adjust chip container to center vertically + '& .MuiInputBase-root': { + display: 'flex', + flexWrap: 'wrap', + gap: '4px', + padding: '4px 8px', + alignItems: 'center', + height: "fit-content", // Match height + }, + + // Rest of the styling remains the same... + }} + value={filters} + onChange={(chips) => { + setFilters(chips); + findWorkflow(chips); + }} + //onAdd={(chip) => { + // console.log("ADd: ", chip); + // addFilter(chip); + //}} + //onDelete={(_, index) => { + // console.log("Remove: ", index); + // removeFilter(index); + //}} + /> + + + + +
    +
    + + navigate("/workflows/debug")} + disabled={currTab === 2} + > + + + + + + { + const newView = view === "grid" ? "list" : "grid"; + localStorage.setItem("workflowView", newView); + setView(newView); + }} + disabled={currTab === 2} + > + {view === "grid" ? : } + + + + + upload.click()} + disabled={currTab === 2} + > + {submitLoading ? : } + + + + (upload = ref)} + onChange={importFiles} + /> + + + exportAllWorkflows(workflows)} + > + + + +
    + +
    + + +
    +
    + { + isLoadingWorkflow ? ( + + ) : ( + view === "grid" && currTab !== 2 ? ( + <> +
    + + + + {filteredWorkflows.map((data, index) => { + // Shouldn't be a part of this list + if (data.public === true) { + return null + } + + if (firstLoad) { + workflowDelay += 75 + } else { + return + } + + return ( + + {/**/} + + {/**/} + + ) + })} +
    + + ) : ( + currTab !== 2 && + ) + ) + } + + { + currTab === 2 && + ( + + + + + ) + } + +
    +
    + + + + ); + }); + + const importWorkflowsFromUrl = (url) => { + console.log("IMPORT WORKFLOWS FROM ", downloadUrl); + + const parsedData = { + url: url, + field_3: downloadBranch || "master", + }; + + if (field1.length > 0) { + parsedData["field_1"] = field1; + } + + if (field2.length > 0) { + parsedData["field_2"] = field2; + } + + toast("Getting specific workflows from your URL."); + fetch(globalUrl + "/api/v1/workflows/download_remote", { + method: "POST", + mode: "cors", + headers: { + Accept: "application/json", + }, + body: JSON.stringify(parsedData), + credentials: "include", + }) + .then((response) => { + if (response.status === 200) { + toast("Successfully loaded workflows from " + downloadUrl); + setTimeout(() => { + getAvailableWorkflows(); + }, 1000); + } + + return response.json(); + }) + .then((responseJson) => { + if (!responseJson.success) { + if (responseJson.reason !== undefined) { + toast("Failed loading: " + responseJson.reason); + } else { + toast("Failed loading"); + } + } + }) + .catch((error) => { + toast(error.toString()); + }); + }; + + const handleGithubValidation = () => { + importWorkflowsFromUrl(downloadUrl); + setLoadWorkflowsModalOpen(false); + } + + const workflowDownloadModalOpen = loadWorkflowsModalOpen ? ( + { }} + PaperProps={{ + style: { + backgroundColor: theme.palette.surfaceColor, + color: "white", + minWidth: "800px", + minHeight: "320px", + }, + }} + > + +
    + Load workflows from github repo +
    + + + +
    +
    +     + + Repository (supported: github, gitlab, bitbucket) + setDownloadUrl(e.target.value)} + placeholder="https://github.com/shuffle/workflows" + fullWidth + /> + + Branch (default value is "main"): + +
    + setDownloadBranch(e.target.value)} + placeholder="master" + fullWidth + /> +
    + + Authentication (optional - private repos etc): + +
    + setField1(e.target.value)} + type="username" + placeholder="Username / APIkey (optional)" + fullWidth + /> + setField2(e.target.value)} + type="password" + placeholder="Password (optional)" + fullWidth + /> +
    +     + + + + +     + ) : null; + + //const + //const [percentDone, setPercentDone] = React.useState(0) + const percentDone = gettingStartedItems.filter((item) => item.done).length / gettingStartedItems.length * 100 + + const GettingStartedItem = ({ item, index }) => { + const [clicked, setClicked] = React.useState(false) + const doneIcon = item.done ? : + + return ( +
    setClicked(true)} + > + + {doneIcon} {index + 1}. {item.name} + + {clicked ? + + + {item.description} + + + + + + : + null + } +
    + ) + } + + const gettingStartedDrawer = true == true ? null : + +
    + + Getting Started + + + { + e.preventDefault(); + setDrawerOpen(false) + + localStorage.setItem(sidebarKey, "closed"); + }} + > + + + +
    +
    + + Setup progress: {isNaN(percentDone) ? 0 : percentDone}% + + + + + + Follow these steps to get you up and running! + + + { + setVideoViewOpen(true) + }}> + Watch 2-min introduction video + +
    +
    + {gettingStartedItems.map((item, index) => { + return ( + + ) + })} +
    +     + + const videoView = + { + setVideoViewOpen(false) + }} + PaperProps={{ + style: { + backgroundColor: theme.palette.surfaceColor, + color: "white", + minWidth: 560, + minHeight: 415, + textAlign: "center", + }, + }} + > + + Welcome to Shuffle! + + + + { + e.preventDefault(); + setVideoViewOpen(false) + }} + > + + + + + + + + const loadedCheck = + isLoaded && isLoggedIn && workflowDone ? ( +
    + {/* + + + + */} + + {/* {modalView} */} + {deleteModal} + {exportVerifyModal} + {publishModal} + {workflowDownloadModalOpen} + + {/*!drawerOpen ? +
    + + { + setDrawerOpen(true) + localStorage.setItem(sidebarKey, "open"); + }}> + + + +
    : null*/} + {isMobile ? null : gettingStartedDrawer} + {videoView} + + {modalOpen === true ? + + : null} + {/*
    + + Need assistance? Ask our support team (it's free!). + + +
    */} +
    + ) : ( +
    + + Loading Workflows +
    + ); + + // Maybe use gridview or something, idk + return
    {loadedCheck}
    ; +}; + + + +export default Workflows2; diff --git a/functions/kubernetes/README.md b/functions/kubernetes/README.md index e8e315af..a54884e8 100644 --- a/functions/kubernetes/README.md +++ b/functions/kubernetes/README.md @@ -34,3 +34,35 @@ Step 2: Open the ```all-in-one.yaml``` file and review the configuration values. Step 3: Now, open ```https://:30008``` or ```http://:30007```. You should be seeing a signup page. NODE_IP should be where the frontend is deployed. +### Dev Mode + +1. Run backend and orborus with the environment variable `IS_KUBERNETES=true`: + +```bash +export IS_KUBERNETES=true +``` + +2. Turn on the k8s engine with minikube: + +```bash +minikube start +``` + +3. To use the worker scale feature, build the image with the following command: + +```bash +$NAME=shuffle-worker-scale +$VERSION=1.2.0 + +minikube build . -t shuffle/shuffle:$NAME -t shuffle/shuffle:$NAME_$VERSION -t docker.pkg.github.com/shuffle/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:nightly -t ghcr.io/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:nightly -t ghcr.io/shuffle/$NAME:latest +``` + +4. To run executions, Make sure to do the following: + +```bash +kubectl create role pod-creator --namespace=default --verb=create --resource=pods +kubectl create rolebinding pod-creator-binding --namespace=default --role=pod-creator --serviceaccount=default:default +``` + + + diff --git a/functions/kubernetes/all-in-one.yaml b/functions/kubernetes/all-in-one.yaml index 746215e9..190a1a29 100644 --- a/functions/kubernetes/all-in-one.yaml +++ b/functions/kubernetes/all-in-one.yaml @@ -1,15 +1,37 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: shuffle + --- +## ONLY for minikube +# apiVersion: storage.k8s.io/v1 +# kind: StorageClass +# metadata: +# name: standard-rwo +# provisioner: k8s.io/minikube-hostpath +# reclaimPolicy: Delete +# volumeBindingMode: Immediate + +# --- apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: shuffle-data + namespace: shuffle provisioner: kubernetes.io/no-provisioner volumeBindingMode: WaitForFirstConsumer --- apiVersion: v1 +metadata: + namespace: shuffle + creationTimestamp: null + labels: + io.kompose.service: backend-env + name: env data: BACKEND_HOSTNAME: shuffle-backend BACKEND_PORT: "5001" @@ -51,11 +73,13 @@ data: SHUFFLE_OPENSEARCH_APIKEY: "" SHUFFLE_OPENSEARCH_CERTIFICATE_FILE: "" SHUFFLE_OPENSEARCH_CLOUDID: "" + KUBERNETES_NAMESPACE: shuffle SHUFFLE_OPENSEARCH_INDEX_PREFIX: "" SHUFFLE_OPENSEARCH_PASSWORD: admin SHUFFLE_OPENSEARCH_PROXY: "" SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY: "true" SHUFFLE_OPENSEARCH_URL: https://opensearch:9200 + SHUFFLE_MEMCACHED: shuffle-memcached:11211 SHUFFLE_OPENSEARCH_USERNAME: admin SHUFFLE_ORBORUS_STARTUP_DELAY: "\t\t" SHUFFLE_PASS_APP_PROXY: "FALSE" @@ -64,18 +88,12 @@ data: SSO_REDIRECT_URL: "" TZ: "Europe/Amsterdam \t\t\t\t\t" IS_KUBERNETES: "true" - REGISTRY_URL: "192.168.29.16:5000" + REGISTRY_URL: "docker-registry:5000" REGISTRY_AUTH: "false" SHUFFLE_KUBERNETES_WORKER: "ghcr.io/shuffle/shuffle-worker:nightly" kind: ConfigMap -metadata: - creationTimestamp: null - labels: - io.kompose.service: backend-env - name: env --- - apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -83,11 +101,17 @@ metadata: name: pod-manager rules: - apiGroups: [""] - resources: ["pods"] + resources: ["pods", "services", "deployments"] verbs: ["get", "list", "create", "update", "delete"] - apiGroups: ["batch"] resources: ["jobs"] verbs: ["create", "get", "list", "watch", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["rolebindings", "roles"] + verbs: ["get", "list", "create"] +- apiGroups: ["apps"] + resources: ["deployments", "pods", "services"] + verbs: ["create", "get", "list", "update", "delete"] --- @@ -107,25 +131,27 @@ roleRef: --- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: shuffle-os-pv -spec: - capacity: - storage: 10Gi # Adjust the storage size as per your requirements - accessModes: - - ReadWriteOnce # This allows read-write access to a single node - persistentVolumeReclaimPolicy: Retain # Adjust the reclaim policy as per your needs - storageClassName: shuffle-data # Set the desired storage class - hostPath: - path: /mnt/shuffle-data/open-search +# apiVersion: v1 +# kind: PersistentVolume +# metadata: +# name: shuffle-os-pv +# namespace: shuffle +# spec: +# capacity: +# storage: 10Gi # Adjust the storage size as per your requirements +# accessModes: +# - ReadWriteOnce # This allows read-write access to a single node +# persistentVolumeReclaimPolicy: Retain # Adjust the reclaim policy as per your needs +# storageClassName: standard-rwo # Set the desired storage class +# hostPath: +# path: /mnt/shuffle-data/open-search ---- +# --- apiVersion: v1 kind: PersistentVolumeClaim metadata: + namespace: shuffle creationTimestamp: null labels: io.kompose.service: opensearch-claim0 @@ -133,7 +159,7 @@ metadata: spec: accessModes: - ReadWriteOnce - storageClassName: shuffle-data + storageClassName: standard-rwo resources: requests: storage: 500Mi @@ -143,6 +169,7 @@ status: {} apiVersion: apps/v1 kind: Deployment metadata: + namespace: shuffle annotations: kompose.cmd: kompose convert -f docker-compose.yml kompose.version: 1.26.0 (40646f47) @@ -222,6 +249,7 @@ status: {} apiVersion: v1 kind: Service metadata: + namespace: shuffle annotations: kompose.cmd: kompose convert -f docker-compose.yml kompose.version: 1.26.0 (40646f47) @@ -241,51 +269,54 @@ status: --- +# apiVersion: v1 +# kind: PersistentVolume +# metadata: +# namespace: shuffle +# name: shuffle-apps-pv +# spec: +# capacity: +# storage: 5Gi +# accessModes: +# - ReadWriteOnce +# persistentVolumeReclaimPolicy: Retain +# storageClassName: shuffle-data +# hostPath: +# path: /mnt/shuffle-data/backend + +# --- +# apiVersion: v1 +# kind: PersistentVolume +# metadata: +# namespace: shuffle +# name: shuffle-files-pv +# spec: +# capacity: +# storage: 5Gi +# accessModes: +# - ReadWriteOnce +# persistentVolumeReclaimPolicy: Retain +# storageClassName: shuffle-data +# hostPath: +# path: /mnt/shuffle-data/backend + +# --- + apiVersion: v1 -kind: PersistentVolume +kind: PersistentVolumeClaim metadata: - name: shuffle-apps-pv + namespace: shuffle + creationTimestamp: null + labels: + io.kompose.service: backend-files-claim + name: backend-files-claim spec: - capacity: - storage: 5Gi - accessModes: - - ReadWriteOnce - persistentVolumeReclaimPolicy: Retain - storageClassName: shuffle-data - hostPath: - path: /mnt/shuffle-data/backend - ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: shuffle-files-pv -spec: - capacity: - storage: 5Gi - accessModes: - - ReadWriteOnce - persistentVolumeReclaimPolicy: Retain - storageClassName: shuffle-data - hostPath: - path: /mnt/shuffle-data/backend - ---- - - apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - creationTimestamp: null - labels: - io.kompose.service: backend-files-claim - name: backend-files-claim - spec: - accessModes: - - ReadWriteOnce - storageClassName: shuffle-data - resources: - requests: - storage: 5Gi + accessModes: + - ReadWriteOnce + storageClassName: standard-rwo + resources: + requests: + storage: 5Gi # status: {} --- @@ -293,6 +324,7 @@ spec: apiVersion: v1 kind: PersistentVolumeClaim metadata: + namespace: shuffle creationTimestamp: null labels: io.kompose.service: backend-apps-claim @@ -300,7 +332,7 @@ metadata: spec: accessModes: - ReadWriteOnce - storageClassName: shuffle-data + storageClassName: standard-rwo resources: requests: storage: 5Gi @@ -311,6 +343,48 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: + name: shuffle-memcached + namespace: shuffle +spec: + replicas: 1 + selector: + matchLabels: + app: shuffle-memcached + template: + metadata: + labels: + app: shuffle-memcached + spec: + containers: + - name: shuffle-memcached + image: memcached:latest + ports: + - containerPort: 11211 + resources: {} + restartPolicy: Always + + +--- + +apiVersion: v1 +kind: Service +metadata: + namespace: shuffle + name: shuffle-memcached +spec: + ports: + - port: 11211 + targetPort: 11211 + selector: + app: shuffle-memcached + type: ClusterIP + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: shuffle annotations: kompose.cmd: kompose convert -f docker-compose.yml kompose.version: 1.26.0 (40646f47) @@ -538,6 +612,11 @@ spec: configMapKeyRef: key: SHUFFLE_OPENSEARCH_APIKEY name: env + - name: SHUFFLE_MEMCACHED + valueFrom: + configMapKeyRef: + key: SHUFFLE_MEMCACHED + name: env - name: SHUFFLE_OPENSEARCH_CERTIFICATE_FILE valueFrom: configMapKeyRef: @@ -642,6 +721,7 @@ status: {} apiVersion: v1 kind: Service metadata: + namespace: shuffle annotations: kompose.cmd: kompose convert -f docker-compose.yml kompose.version: 1.26.0 (40646f47) @@ -660,10 +740,10 @@ status: loadBalancer: {} --- - apiVersion: apps/v1 kind: Deployment metadata: + namespace: shuffle annotations: kompose.cmd: kompose convert -f docker-compose.yml kompose.version: 1.26.0 (40646f47) @@ -688,10 +768,14 @@ spec: io.kompose.service: frontend spec: containers: - - env: - - name: BACKEND_HOSTNAME + - name: shuffle-frontend image: ghcr.io/shuffle/shuffle-frontend:nightly - name: shuffle-frontend + env: + - name: BACKEND_HOSTNAME + valueFrom: + configMapKeyRef: + key: BACKEND_HOSTNAME + name: env ports: - containerPort: 80 - containerPort: 443 @@ -705,6 +789,7 @@ status: {} apiVersion: v1 kind: Service metadata: + namespace: shuffle annotations: kompose.cmd: kompose convert -f docker-compose.yml kompose.version: 1.26.0 (40646f47) @@ -733,6 +818,7 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: + namespace: shuffle annotations: kompose.cmd: kompose convert -f docker-compose.yml kompose.version: 1.26.0 (40646f47) @@ -770,8 +856,8 @@ spec: value: nightly - name: SHUFFLE_SCALE_REPLICAS value: "5" - #- name: SHUFFLE_SWARM_CONFIG - #value: run + - name: SHUFFLE_SWARM_CONFIG + value: run - name: SHUFFLE_WORKER_VERSION value: nightly - name: IS_KUBERNETES @@ -779,6 +865,11 @@ spec: configMapKeyRef: key: IS_KUBERNETES name: env + - name: KUBERNETES_NAMESPACE + valueFrom: + configMapKeyRef: + key: KUBERNETES_NAMESPACE + name: env - name: REGISTRY_URL valueFrom: configMapKeyRef: @@ -789,11 +880,15 @@ spec: configMapKeyRef: key: SHUFFLE_KUBERNETES_WORKER name: env - + - name: SHUFFLE_MEMCACHED + valueFrom: + configMapKeyRef: + key: SHUFFLE_MEMCACHED + name: env image: ghcr.io/shuffle/shuffle-orborus:nightly #imagePullPolicy: Never name: shuffle-orborus resources: {} hostname: shuffle-orborus restartPolicy: Always -status: {} +status: {} \ No newline at end of file diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index cfc374c1..33577275 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -4,23 +4,23 @@ go 1.22.0 toolchain go1.22.2 +// replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared + require ( - github.com/docker/docker v26.1.5+incompatible + github.com/docker/docker v27.0.2+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.6.29 - k8s.io/api v0.30.0 - k8s.io/apimachinery v0.30.0 - k8s.io/client-go v0.30.0 + github.com/shuffle/shuffle-shared v0.6.90 + k8s.io/api v0.30.2 + k8s.io/apimachinery v0.30.2 ) require ( - cloud.google.com/go v0.112.0 // indirect - cloud.google.com/go/compute v1.24.0 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/datastore v1.15.0 // indirect - cloud.google.com/go/iam v1.1.6 // indirect - cloud.google.com/go/storage v1.36.0 // indirect + cloud.google.com/go v0.110.2 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect + cloud.google.com/go/datastore v1.11.0 // indirect + cloud.google.com/go/iam v0.13.0 // indirect + cloud.google.com/go/storage v1.29.0 // indirect dario.cat/mergo v1.0.0 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect @@ -29,7 +29,7 @@ require ( github.com/algolia/algoliasearch-client-go/v3 v3.18.1 // indirect github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 // indirect github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect - github.com/cloudflare/circl v1.3.3 // indirect + github.com/cloudflare/circl v1.3.7 // indirect github.com/containerd/log v0.1.0 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -39,12 +39,12 @@ require ( github.com/emirpasic/gods v1.18.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/kin-openapi v0.41.0 // indirect - github.com/frikky/schemaless v0.0.11 // indirect + github.com/frikky/schemaless v0.0.13 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/go-git/go-git/v5 v5.11.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -56,10 +56,10 @@ require ( github.com/google/go-github/v28 v28.1.1 // indirect github.com/google/go-querystring v1.0.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/s2a-go v0.1.7 // indirect + github.com/google/s2a-go v0.1.4 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.12.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/googleapis/gax-go/v2 v2.11.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -80,42 +80,45 @@ require ( github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/sashabaranov/go-openai v1.19.2 // indirect + github.com/sendgrid/rest v2.6.9+incompatible // indirect + github.com/sendgrid/sendgrid-go v3.14.0+incompatible // indirect github.com/sergi/go-diff v1.1.0 // indirect github.com/skeema/knownhosts v1.2.1 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0 // indirect - go.opentelemetry.io/otel v1.25.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.25.0 // indirect - go.opentelemetry.io/otel/metric v1.25.0 // indirect - go.opentelemetry.io/otel/sdk v1.25.0 // indirect - go.opentelemetry.io/otel/trace v1.25.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect + go.opentelemetry.io/otel v1.30.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0 // indirect + go.opentelemetry.io/otel/metric v1.30.0 // indirect + go.opentelemetry.io/otel/sdk v1.30.0 // indirect + go.opentelemetry.io/otel/trace v1.30.0 // indirect go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/mod v0.15.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/oauth2 v0.17.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.18.0 // indirect - google.golang.org/api v0.162.0 // indirect + golang.org/x/crypto v0.27.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/term v0.24.0 // indirect + golang.org/x/text v0.18.0 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/api v0.126.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect - google.golang.org/grpc v1.63.0 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.1 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.5.1 // indirect + k8s.io/client-go v0.30.2 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 1d07b7a8..02b90f14 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -6,47 +6,24 @@ cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxK cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= -cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= +cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA= +cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= -cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/datastore v1.4.0/go.mod h1:d18825/a9bICdAIJy2EkHs9joU4RlIZ1t6l8WDdbdY0= -cloud.google.com/go/datastore v1.15.0 h1:0P9WcsQeTWjuD1H14JIY7XQscIPQ4Laje8ti96IC5vg= -cloud.google.com/go/datastore v1.15.0/go.mod h1:GAeStMBIt9bPS7jMJA85kgkpsMkvseWWXiaHya9Jes8= -cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= -cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= +cloud.google.com/go/datastore v1.11.0 h1:iF6I/HaLs3Ado8uRKMvZRvF/ZLkWaWE9i8AiHzbC774= +cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= -cloud.google.com/go/storage v1.36.0 h1:P0mOkAcaJxhCTvAkMhxMfrTKiNcub4YmmPBtlhAyTr8= -cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= +cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -67,6 +44,7 @@ github.com/algolia/algoliasearch-client-go/v3 v3.18.1 h1:FP2Xtqqs/sefR5Qluygp+jV github.com/algolia/algoliasearch-client-go/v3 v3.18.1/go.mod h1:i7tLoP7TYDmHX3Q7vkIOL4syVse/k5VJ+k0i8WqFiJk= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-sdk-go v1.42.27/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= @@ -91,16 +69,20 @@ github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7N github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= -github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -111,15 +93,14 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v26.1.5+incompatible h1:NEAxTwEjxV6VbBMBoGG3zPqbiJosIApZjxlbrG9q3/g= -github.com/docker/docker v26.1.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.0.2+incompatible h1:mNhCtgXNV1fIRns102grG7rdzIsGGCq1OlOD0KunZos= +github.com/docker/docker v27.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= -github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -127,25 +108,21 @@ github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FM github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= -github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.41.0 h1:oMmjo+ekGS971lb3KLeZZOqRDZOwWi3+g/OiSWP08+s= github.com/frikky/kin-openapi v0.41.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.11 h1:c4r6CJX30XI+SoJdT9RlUd9qYSQlx6hvwGRtsypu+uM= -github.com/frikky/schemaless v0.0.11/go.mod h1:mooDxY+D6weHjhKvjy3+IE9S7P4g4cpNnidkdRv/cHQ= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/frikky/schemaless v0.0.13 h1:ARiN9V7wr2VZXAr9JK5wvTbyPgpGrgeiL1VhR5MlgaQ= +github.com/frikky/schemaless v0.0.13/go.mod h1:mooDxY+D6weHjhKvjy3+IE9S7P4g4cpNnidkdRv/cHQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= @@ -154,12 +131,9 @@ github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3c github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -170,7 +144,6 @@ github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -185,15 +158,10 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -204,7 +172,6 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -215,12 +182,8 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -235,42 +198,32 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= -github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= -github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= -github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF/w5E9CNxSwbpD6No= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU= +github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= +github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= @@ -299,7 +252,6 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= @@ -313,46 +265,8 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= -github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= -github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= -github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= -github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= -github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= -github.com/onsi/ginkgo/v2 v2.8.1/go.mod h1:N1/NbDngAFcSLdyZ+/aYTYGSlq9qMCS/cNKGJjy+csc= -github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxmrTcxyk= -github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= -github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= -github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= -github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0= -github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= -github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= -github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= -github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= -github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= -github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= -github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw= -github.com/onsi/gomega v1.27.3/go.mod h1:5vG284IBtfDAmDyrK+eGyZmUgUlmi+Wngqo557cZ6Gw= -github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= -github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= -github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4= -github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE= github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -367,15 +281,13 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= @@ -383,12 +295,21 @@ github.com/sashabaranov/go-openai v1.19.2 h1:+dkuCADSnwXV02YVJkdphY8XD9AyHLUWwk6 github.com/sashabaranov/go-openai v1.19.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0= +github.com/sendgrid/rest v2.6.9+incompatible/go.mod h1:kXX7q3jZtJXK5c5qK83bSGMdV6tsOE70KbHoqJls4lE= +github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fcif2i7P7wzISv1sU6DUA= +github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shuffle/shuffle-shared v0.6.29 h1:Vr23Sb2m4l0FA1xMn5n3f/0MQ4ziTipyTWkOukj1jaQ= -github.com/shuffle/shuffle-shared v0.6.29/go.mod h1:rWkh1eWdIx7OqQzJ1+JzF3Hck1X/Ty1WkUtjLrp+CU4= +github.com/shuffle/shuffle-shared v0.6.74 h1:os3BDSFZnl4U8ZgsTAY8IsTDADcMXhbc1rS9UMa0BIY= +github.com/shuffle/shuffle-shared v0.6.74/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus= +github.com/shuffle/shuffle-shared v0.6.79 h1:MIy5kcShHYN05ov/50YJ+la1C2v1rL8IENapOvX9I8U= +github.com/shuffle/shuffle-shared v0.6.79/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus= +github.com/shuffle/shuffle-shared v0.6.83 h1:gceT91WtFqh3h9juzTipDhWpxZLfrdtbcsnK+XNj57g= +github.com/shuffle/shuffle-shared v0.6.83/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus= +github.com/shuffle/shuffle-shared v0.6.90 h1:FzIYtEt44eWgEsW/9tj2ki7qq8FEm/HWXUok+THp72M= +github.com/shuffle/shuffle-shared v0.6.90/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skeema/knownhosts v1.2.1 h1:SHWdIUa82uGZz+F+47k8SY4QhhI291cXCpopT1lK2AQ= @@ -404,68 +325,55 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 h1:UNQQKPfTDe1J81ViolILjTKPr9WetKW6uei2hFgJmFs= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0 h1:cEPbyTSEHlQR89XVlyo78gqluF8Y3oMeBkXGWzQsfXY= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0/go.mod h1:DKdbWcT4GH1D0Y3Sqt/PFXt2naRKDWtU+eE6oLdFNA8= -go.opentelemetry.io/otel v1.25.0 h1:gldB5FfhRl7OJQbUHt/8s0a7cE8fbsPAtdpRaApKy4k= -go.opentelemetry.io/otel v1.25.0/go.mod h1:Wa2ds5NOXEMkCmUou1WA7ZBfLTHWIsp034OVD7AO+Vg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.25.0 h1:dT33yIHtmsqpixFsSQPwNeY5drM9wTcoL8h0FWF4oGM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.25.0/go.mod h1:h95q0LBGh7hlAC08X2DhSeyIG02YQ0UyioTCVAqRPmc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.25.0 h1:Mbi5PKN7u322woPa85d7ebZ+SOvEoPvoiBu+ryHWgfA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.25.0/go.mod h1:e7ciERRhZaOZXVjx5MiL8TK5+Xv7G5Gv5PA2ZDEJdL8= -go.opentelemetry.io/otel/metric v1.25.0 h1:LUKbS7ArpFL/I2jJHdJcqMGxkRdxpPHE0VU/D4NuEwA= -go.opentelemetry.io/otel/metric v1.25.0/go.mod h1:rkDLUSd2lC5lq2dFNrX9LGAbINP5B7WBkC78RXCpH5s= -go.opentelemetry.io/otel/sdk v1.25.0 h1:PDryEJPC8YJZQSyLY5eqLeafHtG+X7FWnf3aXMtxbqo= -go.opentelemetry.io/otel/sdk v1.25.0/go.mod h1:oFgzCM2zdsxKzz6zwpTZYLLQsFwc+K0daArPdIhuxkw= -go.opentelemetry.io/otel/trace v1.25.0 h1:tqukZGLwQYRIFtSQM2u2+yfMVTgGVeqRLPUYx1Dq6RM= -go.opentelemetry.io/otel/trace v1.25.0/go.mod h1:hCCs70XM/ljO+BeQkyFnbK28SBIJ/Emuha+ccrCRT7I= -go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= -go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4ba9kHbBol3Xin3leYJ+252h0zszDtBwyKe2A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0= +go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= +go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 h1:lsInsfvhVIfOI6qHVyysXMNDnjO9Npvl7tlDPJFBVd4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0/go.mod h1:KQsVNh4OjgjTG0G6EiNi1jVpnaeeKsKMRwbLN+f1+8M= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0 h1:umZgi92IyxfXd/l4kaDhnKgY8rnN/cZcF1LKc6I8OQ8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0/go.mod h1:4lVs6obhSVRb1EW5FhOuBTyiQhtRtAnnva9vD3yRfq8= +go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= +go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= +go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= +go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= +go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= -golang.org/x/arch v0.1.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -473,9 +381,7 @@ golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm0 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -487,31 +393,19 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8= -golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -520,79 +414,44 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ= -golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -601,97 +460,52 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -711,49 +525,17 @@ golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= -golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= -golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= -golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= -golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -768,27 +550,13 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQWYWo= -google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.162.0 h1:Vhs54HkaEpkMBdgGdOT2P6F0csGG/vxDS0hWHJzmmps= -google.golang.org/api v0.162.0/go.mod h1:6SulDkfoBIg4NFmCuZ39XeeAgSHCPecfSUuDyYlAHs0= +google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o= +google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -803,37 +571,15 @@ google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= -google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0= -google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1:hjSy6tcFQZ171igDaN5QHOw2n6vx40juYbC/x67CEhc= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -842,17 +588,12 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.34.1/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.63.0 h1:WjKe+dnvABXyPJMD7KDNLxtoGk5tgk+YFWN6cBWjZE8= -google.golang.org/grpc v1.63.0/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.66.1 h1:hO5qAXR19+/Z44hmvIM4dQFMSYX9XcWsByfoxutBpAM= +google.golang.org/grpc v1.66.1/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -861,29 +602,25 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -896,14 +633,12 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.30.0 h1:siWhRq7cNjy2iHssOB9SCGNCl2spiF1dO3dABqZ8niA= -k8s.io/api v0.30.0/go.mod h1:OPlaYhoHs8EQ1ql0R/TsUgaRPhpKNxIMrKQfWUp8QSE= -k8s.io/apimachinery v0.30.0 h1:qxVPsyDM5XS96NIh9Oj6LavoVFYff/Pon9cZeDIkHHA= -k8s.io/apimachinery v0.30.0/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/client-go v0.30.0 h1:sB1AGGlhY/o7KCyCEQ0bPWzYDL0pwOZO4vAtTSh/gJQ= -k8s.io/client-go v0.30.0/go.mod h1:g7li5O5256qe6TYdAMyX/otJqMhIiGgTapdLchhmOaY= +k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= +k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI= +k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= +k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.2 h1:sBIVJdojUNPDU/jObC+18tXWcTJVcwyqS9diGdWHk50= +k8s.io/client-go v0.30.2/go.mod h1:JglKSWULm9xlJLx4KCkfLLQ7XwtlbflV6uFFSHTMgVs= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= @@ -911,7 +646,6 @@ k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVh k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index f4237c43..3054066a 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1,7 +1,7 @@ package main /* - Orborus exists to listen for new workflow executions which are deployed as workers. + Orborus exists to listen for new jobs which are deployed as workers. */ // Potential issues: @@ -9,8 +9,7 @@ package main // Ingress network may not exist (default) import ( - "github.com/shuffle/shuffle-shared" - + "archive/zip" "bytes" "context" "encoding/json" @@ -24,17 +23,23 @@ import ( "net/http" "os" "os/exec" + "path/filepath" "runtime" "strconv" "strings" "sync" "time" + "github.com/shuffle/shuffle-shared" + + "math/rand" //"os/signal" //"syscall" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/api/types/image" "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/network" "github.com/docker/docker/api/types/swarm" @@ -48,15 +53,11 @@ import ( //"github.com/mackerelio/go-osstat/memory" //"github.com/shirou/gopsutil/cpu" - //k8s deps - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/util/homedir" - "path/filepath" - + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) // Starts jobs in bulk, so this could be increased @@ -78,7 +79,6 @@ var isKubernetes = os.Getenv("IS_KUBERNETES") var kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") var maxCPUPercent = 90 - // var baseimagename = "docker.pkg.github.com/shuffle/shuffle" // var baseimagename = "ghcr.io/frikky" // var baseimagename = "shuffle/shuffle" @@ -104,15 +104,26 @@ var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG") var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME") var orborusLabel = os.Getenv("SHUFFLE_ORBORUS_LABEL") var memcached = os.Getenv("SHUFFLE_MEMCACHED") -var tenzirUrl = os.Getenv("SHUFFLE_TENZIR_URL") +var queuePerMinute = os.Getenv("SHUFFLE_EXECUTION_PER_MINIUTE") +var queuePerMinuteInt int + +// For it to download from Sigma? +var apiKey = os.Getenv("AUTH_FOR_ORBORUS") +var pipelineUrl = os.Getenv("SHUFFLE_PIPELINE_URL") var executionIds = []string{} -var namespacemade = false // For K8s +var pipelines = []shuffle.PipelineInfoMini{} +var namespacemade = false // For K8s +var skipPipelineMount = false +var tenzirDisabled = false var dockercli *dockerclient.Client var containerId string var executionCount = 0 +var imagedownloadTimeout = time.Second * 300 +var window = shuffle.NewTimeWindow(1 * time.Minute) + func init() { var err error @@ -180,7 +191,95 @@ func getThisContainerId() { log.Printf(`[INFO] Started with containerId "%s"`, containerId) } +func skipCheckInCleanup(name string) bool { + return strings.HasPrefix(name, "backend") || + strings.HasPrefix(name, "shuffle-backend") || + strings.HasPrefix(name, "frontend") || + strings.HasPrefix(name, "shuffle-frontend") || + strings.HasPrefix(name, "orborus") || + strings.HasPrefix(name, "shuffle-orborus") || + strings.HasPrefix(name, "opensearch") || + strings.HasPrefix(name, "shuffle-opensearch") +} + func cleanupExistingNodes(ctx context.Context) error { + + if isKubernetes == "true" { + // of course, this doesn't clean up "nodes" but + // rather pods, services, roles etc. + + if kubernetesNamespace == "" { + kubernetesNamespace = "default" + } + + clientset, _, err := shuffle.GetKubernetesClient() + if err != nil { + log.Printf("[ERROR] Error getting kubernetes client:", err) + return err + } + + // Delete all pods + pods, err := clientset.CoreV1().Pods(kubernetesNamespace).List(context.Background(), metav1.ListOptions{}) + if err != nil { + log.Printf("[ERROR] Failed listing pods: %s", err) + return err + } + + for _, pod := range pods.Items { + // check if pod.Name starts with: + // "backend-", "frontend-", "orborus-", "opensearch-" or "memcached-" + if skipCheckInCleanup(pod.Name) { + continue + } + + err := clientset.CoreV1().Pods(kubernetesNamespace).Delete(context.Background(), pod.Name, metav1.DeleteOptions{}) + if err != nil { + log.Printf("[ERROR] Failed deleting pod %s: %s", pod.Name, err) + } + } + + // Delete all services + services, err := clientset.CoreV1().Services(kubernetesNamespace).List(context.Background(), metav1.ListOptions{}) + if err != nil { + log.Printf("[ERROR] Failed listing services: %s", err) + return err + } + + for _, service := range services.Items { + if skipCheckInCleanup(service.Name) { + continue + } + + err := clientset.CoreV1().Services(kubernetesNamespace).Delete(context.Background(), service.Name, metav1.DeleteOptions{}) + if err != nil { + log.Printf("[ERROR] Failed deleting service %s: %s", service.Name, err) + } + } + + deployments, err := clientset.AppsV1().Deployments(kubernetesNamespace).List(context.Background(), metav1.ListOptions{}) + if err != nil { + log.Printf("[ERROR] Failed listing deployments: %s", err) + return err + } + + for _, deployment := range deployments.Items { + if skipCheckInCleanup(deployment.Name) { + continue + } + + err := clientset.AppsV1().Deployments(kubernetesNamespace).Delete(context.Background(), deployment.Name, metav1.DeleteOptions{}) + if err != nil { + log.Printf("[ERROR] Failed deleting deployment %s: %s", deployment.Name, err) + } + } + + log.Printf("[INFO] Cleaned up all pods and services in namespace %s. Waiting 10 seconds for cleanup to reflect", kubernetesNamespace) + + time.Sleep(10 * time.Second) + + return nil + } + serviceListOptions := types.ServiceListOptions{} services, err := dockercli.ServiceList( context.Background(), @@ -195,7 +294,6 @@ func cleanupExistingNodes(ctx context.Context) error { //log.Printf("\n\nFound %d contaienrs", len(services)) for _, service := range services { - //log.Printf("[INFO] Service: %#v", service.Spec.Annotations.Name) //portFound := false //for _, endpoint := range service.Spec.EndpointSpec.Ports { @@ -229,393 +327,409 @@ func cleanupExistingNodes(ctx context.Context) error { func deployServiceWorkers(image string) { log.Printf("[DEBUG] Validating deployment of workers as services IF swarmConfig = run (value: %#v)", swarmConfig) - if swarmConfig == "run" || swarmConfig == "swarm" { - ctx := context.Background() - // Looks for and cleans up all existing items in swarm we can't re-use (Shuffle only) + if swarmConfig != "run" && swarmConfig != "swarm" { + log.Printf("[DEBUG] Skipping deployment of workers as services as swarmConfig is not set to run or swarm. Value: %#v", swarmConfig) + return + } + ctx := context.Background() - // frikky@debian:~/git/shuffle/functions/onprem/worker$ docker service create --replicas 5 --name shuffle-workers --env SHUFFLE_SWARM_CONFIG=run --publish published=33333,target=33333 ghcr.io/shuffle/shuffle-worker:nightly + // Looks for and cleans up all existing items in swarm we can't re-use (Shuffle only) - // Get a list of network interfaces - interfaces, err := net.Interfaces() + // frikky@debian:~/git/shuffle/functions/onprem/worker$ docker service create --replicas 5 --name shuffle-workers --env SHUFFLE_SWARM_CONFIG=run --publish published=33333,target=33333 ghcr.io/shuffle/shuffle-worker:nightly + + // Get a list of network interfaces + interfaces, err := net.Interfaces() + if err != nil { + log.Printf("[ERROR] Failed to get network interfaces: %s", err) + } + + mtu := 1500 + if len(dockerSwarmBridgeMTU) == 0 { + mtu, err = strconv.Atoi(dockerSwarmBridgeMTU) // by default if err != nil { - log.Printf("[ERROR] Failed to get network interfaces: %s", err) + log.Printf("[DEBUG] Failed to convert the default MTU to int: %s. Using 1500 instead. Input: %s", err, dockerSwarmBridgeMTU) + mtu = 1500 } + } - mtu := 1500 - if len(dockerSwarmBridgeMTU) == 0 { - mtu, err = strconv.Atoi(dockerSwarmBridgeMTU) // by default - if err != nil { - log.Printf("[DEBUG] Failed to convert the default MTU to int: %s. Using 1500 instead. Input: %s", err, dockerSwarmBridgeMTU) - mtu = 1500 + bridgeName := dockerSwarmBridgeInterface + if bridgeName == "" { + bridgeName = "eth0" + } + + // Check if there is at least one interface + if len(interfaces) < 2 { + // this assumes that the machine should have at least 2 network + // interfaces. If not, we will use the default MTU. + // interface 1 is the loopback interface + // interface 2 is eth0, The eth0 interface inside a + // Docker container corresponds to the virtual Ethernet + // interface that connects the container to the docker0 + log.Printf("[ERROR] Failed to get enough network interfaces") + } else { + // Get the preferred interface + for _, iface := range interfaces { + if strings.Contains(iface.Name, bridgeName) { + targetInterface := iface + mtu = targetInterface.MTU + log.Printf("[INFO] Using MTU %d from interface %s", mtu, targetInterface.Name) + break } } + } - bridgeName := dockerSwarmBridgeInterface - if bridgeName == "" { - bridgeName = "eth0" + // Create the network options with the specified MTU + options := make(map[string]string) + options["com.docker.network.driver.mtu"] = fmt.Sprintf("%d", mtu) + + ingressOptions := types.NetworkCreate{ + Driver: "overlay", + Attachable: false, + Ingress: true, + IPAM: &network.IPAM{ + Driver: "default", + Config: []network.IPAMConfig{ + network.IPAMConfig{ + Subnet: "10.225.225.0/24", + Gateway: "10.225.225.1", + }, + }, + }, + } + + _, err = dockercli.NetworkCreate( + ctx, + "ingress", + ingressOptions, + ) + + if err != nil { + log.Printf("[WARNING] Ingress network may already exist: %s", err) + } + + //docker network create --driver=overlay workers + // Specific subnet? + networkName := "shuffle_swarm_executions" + if len(swarmNetworkName) > 0 { + networkName = swarmNetworkName + } + + networkCreateOptions := types.NetworkCreate{ + Driver: "overlay", + Options: options, + Attachable: true, + Ingress: false, + IPAM: &network.IPAM{ + Driver: "default", + Config: []network.IPAMConfig{ + network.IPAMConfig{ + Subnet: "10.224.224.0/24", + Gateway: "10.224.224.1", + }, + }, + }, + } + _, err = dockercli.NetworkCreate( + ctx, + networkName, + networkCreateOptions, + ) + + if err != nil { + if strings.Contains(fmt.Sprintf("%s", err), "already exists") { + // Try patching for attachable + + } else { + log.Printf("[DEBUG] Failed to create network %s for workers: %s. This is not critical, and containers will still be added", networkName, err) + } + } + + isMemcachedRunning, err := checkMemcached(ctx, dockercli) + if err != nil { + log.Printf("[ERROR] Failed checking memcached: %s", err) + } + if isMemcachedRunning == false { + log.Printf("[ERROR] Memcached is not running. Will try to deploy it.") + deployMemcached(dockercli) + } + + ip := "shuffle-cache" + + os.Setenv("SHUFFLE_MEMCACHED", fmt.Sprintf("%s:11211", ip)) + + defaultNetworkAttach := false + if containerId != "" { + log.Printf("[DEBUG] Should connect orborus container to worker network as it's running in Docker with name %#v!", containerId) + // https://pkg.go.dev/github.com/docker/docker@v20.10.12+incompatible/api/types/network#EndpointSettings + networkConfig := &network.EndpointSettings{} + err := dockercli.NetworkConnect(ctx, networkName, containerId, networkConfig) + if err != nil { + log.Printf("[ERROR] Failed connecting Orborus to docker network %s: %s", networkName, err) } - // Check if there is at least one interface - if len(interfaces) < 2 { - // this assumes that the machine should have at least 2 network - // interfaces. If not, we will use the default MTU. - // interface 1 is the loopback interface - // interface 2 is eth0, The eth0 interface inside a - // Docker container corresponds to the virtual Ethernet - // interface that connects the container to the docker0 - log.Printf("[ERROR] Failed to get enough network interfaces") - } else { - // Get the preferred interface - for _, iface := range interfaces { - if strings.Contains(iface.Name, bridgeName) { - targetInterface := iface - mtu = targetInterface.MTU - log.Printf("[INFO] Using MTU %d from interface %s", mtu, targetInterface.Name) + if len(containerId) == 64 && baseUrl == "http://shuffle-backend:5001" { + log.Printf("[WARNING] Network MAY not work due to backend being %s and container length 64. Will try to attach shuffle_shuffle network", baseUrl) + defaultNetworkAttach = true + } + } + + if len(os.Getenv("DOCKER_HOST")) > 0 { + log.Printf("[DEBUG] Deploying docker socket proxy to the network %s as the DOCKER_HOST variable is set", networkName) + containers, err := dockercli.ContainerList(ctx, container.ListOptions{ + All: true, + }) + + if err == nil { + for _, container := range containers { + if strings.Contains(strings.ToLower(container.Image), "docker-socket-proxy") { + networkConfig := &network.EndpointSettings{} + err := dockercli.NetworkConnect(ctx, networkName, container.ID, networkConfig) + if err != nil { + log.Printf("[ERROR] Failed connecting Docker socket proxy to docker network %s: %s", networkName, err) + } else { + log.Printf("[INFO] Attached the docker socket proxy to the execution network") + } + break } } - } - - // Create the network options with the specified MTU - options := make(map[string]string) - options["com.docker.network.driver.mtu"] = fmt.Sprintf("%d", mtu) - - ingressOptions := types.NetworkCreate{ - Driver: "overlay", - Attachable: false, - Ingress: true, - IPAM: &network.IPAM{ - Driver: "default", - Config: []network.IPAMConfig{ - network.IPAMConfig{ - Subnet: "10.225.225.0/24", - Gateway: "10.225.225.1", - }, - }, - }, - } - - _, err = dockercli.NetworkCreate( - ctx, - "ingress", - ingressOptions, - ) - - if err != nil { - log.Printf("[WARNING] Ingress network may already exist: %s", err) - } - - //docker network create --driver=overlay workers - // Specific subnet? - networkName := "shuffle_swarm_executions" - if len(swarmNetworkName) > 0 { - networkName = swarmNetworkName - } - - networkCreateOptions := types.NetworkCreate{ - Driver: "overlay", - Options: options, - Attachable: true, - Ingress: false, - IPAM: &network.IPAM{ - Driver: "default", - Config: []network.IPAMConfig{ - network.IPAMConfig{ - Subnet: "10.224.224.0/24", - Gateway: "10.224.224.1", - }, - }, - }, - } - _, err = dockercli.NetworkCreate( - ctx, - networkName, - networkCreateOptions, - ) - - if err != nil { - if strings.Contains(fmt.Sprintf("%s", err), "already exists") { - // Try patching for attachable - - } else { - log.Printf("[DEBUG] Failed to create network %s for workers: %s. This is not critical, and containers will still be added", networkName, err) - } - } - - defaultNetworkAttach := false - if containerId != "" { - log.Printf("[DEBUG] Should connect orborus container to worker network as it's running in Docker with name %#v!", containerId) - // https://pkg.go.dev/github.com/docker/docker@v20.10.12+incompatible/api/types/network#EndpointSettings - networkConfig := &network.EndpointSettings{} - err := dockercli.NetworkConnect(ctx, networkName, containerId, networkConfig) - if err != nil { - log.Printf("[ERROR] Failed connecting Orborus to docker network %s: %s", networkName, err) - } - - if len(containerId) == 64 && baseUrl == "http://shuffle-backend:5001" { - log.Printf("[WARNING] Network MAY not work due to backend being %s and container length 64. Will try to attach shuffle_shuffle network", baseUrl) - defaultNetworkAttach = true - } - } - - if len(os.Getenv("DOCKER_HOST")) > 0 { - log.Printf("[DEBUG] Deploying docker socket proxy to the network %s as the DOCKER_HOST variable is set", networkName) - containers, err := dockercli.ContainerList(ctx, container.ListOptions{ - All: true, - }) - - if err == nil { - for _, container := range containers { - if strings.Contains(strings.ToLower(container.Image), "docker-socket-proxy") { - networkConfig := &network.EndpointSettings{} - err := dockercli.NetworkConnect(ctx, networkName, container.ID, networkConfig) - if err != nil { - log.Printf("[ERROR] Failed connecting Docker socket proxy to docker network %s: %s", networkName, err) - } else { - log.Printf("[INFO] Attached the docker socket proxy to the execution network") - } - - break - } - } - } else { - log.Printf("[ERROR] Failed listing containers when deploying socket proxy on swarm: %s", err) - } - //} else { - // log.Printf("[ERROR] Failed listing and finding the right image for docker socket proxy: %s", err) - //} - } - - replicas := uint64(1) - scaleReplicas := os.Getenv("SHUFFLE_SCALE_REPLICAS") - if len(scaleReplicas) > 0 { - tmpInt, err := strconv.Atoi(scaleReplicas) - if err != nil { - log.Printf("[ERROR] %s is not a valid number for replication", scaleReplicas) - } else { - replicas = uint64(tmpInt) - } - - log.Printf("[DEBUG] SHUFFLE_SCALE_REPLICAS set to value %#v. Trying to overwrite default (%d/node)", scaleReplicas, replicas) - } - - innerContainerName := fmt.Sprintf("shuffle-workers") - cnt, err := findActiveSwarmNodes() - if err != nil { - log.Printf("[ERROR] Failed to find active swarm nodes: %s. Defaulting to 1", err) - } - - nodeCount := uint64(1) - if cnt > 0 { - nodeCount = uint64(cnt) - } - - appReplicas := os.Getenv("SHUFFLE_APP_REPLICAS") - appReplicaCnt := 1 - if len(appReplicas) > 0 { - newCnt, err := strconv.Atoi(appReplicas) - if err != nil { - log.Printf("[ERROR] %s is not a valid number for SHUFFLE_APP_REPLICAS", appReplicas) - } else { - appReplicaCnt = newCnt - } - } - - log.Printf("[DEBUG] Found %d node(s) to replicate over. Defaulting to 1 IF we can't auto-discover them.", cnt) - replicatedJobs := uint64(replicas * nodeCount) - - log.Printf("[DEBUG] Deploying %d container(s) for worker with swarm to each node. Service name: %s. Image: %s", replicas, innerContainerName, image) - - if timezone == "" { - timezone = "Europe/Amsterdam" - } - - // FIXME: May not need ingress ports. Could use internal services and DNS of swarm itself - // https://github.com/moby/moby/blob/e2f740de442bac52b280bc485a3ca5b31567d938/api/types/swarm/service.go#L46 - serviceSpec := swarm.ServiceSpec{ - Annotations: swarm.Annotations{ - Name: innerContainerName, - Labels: map[string]string{}, - }, - Mode: swarm.ServiceMode{ - Replicated: &swarm.ReplicatedService{ - Replicas: &replicatedJobs, - }, - }, - Networks: []swarm.NetworkAttachmentConfig{ - swarm.NetworkAttachmentConfig{ - Target: networkName, - }, - swarm.NetworkAttachmentConfig{ - Target: "ingress", - }, - }, - EndpointSpec: &swarm.EndpointSpec{ - Mode: "vip", - Ports: []swarm.PortConfig{ - swarm.PortConfig{ - Protocol: swarm.PortConfigProtocolTCP, - PublishMode: swarm.PortConfigPublishModeIngress, - Name: "worker-port", - PublishedPort: 33333, - TargetPort: 33333, - }, - }, - }, - TaskTemplate: swarm.TaskSpec{ - Resources: &swarm.ResourceRequirements{ - Reservations: &swarm.Resources{}, - }, - LogDriver: &swarm.Driver{ - Name: "json-file", - Options: map[string]string{ - "max-size": "10m", - }, - }, - ContainerSpec: &swarm.ContainerSpec{ - Image: image, - Env: []string{ - fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", os.Getenv("SHUFFLE_SWARM_CONFIG")), - fmt.Sprintf("SHUFFLE_SWARM_NETWORK_NAME=%s", networkName), - fmt.Sprintf("SHUFFLE_APP_REPLICAS=%d", appReplicaCnt), - fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED")), - fmt.Sprintf("DEBUG_MEMORY=%s", os.Getenv("DEBUG_MEMORY")), - fmt.Sprintf("SHUFFLE_APP_SDK_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")), - fmt.Sprintf("SHUFFLE_MAX_SWARM_NODES=%d", os.Getenv("SHUFFLE_MAX_SWARM_NODES")), - fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", os.Getenv("SHUFFLE_BASE_IMAGE_NAME")), - fmt.Sprintf("SHUFFLE_APP_REQUEST_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_REQUEST_TIMEOUT")), - }, - //Hosts: []string{ - // innerContainerName, - //}, - }, - RestartPolicy: &swarm.RestartPolicy{ - Condition: swarm.RestartPolicyConditionOnFailure, - }, - Placement: &swarm.Placement{ - MaxReplicas: replicas, - }, - }, - } - - if defaultNetworkAttach == true || strings.ToLower(os.Getenv("SHUFFLE_DEFAULT_NETWORK_ATTACH")) == "true" { - targetName := "shuffle_shuffle" - log.Printf("[DEBUG] Adding network attach for network %s to worker in swarm", targetName) - serviceSpec.Networks = append(serviceSpec.Networks, swarm.NetworkAttachmentConfig{ - Target: targetName, - }) - - // FIXM: Remove this if deployment fails? - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_SWARM_OTHER_NETWORK=%s", targetName)) - } - - if dockerApiVersion != "" { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("DOCKER_API_VERSION=%s", dockerApiVersion)) - } - - if len(os.Getenv("SHUFFLE_SCALE_REPLICAS")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_SCALE_REPLICAS=%s", os.Getenv("SHUFFLE_SCALE_REPLICAS"))) - } - - if len(os.Getenv("SHUFFLE_MEMCACHED")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_MEMCACHED=%s", os.Getenv("SHUFFLE_MEMCACHED"))) - } - - if strings.ToLower(os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) == "true" { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))) - } - - if len(workerServerUrl) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_WORKER_SERVER_URL=%s", os.Getenv("SHUFFLE_WORKER_SERVER_URL"))) - } - - // Handles backend - if len(os.Getenv("BASE_URL")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("BASE_URL=%s", os.Getenv("BASE_URL"))) - } - - if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_CLOUDRUN_URL=%s", os.Getenv("SHUFFLE_CLOUDRUN_URL"))) - } - - if len(os.Getenv("SHUFFLE_AUTO_IMAGE_DOWNLOAD")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_AUTO_IMAGE_DOWNLOAD=%s", os.Getenv("SHUFFLE_AUTO_IMAGE_DOWNLOAD"))) - } - - if len(os.Getenv("DOCKER_HOST")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("DOCKER_HOST=%s", os.Getenv("DOCKER_HOST"))) } else { - if runtime.GOOS == "windows" { - serviceSpec.TaskTemplate.ContainerSpec.Mounts = []mount.Mount{ - mount.Mount{ - Source: `\\.\pipe\docker_engine`, - Target: `\\.\pipe\docker_engine`, - Type: mount.TypeBind, - }, - } - } else { - serviceSpec.TaskTemplate.ContainerSpec.Mounts = []mount.Mount{ - mount.Mount{ - Source: "/var/run/docker.sock", - Target: "/var/run/docker.sock", - Type: mount.TypeBind, - }, - } - - } + log.Printf("[ERROR] Failed listing containers when deploying socket proxy on swarm: %s", err) } + //} else { + // log.Printf("[ERROR] Failed listing and finding the right image for docker socket proxy: %s", err) + //} + } - // Look for SHUFFLE_VOLUME_BINDS - if len(os.Getenv("SHUFFLE_VOLUME_BINDS")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_VOLUME_BINDS=%s", os.Getenv("SHUFFLE_VOLUME_BINDS"))) - } - - overrideHttpProxy := os.Getenv("SHUFFLE_INTERNAL_HTTP_PROXY") - overrideHttpsProxy := os.Getenv("SHUFFLE_INTERNAL_HTTPS_PROXY") - if len(overrideHttpProxy) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_INTERNAL_HTTP_PROXY=%s", overrideHttpProxy)) - } - - if len(overrideHttpsProxy) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_INTERNAL_HTTPS_PROXY=%s", overrideHttpsProxy)) - } - - serviceOptions := types.ServiceCreateOptions{} - _, err = dockercli.ServiceCreate( - ctx, - serviceSpec, - serviceOptions, - ) - - //dockercli.ServiceUpdate( - - if err == nil { - log.Printf("[DEBUG] Successfully deployed workers with %d replica(s) on %d node(s)", replicas, cnt) - //time.Sleep(time.Duration(10) * time.Second) - //log.Printf("[DEBUG] Servicecreate request: %#v %#v", service, err) + replicas := uint64(1) + scaleReplicas := os.Getenv("SHUFFLE_SCALE_REPLICAS") + if len(scaleReplicas) > 0 { + tmpInt, err := strconv.Atoi(scaleReplicas) + if err != nil { + log.Printf("[ERROR] %s is not a valid number for replication", scaleReplicas) } else { - if !strings.Contains(fmt.Sprintf("%s", err), "Already Exists") && !strings.Contains(fmt.Sprintf("%s", err), "is already in use by service") { - log.Printf("[ERROR] Failed making service: %s", err) - } else { - log.Printf("[WARNING] Failed deploying workers: %s", err) - if len(serviceSpec.Networks) > 1 { - serviceSpec.Networks = []swarm.NetworkAttachmentConfig{ - swarm.NetworkAttachmentConfig{ - Target: "shuffle_shuffle", - }, - } - - _, _ = dockercli.ServiceCreate( - ctx, - serviceSpec, - serviceOptions, - ) - } - } + replicas = uint64(tmpInt) } + log.Printf("[DEBUG] SHUFFLE_SCALE_REPLICAS set to value %#v. Trying to overwrite default (%d/node)", scaleReplicas, replicas) + } + + innerContainerName := fmt.Sprintf("shuffle-workers") + cnt, err := findActiveSwarmNodes() + if err != nil { + log.Printf("[ERROR] Failed to find active swarm nodes: %s. Defaulting to 1", err) + } + + nodeCount := uint64(1) + if cnt > 0 { + nodeCount = uint64(cnt) + } + + appReplicas := os.Getenv("SHUFFLE_APP_REPLICAS") + appReplicaCnt := 1 + if len(appReplicas) > 0 { + newCnt, err := strconv.Atoi(appReplicas) + if err != nil { + log.Printf("[ERROR] %s is not a valid number for SHUFFLE_APP_REPLICAS", appReplicas) + } else { + appReplicaCnt = newCnt + } + } + + log.Printf("[DEBUG] Found %d node(s) to replicate over. Defaulting to 1 IF we can't auto-discover them.", cnt) + replicatedJobs := uint64(replicas * nodeCount) + + log.Printf("[DEBUG] Deploying %d container(s) for worker with swarm to each node. Service name: %s. Image: %s", replicas, innerContainerName, image) + + if timezone == "" { + timezone = "Europe/Amsterdam" + } + + // FIXME: May not need ingress ports. Could use internal services and DNS of swarm itself + // https://github.com/moby/moby/blob/e2f740de442bac52b280bc485a3ca5b31567d938/api/types/swarm/service.go#L46 + serviceSpec := swarm.ServiceSpec{ + Annotations: swarm.Annotations{ + Name: innerContainerName, + Labels: map[string]string{}, + }, + Mode: swarm.ServiceMode{ + Replicated: &swarm.ReplicatedService{ + Replicas: &replicatedJobs, + }, + }, + Networks: []swarm.NetworkAttachmentConfig{ + swarm.NetworkAttachmentConfig{ + Target: networkName, + }, + swarm.NetworkAttachmentConfig{ + Target: "ingress", + }, + }, + EndpointSpec: &swarm.EndpointSpec{ + Mode: "vip", + Ports: []swarm.PortConfig{ + swarm.PortConfig{ + Protocol: swarm.PortConfigProtocolTCP, + PublishMode: swarm.PortConfigPublishModeIngress, + Name: "worker-port", + PublishedPort: 33333, + TargetPort: 33333, + }, + }, + }, + TaskTemplate: swarm.TaskSpec{ + Resources: &swarm.ResourceRequirements{ + Reservations: &swarm.Resources{}, + }, + LogDriver: &swarm.Driver{ + Name: "json-file", + Options: map[string]string{ + "max-size": "10m", + }, + }, + ContainerSpec: &swarm.ContainerSpec{ + Image: image, + Env: []string{ + fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", os.Getenv("SHUFFLE_SWARM_CONFIG")), + fmt.Sprintf("SHUFFLE_SWARM_NETWORK_NAME=%s", networkName), + fmt.Sprintf("SHUFFLE_APP_REPLICAS=%d", appReplicaCnt), + fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED")), + fmt.Sprintf("DEBUG_MEMORY=%s", os.Getenv("DEBUG_MEMORY")), + fmt.Sprintf("SHUFFLE_APP_SDK_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")), + fmt.Sprintf("SHUFFLE_MAX_SWARM_NODES=%d", os.Getenv("SHUFFLE_MAX_SWARM_NODES")), + fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", os.Getenv("SHUFFLE_BASE_IMAGE_NAME")), + fmt.Sprintf("SHUFFLE_APP_REQUEST_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_REQUEST_TIMEOUT")), + }, + //Hosts: []string{ + // innerContainerName, + //}, + }, + RestartPolicy: &swarm.RestartPolicy{ + Condition: swarm.RestartPolicyConditionOnFailure, + }, + Placement: &swarm.Placement{ + Constraints: []string{}, + }, + }, + } + + if defaultNetworkAttach == true || strings.ToLower(os.Getenv("SHUFFLE_DEFAULT_NETWORK_ATTACH")) == "true" { + targetName := "shuffle_shuffle" + log.Printf("[DEBUG] Adding network attach for network %s to worker in swarm", targetName) + serviceSpec.Networks = append(serviceSpec.Networks, swarm.NetworkAttachmentConfig{ + Target: targetName, + }) + + // FIXM: Remove this if deployment fails? + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_SWARM_OTHER_NETWORK=%s", targetName)) + } + + if dockerApiVersion != "" { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("DOCKER_API_VERSION=%s", dockerApiVersion)) + } + + if len(os.Getenv("SHUFFLE_SCALE_REPLICAS")) > 0 { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_SCALE_REPLICAS=%s", os.Getenv("SHUFFLE_SCALE_REPLICAS"))) + } + + if len(os.Getenv("SHUFFLE_MEMCACHED")) > 0 { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_MEMCACHED=%s", os.Getenv("SHUFFLE_MEMCACHED"))) + } + + if strings.ToLower(os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) == "true" { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))) + } + + if len(workerServerUrl) > 0 { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_WORKER_SERVER_URL=%s", os.Getenv("SHUFFLE_WORKER_SERVER_URL"))) + } + + // Handles backend + if len(os.Getenv("BASE_URL")) > 0 { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("BASE_URL=%s", os.Getenv("BASE_URL"))) + } + + if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_CLOUDRUN_URL=%s", os.Getenv("SHUFFLE_CLOUDRUN_URL"))) + } + + if len(os.Getenv("SHUFFLE_AUTO_IMAGE_DOWNLOAD")) > 0 { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_AUTO_IMAGE_DOWNLOAD=%s", os.Getenv("SHUFFLE_AUTO_IMAGE_DOWNLOAD"))) + } + + if len(os.Getenv("DOCKER_HOST")) > 0 { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("DOCKER_HOST=%s", os.Getenv("DOCKER_HOST"))) + } else { + if runtime.GOOS == "windows" { + serviceSpec.TaskTemplate.ContainerSpec.Mounts = []mount.Mount{ + mount.Mount{ + Source: `\\.\pipe\docker_engine`, + Target: `\\.\pipe\docker_engine`, + Type: mount.TypeBind, + }, + } + } else { + serviceSpec.TaskTemplate.ContainerSpec.Mounts = []mount.Mount{ + mount.Mount{ + Source: "/var/run/docker.sock", + Target: "/var/run/docker.sock", + Type: mount.TypeBind, + }, + } + + } + } + + // Look for SHUFFLE_VOLUME_BINDS + if len(os.Getenv("SHUFFLE_VOLUME_BINDS")) > 0 { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_VOLUME_BINDS=%s", os.Getenv("SHUFFLE_VOLUME_BINDS"))) + } + + overrideHttpProxy := os.Getenv("SHUFFLE_INTERNAL_HTTP_PROXY") + overrideHttpsProxy := os.Getenv("SHUFFLE_INTERNAL_HTTPS_PROXY") + if len(overrideHttpProxy) > 0 { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_INTERNAL_HTTP_PROXY=%s", overrideHttpProxy)) + } + + if len(overrideHttpsProxy) > 0 { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_INTERNAL_HTTPS_PROXY=%s", overrideHttpsProxy)) + } + + serviceOptions := types.ServiceCreateOptions{} + _, err = dockercli.ServiceCreate( + ctx, + serviceSpec, + serviceOptions, + ) + + // Force deploy if it's not disabled + deployTenzirNode() + + if err == nil { + log.Printf("[DEBUG] Successfully deployed workers with %d replica(s) on %d node(s)", replicas, cnt) + //time.Sleep(time.Duration(10) * time.Second) + //log.Printf("[DEBUG] Servicecreate request: %#v %#v", service, err) + } else { + if !strings.Contains(fmt.Sprintf("%s", err), "Already Exists") && !strings.Contains(fmt.Sprintf("%s", err), "is already in use by service") { + log.Printf("[ERROR] Failed making service: %s", err) + } else { + log.Printf("[WARNING] Failed deploying workers: %s", err) + if len(serviceSpec.Networks) > 1 { + serviceSpec.Networks = []swarm.NetworkAttachmentConfig{ + swarm.NetworkAttachmentConfig{ + Target: "shuffle_shuffle", + }, + } + + _, _ = dockercli.ServiceCreate( + ctx, + serviceSpec, + serviceOptions, + ) + } + } } } @@ -630,164 +744,502 @@ func buildEnvVars(envMap map[string]string) []corev1.EnvVar { return envVars } - func handleBackendImageDownload(ctx context.Context, images string) error { - // Should use docker to: - // 1. Pull the image & tag it - // 2. Distribute the image by updating service if "run" + + // Replicate images with lowercase, as the name may be wrong + // Most of the time lowercase is correct. Swapping to have that first + originalImages := images + images = strings.ToLower(images) + "," + originalImages + + // Remove the image + handled := []string{} + log.Printf("[DEBUG] Should remove existing image (s): %s. Waiting 30 seconds to ensure backend has the latest images built and ready to distribute.", images) + removeOptions := image.RemoveOptions{} + + time.Sleep(time.Duration(30) * time.Second) + + newImages := []string{} + for _, image := range strings.Split(images, ",") { + image = strings.TrimSpace(image) + if shuffle.ArrayContains(handled, image) { + continue + } + + handled = append(handled, image) + if !strings.Contains(image, "/") { + image = fmt.Sprintf("frikky/shuffle:%s", image) + } + + newImages = append(newImages, image) + + // There is no real point in actual removal. This may however be a good idea, as Worker will force download the new one anyway + resp, err := dockercli.ImageRemove(ctx, image, removeOptions) + if err != nil { + log.Printf("[ERROR] Failed removing image: %s. Resp: %#v", err, resp) + + // Goroutining images that don't already exist, as they are most likely not the correct one + go shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image) + } else { + log.Printf("[DEBUG] Removed image: %s", image) + + err = shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image) + if err != nil { + log.Printf("[ERROR] Failed downloading image: %s", err) + } else { + log.Printf("[DEBUG] Downloaded image: %s", image) + //break + } + } + } + if swarmConfig == "run" || swarmConfig == "swarm" { - log.Printf("[DEBUG] Should update service with new image after updating(s): %s. \n\nNOT IMPLEMENTED: Contact support@shuffler.io for support.\n\n", images) + log.Printf("[DEBUG] Should update service with new image after updating(s): %s. \n\nBETA REPLACEMENT IMPLEMENTATION: Contact support@shuffler.io for support.", strings.Join(newImages, "\n")) // 1. Download the image // 2. Find the existing service using the image // 3. Update the service with the new image in a rolling restart - } else { - log.Printf("[DEBUG] Should remove existing image (s): %s", images) - // Remove the image - removeOptions := types.ImageRemoveOptions{ - } + // Find the existing service + serviceListOptions := types.ServiceListOptions{} + services, err := dockercli.ServiceList( + ctx, + serviceListOptions, + ) - for _, image := range strings.Split(images, ",") { - image = strings.TrimSpace(image) - if !strings.Contains(image, "/") { - image = fmt.Sprintf("frikky/shuffle:%s", image) + if err != nil { + log.Printf("[ERROR] Failed finding containers: %s", err) + } else { + log.Printf("[DEBUG] Found %d services", len(services)) + + for _, service := range services { + + log.Printf("Imagename: %s", service.Spec.TaskTemplate.ContainerSpec.Image) + + for _, image := range newImages { + if !strings.Contains(service.Spec.TaskTemplate.ContainerSpec.Image, image) { + continue + } + + log.Printf("[DEBUG] Found service for image %#v: %#v", service.Spec.Annotations.Name) + + // Update the service to run with the new image + //docker service update --image username/imagename:latest servicename --force + serviceUpdateOptions := types.ServiceUpdateOptions{} + resp, err := dockercli.ServiceUpdate( + ctx, + service.ID, + service.Version, + service.Spec, + serviceUpdateOptions, + ) + + if err != nil { + log.Printf("[ERROR] Failed updating service %s with the new image %s: %s. Resp: %#v", service.Spec.Annotations.Name, image, err, resp) + } else { + log.Printf("[DEBUG] Updated service %s with the new image %s. Resp: %#v", service.Spec.Annotations.Name, image, resp) + + if !strings.Contains(fmt.Sprintf("%s", resp), "error") { + break + } + } + } } - resp, err := dockercli.ImageRemove(ctx, image, removeOptions) - if err != nil { - log.Printf("[ERROR] Failed removing image: %s", err) - } else { - log.Printf("[DEBUG] Removed image: %s", resp) - } } + } return nil } +func fixk8sRoles() { + clientset, _, err := shuffle.GetKubernetesClient() + if err != nil { + log.Printf("[ERROR] Error getting kubernetes client: %s", err) + os.Exit(1) + } + + kubernetesNamespace := "default" + + // Check if namespace exist as variable. If so, make it + if len(os.Getenv("KUBERNETES_NAMESPACE")) > 0 { + kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") + } + + // fix roles + // check if "service-creator" role is assigned to the service account "default" + // roleBindingNames := []string{"service-creator-binding", "pod-creator-binding", "deployment-creator-binding"} + serviceAccountName := "default" + roleBindingName := "creator-all" + + resourceTypes := []string{"services", "pods", "deployments"} + + // Check if the RoleBinding exists + roleBinding, err := clientset.RbacV1().RoleBindings(kubernetesNamespace).Get(context.TODO(), roleBindingName, metav1.GetOptions{}) + if err != nil { + log.Printf("[WARNING] Failed to get RoleBinding %s: %s", roleBindingName, err) + + // create role and rolebinding + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: roleBindingName, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"", "apps"}, + Resources: resourceTypes, + Verbs: []string{"create", "list"}, + }, + }, + } + + ctx := context.TODO() + + _, err := clientset.RbacV1().Roles(kubernetesNamespace).Create(ctx, role, metav1.CreateOptions{}) + if err != nil { + log.Printf("[ERROR] Failed to create Role %s: %s", roleBindingName, err) + if !strings.Contains(fmt.Sprintf("%s", err), "already exists") { + log.Printf("[INFO] role %s already exists", roleBindingName) + } + } + + roleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: roleBindingName, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: serviceAccountName, + Namespace: kubernetesNamespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "Role", + Name: roleBindingName, + }, + } + + _, err = clientset.RbacV1().RoleBindings(kubernetesNamespace).Create(ctx, roleBinding, metav1.CreateOptions{}) + if err != nil { + log.Printf("[ERROR] Failed to create RoleBinding %s: %s", roleBindingName, err) + if strings.Contains(fmt.Sprintf("%s", err), "already exists") { + log.Printf("[INFO] rolebinding %s already exists", roleBindingName) + } + } + + log.Printf("[INFO] Created Role %s and RoleBinding %s", roleBindingName, roleBindingName) + } else { + log.Printf("[INFO] RoleBinding %s exists", roleBindingName) + } + + // Check if the RoleBinding is assigned to the service account + var found bool + for _, subject := range roleBinding.Subjects { + if subject.Kind == "ServiceAccount" && subject.Name == serviceAccountName { + found = true + break + } + } + + if !found { + log.Printf("[WARNING] Service account %s is not assigned to RoleBinding %s\n", serviceAccountName, roleBindingName) + // assign the service account to the rolebinding + roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{ + Kind: "ServiceAccount", + Name: serviceAccountName, + Namespace: kubernetesNamespace, + }) + + ctx := context.TODO() + + _, err := clientset.RbacV1().RoleBindings(kubernetesNamespace).Update(ctx, roleBinding, metav1.UpdateOptions{}) + if err != nil { + log.Printf("[ERROR](ns - %s) Failed to update RoleBinding %s: %s", kubernetesNamespace, roleBindingName, err) + if !strings.Contains(fmt.Sprintf("%s", err), "already exists") { + log.Printf("[INFO] rolebinding %s already exists", roleBindingName) + } + } + } +} + +func int32Ptr(i int32) *int32 { return &i } + +func deployK8sWorker(image string, identifier string, env []string) error { + env = append(env, fmt.Sprintf("IS_KUBERNETES=true")) + env = append(env, fmt.Sprintf("KUBERNETES_NAMESPACE=%s", os.Getenv("KUBERNETES_NAMESPACE"))) + + if len(os.Getenv("KUBERNETES_SERVICE_HOST")) > 0 { + env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_HOST=%s", os.Getenv("KUBERNETES_SERVICE_HOST"))) + } + + if len(os.Getenv("SHUFFLE_MEMCACHED")) > 0 { + env = append(env, fmt.Sprintf("SHUFFLE_MEMCACHED=%s", os.Getenv("SHUFFLE_MEMCACHED"))) + } + + if len(os.Getenv("KUBERNETES_SERVICE_PORT")) > 0 { + env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_PORT=%s", os.Getenv("KUBERNETES_SERVICE_PORT"))) + } + + if len(os.Getenv("REGISTRY_URL")) > 0 { + env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL"))) + } + + if len(os.Getenv("SHUFFLE_USE_GHCR_OVERRIDE_FOR_AUTODEPLOY")) > 0 { + env = append(env, fmt.Sprintf("SHUFFLE_USE_GHCR_OVERRIDE_FOR_AUTODEPLOY=%s", os.Getenv("SHUFFLE_USE_GHCR_OVERRIDE_FOR_AUTODEPLOY"))) + } + + clientset, _, err := shuffle.GetKubernetesClient() + if err != nil { + log.Printf("[ERROR] Error getting kubernetes client:", err) + return err + } + + //env = append(env, fmt.Sprintf("KUBERNETES_CONFIG=%s", config.String())) + + // FIXME: When a service account is used, the account is also mounted in the pod + // The volume mount location is: + // /var/run/secrets/kubernetes.io/serviceaccount + + // Look for if there is a default service account in use + if len(os.Getenv("KUBERNETES_SERVICE_ACCOUNT")) > 0 { + log.Printf("[DEBUG] Using Kubernetes service account %s", os.Getenv("KUBERNETES_SERVICE_ACCOUNT")) + env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_ACCOUNT=%s", os.Getenv("KUBERNETES_SERVICE_ACCOUNT"))) + + // use k8s downward API to find it if we are in a pod + } + + // Check if namespace exist as variable. If so, make it + if len(os.Getenv("KUBERNETES_NAMESPACE")) > 0 && !namespacemade { + kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") + + // Make the namespace + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: os.Getenv("KUBERNETES_NAMESPACE"), + }, + } + + _, err := clientset.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{}) + if err != nil { + if !strings.Contains(strings.ToLower(fmt.Sprintf("%s", err)), "already exists") { + log.Printf("[ERROR] Failed creating Kubernetes namespace: %s", err) + } else { + namespacemade = true + } + } else { + namespacemade = true + } + } + + env = append(env, fmt.Sprintf("BASE_URL=%s", baseUrl)) + env = append(env, fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", swarmConfig)) + env = append(env, fmt.Sprintf("WORKER_HOSTNAME=%s", "shuffle-workers")) + + if len(kubernetesNamespace) == 0 { + foundNamespace, err := shuffle.GetKubernetesNamespace() + if err != nil { + //log.Printf("[ERROR] Failed getting Kubernetes namespace: %s", err) + } + + if len(foundNamespace) > 0 { + kubernetesNamespace = foundNamespace + os.Setenv("KUBERNETES_NAMESPACE", kubernetesNamespace) + } + } + + if len(kubernetesNamespace) == 0 { + kubernetesNamespace = "default" + } + + kubernetesImage := os.Getenv("SHUFFLE_KUBERNETES_WORKER") + if len(kubernetesImage) == 0 { + kubernetesImage = image + } + log.Printf("[DEBUG] Using Kubernetes worker image '%s'", kubernetesImage) + // image = "shuffle-worker:v1" //hard coded image name to test locally + + envMap := make(map[string]string) + for _, envStr := range env { + parts := strings.SplitN(envStr, "=", 2) + if len(parts) == 2 { + envMap[parts[0]] = parts[1] + } + } + + containerLabels := map[string]string{ + "container": "shuffle-worker", + } + + containerAttachment := corev1.Container{ + Name: identifier, + Image: kubernetesImage, + Env: buildEnvVars(envMap), + + //ImagePullPolicy: "Never", + ImagePullPolicy: corev1.PullIfNotPresent, + } + + podname := shuffle.GetPodName() + + ctx := context.Background() + + if len(podname) > 0 { + _, err := shuffle.GetCurrentPodNetworkConfig(ctx, clientset, kubernetesNamespace, podname) + if err != nil { + log.Printf("[ERROR] Failed getting current pod network: %s", err) + } else { + log.Printf("[DEBUG] Current pod found!") + // currentPodStatus = k8s.io/api/core/v1.PodStatus + } + } + + // While testing: + // kubectl delete pods --all --all-namespaces; kubectl delete services --all --all-namespaces + // pod := &corev1.Pod{ + // ObjectMeta: metav1.ObjectMeta{ + // Name: identifier, + // Labels: containerLabels, + // }, + // Spec: corev1.PodSpec{ + // RestartPolicy: "Never", + // // DNSPolicy: "Default", + // DNSPolicy: corev1.DNSClusterFirst, + // // NodeSelector: map[string]string{ + // // "node": "master", + // // }, + // Containers: []corev1.Container{ + // containerAttachment, + // }, + // }, + // } + + // // Check if running on ARM or x86 to download the correct image + + // // Get current pod's network so we can make the pod in it + + // _, err = clientset.CoreV1().Pods(kubernetesNamespace).List(context.Background(), metav1.ListOptions{}) + // if err != nil { + // log.Printf("[ERROR] Failed listing pods: %s", err) + // } + + // createdPod, err := clientset.CoreV1().Pods(kubernetesNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) + // if err != nil { + // //log.Printf("[ERROR] Failed creating pod: %v", err) + // return err + // } + + // log.Printf("[INFO] Created pod %q in namespace %q\n", createdPod.Name, createdPod.Namespace) + + // // kubectl expose pod shuffle-workers --type=LoadBalancer --port=33333 + // service := &corev1.Service{ + // ObjectMeta: metav1.ObjectMeta{ + // Name: identifier, + // }, + // Spec: corev1.ServiceSpec{ + // Selector: map[string]string{ + // "container": "shuffle-workers", + // }, + // Ports: []corev1.ServicePort{ + // { + // Protocol: "TCP", + // Port: 33333, + // TargetPort: intstr.FromInt(33333), + // }, + // }, + // Type: corev1.ServiceTypeLoadBalancer, + // }, + // } + + // _, err = clientset.CoreV1().Services(kubernetesNamespace).Create(context.TODO(), service, metav1.CreateOptions{}) + // if err != nil { + // log.Printf("[ERROR] Failed creating service: %v", err) + // return err + // } + + replicaNumberStr := os.Getenv("SHUFFLE_SCALE_REPLICAS") + replicaNumber := 1 + if len(replicaNumberStr) > 0 { + tmpInt, err := strconv.Atoi(replicaNumberStr) + if err != nil { + log.Printf("[ERROR] %s is not a valid number for replication", replicaNumberStr) + } else { + replicaNumber = tmpInt + + } + } + + replicaNumberInt32 := int32(replicaNumber) + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: identifier, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: int32Ptr(replicaNumberInt32), + Selector: &metav1.LabelSelector{ + MatchLabels: containerLabels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: containerLabels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + containerAttachment, + }, + DNSPolicy: corev1.DNSClusterFirst, + }, + }, + }, + } + + _, err = clientset.AppsV1().Deployments(kubernetesNamespace).Create(context.Background(), deployment, metav1.CreateOptions{}) + if err != nil { + log.Printf("[ERROR] Failed creating deployment: %v", err) + return err + } + + // kubectl expose deployment shuffle-workers --type=NodePort --port=33333 --target-port=33333 + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: identifier, + }, + Spec: corev1.ServiceSpec{ + Selector: containerLabels, + Ports: []corev1.ServicePort{ + { + Protocol: "TCP", + Port: 33333, + TargetPort: intstr.FromInt(33333), + }, + }, + Type: corev1.ServiceTypeNodePort, + }, + } + + _, err = clientset.CoreV1().Services(kubernetesNamespace).Create(context.Background(), service, metav1.CreateOptions{}) + if err != nil { + log.Printf("[ERROR] Failed creating service: %v", err) + return err + } + + return nil +} func deployWorker(image string, identifier string, env []string, executionRequest shuffle.ExecutionRequest) error { if len(os.Getenv("REGISTRY_URL")) > 0 && os.Getenv("REGISTRY_URL") != "" { env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL"))) } - if isKubernetes == "true" { - env = append(env, fmt.Sprintf("IS_KUBERNETES=true")) - env = append(env, fmt.Sprintf("KUBERNETES_NAMESPACE=%s", os.Getenv("KUBERNETES_NAMESPACE"))) + // if isKubernetes == "true" { + // err := deployK8sWorker(image, identifier, env, executionRequest) + // if err != nil { + // log.Printf("[ERROR] Failed deploying Kubernetes worker: %s", err) + // } - if len(os.Getenv("KUBERNETES_SERVICE_HOST")) > 0 { - env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_HOST=%s", os.Getenv("KUBERNETES_SERVICE_HOST"))) - } - - if len(os.Getenv("KUBERNETES_SERVICE_PORT")) > 0 { - env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_PORT=%s", os.Getenv("KUBERNETES_SERVICE_PORT"))) - } - - - clientset, config, err := getKubernetesClient() - if err != nil { - log.Printf("[ERROR] Error getting kubernetes client:", err) - return err - } - - env = append(env, fmt.Sprintf("KUBERNETES_CONFIG=%s", config.String())) - - // FIXME: When a service account is used, the account is also mounted in the pod - // The volume mount location is: - // /var/run/secrets/kubernetes.io/serviceaccount - - // Look for if there is a default service account in use - if len(os.Getenv("KUBERNETES_SERVICE_ACCOUNT")) > 0 { - log.Printf("[DEBUG] Using Kubernetes service account %s", os.Getenv("KUBERNETES_SERVICE_ACCOUNT")) - env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_ACCOUNT=%s", os.Getenv("KUBERNETES_SERVICE_ACCOUNT"))) - - // use k8s downward API to find it if we are in a pod - } - - // Check if namespace exist as variable. If so, make it - if len(os.Getenv("KUBERNETES_NAMESPACE")) > 0 && !namespacemade { - kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") - - // Make the namespace - namespace := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: os.Getenv("KUBERNETES_NAMESPACE"), - }, - } - - _, err := clientset.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{}) - if err != nil { - if !strings.Contains(strings.ToLower(fmt.Sprintf("%s", err)), "already exists") { - log.Printf("[ERROR] Failed creating Kubernetes namespace: %s", err) - } else { - namespacemade = true - } - } else { - namespacemade = true - } - } - - if len(kubernetesNamespace) == 0 { - kubernetesNamespace = "default" - } - - kubernetesImage := os.Getenv("SHUFFLE_KUBERNETES_WORKER") - if len(kubernetesImage) == 0 { - kubernetesImage = image - } - log.Printf("[DEBUG] Using Kubernetes worker image '%s'", kubernetesImage) - // image = "shuffle-worker:v1" //hard coded image name to test locally - - envMap := make(map[string]string) - for _, envStr := range env { - parts := strings.SplitN(envStr, "=", 2) - if len(parts) == 2 { - envMap[parts[0]] = parts[1] - } - } - - // While testing: - // kubectl delete pods --all --all-namespaces; kubectl delete services --all --all-namespaces - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: identifier, - Labels: map[string]string{"app": "shuffle-worker"}, - }, - Spec: corev1.PodSpec{ - RestartPolicy: "Never", - DNSPolicy: "Default", - // NodeSelector: map[string]string{ - // "node": "master", - // }, - Containers: []corev1.Container{ - { - Name: identifier, - Image: kubernetesImage, - Env: buildEnvVars(envMap), - - //ImagePullPolicy: "Never", - ImagePullPolicy: corev1.PullIfNotPresent, - //ImagePullPolicy: "Always", - }, - }, - }, - } - - // Check if running on ARM or x86 to download the correct image - - // Add environment variables - // pod.Spec.Containers[0].Env = buildEnvVars(envMap) - - createdPod, err := clientset.CoreV1().Pods(kubernetesNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) - if err != nil { - //log.Printf("[ERROR] Failed creating pod: %v", err) - return err - } - - log.Printf("[INFO] Created pod %q in namespace %q\n", createdPod.Name, createdPod.Namespace) - return nil - } + // return err + // } // Binds is the actual "-v" volume. // Max 20% CPU every second @@ -805,6 +1257,28 @@ func deployWorker(image string, identifier string, env []string, executionReques Resources: container.Resources{}, } + certPath := "/certs" + + // This is just to test the mounting locally so + // I can control from what source I'm mounting + // the certs to. Default behaviour is: + // /certs:/certs. + if os.Getenv("SHUFFLE_CERT_PATH") != "" { + certPath = os.Getenv("SHUFFLE_CERT_PATH") + } + + _, err := os.ReadDir(certPath) + + if certPath != "" && err == nil { + certVol := mount.Mount{ + Type: mount.TypeBind, + Source: certPath, + Target: "/certs", + } + + hostConfig.Mounts = append(hostConfig.Mounts, certVol) + } + if len(os.Getenv("DOCKER_HOST")) == 0 { if runtime.GOOS == "windows" { hostConfig.Binds = []string{`\\.\pipe\docker_engine:\\.\pipe\docker_engine`} @@ -813,23 +1287,25 @@ func deployWorker(image string, identifier string, env []string, executionReques } } - hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) - if strings.ToLower(cleanupEnv) != "false" { - hostConfig.AutoRemove = true - } - config := &container.Config{ Image: image, Env: env, } + if isKubernetes != "true" { + hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) + if strings.ToLower(cleanupEnv) != "false" { + hostConfig.AutoRemove = true + } + } + //var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG") parsedUuid := uuid.NewV4() - if swarmConfig == "run" || swarmConfig == "swarm" { + if swarmConfig == "run" || swarmConfig == "swarm" || isKubernetes == "true" { // FIXME: Should we handle replies properly? // In certain cases, a workflow may e.g. be aborted already. If it's aborted, that returns // a 401 from the worker, which returns an error here - go sendWorkerRequest(executionRequest) + go sendWorkerRequest(executionRequest, image, env) return nil } @@ -994,7 +1470,7 @@ func initializeImages() { newWorker, } - pullOptions := types.ImagePullOptions{} + pullOptions := image.PullOptions{} for _, image := range images { if isKubernetes == "true" { log.Printf("[DEBUG] Skipping image pull of '%s' because Kubernetes does it in realtime instead", image) @@ -1068,8 +1544,19 @@ func checkSwarmService(ctx context.Context) { // https://docs.docker.com/engine/reference/commandline/swarm_init/ ip := getLocalIP() log.Printf("[DEBUG] Attempting swarm setup on %s", ip) + + info, err := dockercli.Info(ctx) + if err != nil { + log.Printf("[WARNING] Failed to get Docker Info: %s", err) + } + + if info.Swarm.ControlAvailable { + log.Printf("[INFO] Already part of swarm as a manager") + return + } + req := swarm.InitRequest{ - ListenAddr: fmt.Sprintf("0.0.0.0:2377", ip), + ListenAddr: "0.0.0.0:2377", AdvertiseAddr: fmt.Sprintf("%s:2377", ip), } @@ -1174,8 +1661,7 @@ func getOrborusStats(ctx context.Context) shuffle.OrborusStats { newStats.MaxMemory = int(pers.MemTotal) } - - // Get list of all running containers + // Get list of all running containers containers, err := dockercli.ContainerList(ctx, container.ListOptions{}) if err != nil { @@ -1289,51 +1775,8 @@ func getOrborusStats(ctx context.Context) shuffle.OrborusStats { return newStats } -func isRunningInCluster() bool { - _, existsHost := os.LookupEnv("KUBERNETES_SERVICE_HOST") - _, existsPort := os.LookupEnv("KUBERNETES_SERVICE_PORT") - return existsHost && existsPort -} - -func getKubernetesClient() (*kubernetes.Clientset, *rest.Config, error) { - - config := &rest.Config{} - var err error - - if isRunningInCluster() { - config, err := rest.InClusterConfig() - if err != nil { - return nil, config, err - } - - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, config, err - } - - return clientset, config, nil - - } - - home := homedir.HomeDir() - kubeconfigPath := filepath.Join(home, ".kube", "config") - config, err = clientcmd.BuildConfigFromFlags("", kubeconfigPath) - if err != nil { - return nil, config, err - } - - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, config, err - } - - return clientset, config, nil -} - - func sendRemoveRequest(client *http.Client, toBeRemoved shuffle.ExecutionRequestWrapper, baseUrl, environment, auth, org string, sleepTime int) error { confirmUrl := fmt.Sprintf("%s/api/v1/workflows/queue/confirm", baseUrl) - data, err := json.Marshal(toBeRemoved) if err != nil { log.Printf("[WARNING] Failed removal marshalling: %s", err) @@ -1404,10 +1847,14 @@ func main() { //defer cleanup() // Block until a signal is received - if isRunningInCluster() { + if shuffle.IsRunningInCluster() { log.Printf("[INFO] Running inside k8s cluster") } + if isKubernetes == "true" { + fixk8sRoles() + } + startupDelay := os.Getenv("SHUFFLE_ORBORUS_STARTUP_DELAY") if len(startupDelay) > 0 { log.Printf("[DEBUG] Setting startup delay to %#v", startupDelay) @@ -1498,6 +1945,29 @@ func main() { log.Printf("[WARNING] Defaulting to environment name %s. Set environment variable ENVIRONMENT_NAME to change. This should be the same as in the frontend action.", environment) } + if pipelineUrl == "" { + pipelineUrl = "http://localhost:5160" + + // Find the IP in baseUrl. Base format is http://: + if baseUrl != "" && !strings.Contains(baseUrl, "shuffle") && !strings.Contains(baseUrl, "localhost") && !strings.Contains(baseUrl, "run.app") { + urlSplit := strings.Split(baseUrl, "://") + if len(urlSplit) > 1 { + // Find the IP + ipSplit := strings.Split(urlSplit[1], ":") + if len(ipSplit) > 0 { + pipelineUrl = fmt.Sprintf("http://%s:5160", ipSplit[0]) + } + } + } + + if len(containerId) > 0 { + pipelineUrl = "http://tenzir-node:5160" + } + + log.Printf("[WARNING] SHUFFLE_PIPELINE_URL not set, falling back to default URL: %s. If BASE_URL is set, we use the external IP for that", pipelineUrl) + os.Setenv("SHUFFLE_PIPELINE_URL", pipelineUrl) + } + // FIXME - during init, BUILD and/or LOAD worker and app_sdk // Build/load app_sdk so it can be loaded as 127.0.0.1:5000/walkoff_app_sdk log.Printf("[INFO] Setting up Docker environment. Downloading worker and App SDK!") @@ -1508,16 +1978,32 @@ func main() { workerImage = newWorkerImage } - if swarmConfig == "run" || swarmConfig == "swarm" { - checkSwarmService(ctx) + if swarmConfig == "run" || swarmConfig == "swarm" || isKubernetes == "true" { + if isKubernetes != "true" { + checkSwarmService(ctx) + } log.Printf("[DEBUG] Cleaning up containers from previous run") cleanupExistingNodes(ctx) time.Sleep(time.Duration(5) * time.Second) log.Printf("[DEBUG] Deploying worker image %s to swarm", workerImage) - deployServiceWorkers(workerImage) - log.Printf("[DEBUG] Waiting 45 seconds to ensure workers are deployed. Run: \"docker service ls\" for more info") + + runString := "Run: \"docker service ls\" for more info" + + if isKubernetes != "true" { + deployServiceWorkers(workerImage) + } else { + deployK8sWorker(workerImage, "shuffle-workers", []string{}) + runString = "Run: \"kubectl get pods\" for more info" + } + + err := setBackendToSwarmNetwork(ctx) + if err != nil { + log.Printf("[WARNING] Failed setting backend to swarm network: %s", err) + } + + log.Printf("[DEBUG] Waiting 45 seconds to ensure workers are deployed. %s", runString) time.Sleep(time.Duration(45) * time.Second) //deployServiceWorkers(workerImage) @@ -1527,7 +2013,12 @@ func main() { client := shuffle.GetExternalClient(baseUrl) fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl) - log.Printf("[INFO] Finished configuring docker environment. Connecting to %s", fullUrl) + + if isKubernetes == "true" { + log.Printf("[INFO] Finished configuring kubernetes environment. Connecting to %s", fullUrl) + } else { + log.Printf("[INFO] Finished configuring docker environment. Connecting to %s", fullUrl) + } forwardData := bytes.NewBuffer([]byte{}) forwardMethod := "POST" @@ -1593,6 +2084,14 @@ func main() { // Marshal and set body orborusStats := getOrborusStats(ctx) + pipelinePayload, pipelineerr := sendPipelineHealthStatus() + + if pipelineerr != nil { + // Too verbose to be enabled. + //log.Printf("[ERROR] Failed sending pipeline health status: %s", pipelineerr) + } + + orborusStats.DataLake = pipelinePayload jsonData, err := json.Marshal(orborusStats) if err == nil { req.Body = ioutil.NopCloser(bytes.NewBuffer(jsonData)) @@ -1651,6 +2150,9 @@ func main() { log.Printf("[DEBUG] Starting iteration on environment %#v (default = Shuffle). Got statuscode %d from backend on first request", environment, newresp.StatusCode) } + if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" && os.Getenv("SHUFFLE_SCALE_REPLICAS") == "" { + go AutoScale(ctx) + } hasStarted = true } @@ -1668,38 +2170,157 @@ func main() { continue } - if hasStarted && len(executionRequests.Data) > 0 { //log.Printf("[INFO] Body: %s", string(body)) // Type string `json:"type"` } - // FIXME: Add features here for orborus & worker to + // FIXME: Add features here for orborus & worker to // do things on behalf of backend var toBeRemoved shuffle.ExecutionRequestWrapper if len(executionRequests.Data) > 0 { newrequests := []shuffle.ExecutionRequest{} + + // Deduplicating in case same job shows up multiple times + // This is specifically to handle data pipelines better + deduplicatedJobs := []shuffle.ExecutionRequest{} for _, incRequest := range executionRequests.Data { + if !strings.Contains(incRequest.Type, "DOCKER") && !strings.Contains(incRequest.Type, "PIPELINE") && !strings.Contains(incRequest.Type, "SIGMA") && !strings.Contains(incRequest.Type, "TENZIR") { + deduplicatedJobs = append(deduplicatedJobs, incRequest) + continue + } + + found := false + for _, dedupRequest := range deduplicatedJobs { + if incRequest.ExecutionArgument == dedupRequest.ExecutionArgument && incRequest.Type == dedupRequest.Type { + found = true + break + } + } + + if found { + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + continue + } + + deduplicatedJobs = append(deduplicatedJobs, incRequest) + } + + executionRequests.Data = deduplicatedJobs + for _, incRequest := range executionRequests.Data { + // Looking for specific jobs if incRequest.Type == "PIPELINE_CREATE" || incRequest.Type == "PIPELINE_START" || incRequest.Type == "PIPELINE_STOP" || incRequest.Type == "PIPELINE_DELETE" { err := handlePipeline(incRequest) if err != nil { - log.Printf("[ERROR] Failed handling pipeline: %s", err) + + log.Printf("[ERROR] Failed handling pipeline (%s %s): %s. Deleting job anyway.", incRequest.Type, incRequest.ExecutionSource, err) } toBeRemoved.Data = append(toBeRemoved.Data, incRequest) } else if incRequest.Type == "DOCKER_IMAGE_DOWNLOAD" { - log.Printf("[INFO] Should delete -> download new image %#v", incRequest.ExecutionArgument) + log.Printf("[INFO] Should delete -> download new images: %#v", incRequest.ExecutionArgument) if len(incRequest.ExecutionArgument) > 0 { - err = handleBackendImageDownload(ctx, incRequest.ExecutionArgument) - if err != nil { - log.Printf("[ERROR] Failed handling image delete -> download: %s", err) + // FIXME: Wait X seconds before running this as the image build may not be done yet. This is shitty, but may be ok to do in Orborus. Easy fix for the future: Just let it run through jobs 5-10 times before actually picking it up + + // Run after 25 seconds in the goroutine instead + go handleBackendImageDownload(ctx, incRequest.ExecutionArgument) + } else { + log.Printf("[ERROR] No image name provided for download. Removing job from queue.") + } + + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + + } else if incRequest.Type == "CATEGORY_UPDATE" { + + err := deployTenzirNode() + if err != nil { + log.Printf("[ERROR] Failed to run CATEGORY UPDATE, reason: %s", err) + } else { + continue + } + + err = handleFileCategoryChange() + if err != nil { + log.Printf("[ERROR] Failed to download the file category: %s", err) + } else { + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + } + + } else if incRequest.Type == "DISABLE_SIGMA_FILE" { + fileName := incRequest.ExecutionArgument + err := deployTenzirNode() + if err != nil { + log.Printf("[ERROR] Failed to run DISABLE SIGMA FILE, reason: %s", err) + } else { + continue + } + + err = disableRule(fileName) + if err != nil { + log.Printf("[ERROR] Failed to disable the sigma file %s, reason: %s", fileName, err) + } else { + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + } + + } else if incRequest.Type == "ENABLE_SIGMA_FILE" { + fileName := incRequest.ExecutionArgument + err := deployTenzirNode() + if err != nil { + log.Printf("[ERROR] Failed to run ENABLE SIGMA FILE, reason: %s", err) + } else { + continue + } + + err = enableRule(fileName) + if err != nil { + log.Printf("[ERROR] Failed to disable the sigma file %s, reason: %s", fileName, err) + } else { + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + } + + } else if incRequest.Type == "DISABLE_SIGMA_FOLDER" { + err := deployTenzirNode() + if err != nil { + log.Printf("[ERROR] Failed to run DISABLE SIGMA FOLDER, reason: %s", err) + } + + err = removeAllFiles() + if err != nil { + log.Printf("[ERROR] Failed to disable the sigma rules: %s", err) + } else { + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + } + } else if incRequest.Type == "START_TENZIR" { + log.Printf("[INFO] Got job to start tenzir") + + err := deployTenzirNode() + if err != nil { + if strings.Contains(fmt.Sprintf("%s", err), "node available") { + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + } else { + log.Printf("[ERROR] Failed to start tenzir, reason: %s", err) + err = shuffle.CreateOrgNotification( + ctx, + fmt.Sprintf("Failed to start Tenzir: %s", err), + fmt.Sprintf("Tenzir failed to start due to: %s", err), + fmt.Sprintf("/detections/Sigma"), + org, + true, + ) + + if err != nil { + log.Printf("[ERROR] Failed to send notification: %s", err) + return + } } + } else { + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) } - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + } else { newrequests = append(newrequests, incRequest) } @@ -1746,7 +2367,7 @@ func main() { log.Printf("[WARNING] Throttle - Cutting down requests from %d to %d (MAX: %d, CUR: %d)", len(executionRequests.Data), allowed, maxConcurrency, executionCount) executionRequests.Data = executionRequests.Data[0:allowed] } - } else if (swarmControlMode && (swarmConfig == "run" || swarmConfig == "swarm")) { + } else if swarmControlMode && (swarmConfig == "run" || swarmConfig == "swarm") { if len(executionRequests.Data) > 50 { executionRequests.Data = executionRequests.Data[0:50] } @@ -1807,6 +2428,7 @@ func main() { fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", os.Getenv("SHUFFLE_SWARM_CONFIG")), fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED")), fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", os.Getenv("SHUFFLE_BASE_IMAGE_NAME")), + fmt.Sprintf("SHUFFLE_ALLOW_PACKAGE_INSTAL=%s", os.Getenv("SHUFFLE_ALLOW_PACKAGE_INSTALL")), } //log.Printf("Running worker with proxy? %s", os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) @@ -1890,163 +2512,10 @@ func main() { } } - time.Sleep(time.Duration(sleepTime) * time.Second) } } - -// func deployPipeline(image, identifier, command string) error { -// if isKubernetes == "true" { -// return errors.New("Kubernetes not implemented") -// } - -// ctx := context.Background() -// hostConfig := &container.HostConfig{ -// LogConfig: container.LogConfig{ -// Type: "json-file", -// Config: map[string]string{ -// "max-size": "10m", -// }, -// }, -// Resources: container.Resources{}, -// } - -// hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) -// if strings.ToLower(cleanupEnv) != "false" { -// hostConfig.AutoRemove = true -// } - -// envVariables := []string{ -// } - - -// // Add volume binds for storage -// // Want read/write with full access for the container -// //sourceFolder := "/Users/frikky/git/shuffle/shuffle-database" -// //destinationFolder := "/tmp/storage" -// //hostConfig.Mounts = append(hostConfig.Mounts, mount.Mount{ -// // Type: mount.TypeBind, -// // Source: sourceFolder, -// // Target: destinationFolder, -// //}) - -// // FIXME: Is using sigma "automatically" here good? -// // Or is it better to run it as a separate workflow? -// if strings.Contains(command, "sigma") { -// log.Printf("[DEBUG] Should LOAD sigma from backend in realtime and dump it in a folder inside the container") - -// //sourceFolder := "/tmp/tenzir/sigma" -// //sigmaFolder := "/tmp/tenzir/sigma" -// //hostConfig.Mounts = append(hostConfig.Mounts, mount.Mount{ -// // Type: mount.TypeBind, -// // Source: sigmaFolder, -// // Target: sigmaFolder, -// //} -// } - -// config := &container.Config{ -// Image: image, -// Env: envVariables, -// Cmd: []string{ -// command, -// }, -// } - -// // Add label to container in case of zombies -// config.Labels = map[string]string{ -// "name": identifier, -// "shuffle": "shuffle", -// } - - -// cont, err := dockercli.ContainerCreate( -// ctx, -// config, -// hostConfig, -// nil, -// nil, -// identifier, -// ) - -// if err != nil { -// if strings.Contains(fmt.Sprintf("%s", err), "Conflict. The container name ") { -// log.Printf("[DEBUG] Pipeline Container %s already exists, removing it", identifier) -// } else { -// log.Printf("[ERROR] Failed to create pipeline container %s: %s", identifier, err) -// return err -// } -// } - -// containerStartOptions := container.StartOptions{} -// err = dockercli.ContainerStart( -// ctx, -// cont.ID, -// containerStartOptions, -// ) -// if err != nil { -// if strings.Contains(fmt.Sprintf("%s", err), "cannot join network") || strings.Contains(fmt.Sprintf("%s", err), "No such container") { -// hostConfig.NetworkMode = "" -// cont, err = dockercli.ContainerCreate( -// ctx, -// config, -// hostConfig, -// nil, -// nil, -// identifier+"-2", -// ) -// if err != nil { -// log.Printf("[ERROR] Failed to CREATE pipeline container (2): %s", err) -// } - -// err = dockercli.ContainerStart( -// ctx, -// cont.ID, -// containerStartOptions, -// ) -// if err != nil { -// log.Printf("[ERROR] Failed to start pipeline container (2): %s", err) -// return err -// } -// } else { -// log.Printf("[ERROR] Failed initial pipeline container start. Quitting as this is NOT a simple network issue. Err: %s", err) -// } - -// if err != nil { -// log.Printf("[ERROR] Failed to start pipeline container in environment %s: %s", environment, err) -// return err -// } else { -// log.Printf("[INFO] Pipeline Container created (1). Environment %s: docker logs %s", environment, cont.ID) -// } - -// stats, err := dockercli.ContainerInspect(ctx, cont.ID) -// if err != nil { -// log.Printf("[ERROR] Failed checking pipeline with containername '%s'", cont.ID) -// return nil -// } - -// containerStatus := stats.ContainerJSONBase.State.Status -// log.Printf("[DEBUG] Status of pipeline '%s' is %s. Should be running. Will reset", containerName, containerStatus) -// } - -// // Wait for the container to finish -// /* -// statusCh, errCh := dockercli.ContainerWait(ctx, cont.ID, container.WaitConditionNotRunning) -// select { -// case err := <-errCh: -// if err != nil { -// log.Printf("[ERROR] Failed to wait for container: %s", err) -// } -// case <-statusCh: -// log.Printf("[INFO] Container finished") -// } -// */ - -// return nil -// } - - - // Tenzir command samples // docker pull ghcr.io/dominiklohmann/tenzir-arm64:latest // docker tag ghcr.io/dominiklohmann/tenzir-arm64:latest tenzir/tenzir:latest @@ -2054,15 +2523,12 @@ func main() { // Read from Cache and send it to a webhook // docker run tenzir/tenzir:latest 'from http://192.168.86.44:5002/api/v1/orgs/7e9b9007-5df2-4b47-bca5-c4d267ef2943/cache/CIDR%20ranges?type=text&authorization=cec9d01f-09b2-4419-8a0a-76c6046e3fef read lines | to http://192.168.86.44:5002/api/v1/hooks/webhook_665ace5f-f27b-496a-a365-6e07eb61078c write lines' func handlePipeline(incRequest shuffle.ExecutionRequest) error { - - if tenzirUrl == "" { - tenzirUrl = "http://localhost:5160" - log.Printf("[WARNING] SHUFFLE_TENZIR_URL not set, falling back to default URL: %s",tenzirUrl) - } + + log.Printf("[INFO] Pipeline: %s to %s", incRequest.Type, incRequest.ExecutionSource) err := deployTenzirNode() - if err != nil{ - log.Printf("[ERROR] failed to deploy the pipeline, reason: %s", err) + if err != nil { + log.Printf("[ERROR] Failed to deploy the pipeline, reason: %s", err) return err } @@ -2072,10 +2538,13 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { return errors.New("no execution argument found for pipeline create. Skipping") } - //image := "tenzir/tenzir:latest" - identifier := fmt.Sprintf("shuffle-%s", strings.ToLower(strings.ReplaceAll(incRequest.ExecutionSource, " ", "-"))) - command := incRequest.ExecutionArgument + identifier := strings.ToLower(strings.ReplaceAll(incRequest.ExecutionSource, " ", "-")) + if !strings.HasPrefix(strings.ToLower(incRequest.ExecutionSource), "shuffle") { + identifier = fmt.Sprintf("shuffle-%s", strings.ToLower(strings.ReplaceAll(incRequest.ExecutionSource, " ", "-"))) + } + + command := incRequest.ExecutionArgument if incRequest.Type == "PIPELINE_CREATE" { log.Printf("[INFO] Should delete -> recreate new pipeline with id %#v", identifier) //err := deployPipeline(image, identifier, command) @@ -2084,51 +2553,59 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { log.Printf("[ERROR] Failed to create pipeline: %s", err) return err } - } else if incRequest.Type == "PIPELINE_DELETE" { - log.Printf("[INFO] Should delete pipeline %#v", identifier) - pipelineId, err := searchPipeline(identifier) - if err != nil { - log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) - return err - } - err = deletePipeline(pipelineId) - if err != nil { - log.Printf("[ERROR] Failed Deleting Pipeline %s", err) - return err - } - } else if incRequest.Type == "PIPELINE_STOP" { - log.Printf("[INFO] Should stop the pipeline %#v", identifier) - pipelineId, err := searchPipeline(identifier) - if err != nil { - log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) - return err - } - _, err = updatePipelineState(pipelineId, "stop") - if err != nil { - log.Printf("[ERROR] Failed to stop Pipeline: %s reason:%s ", pipelineId, err) - return err - } else { - log.Printf("[INFO] successfully stopped the Pipeline: %s", pipelineId) + } else if incRequest.Type == "PIPELINE_DELETE" || incRequest.Type == "PIPELINE_STOP" { + { + log.Printf("[INFO] Should delete pipeline %#v", identifier) + pipelineId, err := searchPipeline(identifier) + if err != nil { + log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) + return err + } + + err = deletePipeline(pipelineId) + if err != nil { + log.Printf("[ERROR] Failed Deleting Pipeline %s", err) + return err + } } - } else if incRequest.Type == "PIPELINE_START" { + /* + } else if incRequest.Type == "PIPELINE_STOP" { + log.Printf("[INFO] Should stop the pipeline %#v", identifier) + pipelineId, err := searchPipeline(identifier) + if err != nil { + log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + return err + } + _, err = updatePipelineState(command, pipelineId, "stop") + if err != nil { + log.Printf("[ERROR] Failed to stop Pipeline: %s reason:%s ", pipelineId, err) + return err + } else { + log.Printf("[INFO] Successfully stopped the Pipeline: %s", pipelineId) + } + */ + + } else if incRequest.Type == "PIPELINE_START" { log.Printf("[INFO] Should start the pipeline %#v", identifier) pipelineId, err := searchPipeline(identifier) - if err != nil { + if err != nil { if err.Error() == "no existing pipeline found with name" { - log.Printf("[WARNING] no pipeline found for %s, creating a new one", identifier) + log.Printf("[WARNING] No pipeline found for '%s', creating a new one", identifier) _, CreateErr := createPipeline(command, identifier) return CreateErr } + log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) return err } - _, err = updatePipelineState(pipelineId, "start") + _, err = updatePipelineState(command, pipelineId, "start") if err != nil { log.Printf("[ERROR] Failed to start Pipeline: %s reason:%s ", pipelineId, err) return err } else { - log.Printf("[INFO] successfully started the Pipeline: %s", pipelineId) + log.Printf("[INFO] Successfully started the Pipeline: %s", pipelineId) } } else { @@ -2140,203 +2617,381 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { } func deployTenzirNode() error { - if isKubernetes == "true" { - return errors.New("kubernetes not implemented") - } + if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "true" { + return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") + } - ctx := context.Background() - cacheKey := "tenzir-key" + if isKubernetes == "true" { + return errors.New("Kubernetes not implemented for Tenzir node") + } - imageName := "tenzir/tenzir:latest" - containerName := "tenzir-node" - containerStartOptions := container.StartOptions{} + err := checkTenzirNode() + if err == nil { + return nil + } - _, err := shuffle.GetCache(ctx, cacheKey) - if err == nil { - return nil - } + ctx := context.Background() + cacheKey := "tenzir-key" - containerInfo, err := dockercli.ContainerInspect(ctx, containerName) - if err != nil { - if dockerclient.IsErrNotFound(err) { + imageName := "tenzir/tenzir:latest" + containerName := "tenzir-node" + containerStartOptions := container.StartOptions{} + _, err = shuffle.GetCache(ctx, cacheKey) + if err == nil { + return nil + } - // Check if image exists - _, _, err := dockercli.ImageInspectWithRaw(ctx, imageName) - if dockerclient.IsErrNotFound(err) { - log.Printf("[DEBUG] pulling image %s", imageName) - pullOptions := types.ImagePullOptions{} - out, err := dockercli.ImagePull(ctx, imageName, pullOptions) - if err != nil { - log.Printf("[ERROR] Failed to pull the Tenzir image: %s", err) - return err - } - defer out.Close() + containerInfo, err := dockercli.ContainerInspect(ctx, containerName) + if err != nil { + if dockerclient.IsErrNotFound(err) { + // Create network if it doesn't exist + networkName := "tenzir-network" + networkSubnet := "192.168.1.0/24" + networkGateway := "192.168.1.1" - io.Copy(io.Discard, out) - } else if err != nil { - return err - } + err = createNetworkIfNotExists(ctx, networkName, networkSubnet, networkGateway) + if err != nil { + log.Printf("[ERROR] Failed to create network: %s", err) + return err + } - err = createAndStartTenzirNode(ctx, containerName, imageName, containerStartOptions) - if err != nil { - return err - } - } else { - return err - } - } else { - if !containerInfo.State.Running { - log.Printf("[DEBUG] Tenzir Node exists but is not running") - err := dockercli.ContainerStart(ctx, containerName, containerStartOptions) - if err != nil { - log.Printf("[ERROR] Failed to start Tenzir Node container: %v", err) - return err - } + // Check if image exists + _, _, err := dockercli.ImageInspectWithRaw(ctx, imageName) + if dockerclient.IsErrNotFound(err) { + log.Printf("[DEBUG] Pulling image %s. This may take a while.", imageName) + pullOptions := image.PullOptions{} + out, err := dockercli.ImagePull(ctx, imageName, pullOptions) + if err != nil { + log.Printf("[ERROR] Failed to pull the Tenzir image: %s", err) + return err + } + defer out.Close() - log.Printf("[INFO] Waiting for Tenzir to become available ...") - err = checkTenzirNode() - if err != nil { - return err - } - } - } + io.Copy(io.Discard, out) + } else if err != nil { + return err + } - tenzirStatus := struct { - ContainerStatus string `json:"container_status"` - }{ - ContainerStatus: "running", - } + err = createAndStartTenzirNode(ctx, containerName, imageName, containerStartOptions) + if err != nil { + return err + } + } else { + return err + } + } else { + if !containerInfo.State.Running { + log.Printf("[DEBUG] Tenzir Node exists but is not running. Restarting it.") + err := dockercli.ContainerStart(ctx, containerName, containerStartOptions) + if err != nil { + log.Printf("[ERROR] Failed to start Tenzir Node container: %v", err) + return err + } - cacheData, err := json.Marshal(tenzirStatus) - if err != nil { - log.Printf("[WARNING] Failed marshalling execution: %s", err) - } - err = shuffle.SetCache(ctx, cacheKey, cacheData, 1) - if err != nil { - log.Printf("[WARNING] Failed updating cache for tenzir: %s", err) - } + time.Sleep(10 * time.Second) + log.Printf("[INFO] Waiting for Tenzir to become available ...") + err = checkTenzirNode() + if err != nil { + return err + } + } + } - return nil + tenzirStatus := struct { + ContainerStatus string `json:"container_status"` + }{ + ContainerStatus: "running", + } + + cacheData, err := json.Marshal(tenzirStatus) + if err != nil { + log.Printf("[WARNING] Failed marshalling execution: %s", err) + } + err = shuffle.SetCache(ctx, cacheKey, cacheData, 1) + if err != nil { + log.Printf("[WARNING] Failed updating cache for tenzir: %s", err) + } + + return nil +} + +func createAndStartTenzirNode(ctx context.Context, containerName, imageName string, containerStartOptions container.StartOptions) error { + healthconfig := &container.HealthConfig{ + Test: []string{"tenzir --connection-timeout=30s --connection-retry-delay=1s 'api /ping'"}, + Interval: 30 * time.Second, + Retries: 1, + } + + // Ensure restart policy is there + config := &container.Config{ + Hostname: containerName, + Cmd: []string{"--commands=web server --mode=dev --bind=0.0.0.0"}, + Image: imageName, + Healthcheck: healthconfig, + ExposedPorts: nat.PortSet{ + "5160/tcp": struct{}{}, + "514/udp": struct{}{}, + "514/tcp": struct{}{}, + }, + Entrypoint: []string{containerName}, + Env: []string{}, + } + + tenzirApikey := os.Getenv("TENZIR_PLUGINS__PLATFORM__API_KEY") + tenzirControlEndpoint := os.Getenv("TENZIR_PLUGINS__PLATFORM__CONTROL_ENDPOINT") + tenzirPluginsPlatform := os.Getenv("TENZIR_PLUGINS__PLATFORM__TENANT_ID") + + anyFound := false + if len(tenzirApikey) > 0 { + config.Env = append(config.Env, fmt.Sprintf("TENZIR_PLUGINS__PLATFORM__API_KEY=%s", tenzirApikey)) + anyFound = true + } + + if len(tenzirControlEndpoint) > 0 { + config.Env = append(config.Env, fmt.Sprintf("TENZIR_PLUGINS__PLATFORM__CONTROL_ENDPOINT=%s", tenzirControlEndpoint)) + anyFound = true + } + + if len(tenzirPluginsPlatform) > 0 { + config.Env = append(config.Env, fmt.Sprintf("TENZIR_PLUGINS__PLATFORM__TENANT_ID=%s", tenzirPluginsPlatform)) + anyFound = true + } + + tenzirStorageFolder := os.Getenv("SHUFFLE_STORAGE_FOLDER") + if len(tenzirStorageFolder) > 0 { + tenzirStorageFolder = tenzirStorageFolder + + if !strings.HasSuffix(tenzirStorageFolder, "/") { + tenzirStorageFolder = tenzirStorageFolder + "/" + } + } else { + tenzirStorageFolder = "/tmp/" + log.Printf("[DEBUG] Using folder %s for Tenzir storage. Change it using SHUFFLE_STORAGE_FOLDER", tenzirStorageFolder) + } + + if !anyFound { + //log.Printf("[DEBUG] No Tenzir Plugin environment variables found.") + } else { + //log.Printf("[DEBUG] Attempting Tenzir connection with app.tenzir.com tenant '%s'", tenzirPluginsPlatform) + } + + hostConfig := &container.HostConfig{ + PortBindings: nat.PortMap{ + "514/tcp": []nat.PortBinding{{HostPort: "514"}}, + "514/udp": []nat.PortBinding{{HostPort: "514"}}, + "5160/tcp": []nat.PortBinding{{HostPort: "5160"}}, + }, + Mounts: []mount.Mount{ + { + Type: "bind", + Source: tenzirStorageFolder, + Target: "/var/lib/tenzir/", + }, + { + Type: "bind", + Source: tenzirStorageFolder, + Target: "/var/log/tenzir/", + }, + { + Type: "bind", + Source: tenzirStorageFolder, + Target: "/var/cache/tenzir/", + }, + }, + VolumeDriver: "local", + RestartPolicy: container.RestartPolicy{ + Name: "always", + }, + } + + if skipPipelineMount { + hostConfig.Mounts = []mount.Mount{} + } + + networkingConfig := &network.NetworkingConfig{ + EndpointsConfig: map[string]*network.EndpointSettings{ + "tenzir-network": { + IPAMConfig: &network.EndpointIPAMConfig{ + IPv4Address: "192.168.1.100", + }, + }, + }, + } + + if isKubernetes != "true" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" { + hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) + } + + resp, err := dockercli.ContainerCreate(ctx, config, hostConfig, networkingConfig, nil, containerName) + if err != nil { + if strings.Contains(err.Error(), "path does not exist") { + log.Printf("[ERROR] Not using permanent pipeline storage as storage folder %s does not exist. If you want permanent storage, create the %s folder then restart Orborus (1). Raw: %s", tenzirStorageFolder, tenzirStorageFolder, err) + skipPipelineMount = true + } else { + log.Printf("[ERROR] Failed to create Tenzir Node container: %v", err) + } + + return err + } + + if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" { + networkName := "shuffle_swarm_executions" + err = dockercli.NetworkConnect(ctx, networkName, resp.ID, nil) + if err != nil { + log.Printf("[ERROR] Error connecting tenzir container to network: %s", err) + } + } + + err = dockercli.ContainerStart(ctx, containerName, containerStartOptions) + if err != nil { + if strings.Contains(err.Error(), "path does not exist") { + log.Printf("[ERROR] Not using permanent pipeline storage as storage folder %s does not exist. If you want permanent storage, create the %s folder then restart Orborus (2). Raw: %s", tenzirStorageFolder, tenzirStorageFolder, err) + skipPipelineMount = true + } else { + log.Printf("[ERROR] Failed to START Tenzir Node container: %v", err) + } + + return err + } + + log.Printf("[INFO] Tenzir Node container started successfully. Waiting for it to become available..") + time.Sleep(20 * time.Second) + err = checkTenzirNode() + if err != nil { + log.Printf("[ERROR] Tenzir node is not available during deployment: %s", err) + return err + } + + log.Printf("[INFO] Successfully deployed Tenzir Node! Setting up default syslog listener on UDP 514") + + command := "from udp://0.0.0.0:514 read syslog | import" + _, err = createPipeline(command, "default-syslog-514") + if err != nil { + log.Printf("[ERROR] Failed to create default syslog pipeline: %s", err) + return nil + } + + return nil +} + +func createNetworkIfNotExists(ctx context.Context, networkName, subnet, gateway string) error { + networks, err := dockercli.NetworkList(ctx, types.NetworkListOptions{}) + if err != nil { + return err + } + + for _, network := range networks { + if network.Name == networkName { + // Network exists + return nil + } + } + + ipamConfig := &network.IPAM{ + Config: []network.IPAMConfig{ + { + Subnet: subnet, + Gateway: gateway, + }, + }, + } + + networkCreate := types.NetworkCreate{ + //CheckDuplicate: true, + Driver: "bridge", + IPAM: ipamConfig, + } + + _, err = dockercli.NetworkCreate(ctx, networkName, networkCreate) + if err != nil { + return err + } + + return nil } func checkTenzirNode() error { - retries := 20 - retryInterval := 3 * time.Second - url := fmt.Sprintf("%s/api/v0/ping",tenzirUrl) + if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "true" { + return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") + } + + url := fmt.Sprintf("%s/api/v0/ping", pipelineUrl) forwardMethod := "POST" - client := http.Client{} - req, err := http.NewRequest(forwardMethod, url, nil) + client := http.Client{ + Timeout: 1 * time.Second, + } + req, err := http.NewRequest(forwardMethod, url, nil) if err != nil { log.Printf("[ERROR] Failed to create HTTP request: %s", err) return err } - for i := 0; i < retries; i++ { - resp, err := client.Do(req) - if err == nil && resp.StatusCode == http.StatusOK { - return nil - } - time.Sleep(retryInterval) - } + resp, err := client.Do(req) + if err == nil && resp.StatusCode == http.StatusOK { + return nil + } - return fmt.Errorf("tenzir node is not available") -} - -func createAndStartTenzirNode(ctx context.Context, containerName, imageName string, containerStartOptions container.StartOptions) error { - healthconfig := &container.HealthConfig{ - Test: []string{"tenzir --connection-timeout=30s --connection-retry-delay=1s 'api /ping'"}, - Interval: 30 * time.Second, - Retries: 1, - } - - config := &container.Config{ - Cmd: []string{"--commands=web server --mode=dev --bind=0.0.0.0"}, - Image: imageName, - Healthcheck: healthconfig, - ExposedPorts: nat.PortSet{"5160/tcp": struct{}{}}, - Entrypoint: []string{containerName}, - } - - hostConfig := &container.HostConfig{ - PortBindings: nat.PortMap{ - "5160/tcp": []nat.PortBinding{{HostPort: "5160"}}, - }, - Mounts: []mount.Mount{ - { - Type: mount.TypeVolume, - Source: containerName, - Target: "/var/lib/tenzir/", - }, - }, - VolumeDriver: "local", - } - _, err := dockercli.ContainerCreate(ctx, config, hostConfig, nil, nil, containerName) - if err != nil { - return err - } - - err = dockercli.ContainerStart(ctx, containerName, containerStartOptions) - if err != nil { - log.Printf("[ERROR] Failed to start Tenzir Node container: %v", err) - return err - } - log.Printf("[INFO] Tenzir Node container started successfully") - - log.Printf("[INFO] Waiting for Tenzir to become available ...") - err = checkTenzirNode() - if err != nil { - return err - } - log.Printf("[INFO] Successfully deployed Tenzir Node !") - - return nil + return fmt.Errorf("Tenzir node is not available due to: %s", err) } func createPipeline(command, identifier string) (string, error) { - toBeDeleted := false - pipelineId, err := searchPipeline(identifier) + //toBeDeleted := false + /* + // Pre-checked. No point here + pipelineId, err := searchPipeline(identifier) + if err != nil { + return "", err + } + */ - url := fmt.Sprintf("%s/api/v0/pipeline/create", tenzirUrl) + url := fmt.Sprintf("%s/api/v0/pipeline/create", pipelineUrl) forwardMethod := "POST" - if err != nil { - if strings.Contains(fmt.Sprintf("%s", err), "no existing pipeline found") { - log.Printf("[INFO] No existing pipeline found with id: %s. Creating a new one!", identifier) + /* + if err != nil { + if strings.Contains(fmt.Sprintf("%s", err), "no existing pipeline found") { + log.Printf("[INFO] No existing pipeline found with id: %s. Creating a new one!", identifier) + } else { + log.Printf("[ERROR] Failed to search for existing pipeline but continuing anyway : %s", err) + } } else { - log.Printf("[ERROR] Failed to search for existing pipeline but continuing anyway : %s", err) + log.Printf("[INFO] an existing pipeline found with ID: %s. it will be deleted", pipelineId) + toBeDeleted = true } - } else { - log.Printf("[INFO] an existing pipeline found with ID: %s. it will be deleted", pipelineId) - toBeDeleted = true - } - if strings.Contains(command, "shuffler.io") { + */ - } else { - var scheme string - if strings.Contains(command, "http://") { - scheme = "http://" - } else if strings.Contains(command, "https://") { - scheme = "https://" - } + // if strings.Contains(command, "shuffler.io") { + + // } else { + // var scheme string + // if strings.Contains(command, "http://") { + // scheme = "http://" + // } else if strings.Contains(command, "https://") { + // scheme = "https://" + // } + + // startIndex := strings.Index(command, scheme) + // if startIndex != -1 { + // endIndex := startIndex + len(scheme) + // endIndex += strings.Index(command[endIndex:], "/") + + // command = command[:startIndex] + baseUrl + command[endIndex:] + // } + // } + + //command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | sigma /var/lib/tenzir/rule.yaml" + //command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | import" - startIndex := strings.Index(command, scheme) - if startIndex != -1 { - endIndex := startIndex + len(scheme) - endIndex += strings.Index(command[endIndex:], "/") - - command = command[:startIndex] + baseUrl + command[endIndex:] - } - } requestBody := map[string]interface{}{ "definition": command, "name": identifier, "hidden": false, "autostart": map[string]bool{ "created": true, - "completed": true, - "failed": true, + "completed": false, + "failed": false, }, "autodelete": map[string]bool{ "completed": false, @@ -2395,21 +3050,22 @@ func createPipeline(command, identifier string) (string, error) { id := response.ID - if toBeDeleted { - go deletePipeline(pipelineId) - } + //if toBeDeleted { + // go deletePipeline(pipelineId) + //} return id, nil } -func updatePipelineState(pipelineId, action string) (string, error) { +func updatePipelineState(command, pipelineId, action string) (string, error) { - url := fmt.Sprintf("%s/api/v0/pipeline/update", tenzirUrl) + url := fmt.Sprintf("%s/api/v0/pipeline/update", pipelineUrl) forwardMethod := "POST" requestBody := map[string]interface{}{ - "id": pipelineId, - "action": action, + "id": pipelineId, + "definition": command, + "action": action, "autostart": map[string]bool{ "created": true, "completed": true, @@ -2473,7 +3129,7 @@ func deletePipeline(pipelineId string) error { "id": pipelineId, } - url := fmt.Sprintf("%s/api/v0/pipeline/delete", tenzirUrl) + url := fmt.Sprintf("%s/api/v0/pipeline/delete", pipelineUrl) forwardMethod := "POST" requestBodyJSON, err := json.Marshal(requestBody) @@ -2508,44 +3164,48 @@ func deletePipeline(pipelineId string) error { return fmt.Errorf("got the status code %d instead of 200", resp.StatusCode) } - log.Printf("[INFO] pipeline with ID: %s deleted successfully", pipelineId) + log.Printf("[INFO] Pipeline with ID: %s deleted successfully", pipelineId) + + pipelines = []shuffle.PipelineInfoMini{} return nil } -func searchPipeline(identifier string) (string, error) { - - type pipelineInfo struct { - ID string `json:"id"` - Name string `json:"name"` - } - - var reqBody []byte - - url := fmt.Sprintf("%s/api/v0/pipeline/list", tenzirUrl) +// Lists the pipelines from the API exactly as they are. Definition is set up in Shuffle structs +func listPipelines() ([]shuffle.PipelineInfo, error) { + responseData := shuffle.PipelineInfoWrapper{} + var reqBody []byte + url := fmt.Sprintf("%s/api/v0/pipeline/list", pipelineUrl) resp, err := http.Post(url, "application/json", bytes.NewBuffer(reqBody)) - if err != nil { - return "", err - } - defer resp.Body.Close() + if err != nil { + return responseData.Pipelines, err + } + + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("got the status code %d instead of 200", resp.StatusCode) + return responseData.Pipelines, fmt.Errorf("Got the status code %d instead of 200 from Pipeline node", resp.StatusCode) } body, err := ioutil.ReadAll(resp.Body) if err != nil { - return "", err + return responseData.Pipelines, err } - var responseData struct { - Pipelines []pipelineInfo `json:"pipelines"` - } if err := json.Unmarshal(body, &responseData); err != nil { + return responseData.Pipelines, err + } + + return responseData.Pipelines, nil +} + +func searchPipeline(identifier string) (string, error) { + allPipelines, err := listPipelines() + if err != nil { return "", err } - for _, pipeline := range responseData.Pipelines { + for _, pipeline := range allPipelines { if pipeline.Name == identifier { return pipeline.ID, nil } @@ -2554,65 +3214,324 @@ func searchPipeline(identifier string) (string, error) { return "", errors.New("no existing pipeline found with name") } -// func savePipelineData(pipelineId, identifier, status string) error { +func handleFileCategoryChange() error { + apiEndpoint := baseUrl + "/api/v1/files/namespaces/sigma" + req, err := http.NewRequest("GET", apiEndpoint, nil) + if err != nil { + return err + } -// url := fmt.Sprintf("%s/api/v1/triggers/pipeline/save", baseUrl) -// identifierWithoutPrefix := strings.TrimPrefix(identifier, "shuffle-") + req.Header.Add("Authorization", "Bearer "+apiKey) -// forwardMethod := "PUT" + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() -// payload := map[string]interface{}{ -// "pipeline_id": pipelineId, -// "trigger_id": identifierWithoutPrefix, -// "status": status, -// } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("received non-200 response: %s", resp.Status) + } -// payloadBytes, err := json.Marshal(payload) -// if err != nil { -// log.Printf("[ERROR] Failed to marshal payload: %s", err) -// return err -// } + out, err := os.Create("files.zip") + if err != nil { + return err + } -// forwardData := bytes.NewBuffer(payloadBytes) + defer out.Close() + defer os.Remove("files.zip") -// req, err := http.NewRequest( -// forwardMethod, -// url, -// forwardData, -// ) -// if err != nil { -// log.Printf("[ERROR] Failed to create HTTP request: %s", err) -// return err -// } -// req.Header.Set("Content-Type", "application/json") + _, err = io.Copy(out, resp.Body) + if err != nil { + return err + } -// client := &http.Client{Timeout: 10 * time.Second} -// resp, err := client.Do(req) -// if err != nil { -// log.Printf("[ERROR] Failed to send HTTP request: %s", err) -// return err -// } -// defer resp.Body.Close() + log.Println("ZIP file downloaded successfully.") -// if resp.StatusCode != 200 { -// log.Printf("[ERROR] Received non-successful HTTP status code: %d", resp.StatusCode) -// return fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode) -// } + err = extractZIP("files.zip", "sigma_rules") + if err != nil { + return err + } -// return nil -// } + destPath := "/var/lib/tenzir/sigma_rules" + + err = copyToTenzir("sigma_rules", destPath) + if err != nil { + return err + } + + log.Println("Files copied to container successfully.") + + checkDisabledDirCmd := exec.Command("docker", "exec", "tenzir-node", "sh", "-c", "test -d /var/lib/tenzir/disabled_rules") + if err := checkDisabledDirCmd.Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { + // Directory does not exist, nothing to do + log.Println("[DEBUG] /var/lib/tenzir/disabled_rules does not exist.") + return nil + } + + return fmt.Errorf("error checking disabled rules directory: %v", err) + } + + // List files in /var/lib/tenzir/disabled_rules + listFilesCmd := exec.Command("docker", "exec", "tenzir-node", "sh", "-c", "ls /var/lib/tenzir/disabled_rules") + output, err := listFilesCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error listing files in disabled rules directory: %v, output: %s", err, output) + } + + files := strings.Split(strings.TrimSpace(string(output)), "\n") + for _, file := range files { + disabledFilePath := fmt.Sprintf("/var/lib/tenzir/sigma_rules/%s", file) + checkFileCmd := exec.Command("docker", "exec", "tenzir-node", "sh", "-c", fmt.Sprintf("test -f %s", disabledFilePath)) + if err := checkFileCmd.Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { + log.Printf("[ERROR] File does not exist: %s, moving on.\n", disabledFilePath) + continue + } + return fmt.Errorf("error checking file: %v", err) + } + + deleteFileCmd := exec.Command("docker", "exec", "-u", "root", "tenzir-node", "sh", "-c", fmt.Sprintf("rm -f %s", disabledFilePath)) + if err := deleteFileCmd.Run(); err != nil { + return fmt.Errorf("error deleting file: %v", err) + } + log.Printf("[INFO] Deleted file: %s\n", disabledFilePath) + } + + return nil +} + +func extractZIP(zipFile, destDir string) error { + r, err := zip.OpenReader(zipFile) + if err != nil { + return err + } + defer r.Close() + + if err := os.MkdirAll(destDir, 0755); err != nil { + return err + } + + for _, f := range r.File { + err := extractFile(f, destDir) + if err != nil { + return err + } + } + + return nil +} + +func extractFile(f *zip.File, destDir string) error { + rc, err := f.Open() + if err != nil { + return err + } + defer rc.Close() + + path := filepath.Join(destDir, f.Name) + + out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, rc) + return err +} + +func copyToTenzir(srcPath, destPath string) error { + containerName := "tenzir-node" + + checkCmd := exec.Command("docker", "exec", containerName, "test", "-d", destPath) + if err := checkCmd.Run(); err == nil { + rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", "-rf", destPath) + if err := rmCmd.Run(); err != nil { + return fmt.Errorf("error removing existing directory in container: %v", err) + } + } + + cpCmd := exec.Command("docker", "cp", srcPath, fmt.Sprintf("%s:%s", containerName, destPath)) + var out bytes.Buffer + cpCmd.Stdout = &out + cpCmd.Stderr = &out + + err := cpCmd.Run() + if err != nil { + return fmt.Errorf("error copying files: %v, output: %s", err, out.String()) + } + + return nil +} + +func removeAllFiles() error { + containerName := "tenzir-node" + sigmaPath := "/var/lib/tenzir/sigma_rules/*" + + checkCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("ls %s", sigmaPath)) + checkOutput, checkErr := checkCmd.CombinedOutput() + if checkErr != nil { + if strings.Contains(string(checkOutput), "No such file or directory") { + return nil // nothing to delete + } + return fmt.Errorf("error checking files: %v, output: %s", checkErr, checkOutput) + } + + cmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("rm -rf %s", sigmaPath)) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error removing files: %v, output: %s", err, output) + } + return nil +} + +func removeFile(fileName string) error { + containerName := "tenzir-node" + srcPath := fmt.Sprintf("/var/lib/tenzir/sigma_rules/%s", fileName) + + checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) + if err := checkSrcCmd.Run(); err != nil { + // If the file does not exist, simply return nil + if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { + log.Printf("[ERROR] No such file: %s, nothing to delete\n", srcPath) + return nil + } + return fmt.Errorf("error checking source file: %v", err) + } + + return removePath(containerName, srcPath) +} + +func removePath(containerName, path string) error { + rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("rm -rf %s", path)) + output, err := rmCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error removing path: %v, output: %s", err, output) + } + return nil +} + +func sendPipelineHealthStatus() (shuffle.LakeConfig, error) { + pipelinePayload := shuffle.LakeConfig{ + Enabled: false, + Pipelines: []shuffle.PipelineInfoMini{}, + } + + // To not spam down the list API too much + randint := rand.Intn(5) + if len(pipelines) == 0 || randint == 0 { + pipelineDef, err := listPipelines() + + if err == nil { + for _, pipeline := range pipelineDef { + pipelinePayload.Pipelines = append(pipelinePayload.Pipelines, shuffle.PipelineInfoMini{ + ID: pipeline.ID, + Name: pipeline.Name, + Definition: pipeline.Definition, + TotalRuns: pipeline.TotalRuns, + CreatedAt: pipeline.CreatedAt, + }) + } + + pipelines = pipelinePayload.Pipelines + } + } else { + pipelinePayload.Pipelines = pipelines + } + + if tenzirDisabled { + return pipelinePayload, nil + } + + err := deployTenzirNode() + if err != nil { + if (!strings.Contains(err.Error(), "SHUFFLE_SKIP_PIPELINES") && !strings.Contains(err.Error(), "Kubernetes not implemented for Tenzir node")) && !strings.Contains(err.Error(), "Tenzir Node is already running") && !strings.Contains(err.Error(), "docker daemon") { + + log.Printf("[ERROR] Tenzir node connection problem: %s", err) + } else { + tenzirDisabled = true + log.Printf("[ERROR] Disabling pipelines: %s. You will need to restart the Orborus to fix this.", err) + } + + return pipelinePayload, err + } + + pipelinePayload.Enabled = true + + // No direct sending. + return pipelinePayload, nil +} + +func disableRule(fileName string) error { + containerName := "tenzir-node" + srcPath := fmt.Sprintf("/var/lib/tenzir/sigma_rules/%s", fileName) + destDir := "/var/lib/tenzir/disabled_rules" + destPath := fmt.Sprintf("%s/%s", destDir, fileName) + + checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) + if err := checkSrcCmd.Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { + fmt.Printf("File does not exist: %s\n", srcPath) + return nil // Nothing to disable + } + return fmt.Errorf("error checking source file: %v", err) + } + + checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mkdir -p %s", destDir)) + if err := checkDestDirCmd.Run(); err != nil { + return fmt.Errorf("error ensuring destination directory exists: %v", err) + } + + moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mv %s %s", srcPath, destPath)) + if err := moveCmd.Run(); err != nil { + return fmt.Errorf("error moving file: %v", err) + } + + fmt.Printf("File %s moved to %s successfully.\n", fileName, destDir) + return nil +} + +func enableRule(fileName string) error { + containerName := "tenzir-node" + srcPath := fmt.Sprintf("/var/lib/tenzir/disabled_rules/%s", fileName) + destDir := "/var/lib/tenzir/sigma_rules" + destPath := fmt.Sprintf("%s/%s", destDir, fileName) + + checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) + if err := checkSrcCmd.Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { + fmt.Printf("File does not exist: %s\n", srcPath) + return nil // Nothing to enable + } + return fmt.Errorf("error checking source file: %v", err) + } + + checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mkdir -p %s", destDir)) + if err := checkDestDirCmd.Run(); err != nil { + return fmt.Errorf("error ensuring destination directory exists: %v", err) + } + moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mv %s %s", srcPath, destPath)) + if err := moveCmd.Run(); err != nil { + return fmt.Errorf("error moving file: %v", err) + } + + fmt.Printf("[DEBUG] File %s moved to %s successfully.\n", fileName, destDir) + return nil +} // Is this ok to do with Docker? idk :) func getRunningWorkers(ctx context.Context, workerTimeout int) int { //log.Printf("[DEBUG] Getting running workers with API version %s", dockerApiVersion) counter := 0 - if isKubernetes == "true" { + if isKubernetes == "true" { log.Printf("[INFO] Getting running workers in kubernetes") - thresholdTime := time.Now().Add(time.Duration(-workerTimeout) * time.Second) - clientset, _, err := getKubernetesClient() + clientset, _, err := shuffle.GetKubernetesClient() if err != nil { log.Printf("[ERROR] Failed getting kubernetes client: %s", err) return 0 @@ -2644,7 +3563,7 @@ func getRunningWorkers(ctx context.Context, workerTimeout int) int { // Automatically updates the version if err != nil { - log.Printf("[ERROR] Error getting containers: %s", err) + log.Printf("[ERROR] Error getting containers from Docker: %s", err) newVersionSplit := strings.Split(fmt.Sprintf("%s", err), "version is") if len(newVersionSplit) > 1 { @@ -2736,7 +3655,7 @@ func zombiecheck(ctx context.Context, workerTimeout int) error { // Check image name if !shuffleFound { - log.Printf("[WARNING] Zombie container skip: %#v, %s", container.Labels, container.Image) + //log.Printf("[WARNING] Zombie container skip: %#v, %s", container.Labels, container.Image) continue } //} else { @@ -2792,7 +3711,7 @@ func zombiecheck(ctx context.Context, workerTimeout int) error { return nil } -func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest) error { +func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, env []string) error { parsedRequest := shuffle.OrborusExecutionRequest{ ExecutionId: workflowExecution.ExecutionId, Authorization: workflowExecution.Authorization, @@ -2825,15 +3744,37 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest) error { streamUrl = fmt.Sprintf("%s:33333/api/v1/execute", parsedBaseurl) } - if len(workerServerUrl) > 0 { - streamUrl = fmt.Sprintf("%s:33333/api/v1/execute", workerServerUrl) + identifier := "shuffle-workers" + + if isKubernetes == "true" { + if shuffle.IsRunningInCluster() { + log.Printf("[INFO] Running in Kubernetes cluster") + // try getting the k8s worker server url + } } - if strings.Contains(streamUrl, "shuffler.io") || strings.Contains(streamUrl, "localhost") || strings.Contains(streamUrl, "shuffle-backend") { - log.Printf("[INFO] Using default worker server url as previous is invalid: %s", streamUrl) + if strings.Contains(streamUrl, "shuffler.io") || strings.Contains(streamUrl, "localhost") || strings.Contains(streamUrl, "127.0.0.1") || strings.Contains(streamUrl, "shuffle-backend") { + + // Specific to debugging + if len(workerServerUrl) == 0 { + log.Printf("[INFO] Using default worker server url as previous is invalid: %s", streamUrl) + } + streamUrl = fmt.Sprintf("http://shuffle-workers:33333/api/v1/execute") } + if len(workerServerUrl) > 0 { + // Check if a port is supplied or not + if strings.Contains(workerServerUrl, "/api/v1/execute") { + streamUrl = workerServerUrl + } else { + streamUrl = fmt.Sprintf("%s/api/v1/execute", workerServerUrl) + if !strings.Contains(workerServerUrl, ":") { + streamUrl = fmt.Sprintf("%s:33333/api/v1/execute", workerServerUrl) + } + } + } + client := &http.Client{} req, err := http.NewRequest( "POST", @@ -2849,7 +3790,12 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest) error { if len(newWorkerImage) > 0 { workerImage = newWorkerImage } - deployServiceWorkers(workerImage) + + if isKubernetes == "true" { + deployK8sWorker(workerImage, identifier, env) + } else { + deployServiceWorkers(workerImage) + } time.Sleep(time.Duration(10) * time.Second) //err = sendWorkerRequest(executionRequest) @@ -2860,7 +3806,9 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest) error { newresp, err := client.Do(req) if err != nil { + // Connection refused? log.Printf("[ERROR] Error running worker request to %s (1): %s", streamUrl, err) + if strings.Contains(fmt.Sprintf("%s", err), "connection refused") || strings.Contains(fmt.Sprintf("%s", err), "EOF") { workerImage := fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion) @@ -2868,7 +3816,11 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest) error { workerImage = newWorkerImage } - deployServiceWorkers(workerImage) + if isKubernetes == "true" { + deployK8sWorker(workerImage, identifier, env) + } else { + deployServiceWorkers(workerImage) + } time.Sleep(time.Duration(10) * time.Second) //err = sendWorkerRequest(executionRequest) @@ -2883,6 +3835,7 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest) error { log.Printf("[ERROR] Failed reading body in worker request body to worker on %s: %s", streamUrl, err) return err } + window.AddEvent(time.Now()) if newresp.StatusCode != 200 { log.Printf("[WARNING] POTENTIAL error running worker request (2) - status code is %d for %s, not 200. Body: %s", newresp.StatusCode, streamUrl, string(body)) @@ -2897,6 +3850,329 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest) error { _ = body - log.Printf("[DEBUG] Ran worker from request with execution ID: %s. Worker URL: %s. DEBUGGING:\ndocker service logs shuffle-workers 2>&1 -f | grep %s", workflowExecution.ExecutionId, streamUrl, workflowExecution.ExecutionId) + debugCommand := fmt.Sprintf("docker service logs shuffle-workers 2>&1 -f | grep %s", workflowExecution.ExecutionId) + if isKubernetes == "true" { + debugCommand = fmt.Sprintf("kubectl logs -n %s container=shuffle-worker | grep %s", kubernetesNamespace, workflowExecution.ExecutionId) + } + + log.Printf("[DEBUG] Ran worker from request with execution ID: %s. Worker URL: %s. DEBUGGING:\n%s", workflowExecution.ExecutionId, streamUrl, debugCommand) + return nil +} + +func AutoScale(ctx context.Context) { + if os.Getenv("SHUFFLE_SCALE_REPLICAS") != "" { + return + } + + ticker := time.NewTicker(1 * time.Second) + coolDownPeriod := 10 * time.Second + queuePerMinuteInt = 20 + if os.Getenv("SHUFFLE_QUEUE_PER_MINUTE") != "" { + var err error + queuePerMinuteInt, err = strconv.Atoi(os.Getenv("SHUFFLE_QUEUE_PER_MINUTE")) + if err != nil { + log.Printf("[WARNING] Cannot convert %s to int. Using default value for it: %d", queuePerMinute, queuePerMinuteInt) + } + } + + lastScaleTime := time.Now() + currentWorkers := currentWokerCount(ctx, dockercli) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if time.Since(lastScaleTime) < (coolDownPeriod) { + continue + } + currentRequestCount := window.CountEvents(time.Now()) + requiredReplicas := 0 + if currentRequestCount >= queuePerMinuteInt*currentWorkers { + // FIXME: Hardcoded Max Replicas should be 6 + requiredReplicas = int(math.Min(float64(6), float64(currentRequestCount/queuePerMinuteInt)+1)) + } + + if requiredReplicas > 0 { + err := scaleService(ctx, dockercli, uint64(requiredReplicas)) + if err != nil { + log.Printf("[ERROR] Failed to scale service: %s", err) + } else { + lastScaleTime = time.Now() + currentWorkers = currentWokerCount(ctx, dockercli) + } + } + } + } +} + +func scaleService(ctx context.Context, client *dockerclient.Client, replicas uint64) error { + service, _, err := client.ServiceInspectWithRaw(ctx, "shuffle-workers", types.ServiceInspectOptions{}) + if err != nil { + return err + } + + if service.Spec.Mode.Replicated == nil { + return errors.New("Service cannot be replicated") + } + + if *service.Spec.Mode.Replicated.Replicas >= replicas { + return nil + } + + service.Spec.Mode.Replicated.Replicas = &replicas + + _, err = dockercli.ServiceUpdate(ctx, service.ID, service.Version, service.Spec, types.ServiceUpdateOptions{}) + if err != nil { + return err + } + + log.Printf("[INFO] Scaled shuffle-worker to %d replicas", replicas) + return nil +} + +func currentWokerCount(ctx context.Context, client *dockerclient.Client) int { + service, _, err := client.ServiceInspectWithRaw(ctx, "shuffle-workers", types.ServiceInspectOptions{}) + if err != nil { + return 0 + } + + if service.Spec.Mode.Replicated == nil { + return 0 + } + + return int(*service.Spec.Mode.Replicated.Replicas) +} + +func queueScaleFactor(numQueue int, queuePerMin int) float64 { + if numQueue > queuePerMin { + queuePressure := float64(numQueue) / float64(queuePerMin) + return 1.0 + math.Min(queuePressure-1.0, 1.0) + } + + return 1.0 +} + +func checkMemcached(ctx context.Context, dockercli *dockerclient.Client) (bool, error) { + containerName := "shuffle-cache" + continer, err := dockercli.ContainerInspect(context.Background(), containerName) + if err != nil { + if dockerclient.IsErrNotFound(err) { + return false, nil + } + return false, err + } + networkName := "shuffle_swarm_executions" + err = dockercli.NetworkConnect(ctx, networkName, containerName, nil) + if err != nil { + log.Printf("[WARNING] Failed connecting memcached container to network: %s", err) + } + + if continer.State.Running == false { + log.Printf("[INFO] Container %s exists but is not running. Attempting to start it.", containerName) + err = dockercli.ContainerStart(ctx, containerName, container.StartOptions{}) + if err != nil { + log.Printf("[ERROR] Failed to start container %s: %v", containerName, err) + return false, err + } + log.Printf("[INFO] Successfully started container %s.", containerName) + return true, nil + } + + return continer.State.Running, nil +} + +func deployMemcached(dockercli *dockerclient.Client) error { + if os.Getenv("SHUFFLE_MEMCACHED") != "" { + return errors.New("Memcached already running") + } + + defaultMem := "1024" + log.Printf("[INFO] Spanning a default memcached container to handle the distribution between cache across different workers. Default memory assigned %s", defaultMem) + + ctx := context.Background() + + memcachedImage := "docker.io/library/memcached:latest" + containerConfig := &container.Config{ + Image: memcachedImage, + Cmd: []string{"-m", defaultMem}, + } + + hostConfig := &container.HostConfig{ + PortBindings: nat.PortMap{ + "11211/tcp": []nat.PortBinding{{HostPort: "11211"}}, + }, + } + + _, _, err := dockercli.ImageInspectWithRaw(ctx, memcachedImage) + if dockerclient.IsErrNotFound(err) { + log.Printf("[DEBUG] Pulling image %s. This may take a while.", memcachedImage) + pullOptions := image.PullOptions{} + out, err := dockercli.ImagePull(ctx, memcachedImage, pullOptions) + if err != nil { + log.Printf("[ERROR] Failed to pull the memcached image: %s", err) + return err + } + defer out.Close() + + io.Copy(io.Discard, out) + } else if err != nil { + return err + } + + containerName := "shuffle-cache" + resp, err := dockercli.ContainerCreate(ctx, containerConfig, hostConfig, nil, nil, containerName) + if err != nil { + log.Printf("[ERROR] Error spanning memcached continer: %s", err) + return err + } + + if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" { + networkName := "shuffle_swarm_executions" + err = dockercli.NetworkConnect(ctx, networkName, resp.ID, nil) + if err != nil { + log.Printf("[ERROR] Error connecting tenzir container to network: %s", err) + } + } + + err = dockercli.ContainerStart(ctx, resp.ID, container.StartOptions{}) + if err != nil { + log.Printf("[ERROR] Error starting memcached continer: %s", err) + return err + } + + networkName := "shuffle_swarm_executions" + err = dockercli.NetworkConnect(ctx, networkName, resp.ID, nil) + if err != nil { + log.Printf("[ERROR] Error connecting memcached container to network: %s", err) + } + + log.Printf("[INFO] Memcached container started successfully at port 11211") + + return nil +} + +// How do we get the cpu usage? maybe just get the number of requests (much more useful for apps) +/* +func nodesResourceUsage(ctx context.Context, client *dockerclient.Client) error { + nodes, err := client.NodeList(ctx, types.NodeListOptions{}) + if err != nil { + return err + } + for _, node := range nodes { + res := node.Description.Resources + } + + return nil +} +*/ + +/* +func numberOfReplicas(ctx context.Context, queueLength int, config shuffle.ScalingConfig) (int, int) { + queueScaleFactor := queueScaleFactor(queueLength, config) + numReplicas := int(float64(queueLength) * queueScaleFactor) + serviceName := "shuffle-workers" + nodes, err := dockercli.NodeList(ctx, types.NodeListOptions{}) + if err != nil { + log.Printf("[ERROR] Cannot find any nodes in the swarm network") + } + + filterArgs := filters.NewArgs() + filterArgs.Add("service", serviceName) + filterArgs.Add("desired-state", "running") + + tasks, err := dockercli.TaskList(context.Background(), types.TaskListOptions{ + Filters: filterArgs, + }) + if err != nil { + log.Fatalf("[WARNING] Failed to list tasks for service %s: %s", serviceName, err) + } + + runningReplicas := len(tasks) + if numReplicas > runningReplicas*len(nodes) { + maxIncrease := config.MaxScaleUpStep + if numReplicas > (runningReplicas*len(nodes) + maxIncrease) { + numReplicas = runningReplicas + maxIncrease + } + } + + if numReplicas < config.MinReplicas { + numReplicas = config.MinReplicas + } + if numReplicas > config.MaxReplicas { + numReplicas = config.MaxReplicas + } + + return numReplicas, runningReplicas +} +*/ + +// TODO: Currently we use number of request made for the worker to run a execution as it is much +// easier to track in a window time frame. But this could be useful. +func collectMetrics(ctx context.Context, dockerClient *dockerclient.Client) (int, error) { + client := shuffle.GetExternalClient(baseUrl) + fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl) + req, err := http.NewRequest("GET", fullUrl, nil) + if err != nil { + log.Printf("[ERROR] Failed to send a request to %s: %s", fullUrl, err) + return 0, err + } + + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Org-Id", environment) + if len(auth) > 0 { + req.Header.Add("Authorization", auth) + } + + if len(org) > 0 { + req.Header.Add("Org", org) + } + + if len(orborusLabel) > 0 { + log.Printf("[DEBUG] Sending with Label '%s'", orborusLabel) + req.Header.Add("X-Orborus-Label", orborusLabel) + } + + if swarmConfig != "run" && swarmConfig != "swarm" { + req.Header.Add("X-Orborus-Runmode", "Default") + } else { + req.Header.Add("X-Orborus-Runmode", "Docker Swarm") + } + + resp, err := client.Do(req) + if err != nil { + return 0, err + } + + var executionRequests shuffle.ExecutionRequestWrapper + body, err := ioutil.ReadAll(resp.Body) + + json.Unmarshal(body, &executionRequests) + + return len(executionRequests.Data), nil +} + +func setBackendToSwarmNetwork(ctx context.Context) error { + containerId := "" + filterArgs := filters.NewArgs() + filterArgs.Add("name", "shuffle-backend") + + containers, err := dockercli.ContainerList(ctx, container.ListOptions{ + All: true, + Filters: filterArgs, + }) + if err != nil { + return err + } + if len(containers) == 0 { + return errors.New("No containers found with name shuffle-backend") + } + + containerId = containers[0].ID + networkName := "shuffle_swarm_executions" + err = dockercli.NetworkConnect(ctx, networkName, containerId, nil) + if err != nil { + log.Printf("[ERROR] Error connecting backend container to network: %s", err) + } + return nil } diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index e038ad2d..25043bb9 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -6,10 +6,10 @@ require ( github.com/docker/docker v26.1.5+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.6.30 - k8s.io/api v0.30.0 - k8s.io/apimachinery v0.30.0 - k8s.io/client-go v0.30.0 + github.com/shuffle/shuffle-shared v0.6.90 + k8s.io/api v0.30.2 + k8s.io/apimachinery v0.30.2 + k8s.io/client-go v0.30.2 ) require ( @@ -27,7 +27,7 @@ require ( github.com/algolia/algoliasearch-client-go/v3 v3.18.1 // indirect github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 // indirect github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect - github.com/cloudflare/circl v1.3.3 // indirect + github.com/cloudflare/circl v1.3.7 // indirect github.com/containerd/log v0.1.0 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -38,7 +38,7 @@ require ( github.com/emirpasic/gods v1.18.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/kin-openapi v0.41.0 // indirect - github.com/frikky/schemaless v0.0.11 // indirect + github.com/frikky/schemaless v0.0.13 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect @@ -79,6 +79,8 @@ require ( github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/sashabaranov/go-openai v1.19.2 // indirect + github.com/sendgrid/rest v2.6.9+incompatible // indirect + github.com/sendgrid/sendgrid-go v3.14.0+incompatible // indirect github.com/sergi/go-diff v1.1.0 // indirect github.com/skeema/knownhosts v1.2.1 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index 76255634..503b0854 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -6,45 +6,24 @@ cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxK cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/datastore v1.4.0/go.mod h1:d18825/a9bICdAIJy2EkHs9joU4RlIZ1t6l8WDdbdY0= cloud.google.com/go/datastore v1.15.0 h1:0P9WcsQeTWjuD1H14JIY7XQscIPQ4Laje8ti96IC5vg= cloud.google.com/go/datastore v1.15.0/go.mod h1:GAeStMBIt9bPS7jMJA85kgkpsMkvseWWXiaHya9Jes8= cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= cloud.google.com/go/storage v1.36.0 h1:P0mOkAcaJxhCTvAkMhxMfrTKiNcub4YmmPBtlhAyTr8= cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= @@ -95,10 +74,10 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= @@ -119,7 +98,6 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= -github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -127,7 +105,6 @@ github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FM github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= @@ -135,17 +112,14 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.41.0 h1:oMmjo+ekGS971lb3KLeZZOqRDZOwWi3+g/OiSWP08+s= github.com/frikky/kin-openapi v0.41.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.11 h1:c4r6CJX30XI+SoJdT9RlUd9qYSQlx6hvwGRtsypu+uM= -github.com/frikky/schemaless v0.0.11/go.mod h1:mooDxY+D6weHjhKvjy3+IE9S7P4g4cpNnidkdRv/cHQ= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/frikky/schemaless v0.0.13 h1:ARiN9V7wr2VZXAr9JK5wvTbyPgpGrgeiL1VhR5MlgaQ= +github.com/frikky/schemaless v0.0.13/go.mod h1:mooDxY+D6weHjhKvjy3+IE9S7P4g4cpNnidkdRv/cHQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= @@ -154,10 +128,7 @@ github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3c github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -170,7 +141,6 @@ github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -185,26 +155,19 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -215,12 +178,8 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -235,21 +194,11 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= @@ -270,9 +219,7 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= @@ -301,7 +248,6 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= @@ -315,46 +261,8 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= -github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= -github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= -github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= -github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= -github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= -github.com/onsi/ginkgo/v2 v2.8.1/go.mod h1:N1/NbDngAFcSLdyZ+/aYTYGSlq9qMCS/cNKGJjy+csc= -github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxmrTcxyk= -github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= -github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= -github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= -github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0= -github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= -github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= -github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= -github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= -github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= -github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= -github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw= -github.com/onsi/gomega v1.27.3/go.mod h1:5vG284IBtfDAmDyrK+eGyZmUgUlmi+Wngqo557cZ6Gw= -github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= -github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= -github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4= -github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE= github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -369,15 +277,12 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= @@ -385,12 +290,17 @@ github.com/sashabaranov/go-openai v1.19.2 h1:+dkuCADSnwXV02YVJkdphY8XD9AyHLUWwk6 github.com/sashabaranov/go-openai v1.19.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0= +github.com/sendgrid/rest v2.6.9+incompatible/go.mod h1:kXX7q3jZtJXK5c5qK83bSGMdV6tsOE70KbHoqJls4lE= +github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fcif2i7P7wzISv1sU6DUA= +github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shuffle/shuffle-shared v0.6.30 h1:EdM2mKgGIK2PJWqZ1iFxwQyzwcmHg/3WFBiAexXrudM= -github.com/shuffle/shuffle-shared v0.6.30/go.mod h1:rWkh1eWdIx7OqQzJ1+JzF3Hck1X/Ty1WkUtjLrp+CU4= +github.com/shuffle/shuffle-shared v0.6.74 h1:os3BDSFZnl4U8ZgsTAY8IsTDADcMXhbc1rS9UMa0BIY= +github.com/shuffle/shuffle-shared v0.6.74/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus= +github.com/shuffle/shuffle-shared v0.6.90 h1:FzIYtEt44eWgEsW/9tj2ki7qq8FEm/HWXUok+THp72M= +github.com/shuffle/shuffle-shared v0.6.90/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skeema/knownhosts v1.2.1 h1:SHWdIUa82uGZz+F+47k8SY4QhhI291cXCpopT1lK2AQ= @@ -406,29 +316,22 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 h1:UNQQKPfTDe1J81ViolILjTKPr9WetKW6uei2hFgJmFs= @@ -451,7 +354,6 @@ go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxi go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= -golang.org/x/arch v0.1.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -459,13 +361,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -475,9 +372,7 @@ golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm0 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -489,31 +384,19 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -522,48 +405,22 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -571,10 +428,6 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ= golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -583,18 +436,12 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -603,71 +450,32 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -675,23 +483,16 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -713,47 +514,15 @@ golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= -golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= -golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= -golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -770,18 +539,6 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQWYWo= -google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.162.0 h1:Vhs54HkaEpkMBdgGdOT2P6F0csGG/vxDS0hWHJzmmps= google.golang.org/api v0.162.0/go.mod h1:6SulDkfoBIg4NFmCuZ39XeeAgSHCPecfSUuDyYlAHs0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -789,8 +546,6 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -805,31 +560,8 @@ google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0= @@ -844,15 +576,7 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.34.1/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.63.0 h1:WjKe+dnvABXyPJMD7KDNLxtoGk5tgk+YFWN6cBWjZE8= google.golang.org/grpc v1.63.0/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -863,11 +587,9 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -876,16 +598,13 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -898,14 +617,12 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.30.0 h1:siWhRq7cNjy2iHssOB9SCGNCl2spiF1dO3dABqZ8niA= -k8s.io/api v0.30.0/go.mod h1:OPlaYhoHs8EQ1ql0R/TsUgaRPhpKNxIMrKQfWUp8QSE= -k8s.io/apimachinery v0.30.0 h1:qxVPsyDM5XS96NIh9Oj6LavoVFYff/Pon9cZeDIkHHA= -k8s.io/apimachinery v0.30.0/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/client-go v0.30.0 h1:sB1AGGlhY/o7KCyCEQ0bPWzYDL0pwOZO4vAtTSh/gJQ= -k8s.io/client-go v0.30.0/go.mod h1:g7li5O5256qe6TYdAMyX/otJqMhIiGgTapdLchhmOaY= +k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= +k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI= +k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= +k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.2 h1:sBIVJdojUNPDU/jObC+18tXWcTJVcwyqS9diGdWHk50= +k8s.io/client-go v0.30.2/go.mod h1:JglKSWULm9xlJLx4KCkfLLQ7XwtlbflV6uFFSHTMgVs= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= @@ -913,7 +630,6 @@ k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVh k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index fe145066..189a619e 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -25,19 +25,24 @@ import ( "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/mount" dockerclient "github.com/docker/docker/client" + // This is for automatic removal of certain code :) + /*** STARTREMOVE ***/ + "math/rand" + + "github.com/docker/docker/api/types/swarm" + uuid "github.com/satori/go.uuid" + + /*** ENDREMOVE ***/ "github.com/gorilla/mux" - "github.com/satori/go.uuid" //k8s deps + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/util/homedir" - "path/filepath" ) // This is getting out of hand :) @@ -53,10 +58,10 @@ var dockerApiVersion = strings.ToLower(os.Getenv("DOCKER_API_VERSION")) var baseimagename = "frikky/shuffle" var kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") +var executionCount int64 // var baseimagename = os.Getenv("SHUFFLE_BASE_IMAGE_NAME") - // var baseimagename = "registry.hub.docker.com/frikky/shuffle" var registryName = "registry.hub.docker.com" var sleepTime = 2 @@ -66,6 +71,7 @@ var requestsSent = 0 var appsInitialized = false var hostname string +var maxReplicas = uint64(12) /* var environments []string @@ -92,14 +98,18 @@ type ImageRequest struct { var finishedExecutions []string var imagesDistributed []string +var imagedownloadTimeout = time.Second * 300 + +var window = shuffle.NewTimeWindow(10 * time.Second) // Images to be autodeployed in the latest version of Shuffle. var autoDeploy = map[string]string{ - "http:1.4.0": "frikky/shuffle:http_1.4.0", - "http:1.3.0": "frikky/shuffle:http_1.3.0", - "shuffle-tools:1.2.0": "frikky/shuffle:shuffle-tools_1.2.0", - "shuffle-subflow:1.0.0": "frikky/shuffle:shuffle-subflow_1.0.0", - "shuffle-subflow:1.1.0": "frikky/shuffle:shuffle-subflow_1.1.0", + "http:1.4.0": "frikky/shuffle:http_1.4.0", + "http:1.3.0": "frikky/shuffle:http_1.3.0", + "shuffle-tools:1.2.0": "frikky/shuffle:shuffle-tools_1.2.0", + "shuffle-subflow:1.0.0": "frikky/shuffle:shuffle-subflow_1.0.0", + "shuffle-subflow:1.1.0": "frikky/shuffle:shuffle-subflow_1.1.0", + "shuffle-tools-fork:1.0.0": "frikky/shuffle:shuffle-tools-fork_1.0.0", } //"testing:1.0.0": "frikky/shuffle:testing_1.0.0", @@ -135,7 +145,7 @@ func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.Workflo err = shuffle.SetCache(ctx, cacheKey, execData, 30) if err != nil { - log.Printf("[ERROR][%s] Failed adding to cache during setexecution", workflowExecution) + log.Printf("[ERROR][%s] Failed adding to cache during setexecution", workflowExecution.ExecutionId) return err } @@ -311,6 +321,11 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason */ } else { + /*** STARTREMOVE ***/ + if os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm" { + log.Printf("[DEBUG][%s] NOT cleaning up containers. IDS: %d, CLEANUP env: %s", workflowExecution.ExecutionId, 0, cleanupEnv) + } + /*** ENDREMOVE ***/ } if len(reason) > 0 && len(nodeId) > 0 { @@ -368,7 +383,7 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason //Finished shutdown (after %d seconds). ", sleepDuration) // Allows everything to finish in subprocesses (apps) - if os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm" { + if os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm" && isKubernetes != "true" { time.Sleep(time.Duration(sleepDuration) * time.Second) os.Exit(3) } else { @@ -376,37 +391,65 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason } } -// Deploys the internal worker whenever something happens -func deployApp(cli *dockerclient.Client, image string, identifier string, env []string, workflowExecution shuffle.WorkflowExecution, action shuffle.Action) error { - if isKubernetes == "true" { - if len(os.Getenv("KUBERNETES_NAMESPACE")) > 0 { - kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") - } else { - kubernetesNamespace = "default" +func int32Ptr(i int32) *int32 { return &i } + +// ** STARTREMOVE ***/ +func deployk8sApp(image string, identifier string, env []string) error { + if len(os.Getenv("KUBERNETES_NAMESPACE")) > 0 { + kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") + } else { + kubernetesNamespace = "default" + } + + envMap := make(map[string]string) + for _, envStr := range env { + parts := strings.SplitN(envStr, "=", 2) + if len(parts) == 2 { + envMap[parts[0]] = parts[1] } + } - envMap := make(map[string]string) - for _, envStr := range env { - parts := strings.SplitN(envStr, "=", 2) - if len(parts) == 2 { - envMap[parts[0]] = parts[1] - } + // add to env + // fmt.Sprintf("SHUFFLE_APP_EXPOSED_PORT=%d", deployport), + // fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", os.Getenv("SHUFFLE_SWARM_CONFIG")), + envMap["SHUFFLE_APP_EXPOSED_PORT"] = "80" + envMap["SHUFFLE_SWARM_CONFIG"] = os.Getenv("SHUFFLE_SWARM_CONFIG") + envMap["BASE_URL"] = "http://shuffle-workers:33333" + + clientset, _, err := shuffle.GetKubernetesClient() + if err != nil { + log.Printf("[ERROR] Failed getting kubernetes: %s", err) + return err + } + + // str := strings.ToLower(identifier) + // strSplit := strings.Split(str, "_") + // value := strSplit[0] + // value = strings.ReplaceAll(value, "_", "-") + value := identifier + + baseDeployMode := false + + // check if autoDeploy contains a value + // that is equal to the image being deployed. + for _, value := range autoDeploy { + if value == image { + baseDeployMode = true } + } - clientset, err := getKubernetesClient() - if err != nil { - log.Printf("[ERROR] Failed getting kubernetes: %s", err) - return err - } + autoDeployOverride := os.Getenv("SHUFFLE_USE_GHCR_OVERRIDE_FOR_AUTODEPLOY") == "true" - str := strings.ToLower(identifier) - strSplit := strings.Split(str, "_") - value := strSplit[0] - value = strings.ReplaceAll(value, "_", "-") + localRegistry := "" - // Checking if app is generated or not - localRegistry := os.Getenv("REGISTRY_URL") - /* + // Checking if app is generated or not + if !(baseDeployMode && autoDeployOverride) { + localRegistry = os.Getenv("REGISTRY_URL") + } else { + log.Printf("[DEBUG] Detected baseDeploy image (%s) and ghcr override. Resorting to using ghcr instead of registry", image) + } + + /* appDetails := strings.Split(image, ":")[1] appDetailsSplit := strings.Split(appDetails, "_") appName := strings.Join(appDetailsSplit[:len(appDetailsSplit)-1], "_") @@ -424,63 +467,278 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] } } } - */ + */ - if len(localRegistry) == 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0 { - localRegistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY") + if (len(localRegistry) == 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0) && !(baseDeployMode && autoDeployOverride) { + localRegistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY") + } + + if (len(localRegistry) > 0 && strings.Count(image, "/") <= 2) && !(baseDeployMode && autoDeployOverride) { + log.Printf("[DEBUG] Using REGISTRY_URL %s", localRegistry) + image = fmt.Sprintf("%s/%s", localRegistry, image) + } else { + if strings.Count(image, "/") <= 2 && !strings.HasPrefix(image, "frikky/shuffle:") { + image = fmt.Sprintf("frikky/shuffle:%s", image) } + } - if len(localRegistry) > 0 && strings.Count(image, "/") <= 2 { - log.Printf("[DEBUG] Using REGISTRY_URL %s", localRegistry) - image = fmt.Sprintf("%s/%s", localRegistry, image) + log.Printf("[DEBUG] Got kubernetes with namespace %#v to run image '%s'", kubernetesNamespace, image) + + //fix naming convention + // podUuid := uuid.NewV4().String() + // podName := fmt.Sprintf("%s-%s", value, podUuid) + // replace identifier "_" with "-" + podName := strings.ReplaceAll(identifier, "_", "-") + + // pod := &corev1.Pod{ + // ObjectMeta: metav1.ObjectMeta{ + // Name: podName, + // Labels: map[string]string{ + // "app": podName, + // // "executionId": workflowExecution.ExecutionId, + // }, + // }, + // Spec: corev1.PodSpec{ + // RestartPolicy: "Never", // As a crash is not useful in this context + // // DNSPolicy: "Default", + // DNSPolicy: corev1.DNSClusterFirst, + // // NodeName: "worker1" + // Containers: []corev1.Container{ + // { + // Name: value, + // Image: image, + // Env: buildEnvVars(envMap), + + // // Pull if not available + // ImagePullPolicy: corev1.PullIfNotPresent, + // }, + // }, + // }, + // } + + // createdPod, err := clientset.CoreV1().Pods(kubernetesNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) + // if err != nil { + // log.Printf("[ERROR] Failed creating pod: %v", err) + // // os.Exit(1) + // } else { + // log.Printf("[DEBUG] Created pod %#v in namespace %#v", createdPod.Name, kubernetesNamespace) + // } + + // service := &corev1.Service{ + // ObjectMeta: metav1.ObjectMeta{ + // Name: identifier, + // }, + // Spec: corev1.ServiceSpec{ + // Selector: map[string]string{ + // "app": podName, + // }, + // Ports: []corev1.ServicePort{ + // { + // Protocol: "TCP", + // Port: 80, + // TargetPort: intstr.FromInt(80), + // }, + // }, + // Type: corev1.ServiceTypeNodePort, + // }, + // } + + // _, err = clientset.CoreV1().Services(kubernetesNamespace).Create(context.TODO(), service, metav1.CreateOptions{}) + // if err != nil { + // log.Printf("[ERROR] Failed creating service: %v", err) + // return err + // } + + // use deployment instead of pod + // then expose a service similarly. + // number of replicas can be set to os.Getenv("SHUFFLE_SCALE_REPLICAS") + replicaNumberStr := os.Getenv("SHUFFLE_SCALE_REPLICAS") + replicaNumber := 1 + if len(replicaNumberStr) > 0 { + tmpInt, err := strconv.Atoi(replicaNumberStr) + if err != nil { + log.Printf("[ERROR] %s is not a valid number for replication", replicaNumberStr) } else { - if strings.Count(image, "/") <= 2 { - image = fmt.Sprintf("frikky/shuffle:%s", image) - } + replicaNumber = tmpInt + } + } - log.Printf("[DEBUG] Got kubernetes with namespace %#v to run image '%s'", kubernetesNamespace, image) + replicaNumberInt32 := int32(replicaNumber) - //fix naming convention - podUuid := uuid.NewV4().String() - podName := fmt.Sprintf("%s-%s", value, podUuid) - - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Labels: map[string]string{ - "app": "shuffle-app", - "executionId": workflowExecution.ExecutionId, + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: int32Ptr(replicaNumberInt32), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": podName, }, }, - Spec: corev1.PodSpec{ - RestartPolicy: "Never", // As a crash is not useful in this context - DNSPolicy: "Default", - // NodeName: "worker1" - Containers: []corev1.Container{ - { - Name: value, - Image: image, - Env: buildEnvVars(envMap), - - // Pull if not available - ImagePullPolicy: corev1.PullIfNotPresent, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": podName, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: value, + Image: image, + Env: buildEnvVars(envMap), + }, }, }, }, - } - - createdPod, err := clientset.CoreV1().Pods(kubernetesNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) - if err != nil { - log.Printf("[ERROR] Failed creating pod: %v", err) - // os.Exit(1) - } else { - log.Printf("[DEBUG] Created pod %#v in namespace %#v", createdPod.Name, kubernetesNamespace) - } - - return nil + }, } + _, err = clientset.AppsV1().Deployments(kubernetesNamespace).Create(context.Background(), deployment, metav1.CreateOptions{}) + if err != nil { + log.Printf("[ERROR] Failed creating deployment: %v", err) + return err + } + + // kubectl expose deployment {podName} --type=NodePort --port=80 --target-port=80 + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app": podName, + }, + Ports: []corev1.ServicePort{ + { + Protocol: "TCP", + Port: 80, + TargetPort: intstr.FromInt(80), + }, + }, + Type: corev1.ServiceTypeNodePort, + }, + } + + _, err = clientset.CoreV1().Services(kubernetesNamespace).Create(context.TODO(), service, metav1.CreateOptions{}) + if err != nil { + log.Printf("[ERROR] Failed creating service: %v", err) + return err + } + + return nil +} + +//** ENDREMOVE ***/ + +// Deploys the internal worker whenever something happens +func deployApp(cli *dockerclient.Client, image string, identifier string, env []string, workflowExecution shuffle.WorkflowExecution, action shuffle.Action) error { + // if isKubernetes == "true" { + // if len(os.Getenv("KUBERNETES_NAMESPACE")) > 0 { + // kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") + // } else { + // kubernetesNamespace = "default" + // } + + // envMap := make(map[string]string) + // for _, envStr := range env { + // parts := strings.SplitN(envStr, "=", 2) + // if len(parts) == 2 { + // envMap[parts[0]] = parts[1] + // } + // } + + // clientset, _, err := shuffle.GetKubernetesClient() + // if err != nil { + // log.Printf("[ERROR] Failed getting kubernetes: %s", err) + // return err + // } + + // str := strings.ToLower(identifier) + // strSplit := strings.Split(str, "_") + // value := strSplit[0] + // value = strings.ReplaceAll(value, "_", "-") + + // // Checking if app is generated or not + // localRegistry := os.Getenv("REGISTRY_URL") + // /* + // appDetails := strings.Split(image, ":")[1] + // appDetailsSplit := strings.Split(appDetails, "_") + // appName := strings.Join(appDetailsSplit[:len(appDetailsSplit)-1], "_") + // appVersion := appDetailsSplit[len(appDetailsSplit)-1] + // for _, app := range workflowExecution.Workflow.Actions { + // // log.Printf("[DEBUG] App: %s, Version: %s", appName, appVersion) + // // log.Printf("[DEBUG] Checking app %s with version %s", app.AppName, app.AppVersion) + // if app.AppName == appName && app.AppVersion == appVersion { + // if app.Generated == true { + // log.Printf("[DEBUG] Generated app, setting local registry") + // image = fmt.Sprintf("%s/%s", localRegistry, image) + // break + // } else { + // log.Printf("[DEBUG] Not generated app, setting shuffle registry") + // } + // } + // } + // */ + + // if len(localRegistry) == 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0 { + // localRegistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY") + // } + + // if len(localRegistry) > 0 && strings.Count(image, "/") <= 2 { + // log.Printf("[DEBUG] Using REGISTRY_URL %s", localRegistry) + // image = fmt.Sprintf("%s/%s", localRegistry, image) + // } else { + // if strings.Count(image, "/") <= 2 { + // image = fmt.Sprintf("frikky/shuffle:%s", image) + // } + // } + + // log.Printf("[DEBUG] Got kubernetes with namespace %#v to run image '%s'", kubernetesNamespace, image) + + // //fix naming convention + // podUuid := uuid.NewV4().String() + // podName := fmt.Sprintf("%s-%s", value, podUuid) + + // pod := &corev1.Pod{ + // ObjectMeta: metav1.ObjectMeta{ + // Name: podName, + // Labels: map[string]string{ + // "app": "shuffle-app", + // "executionId": workflowExecution.ExecutionId, + // }, + // }, + // Spec: corev1.PodSpec{ + // RestartPolicy: "Never", // As a crash is not useful in this context + // // DNSPolicy: "Default", + // DNSPolicy: corev1.DNSClusterFirst, + // // NodeName: "worker1" + // Containers: []corev1.Container{ + // { + // Name: value, + // Image: image, + // Env: buildEnvVars(envMap), + + // // Pull if not available + // ImagePullPolicy: corev1.PullIfNotPresent, + // }, + // }, + // }, + // } + + // createdPod, err := clientset.CoreV1().Pods(kubernetesNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) + // if err != nil { + // log.Printf("[ERROR] Failed creating pod: %v", err) + // // os.Exit(1) + // } else { + // log.Printf("[DEBUG] Created pod %#v in namespace %#v", createdPod.Name, kubernetesNamespace) + // } + + // return nil + // } + // form basic hostConfig ctx := context.Background() @@ -514,6 +772,88 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] } } + /*** STARTREMOVE ***/ + if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || os.Getenv("SHUFFLE_SWARM_CONFIG") == "swarm" { + + appName := strings.Replace(identifier, fmt.Sprintf("_%s", action.ID), "", -1) + appName = strings.Replace(appName, fmt.Sprintf("_%s", workflowExecution.ExecutionId), "", -1) + appName = strings.ToLower(appName) + //log.Printf("[INFO][%s] New appname: %s, image: %s", workflowExecution.ExecutionId, appName, image) + + if !shuffle.ArrayContains(downloadedImages, image) && isKubernetes != "true" { + log.Printf("[DEBUG] Downloading image %s from backend as it's first iteration for this image on the worker. Timeout: 60", image) + // FIXME: Not caring if it's ok or not. Just continuing + // This is working as intended, just designed to download an updated + // image on every Orborus/new worker restart. + + // Running as coroutine for eventual completeness + // FIXME: With goroutines it got too much trouble of deploying with an older version + // Allowing slow startups, as long as it's eventually fast, and uses the same registry as on host. + shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image) + } + + var exposedPort int + var err error + + if isKubernetes != "true" { + exposedPort, err = findAppInfo(image, appName) + if err != nil { + log.Printf("[ERROR] Failed finding and creating port for %s: %s", appName, err) + return err + } + } else { + // ** STARTREMOVE ***/ + exposedPort = 80 + err = findAppInfoKubernetes(image, appName, env) + if err != nil { + log.Printf("[ERROR] Failed finding and creating port for %s: %s", appName, err) + return err + } + // ** ENDREMOVE ***/ + } + + /* + // Makes it not run at all. + cacheData := []byte("1") + newExecId := fmt.Sprintf("%s_%s", workflowExecution.ExecutionId, action.ID) + err = shuffle.SetCache(ctx, newExecId, cacheData, 30) + if err != nil { + log.Printf("[WARNING] (1) Failed setting cache for action %s: %s", newExecId, err) + } else { + log.Printf("[DEBUG][%s] (1) Adding %s to cache (%#v)", workflowExecution.ExecutionId, newExecId, action.Name) + } + */ + + log.Printf("[DEBUG][%s] Should run towards port %d for app %s. DELAY: %d", workflowExecution.ExecutionId, exposedPort, appName, action.ExecutionDelay) + ctx := context.Background() + if action.ExecutionDelay > 0 { + //log.Printf("[DEBUG] Running app %s with delay of %d", action.Name, action.ExecutionDelay) + waitTime := time.Duration(action.ExecutionDelay) * time.Second + + time.AfterFunc(waitTime, func() { + err = sendAppRequest(ctx, baseUrl, appName, exposedPort, &action, &workflowExecution) + if err != nil { + log.Printf("[ERROR] Failed sending SCHEDULED request to app %s on port %d: %s", appName, exposedPort, err) + } + }) + + } else { + rand.Seed(time.Now().UnixNano()) + waitTime := time.Duration(rand.Intn(500)) * time.Millisecond + + // Added a random delay + context timeout to ensure that the function returns, and only once + time.AfterFunc(waitTime, func() { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() // Cancel the context to release resources even if not used + + go sendAppRequest(ctx, baseUrl, appName, exposedPort, &action, &workflowExecution) + }) + } + + return nil + } + /*** ENDREMOVE ***/ + // Max 10% CPU every second //CPUShares: 128, //CPUQuota: 10000, @@ -615,8 +955,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] } func cleanupKubernetesExecution(clientset *kubernetes.Clientset, workflowExecution shuffle.WorkflowExecution, namespace string) error { - - workerName := fmt.Sprintf("worker-%s", workflowExecution.ExecutionId) + // workerName := fmt.Sprintf("worker-%s", workflowExecution.ExecutionId) labelSelector := fmt.Sprintf("app=shuffle-app,executionId=%s", workflowExecution.ExecutionId) podList, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{ @@ -634,11 +973,11 @@ func cleanupKubernetesExecution(clientset *kubernetes.Clientset, workflowExecuti log.Printf("App %s in namespace %s deleted.", pod.Name, namespace) } - podErr := clientset.CoreV1().Pods(namespace).Delete(context.TODO(), workerName, metav1.DeleteOptions{}) - if podErr != nil { - return fmt.Errorf("[ERROR] failed to delete the worker %s in namespace %s: %v", workerName, namespace, podErr) - } - log.Printf("[DEBUG] %s in namespace %s deleted.", workerName, namespace) + // podErr := clientset.CoreV1().Pods(namespace).Delete(context.TODO(), workerName, metav1.DeleteOptions{}) + // if podErr != nil { + // return fmt.Errorf("[ERROR] failed to delete the worker %s in namespace %s: %v", workerName, namespace, podErr) + // } + // log.Printf("[DEBUG] %s in namespace %s deleted.", workerName, namespace) return nil } @@ -821,6 +1160,48 @@ func removeIndex(s []string, i int) []string { func getWorkerURLs() ([]string, error) { workerUrls := []string{} + if isKubernetes == "true" { + workerUrls = append(workerUrls, "http://shuffle-workers:33333") + // workerUrls = append(workerUrls, "http://192.168.29.16:33333") + + // get service "shuffle-workers" "Endpoints" + // serviceName := "shuffle-workers" + // clientset, _, err := shuffle.GetKubernetesClient() + // if err != nil { + // log.Println("[ERROR] Failed to get Kubernetes client:", err) + // return workerUrls, err + // } + + // services, err := clientset.CoreV1().Services("default").List(context.Background(), metav1.ListOptions{}) + // if err != nil { + // log.Println("[ERROR] Failed to list services:", err) + // return workerUrls, err + // } + + // for _, service := range services.Items { + // if service.Name == serviceName { + // endpoints, err := clientset.CoreV1().Endpoints("default").Get(context.Background(), serviceName, metav1.GetOptions{}) + // if err != nil { + // log.Println("[ERROR] Failed to get endpoints for service:", err) + // return workerUrls, err + // } + + // for _, subset := range endpoints.Subsets { + // for _, address := range subset.Addresses { + // for _, port := range subset.Ports { + // url := fmt.Sprintf("http://%s:%d", address.IP, port.Port) + // workerUrls = append(workerUrls, url) + // } + // } + // } + // } + // } + + log.Printf("[DEBUG] Worker URLs for k8s: %#v", workerUrls) + + return workerUrls, nil + } + // Create a new Docker client cli, err := dockerclient.NewEnvClient() if err != nil { @@ -873,13 +1254,12 @@ func askOtherWorkersToDownloadImage(image string) { return } - if len(urls) < 2{ + if len(urls) < 2 { return } - httpClient := &http.Client{} - distributed := false + distributed := false for _, url := range urls { //log.Printf("[DEBUG] Trying to speak to: %s", url) imagesRequest := ImageRequest{ @@ -935,13 +1315,17 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { startAction, extra, children, parents, visited, executed, nextActions, environments := shuffle.GetExecutionVariables(ctx, workflowExecution.ExecutionId) - dockercli, err := dockerclient.NewEnvClient() - if err != nil { - log.Printf("[ERROR] Unable to create docker client (3): %s", err) - return - } + var dockercli *dockerclient.Client + var err error - defer dockercli.Close() + if isKubernetes != "true" { + dockercli, err = dockerclient.NewEnvClient() + if err != nil { + log.Printf("[ERROR] Unable to create docker client (3): %s", err) + return + } + defer dockercli.Close() + } for _, action := range relevantActions { appname := action.AppName @@ -972,22 +1356,24 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { //executed = append(executed, action.ID) // FIXME - check whether it's running locally yet too - - stats, err := dockercli.ContainerInspect(context.Background(), identifier) - if err != nil || stats.ContainerJSONBase.State.Status != "running" { - // REMOVE - if err == nil { - log.Printf("[DEBUG][%s] Docker Container Status: %s, should kill: %s", workflowExecution.ExecutionId, stats.ContainerJSONBase.State.Status, identifier) - err = removeContainer(identifier) - if err != nil { - log.Printf("Error killing container: %s", err) + // take care of auto clean up later on for k8s + if isKubernetes != "true" { + stats, err := dockercli.ContainerInspect(context.Background(), identifier) + if err != nil || stats.ContainerJSONBase.State.Status != "running" { + // REMOVE + if err == nil { + log.Printf("[DEBUG][%s] Docker Container Status: %s, should kill: %s", workflowExecution.ExecutionId, stats.ContainerJSONBase.State.Status, identifier) + err = removeContainer(identifier) + if err != nil { + log.Printf("[ERROR] Error killing container: %s", err) + } + } else { + //log.Printf("WHAT TO DO HERE?: %s", err) } - } else { - //log.Printf("WHAT TO DO HERE?: %s", err) + } else if stats.ContainerJSONBase.State.Status == "running" { + //log.Printf(" + continue } - } else if stats.ContainerJSONBase.State.Status == "running" { - //log.Printf(" - continue } if len(action.Parameters) == 0 { @@ -1127,7 +1513,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { return } - err := downloadDockerImageBackend(&http.Client{Timeout: 60 * time.Second}, image) + err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image) executed := false if err == nil { log.Printf("[DEBUG] Downloaded image %s from backend (CLEANUP)", image) @@ -1240,13 +1626,15 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { } log.Printf("[DEBUG][%s] Failed deploy. Downloading image %s: %s", workflowExecution.ExecutionId, image, err) - err := downloadDockerImageBackend(&http.Client{Timeout: 60 * time.Second}, image) + err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image) + executed := false if err == nil { log.Printf("[DEBUG] Downloaded image %s from backend (CLEANUP)", image) //err = deployApp(dockercli, image, identifier, env, workflow, action) err = deployApp(dockercli, image, identifier, env, workflowExecution, action) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { + log.Printf("[ERROR] Err: %s", err) if strings.Contains(err.Error(), "exited prematurely") { log.Printf("[DEBUG] Shutting down (40)") shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) @@ -1261,6 +1649,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { image = images[2] err = deployApp(dockercli, image, identifier, env, workflowExecution, action) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { + log.Printf("[ERROR] Err: %s", err) if strings.Contains(err.Error(), "exited prematurely") { log.Printf("[DEBUG] Shutting down (11)") shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) @@ -1269,6 +1658,11 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { log.Printf("[WARNING] Failed deploying image THREE TIMES. Attempting to download %s as last resort from backend and dockerhub: %s", image, err) + if isKubernetes == "true" { + log.Printf("[ERROR] Image %s doesn't exist. Returning error for now") + return + } + reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { log.Printf("[ERROR] Failed getting %s. The couldn't be find locally, AND is missing.", image) @@ -1378,7 +1772,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { log.Printf("[DEBUG][%s] Shutting down (17)", workflowExecution.ExecutionId) if isKubernetes == "true" { // log.Printf("workflow execution: %#v", workflowExecution) - clientset, err := getKubernetesClient() + clientset, _, err := shuffle.GetKubernetesClient() if err != nil { log.Println("[ERROR] Error getting kubernetes client (1):", err) os.Exit(1) @@ -1587,7 +1981,7 @@ func handleSubflowPoller(ctx context.Context, workflowExecution shuffle.Workflow log.Printf("[DEBUG] Shutting down (20)") if isKubernetes == "true" { // log.Printf("workflow execution: %#v", workflowExecution) - clientset, err := getKubernetesClient() + clientset, _, err := shuffle.GetKubernetesClient() if err != nil { log.Println("[ERROR] Error getting kubernetes client (2):", err) os.Exit(1) @@ -1623,9 +2017,8 @@ func handleSubflowPoller(ctx context.Context, workflowExecution shuffle.Workflow shutdown(workflowExecution, "", "", true) } - log.Printf("[INFO][%s] (2) Status: %s, Results: %d, actions: %d. Userinput: %#v", workflowExecution.ExecutionId, workflowExecution.Status, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extra, hasUserinput) - return errors.New("Subflow status not found yet") + return errors.New("Subflow status not found yet") } func handleDefaultExecutionWrapper(ctx context.Context, workflowExecution shuffle.WorkflowExecution, streamResultUrl string, extra int) error { @@ -1683,7 +2076,7 @@ func handleDefaultExecutionWrapper(ctx context.Context, workflowExecution shuffl log.Printf("[DEBUG] Shutting down (20)") if isKubernetes == "true" { // log.Printf("workflow execution: %#v", workflowExecution) - clientset, err := getKubernetesClient() + clientset, _, err := shuffle.GetKubernetesClient() if err != nil { log.Println("[ERROR] Error getting kubernetes client (2):", err) os.Exit(1) @@ -1700,7 +2093,7 @@ func handleDefaultExecutionWrapper(ctx context.Context, workflowExecution shuffl log.Printf("[DEBUG] Shutting down (21)") if isKubernetes == "true" { // log.Printf("workflow execution: %#v", workflowExecution) - clientset, err := getKubernetesClient() + clientset, _, err := shuffle.GetKubernetesClient() if err != nil { log.Println("[ERROR] Error getting kubernetes client (3):", err) os.Exit(1) @@ -1888,143 +2281,91 @@ func buildEnvVars(envMap map[string]string) []corev1.EnvVar { return envVars } -func getKubernetesClient() (*kubernetes.Clientset, error) { - - // Gets the config content from Orborus. - kubeconfigContent := os.Getenv("KUBERNETES_CONFIG") - if len(kubeconfigContent) > 0 { - log.Printf("[INFO] Using KUBERNETES_CONFIG to set up Kubernetes client: %#v", os.Getenv("KUBERNETES_CONFIG")) - config, err := rest.InClusterConfig() - if err != nil { - log.Printf("[ERROR] Failed to create Kubernetes client from in-cluster config: %s", err) - } else { - // Replace client configuration with kubeconfig content - config, err = clientcmd.RESTConfigFromKubeConfig([]byte(kubeconfigContent)) - if err != nil { - log.Printf("[ERROR] Failed to create Kubernetes client from KUBERNETES_CONFIG: %s", err) - } else { - // Create Kubernetes client - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, err - } - - return clientset, nil - } - } - } - - // Fallback - if isRunningInCluster() { - config, err := rest.InClusterConfig() - if err != nil { - return nil, err - } - - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, err - } - - return clientset, nil - } - - home := homedir.HomeDir() - kubeconfigPath := filepath.Join(home, ".kube", "config") - config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) - if err != nil { - return nil, err - } - - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, err - } - - return clientset, nil -} - func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { -if request.Body == nil { - resp.WriteHeader(http.StatusBadRequest) - return -} - -defer request.Body.Close() -body, err := ioutil.ReadAll(request.Body) -if err != nil { - log.Printf("[WARNING] (3) Failed reading body for workflowqueue") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return -} - -var actionResult shuffle.ActionResult -err = json.Unmarshal(body, &actionResult) -if err != nil { - log.Printf("[ERROR] Failed shuffle.ActionResult unmarshaling (2): %s", err) - //resp.WriteHeader(401) - //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - //return -} - -if len(actionResult.ExecutionId) == 0 { - log.Printf("[ERROR] No workflow execution id in action result. Data: %s", string(body)) - resp.WriteHeader(400) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No workflow execution id in action result"}`))) - return -} - -// 1. Get the shuffle.WorkflowExecution(ExecutionId) from the database -// 2. if shuffle.ActionResult.Authentication != shuffle.WorkflowExecution.Authentication -> exit -// 3. Add to and update actionResult in workflowExecution -// 4. Push to db -// IF FAIL: Set executionstatus: abort or cancel -ctx := context.Background() -workflowExecution, err := shuffle.GetWorkflowExecution(ctx, actionResult.ExecutionId) -if err != nil { - log.Printf("[ERROR][%s] Failed getting execution (workflowqueue) %s: %s", actionResult.ExecutionId, actionResult.ExecutionId, err) - resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution ID %s because it doesn't exist locally."}`, actionResult.ExecutionId))) - return -} - -if workflowExecution.Authorization != actionResult.Authorization { - log.Printf("[ERROR][%s] Bad authorization key when updating node (workflowQueue). Want: %s, Have: %s", actionResult.ExecutionId, workflowExecution.Authorization, actionResult.Authorization) - resp.WriteHeader(403) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key"}`))) - return -} - -if workflowExecution.Status == "FINISHED" { - log.Printf("[DEBUG][%s] Workflowexecution is already FINISHED. No further action can be taken", workflowExecution.ExecutionId) - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is already finished because it has status %s. Lastnode: %s"}`, workflowExecution.Status, workflowExecution.LastNode))) - return -} - -if workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" { - log.Printf("[WARNING][%s] Workflowexecution already has status %s. No further action can be taken", workflowExecution.ExecutionId, workflowExecution.Status) - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is aborted because of %s with result %s and status %s"}`, workflowExecution.LastNode, workflowExecution.Result, workflowExecution.Status))) - return -} - -retries := 0 -retry, retriesok := request.URL.Query()["retries"] -if retriesok && len(retry) > 0 { - val, err := strconv.Atoi(retry[0]) - if err == nil { - retries = val + if request.Body == nil { + resp.WriteHeader(http.StatusBadRequest) + return } -} -log.Printf("[DEBUG][%s] Action: Received, Label: '%s', Action: '%s', Status: %s, Run status: %s, Extra=Retry:%d", workflowExecution.ExecutionId, actionResult.Action.Label, actionResult.Action.AppName, actionResult.Status, workflowExecution.Status, retries) + defer request.Body.Close() + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Printf("[WARNING] (3) Failed reading body for workflowqueue") + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } -//results = append(results, actionResult) -//log.Printf("[INFO][%s] Time to execute %s (%s) with app %s:%s, function %s, env %s with %d parameters.", workflowExecution.ExecutionId, action.ID, action.Label, action.AppName, action.AppVersion, action.Name, action.Environment, len(action.Parameters)) -//log.Printf("[DEBUG][%s] In workflowQueue with transaction", workflowExecution.ExecutionId) -runWorkflowExecutionTransaction(ctx, 0, workflowExecution.ExecutionId, actionResult, resp) + var actionResult shuffle.ActionResult + err = json.Unmarshal(body, &actionResult) + if err != nil { + log.Printf("[ERROR] Failed shuffle.ActionResult unmarshaling (2): %s", err) + //resp.WriteHeader(401) + //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + //return + } + + if len(actionResult.ExecutionId) == 0 { + log.Printf("[ERROR] No workflow execution id in action result. Data: %s", string(body)) + resp.WriteHeader(400) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No workflow execution id in action result"}`))) + return + } + + // 1. Get the shuffle.WorkflowExecution(ExecutionId) from the database + // 2. if shuffle.ActionResult.Authentication != shuffle.WorkflowExecution.Authentication -> exit + // 3. Add to and update actionResult in workflowExecution + // 4. Push to db + // IF FAIL: Set executionstatus: abort or cancel + ctx := context.Background() + if actionResult.ExecutionId == "TBD" { + return + } + + workflowExecution, err := shuffle.GetWorkflowExecution(ctx, actionResult.ExecutionId) + if err != nil { + log.Printf("[ERROR][%s] Failed getting execution (workflowqueue) %s: %s", actionResult.ExecutionId, actionResult.ExecutionId, err) + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution ID %s because it doesn't exist locally."}`, actionResult.ExecutionId))) + return + } + + if workflowExecution.Authorization != actionResult.Authorization { + log.Printf("[ERROR][%s] Bad authorization key when updating node (workflowQueue). Want: %s, Have: %s", actionResult.ExecutionId, workflowExecution.Authorization, actionResult.Authorization) + resp.WriteHeader(403) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key"}`))) + return + } + + if workflowExecution.Status == "FINISHED" { + log.Printf("[DEBUG][%s] Workflowexecution is already FINISHED. No further action can be taken", workflowExecution.ExecutionId) + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is already finished because it has status %s. Lastnode: %s"}`, workflowExecution.Status, workflowExecution.LastNode))) + return + } + + if workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" { + log.Printf("[WARNING][%s] Workflowexecution already has status %s. No further action can be taken", workflowExecution.ExecutionId, workflowExecution.Status) + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is aborted because of %s with result %s and status %s"}`, workflowExecution.LastNode, workflowExecution.Result, workflowExecution.Status))) + return + } + + retries := 0 + retry, retriesok := request.URL.Query()["retries"] + if retriesok && len(retry) > 0 { + val, err := strconv.Atoi(retry[0]) + if err == nil { + retries = val + } + } + + log.Printf("[DEBUG][%s] Action: Received, Label: '%s', Action: '%s', Status: %s, Run status: %s, Extra=Retry:%d", workflowExecution.ExecutionId, actionResult.Action.Label, actionResult.Action.AppName, actionResult.Status, workflowExecution.Status, retries) + + // results = append(results, actionResult) + // log.Printf("[INFO][%s] Time to execute %s (%s) with app %s:%s, function %s, env %s with %d parameters.", workflowExecution.ExecutionId, action.ID, action.Label, action.AppName, action.AppVersion, action.Name, action.Environment, len(action.Parameters)) + // log.Printf("[DEBUG][%s] In workflowQueue with transaction", workflowExecution.ExecutionId) + runWorkflowExecutionTransaction(ctx, 0, workflowExecution.ExecutionId, actionResult, resp) } // Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times @@ -2063,6 +2404,19 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl return } + /*** STARTREMOVE ***/ + if workflowExecution.Status == "WAITING" && (os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || os.Getenv("SHUFFLE_SWARM_CONFIG") == "swarm") { + log.Printf("[INFO][%s] Workflow execution is waiting while in swarm. Sending info to backend to ensure execution stops.", workflowExecution.ExecutionId) + + shutdownData, err := json.Marshal(workflowExecution) + if err != nil { + log.Printf("[ERROR][%s] Failed marshalling execution (36) - not sending backend WAITING: %s", workflowExecution.ExecutionId, err) + } else { + sendResult(*workflowExecution, shutdownData) + shutdown(*workflowExecution, "", "", false) + } + } + /*** ENDREMOVE ***/ } else { if strings.Contains(strings.ToLower(fmt.Sprintf("%s", err)), "already been ran") || strings.Contains(strings.ToLower(fmt.Sprintf("%s", err)), "already finished") { log.Printf("[ERROR][%s] Skipping rerun of action result as it's already been ran: %s", workflowExecution.ExecutionId) @@ -2101,30 +2455,30 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl //log.Printf(`[DEBUG][%s] Got result %s from %s. Execution status: %s. Save: %#v. Parent: %#v`, actionResult.ExecutionId, actionResult.Status, actionResult.Action.ID, workflowExecution.Status, dbSave, workflowExecution.ExecutionParent) //dbSave := false - + //if len(results) != len(workflowExecution.Results) { // log.Printf("[DEBUG][%s] There may have been an issue in transaction queue. Result lengths: %d vs %d. Should check which exists the base results, but not in entire execution, then append.", workflowExecution.ExecutionId, len(results), len(workflowExecution.Results)) //} - + // Validating that action results hasn't changed // Handled using cachhing, so actually pretty fast cacheKey := fmt.Sprintf("workflowexecution_%s", workflowExecution.ExecutionId) cache, err := shuffle.GetCache(ctx, cacheKey) if err == nil { //parsedValue := value.(*shuffle.WorkflowExecution) - + parsedValue := &shuffle.WorkflowExecution{} cacheData := []byte(cache.([]uint8)) err = json.Unmarshal(cacheData, &workflowExecution) if err != nil { log.Printf("[ERROR][%s] Failed unmarshalling workflowexecution: %s", workflowExecution.ExecutionId, err) } - + if len(parsedValue.Results) > 0 && len(parsedValue.Results) != resultLength { setExecution = false if attempts > 5 { } - + attempts += 1 log.Printf("[DEBUG][%s] Rerunning transaction as results has changed. %d vs %d", workflowExecution.ExecutionId, len(parsedValue.Results), resultLength) /* @@ -2136,7 +2490,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl */ } } - + if setExecution || workflowExecution.Status == "FINISHED" || workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" { log.Printf("[DEBUG][%s] Running setexec with status %s and %d/%d results", workflowExecution.ExecutionId, workflowExecution.Status, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)) //result(s)", workflowExecution.ExecutionId, workflowExecution.Status, len(workflowExecution.Results)) @@ -2146,62 +2500,86 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution actionresult: %s"}`, err))) return } - + + /*** STARTREMOVE ***/ + if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || os.Getenv("SHUFFLE_SWARM_CONFIG") == "swarm" { + finished := shuffle.ValidateFinished(ctx, -1, *workflowExecution) + if !finished { + log.Printf("[DEBUG][%s] Handling next node since it's not finished!", workflowExecution.ExecutionId) + handleExecutionResult(*workflowExecution) + } else { + shutdownData, err := json.Marshal(workflowExecution) + if err != nil { + log.Printf("[ERROR] Failed marshalling shutdowndata during set: %s", err) + } + + sendResult(*workflowExecution, shutdownData) + } + } + /*** ENDREMOVE ***/ } else { log.Printf("[INFO][%s] Skipping setexec with status %s", workflowExecution.ExecutionId, workflowExecution.Status) - + // Just in case. Should MAYBE validate finishing another time as well. // This fixes issues with e.g. shuffle.Action -> shuffle.Trigger -> shuffle.Action. handleExecutionResult(*workflowExecution) } - + //if newExecutions && len(nextActions) > 0 { // log.Printf("[DEBUG][%s] New execution: %#v. NextActions: %#v", newExecutions, nextActions) // //handleExecutionResult(*workflowExecution) //} - + resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } func sendSelfRequest(actionResult shuffle.ActionResult) { + + /*** STARTREMOVE ***/ + if os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm" { + log.Printf("[INFO][%s] Not sending self request info since source is default (not swarm)", actionResult.ExecutionId) + return + } + /*** ENDREMOVE ***/ + data, err := json.Marshal(actionResult) if err != nil { log.Printf("[ERROR][%s] Shutting down (24): Failed to unmarshal data for backend: %s", actionResult.ExecutionId, err) return } - + if actionResult.ExecutionId == "TBD" { return } - + log.Printf("[DEBUG][%s] Sending FAILURE to self to stop the workflow execution. Action: %s (%s), app %s:%s", actionResult.ExecutionId, actionResult.Action.Label, actionResult.Action.ID, actionResult.Action.AppName, actionResult.Action.AppVersion) - + // Literally sending to same worker to run it as a new request streamUrl := fmt.Sprintf("http://localhost:33333/api/v1/streams") hostenv := os.Getenv("WORKER_HOSTNAME") if len(hostenv) > 0 { streamUrl = fmt.Sprintf("http://%s:33333/api/v1/streams", hostenv) } - + req, err := http.NewRequest( "POST", streamUrl, bytes.NewBuffer([]byte(data)), ) - + if err != nil { log.Printf("[ERROR][%s] Failed creating self request (1): %s", actionResult.ExecutionId, err) return } - + client := shuffle.GetExternalClient(streamUrl) newresp, err := client.Do(req) if err != nil { log.Printf("[ERROR][%s] Error running finishing request (2): %s", actionResult.ExecutionId, err) return } - + defer newresp.Body.Close() if newresp.Body != nil { body, err := ioutil.ReadAll(newresp.Body) @@ -2220,7 +2598,7 @@ func sendResult(workflowExecution shuffle.WorkflowExecution, data []byte) { //return } else { } - + // Basically to reduce backend strain /* if shuffle.ArrayContains(finishedExecutions, workflowExecution.ExecutionId) { @@ -2228,32 +2606,31 @@ func sendResult(workflowExecution shuffle.WorkflowExecution, data []byte) { return } */ - + // Take it down again /* - if len(finishedExecutions) > 100 { - log.Printf("[DEBUG][%s] Removing old execution from finishedExecutions: %s", workflowExecution.ExecutionId, finishedExecutions[0]) - finishedExecutions = finishedExecutions[99:] - } - - finishedExecutions = append(finishedExecutions, workflowExecution.ExecutionId) + if len(finishedExecutions) > 100 { + log.Printf("[DEBUG][%s] Removing old execution from finishedExecutions: %s", workflowExecution.ExecutionId, finishedExecutions[0]) + finishedExecutions = finishedExecutions[99:] + } + finishedExecutions = append(finishedExecutions, workflowExecution.ExecutionId) */ - + streamUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl) req, err := http.NewRequest( "POST", streamUrl, bytes.NewBuffer([]byte(data)), ) - + if err != nil { log.Printf("[ERROR][%s] Failed creating finishing request: %s", workflowExecution.ExecutionId, err) log.Printf("[DEBUG][%s] Shutting down (22)", workflowExecution.ExecutionId) shutdown(workflowExecution, "", "", false) return } - + client := shuffle.GetExternalClient(streamUrl) newresp, err := client.Do(req) if err != nil { @@ -2262,7 +2639,7 @@ func sendResult(workflowExecution shuffle.WorkflowExecution, data []byte) { shutdown(workflowExecution, "", "", false) return } - + defer newresp.Body.Close() if newresp.Body != nil { body, err := ioutil.ReadAll(newresp.Body) @@ -2273,11 +2650,11 @@ func sendResult(workflowExecution shuffle.WorkflowExecution, data []byte) { log.Printf("[DEBUG][%s] NEWRESP (from backend): %s", workflowExecution.ExecutionId, string(body)) } } - } - - func validateFinished(workflowExecution shuffle.WorkflowExecution) bool { +} + +func validateFinished(workflowExecution shuffle.WorkflowExecution) bool { ctx := context.Background() - + newexec, err := shuffle.GetWorkflowExecution(ctx, workflowExecution.ExecutionId) if err != nil { log.Printf("[ERROR][%s] Failed getting workflow execution: %s", workflowExecution.ExecutionId, err) @@ -2285,15 +2662,14 @@ func sendResult(workflowExecution shuffle.WorkflowExecution, data []byte) { } else { workflowExecution = *newexec } - + //startAction, extra, children, parents, visited, executed, nextActions, environments := shuffle.GetExecutionVariables(ctx, workflowExecution.ExecutionId) workflowExecution, _ = shuffle.Fixexecution(ctx, workflowExecution) _, extra, _, _, _, _, _, environments := shuffle.GetExecutionVariables(ctx, workflowExecution.ExecutionId) - + log.Printf("[INFO][%s] VALIDATION. Status: %s, shuffle.Actions: %d, Extra: %d, Results: %d. Parent: %#v", workflowExecution.ExecutionId, workflowExecution.Status, len(workflowExecution.Workflow.Actions), extra, len(workflowExecution.Results), workflowExecution.ExecutionParent) - - if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "ABORTED" || (len(environments) == 1 && requestsSent == 0 && len(workflowExecution.Results) >= 1 && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm") || (len(workflowExecution.Results) >= len(workflowExecution.Workflow.Actions)+extra && len(workflowExecution.Workflow.Actions) > 0) { - + + if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "ABORTED" || (len(environments) == 1 && requestsSent == 0 && len(workflowExecution.Results) >= 1 && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm") || (len(workflowExecution.Results) >= len(workflowExecution.Workflow.Actions)+extra && len(workflowExecution.Workflow.Actions) > 0) { if workflowExecution.Status == "FINISHED" { for _, result := range workflowExecution.Results { @@ -2304,15 +2680,21 @@ func sendResult(workflowExecution shuffle.WorkflowExecution, data []byte) { } } + /*** STARTREMOVE ***/ + if os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm" { + requestsSent += 1 + } + /*** ENDREMOVE ***/ + log.Printf("[DEBUG][%s] Should send full result to %s", workflowExecution.ExecutionId, baseUrl) - + //data = fmt.Sprintf(`{"execution_id": "%s", "authorization": "%s"}`, executionId, authorization) shutdownData, err := json.Marshal(workflowExecution) if err != nil { log.Printf("[ERROR][%s] Shutting down (32): Failed to unmarshal data for backend: %s", workflowExecution.ExecutionId, err) shutdown(workflowExecution, "", "", true) } - + cacheKey := fmt.Sprintf("workflowexecution_%s", workflowExecution.ExecutionId) if len(workflowExecution.Authorization) > 0 { err = shuffle.SetCache(ctx, cacheKey, shutdownData, 31) @@ -2320,12 +2702,12 @@ func sendResult(workflowExecution shuffle.WorkflowExecution, data []byte) { log.Printf("[ERROR][%s] Failed adding to cache during ValidateFinished", workflowExecution) } } - + shuffle.RunCacheCleanup(ctx, workflowExecution) sendResult(workflowExecution, shutdownData) return true } - + return false } @@ -2338,7 +2720,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } - + var actionResult shuffle.ActionResult err = json.Unmarshal(body, &actionResult) if err != nil { @@ -2347,14 +2729,14 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) //return } - + if len(actionResult.ExecutionId) == 0 { log.Printf("[WARNING] No workflow execution id in action result (2). Data: %s", string(body)) resp.WriteHeader(400) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No workflow execution id in action result"}`))) return } - + ctx := context.Background() workflowExecution, err := shuffle.GetWorkflowExecution(ctx, actionResult.ExecutionId) if err != nil { @@ -2363,7 +2745,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key or execution_id might not exist."}`))) return } - + // Authorization is done here if workflowExecution.Authorization != actionResult.Authorization { log.Printf("[ERROR] Bad authorization key when getting stream results from cache %s.", actionResult.ExecutionId) @@ -2371,14 +2753,14 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key or execution_id might not exist."}`))) return } - + newjson, err := json.Marshal(workflowExecution) if err != nil { resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflow execution"}`))) return } - + resp.WriteHeader(200) resp.Write(newjson) @@ -2387,12 +2769,80 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { // GetLocalIP returns the non loopback local IP of the host func getLocalIP() string { + /*** STARTREMOVE ***/ + if os.Getenv("IS_KUBERNETES") == "true" { + return "shuffle-workers" + } + + if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || os.Getenv("SHUFFLE_SWARM_CONFIG") == "swarm" { + name, err := os.Hostname() + if err != nil { + log.Printf("[ERROR] Couldn't find hostname of worker: %s", err) + os.Exit(3) + } + + log.Printf("[DEBUG] Found hostname %s since worker is running with \"run\" command", name) + return name + + /** + Everything below was a test to see if we needed to match directly to a network interface. May require docker network API. + **/ + + log.Printf("[DEBUG] Looking for IP for the external docker-network %s", swarmNetworkName) + // Different process to ensure we find the right IP. + // Necessary due to Ingress being added to docker ser + ifaces, err := net.Interfaces() + if err != nil { + log.Printf("[ERROR] FATAL: networks the container is listening in %s: %s", swarmNetworkName, err) + os.Exit(3) + } + + foundIP := "" + for _, i := range ifaces { + log.Printf("NETWORK: %s", i.Name) + //If i.Name != swarmNetworkName { + // continue + //} + + addrs, err := i.Addrs() + if err != nil { + log.Printf("[ERROR] FATAL: Failed getting address for listener in network %s: %s", swarmNetworkName, err) + continue + } + + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + + log.Printf("%s: IP: %#v", i.Name, ip) + + // FIXME: Allow for IPv6 too! + //if strings.Count(ip.String(), ".") == 3 { + // foundIP = ip.String() + // break + //} + // process IP address + } + } + + if len(foundIP) == 0 { + log.Printf("[ERROR] FATAL: No valid IP found for network %s. Defaulting to base IP", swarmNetworkName) + } else { + return foundIP + } + } + /*** ENDREMOVE ***/ addrs, err := net.InterfaceAddrs() if err != nil { return "" } - + for _, address := range addrs { // check the address type and if it is not a loopback the display it if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { @@ -2401,7 +2851,7 @@ func getLocalIP() string { } } } - + return "" } @@ -2412,17 +2862,21 @@ func getAvailablePort() (net.Listener, error) { //return ":5001" return nil, err } - + //defer listener.Close() - + return listener, nil //return fmt.Sprintf(":%d", port) } func webserverSetup(workflowExecution shuffle.WorkflowExecution) net.Listener { hostname = getLocalIP() - os.Setenv("WORKER_HOSTNAME", hostname) - + if isKubernetes == "true" { + os.Setenv("WORKER_HOSTNAME", "shuffle-workers") + } else { + os.Setenv("WORKER_HOSTNAME", hostname) + } + // FIXME: This MAY not work because of speed between first // container being launched and port being assigned to webserver listener, err := getAvailablePort() @@ -2430,136 +2884,49 @@ func webserverSetup(workflowExecution shuffle.WorkflowExecution) net.Listener { log.Printf("[ERROR] Failed to create init listener: %s", err) return listener } - + log.Printf("[DEBUG] OLD HOSTNAME: %s", appCallbackUrl) - + + /*** STARTREMOVE ***/ + if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || os.Getenv("SHUFFLE_SWARM_CONFIG") == "swarm" { + log.Printf("[DEBUG] Starting webserver (1) on port %d with hostname: %s", baseport, hostname) + + os.Setenv("WORKER_PORT", fmt.Sprintf("%d", baseport)) + appCallbackUrl = fmt.Sprintf("http://%s:%d", hostname, baseport) + if os.Getenv("IS_KUBERNETES") == "true" { + appCallbackUrl = fmt.Sprintf("http://%s:%d", "shuffle-workers", baseport) + log.Printf("[DEBUG] NEW WORKER APP: %s", appCallbackUrl) + hostname = "shuffle-workers" + } + + listener, err = net.Listen("tcp", fmt.Sprintf(":%d", baseport)) + if err != nil { + log.Printf("[ERROR] Failed to assign port to %d: %s", baseport, err) + return nil + } + + return listener + } + /*** ENDREMOVE ***/ + port := listener.Addr().(*net.TCPAddr).Port // Set the port environment variable os.Setenv("WORKER_PORT", fmt.Sprintf("%d", port)) - + log.Printf("[DEBUG] Starting webserver (2) on port %d with hostname: %s", port, hostname) appCallbackUrl = fmt.Sprintf("http://%s:%d", hostname, port) - + log.Printf("[INFO] NEW WORKER HOSTNAME: %s", appCallbackUrl) return listener } -func downloadDockerImageBackend(client *http.Client, imageName string) error { - // Check environment SHUFFLE_AUTO_IMAGE_DOWNLOAD - if os.Getenv("SHUFFLE_AUTO_IMAGE_DOWNLOAD") == "false" { - //log.Printf("[DEBUG] SHUFFLE_AUTO_IMAGE_DOWNLOAD is false. Not downloading image %s", imageName) - return nil - } - - if arrayContains(downloadedImages, imageName) { - log.Printf("[DEBUG] Image %s already downloaded", imageName) - return nil - } - - - downloadedImages = append(downloadedImages, imageName) - - data := fmt.Sprintf(`{"name": "%s"}`, imageName) - dockerImgUrl := fmt.Sprintf("%s/api/v1/get_docker_image", baseUrl) - - log.Printf("[DEBUG] Trying to download image %s from backend %s as it doesn't exist. Data sent: %#v, All images: %#v", imageName, baseUrl, data, downloadedImages) - - req, err := http.NewRequest( - "POST", - dockerImgUrl, - bytes.NewBuffer([]byte(data)), - ) - - authorization := os.Getenv("AUTHORIZATION") - if len(authorization) > 0 { - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", authorization)) - } else { - log.Printf("[WARNING] No auth found - running backend download without it.") - //return - } - - newresp, err := topClient.Do(req) - if err != nil { - log.Printf("[ERROR] Failed download request for %s: %s", imageName, err) - return err - } - - defer newresp.Body.Close() - if newresp.StatusCode != 200 { - log.Printf("[ERROR] Docker download for image %s (backend) StatusCode (1): %d", imageName, newresp.StatusCode) - return errors.New(fmt.Sprintf("Failed to get image - status code %d", newresp.StatusCode)) - } - - newImageName := strings.Replace(imageName, "/", "_", -1) - newFileName := newImageName + ".tar" - - tar, err := os.Create(newFileName) - if err != nil { - log.Printf("[WARNING] Failed creating file: %s", err) - return err - } - - defer tar.Close() - _, err = io.Copy(tar, newresp.Body) - if err != nil { - log.Printf("[WARNING] Failed response body copying: %s", err) - return err - } - tar.Seek(0, 0) - - dockercli, err := dockerclient.NewEnvClient() - if err != nil { - log.Printf("[ERROR] Unable to create docker client (3): %s", err) - return err - } - - defer dockercli.Close() - - imageLoadResponse, err := dockercli.ImageLoad(context.Background(), tar, true) - if err != nil { - log.Printf("[ERROR] Error loading images: %s", err) - return err - } - - defer imageLoadResponse.Body.Close() - body, err := ioutil.ReadAll(imageLoadResponse.Body) - if err != nil { - log.Printf("[ERROR] Error reading: %s", err) - return err - } - - if strings.Contains(string(body), "no such file") { - return errors.New(string(body)) - } - - baseTag := strings.Split(imageName, ":") - if len(baseTag) > 1 { - tag := baseTag[1] - log.Printf("[DEBUG] Creating tag copies of downloaded containers from tag %s", tag) - - // Remapping - ctx := context.Background() - dockercli.ImageTag(ctx, imageName, fmt.Sprintf("frikky/shuffle:%s", tag)) - dockercli.ImageTag(ctx, imageName, fmt.Sprintf("registry.hub.docker.com/frikky/shuffle:%s", tag)) - - downloadedImages = append(downloadedImages, fmt.Sprintf("frikky/shuffle:%s", tag)) - downloadedImages = append(downloadedImages, fmt.Sprintf("registry.hub.docker.com/frikky/shuffle:%s", tag)) - - } - - os.Remove(newFileName) - - log.Printf("[INFO] Successfully loaded image %s: %s", imageName, string(body)) - return nil - } - - func findActiveSwarmNodes(dockercli *dockerclient.Client) (int64, error) { +func findActiveSwarmNodes(dockercli *dockerclient.Client) (int64, error) { ctx := context.Background() nodes, err := dockercli.NodeList(ctx, types.NodeListOptions{}) if err != nil { return 1, err } - + nodeCount := int64(0) for _, node := range nodes { //log.Printf("ID: %s - %#v", node.ID, node.Status.State) @@ -2567,7 +2934,7 @@ func downloadDockerImageBackend(client *http.Client, imageName string) error { nodeCount += 1 } } - + // Check for SHUFFLE_MAX_NODES maxNodesString := os.Getenv("SHUFFLE_MAX_SWARM_NODES") // Make it into a number and check if it's lower than nodeCount @@ -2576,14 +2943,14 @@ func downloadDockerImageBackend(client *http.Client, imageName string) error { if err != nil { return nodeCount, err } - + if nodeCount > maxNodes { nodeCount = maxNodes } } - + return nodeCount, nil - + /* containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{ All: true, @@ -2591,159 +2958,496 @@ func downloadDockerImageBackend(client *http.Client, imageName string) error { */ } -// Runs data discovery +/*** STARTREMOVE ***/ +func deploySwarmService(dockercli *dockerclient.Client, name, image string, deployport int) error { + log.Printf("[DEBUG] Deploying service for %s to swarm on port %d", name, deployport) + //containerName := fmt.Sprintf("shuffle-worker-%s", parsedUuid) -func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, action *shuffle.Action, workflowExecution *shuffle.WorkflowExecution) error { -parsedRequest := shuffle.OrborusExecutionRequest{ - Cleanup: cleanupEnv, - ExecutionId: workflowExecution.ExecutionId, - Authorization: workflowExecution.Authorization, - EnvironmentName: os.Getenv("ENVIRONMENT_NAME"), - Timezone: os.Getenv("TZ"), - HTTPProxy: os.Getenv("HTTP_PROXY"), - HTTPSProxy: os.Getenv("HTTPS_PROXY"), - ShufflePassProxyToApp: os.Getenv("SHUFFLE_PASS_APP_PROXY"), - Url: baseUrl, - BaseUrl: baseUrl, - Action: *action, - FullExecution: *workflowExecution, -} -// Sometimes makes it have the wrong data due to timing - -// Specific for subflow to ensure worker matches the backend correctly - -parsedBaseurl := incomingUrl -if strings.Count(baseUrl, ":") >= 2 { - baseUrlSplit := strings.Split(baseUrl, ":") - if len(baseUrlSplit) >= 3 { - parsedBaseurl = strings.Join(baseUrlSplit[0:2], ":") - //parsedRequest.BaseUrl = fmt.Sprintf("%s:33333", parsedBaseurl) - } -} - -if len(parsedRequest.Url) == 0 { - // Fixed callback url to the worker itself - if strings.Count(parsedBaseurl, ":") >= 2 { - parsedRequest.Url = parsedBaseurl - } else { - // Callback to worker - parsedRequest.Url = fmt.Sprintf("%s:%d", parsedBaseurl, baseport) - - //parsedRequest.Url + if len(baseimagename) == 0 || baseimagename == "/" { + baseimagename = "frikky/shuffle" + //var baseimagename = "frikky/shuffle" + //var registryName = "registry.hub.docker.com" } - //log.Printf("[DEBUG][%s] Should add a baseurl for the app to get back to: %s", workflowExecution.ExecutionId, parsedRequest.Url) -} - -// Swapping because this was confusing during dev -// No real reason, just variable names -tmp := parsedRequest.Url -parsedRequest.Url = parsedRequest.BaseUrl -parsedRequest.BaseUrl = tmp - -// Run with proper hostname, but set to shuffle-worker to avoid specific host target. -// This means running with VIP instead. -if len(hostname) > 0 { - parsedRequest.BaseUrl = fmt.Sprintf("http://%s:%d", hostname, baseport) - //parsedRequest.BaseUrl = fmt.Sprintf("http://shuffle-workers:%d", baseport) - //log.Printf("[DEBUG][%s] Changing hostname to local hostname in Docker network for WORKER URL: %s", workflowExecution.ExecutionId, parsedRequest.BaseUrl) - - if parsedRequest.Action.AppName == "shuffle-subflow" || parsedRequest.Action.AppName == "shuffle-subflow-v2" || parsedRequest.Action.AppName == "User Input" { - parsedRequest.BaseUrl = fmt.Sprintf("http://%s:%d", hostname, baseport) - //parsedRequest.Url = parsedRequest.BaseUrl + //image := fmt.Sprintf("%s:%s", baseimagename, name) + networkName := "shuffle-executions" + if len(swarmNetworkName) > 0 { + networkName = swarmNetworkName } -} -// Making sure to get the LATEST execution data -// This is due to cache timing issues -exec, err := shuffle.GetWorkflowExecution(ctx, workflowExecution.ExecutionId) -if err == nil && len(exec.ExecutionId) > 0 { - parsedRequest.FullExecution = *exec -} + replicas := uint64(1) -data, err := json.Marshal(parsedRequest) -if err != nil { - log.Printf("[ERROR] Failed marshalling worker request: %s", err) - return err -} + // Sent from Orborus + // Should be equal to + scaleReplicas := os.Getenv("SHUFFLE_APP_REPLICAS") + if len(scaleReplicas) > 0 { + tmpInt, err := strconv.Atoi(scaleReplicas) + if err != nil { + log.Printf("[ERROR] %s is not a valid number for replication", scaleReplicas) + } else { + replicas = uint64(tmpInt) + } -streamUrl := fmt.Sprintf("http://%s:%d/api/v1/run", appName, port) -//log.Printf("[DEBUG][%s] Worker URL: %s, Backend URL: %s, Target App: %s", workflowExecution.ExecutionId, parsedRequest.BaseUrl, parsedRequest.Url, streamUrl) -req, err := http.NewRequest( - "POST", - streamUrl, - bytes.NewBuffer([]byte(data)), -) + log.Printf("[DEBUG] SHUFFLE_APP_REPLICAS set to value %#v. Trying to overwrite default (%d/node)", scaleReplicas, replicas) + } + + cnt, err := findActiveSwarmNodes(dockercli) + if err != nil { + log.Printf("[ERROR] Unable to find active swarm nodes: %s", err) + } + + nodeCount := uint64(1) + if cnt > 0 { + nodeCount = uint64(cnt) + } + + replicatedJobs := uint64(replicas * nodeCount) + log.Printf("[DEBUG] Deploying app with name %s with image %s", name, image) + + containerName := fmt.Sprintf(strings.Replace(name, ".", "-", -1)) + serviceSpec := swarm.ServiceSpec{ + Annotations: swarm.Annotations{ + Name: containerName, + Labels: map[string]string{}, + }, + Mode: swarm.ServiceMode{ + Replicated: &swarm.ReplicatedService{ + // Max replicas total (?) + Replicas: &replicatedJobs, + }, + }, + Networks: []swarm.NetworkAttachmentConfig{ + swarm.NetworkAttachmentConfig{ + Target: networkName, + }, + }, + EndpointSpec: &swarm.EndpointSpec{ + Ports: []swarm.PortConfig{ + swarm.PortConfig{ + Protocol: swarm.PortConfigProtocolTCP, + PublishMode: swarm.PortConfigPublishModeIngress, + Name: "app-port", + PublishedPort: uint32(deployport), + TargetPort: uint32(deployport), + }, + }, + }, + TaskTemplate: swarm.TaskSpec{ + Resources: &swarm.ResourceRequirements{ + Reservations: &swarm.Resources{}, + }, + LogDriver: &swarm.Driver{ + Name: "json-file", + Options: map[string]string{ + "max-size": "10m", + }, + }, + ContainerSpec: &swarm.ContainerSpec{ + Image: image, + Env: []string{ + fmt.Sprintf("SHUFFLE_APP_EXPOSED_PORT=%d", deployport), + fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", os.Getenv("SHUFFLE_SWARM_CONFIG")), + fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", logsDisabled), + }, + Hosts: []string{ + containerName, + }, + }, + RestartPolicy: &swarm.RestartPolicy{ + Condition: swarm.RestartPolicyConditionAny, + }, + Placement: &swarm.Placement{ + Constraints: []string{}, + }, + }, + } + + if len(os.Getenv("SHUFFLE_SWARM_OTHER_NETWORK")) > 0 { + serviceSpec.Networks = append(serviceSpec.Networks, swarm.NetworkAttachmentConfig{ + Target: "shuffle_shuffle", + }) + } + + if strings.ToLower(os.Getenv("SHUFFLE_PASS_APP_PROXY")) == "true" { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))) + } + + overrideHttpProxy := os.Getenv("SHUFFLE_INTERNAL_HTTP_PROXY") + overrideHttpsProxy := os.Getenv("SHUFFLE_INTERNAL_HTTPS_PROXY") + if overrideHttpProxy != "" { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_INTERNAL_HTTP_PROXY=%s", overrideHttpProxy)) + + } + + if overrideHttpsProxy != "" { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_INTERNAL_HTTPS_PROXY=%s", overrideHttpsProxy)) + } + + /* + Mounts: []mount.Mount{ + mount.Mount{ + Source: "/var/run/docker.sock", + Target: "/var/run/docker.sock", + Type: mount.TypeBind, + }, + }, + */ + + if dockerApiVersion != "" { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("DOCKER_API_VERSION=%s", dockerApiVersion)) + } + + if len(os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")) > 0 { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_APP_SDK_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_SDK_TIMEOUT"))) + } + + // Required for certain apps + if timezone == "" { + timezone = "Europe/Amsterdam" + } + + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("TZ=%s", timezone)) + + serviceOptions := types.ServiceCreateOptions{} + service, err := dockercli.ServiceCreate( + context.Background(), + serviceSpec, + serviceOptions, + ) + _ = service + + if err != nil { + log.Printf("[DEBUG] Failed deploying %s with image %s: %s", name, image, err) + return err + } + + log.Printf("[DEBUG] Successfully deployed service %s with image %s on port %d", name, image, deployport) -// Checking as LATE as possible, ensuring we don't rerun what's already ran -//ctx = context.Background() -newExecId := fmt.Sprintf("%s_%s", workflowExecution.ExecutionId, action.ID) -_, err = shuffle.GetCache(ctx, newExecId) -if err == nil { - log.Printf("[DEBUG] Result for %s already found (PRE REQUEST) - returning", newExecId) return nil } -cacheData := []byte("1") -err = shuffle.SetCache(ctx, newExecId, cacheData, 30) -if err != nil { - log.Printf("[WARNING] Failed setting cache for action %s: %s", newExecId, err) -} else { - //log.Printf("[DEBUG][%s] Adding %s to cache (%#v)", workflowExecution.ExecutionId, newExecId, action.Name) -} +/*** ENDREMOVE ***/ -client := shuffle.GetExternalClient(streamUrl) -customTimeout := os.Getenv("SHUFFLE_APP_REQUEST_TIMEOUT") -if len(customTimeout) > 0 { - // convert to int - timeoutInt, err := strconv.Atoi(customTimeout) +// Runs data discovery +/*** STARTREMOVE ***/ + +func findAppInfoKubernetes(image, name string, env []string) error { + clientset, _, err := shuffle.GetKubernetesClient() if err != nil { - log.Printf("[ERROR] Failed converting SHUFFLE_APP_REQUEST_TIMEOUT to int: %s", err) - } else { - log.Printf("[DEBUG] Setting client timeout to %d seconds for app request", timeoutInt) - client.Timeout = time.Duration(timeoutInt) * time.Second + log.Printf("[ERROR] Failed getting kubernetes: %s", err) + return err } + + // Check if it exists as a pod + namespace := "default" + if len(kubernetesNamespace) > 0 { + namespace = kubernetesNamespace + } + + // check deployments + deployments, err := clientset.AppsV1().Deployments(namespace).List(context.Background(), metav1.ListOptions{}) + if err != nil { + log.Printf("[ERROR] Failed listing deployments: %s", err) + return err + } + + name = strings.Replace(name, "_", "-", -1) + + // check if it exists as a pod + // for _, pod := range pods.Items { + // if pod.Name == name { + // log.Printf("[INFO] Found pod %s - no need to deploy another", name) + // return nil + // } + // } + + for _, deployment := range deployments.Items { + if deployment.Name == name { + log.Printf("[INFO] Found deployment %s - no need to deploy another", name) + return nil + } + } + + err = deployk8sApp(image, name, env) + return err } -newresp, err := client.Do(req) -if err != nil { - // Another timeout issue here somewhere - // context deadline - if strings.Contains(fmt.Sprintf("%s", err), "context deadline exceeded") || strings.Contains(fmt.Sprintf("%s", err), "Client.Timeout exceeded") { - return nil +func findAppInfo(image, name string) (int, error) { + dockercli, err := dockerclient.NewEnvClient() + if err != nil { + log.Printf("[ERROR] Unable to create docker client (2): %s", err) + return -1, err } - if strings.Contains(fmt.Sprintf("%s", err), "timeout awaiting response") { - return nil - } + highest := baseport + exposedPort := -1 - newerr := fmt.Sprintf("%s", err) - if strings.Contains(newerr, "connection refused") || strings.Contains(newerr, "no such host") { - newerr = fmt.Sprintf("Failed connecting to app %s. Is the Docker image available?", appName) + // Exists as a "cache" layer + if portMappings != nil { + for key, value := range portMappings { + if value > highest { + highest = value + } + + if key == name { + exposedPort = value + break + } + } } else { - // escape quotes and newlines - newerr = strings.ReplaceAll(strings.ReplaceAll(newerr, "\"", "\\\""), "\n", "\\n") + portMappings = make(map[string]int) } - if strings.Contains(fmt.Sprintf("%s", err), "no such host") { - log.Printf("[DEBUG] SHOULD be Removing references to location for app %s as to be rediscovered", action.AppName) + //Filters: + if exposedPort == -1 { + serviceListOptions := types.ServiceListOptions{} + services, err := dockercli.ServiceList( + context.Background(), + serviceListOptions, + ) - //for k, v := range portMappings { - // if strings.Contains(strings.ToLower(strings.ReplaceAll(action.AppName, " ", "_"))) { - // } - //} + // Basic self-correction + if err != nil { + log.Printf("[ERROR] Unable to list services: %s (may continue anyway?)", err) + if strings.Contains(fmt.Sprintf("%s", err), "is too new") { + // Static for some reason + defaultVersion := "1.40" + dockerApiVersion = defaultVersion + os.Setenv("DOCKER_API_VERSION", defaultVersion) + log.Printf("[DEBUG] Setting Docker API to %s default and retrying listing requests", defaultVersion) + } else { + return -1, err + } - //var portMappings map[string]int + services, err = dockercli.ServiceList( + context.Background(), + serviceListOptions, + ) + + if err != nil { + log.Printf("[ERROR] Unable to list services (2): %s", err) + return -1, err + } + } + + for _, service := range services { + //log.Printf("[INFO] Service: %#v", service.Spec.Annotations.Name) + + for _, endpoint := range service.Spec.EndpointSpec.Ports { + if strings.Contains(endpoint.Name, "port") { + portMappings[service.Spec.Annotations.Name] = int(endpoint.PublishedPort) + if int(endpoint.PublishedPort) > highest { + highest = int(endpoint.PublishedPort) + } + + if service.Spec.Annotations.Name == name || service.Spec.Annotations.Name == strings.Replace(name, ".", "-", -1) { + exposedPort = int(endpoint.PublishedPort) + //break + } + } + } + + //log.Printf("%s - %s", service.Spec.Annotations.Name, strings.Replace(name, ".", "-", -1)) + if service.Spec.Annotations.Name != name && service.Spec.Annotations.Name != strings.Replace(name, ".", "-", -1) { + continue + } + + // Break if it's the correct port, as it's the right service + if exposedPort >= 0 { + break + } + } } - log.Printf("[ERROR][%s] Error running app run request: %s", workflowExecution.ExecutionId, err) - actionResult := shuffle.ActionResult{ - Action: *action, - ExecutionId: workflowExecution.ExecutionId, - Authorization: workflowExecution.Authorization, - Result: fmt.Sprintf(`{"success": false, "reason": "Failed to connect to app %s in swarm. Try the action again, restart Orborus if this is recurring, or contact support@shuffler.io.", "details": "%s"}`, streamUrl, newerr), - StartedAt: int64(time.Now().Unix()), - CompletedAt: int64(time.Now().Unix()), + //log.Printf("[DEBUG] Portmappings: %#v", portMappings) + + if exposedPort >= 0 { + //log.Printf("[INFO] Found service %s on port %d - no need to deploy another", name, exposedPort) + } else { + // Increment by 1 for highest port + if highest <= baseport { + highest = baseport + } + + highest += 1 + err = deploySwarmService(dockercli, name, image, highest) + if err != nil { + log.Printf("[WARNING] NOT Found service: %s. error: %s", name, err) + return highest, err + } else { + log.Printf("[DEBUG] Deployed app with name %s", name) + } + + exposedPort = highest + + if appsInitialized { + log.Printf("[DEBUG] Waiting 30 seconds before moving on to let app start") + time.Sleep(time.Duration(30) * time.Second) + } + } + + return exposedPort, nil +} + +/*** ENDREMOVE ***/ + +func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, action *shuffle.Action, workflowExecution *shuffle.WorkflowExecution) error { + parsedRequest := shuffle.OrborusExecutionRequest{ + Cleanup: cleanupEnv, + ExecutionId: workflowExecution.ExecutionId, + Authorization: workflowExecution.Authorization, + EnvironmentName: os.Getenv("ENVIRONMENT_NAME"), + Timezone: os.Getenv("TZ"), + HTTPProxy: os.Getenv("HTTP_PROXY"), + HTTPSProxy: os.Getenv("HTTPS_PROXY"), + ShufflePassProxyToApp: os.Getenv("SHUFFLE_PASS_APP_PROXY"), + Url: baseUrl, + BaseUrl: baseUrl, + Action: *action, + FullExecution: *workflowExecution, + } + // Sometimes makes it have the wrong data due to timing + + // Specific for subflow to ensure worker matches the backend correctly + + parsedBaseurl := incomingUrl + if strings.Count(baseUrl, ":") >= 2 { + baseUrlSplit := strings.Split(baseUrl, ":") + if len(baseUrlSplit) >= 3 { + parsedBaseurl = strings.Join(baseUrlSplit[0:2], ":") + //parsedRequest.BaseUrl = fmt.Sprintf("%s:33333", parsedBaseurl) + } + } + + if len(parsedRequest.Url) == 0 { + // Fixed callback url to the worker itself + if strings.Count(parsedBaseurl, ":") >= 2 { + parsedRequest.Url = parsedBaseurl + } else { + // Callback to worker + parsedRequest.Url = fmt.Sprintf("%s:%d", parsedBaseurl, baseport) + + //parsedRequest.Url + } + + //log.Printf("[DEBUG][%s] Should add a baseurl for the app to get back to: %s", workflowExecution.ExecutionId, parsedRequest.Url) + } + + // Swapping because this was confusing during dev + // No real reason, just variable names + tmp := parsedRequest.Url + parsedRequest.Url = parsedRequest.BaseUrl + parsedRequest.BaseUrl = tmp + + // Run with proper hostname, but set to shuffle-worker to avoid specific host target. + // This means running with VIP instead. + if len(hostname) > 0 { + parsedRequest.BaseUrl = fmt.Sprintf("http://%s:%d", hostname, baseport) + //parsedRequest.BaseUrl = fmt.Sprintf("http://shuffle-workers:%d", baseport) + //log.Printf("[DEBUG][%s] Changing hostname to local hostname in Docker network for WORKER URL: %s", workflowExecution.ExecutionId, parsedRequest.BaseUrl) + + if parsedRequest.Action.AppName == "shuffle-subflow" || parsedRequest.Action.AppName == "shuffle-subflow-v2" || parsedRequest.Action.AppName == "User Input" { + parsedRequest.BaseUrl = fmt.Sprintf("http://%s:%d", hostname, baseport) + //parsedRequest.Url = parsedRequest.BaseUrl + } + } + + // Making sure to get the LATEST execution data + // This is due to cache timing issues + exec, err := shuffle.GetWorkflowExecution(ctx, workflowExecution.ExecutionId) + if err == nil && len(exec.ExecutionId) > 0 { + parsedRequest.FullExecution = *exec + } + + data, err := json.Marshal(parsedRequest) + if err != nil { + log.Printf("[ERROR] Failed marshalling worker request: %s", err) + return err + } + + if isKubernetes == "true" { + appName = strings.Replace(appName, "_", "-", -1) + } + + streamUrl := fmt.Sprintf("http://%s:%d/api/v1/run", appName, port) + // log.Printf("[DEBUG][%s] Worker URL: %s, Backend URL: %s, Target App: %s", workflowExecution.ExecutionId, parsedRequest.BaseUrl, parsedRequest.Url, streamUrl) + req, err := http.NewRequest( + "POST", + streamUrl, + bytes.NewBuffer([]byte(data)), + ) + + // Checking as LATE as possible, ensuring we don't rerun what's already ran + // ctx = context.Background() + newExecId := fmt.Sprintf("%s_%s", workflowExecution.ExecutionId, action.ID) + _, err = shuffle.GetCache(ctx, newExecId) + if err == nil { + log.Printf("[DEBUG] Result for %s already found (PRE REQUEST) - returning", newExecId) + return nil + } + + cacheData := []byte("1") + err = shuffle.SetCache(ctx, newExecId, cacheData, 30) + if err != nil { + log.Printf("[WARNING] Failed setting cache for action %s: %s", newExecId, err) + } else { + //log.Printf("[DEBUG][%s] Adding %s to cache (%#v)", workflowExecution.ExecutionId, newExecId, action.Name) + } + + client := shuffle.GetExternalClient(streamUrl) + customTimeout := os.Getenv("SHUFFLE_APP_REQUEST_TIMEOUT") + if len(customTimeout) > 0 { + // convert to int + timeoutInt, err := strconv.Atoi(customTimeout) + if err != nil { + log.Printf("[ERROR] Failed converting SHUFFLE_APP_REQUEST_TIMEOUT to int: %s", err) + } else { + log.Printf("[DEBUG] Setting client timeout to %d seconds for app request", timeoutInt) + client.Timeout = time.Duration(timeoutInt) * time.Second + } + } + + newresp, err := client.Do(req) + if err != nil { + // Another timeout issue here somewhere + // context deadline + if strings.Contains(fmt.Sprintf("%s", err), "context deadline exceeded") || strings.Contains(fmt.Sprintf("%s", err), "Client.Timeout exceeded") { + return nil + } + + if strings.Contains(fmt.Sprintf("%s", err), "timeout awaiting response") { + return nil + } + + newerr := fmt.Sprintf("%s", err) + if strings.Contains(newerr, "connection refused") || strings.Contains(newerr, "no such host") { + newerr = fmt.Sprintf("Failed connecting to app %s. Is the Docker image available?", appName) + } else { + // escape quotes and newlines + newerr = strings.ReplaceAll(strings.ReplaceAll(newerr, "\"", "\\\""), "\n", "\\n") + } + + if strings.Contains(fmt.Sprintf("%s", err), "no such host") { + log.Printf("[DEBUG] SHOULD be Removing references to location for app %s as to be rediscovered", action.AppName) + + //for k, v := range portMappings { + // if strings.Contains(strings.ToLower(strings.ReplaceAll(action.AppName, " ", "_"))) { + // } + //} + + //var portMappings map[string]int + } + + log.Printf("[ERROR][%s] Error running app run request: %s", workflowExecution.ExecutionId, err) + actionResult := shuffle.ActionResult{ + Action: *action, + ExecutionId: workflowExecution.ExecutionId, + Authorization: workflowExecution.Authorization, + Result: fmt.Sprintf(`{"success": false, "reason": "Failed to connect to app %s in swarm. Try the action again, restart Orborus if this is recurring, or contact support@shuffler.io.", "details": "%s"}`, streamUrl, newerr), + StartedAt: int64(time.Now().Unix()), + CompletedAt: int64(time.Now().Unix()), Status: "FAILURE", } @@ -2768,13 +3472,17 @@ if err != nil { // Has some issues with loading when running multiple workers and such. func baseDeploy() { - cli, err := dockerclient.NewEnvClient() - if err != nil { - log.Printf("[ERROR] Unable to create docker client (3): %s", err) - return - } + var cli *dockerclient.Client + var err error - defer cli.Close() + if isKubernetes != "true" { + cli, err := dockerclient.NewEnvClient() + if err != nil { + log.Printf("[ERROR] Unable to create docker client (3): %s", err) + return + } + defer cli.Close() + } for key, value := range autoDeploy { newNameSplit := strings.Split(key, ":") @@ -2803,6 +3511,10 @@ func baseDeploy() { fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", logsDisabled), } + if key == "shuffle-tools-fork:1.0.0" { + env = append(env, fmt.Sprintf("SHUFFLE_ALLOW_PACKAGE_INSTALL=%s", "true")) + } + if strings.ToLower(os.Getenv("SHUFFLE_PASS_APP_PROXY")) == "true" { //log.Printf("APPENDING PROXY TO THE APP!") env = append(env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) @@ -2894,7 +3606,7 @@ func getStreamResultsWrapper(client *http.Client, req *http.Request, workflowExe } // Checks if a subflow is child of the startnode, as sub-subflows aren't working properly yet - childNodes := shuffle.FindChildNodes(workflowExecution, workflowExecution.Start, []string{}, []string{}) + childNodes := shuffle.FindChildNodes(workflowExecution.Workflow, workflowExecution.Start, []string{}, []string{}) log.Printf("[DEBUG] Looking for subflow in %#v to check execution pattern as child of %s", childNodes, workflowExecution.Start) subflowFound := false for _, childNode := range childNodes { @@ -2979,6 +3691,11 @@ func getStreamResultsWrapper(client *http.Client, req *http.Request, workflowExe // Initial loop etc func main() { + /*** STARTREMOVE ***/ + if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || os.Getenv("SHUFFLE_SWARM_CONFIG") == "swarm" { + logsDisabled = "true" + } + /*** ENDREMOVE ***/ // Elasticsearch necessary to ensure we'ren ot running with Datastore configurations for minimal/maximal data sizes // Recursive import kind of :) _, err := shuffle.RunInit(*shuffle.GetDatastore(), *shuffle.GetStorage(), "", "worker", true, "elasticsearch", false, 0) @@ -2987,7 +3704,11 @@ func main() { log.Printf("[ERROR] Failed to run worker init: %s", err) } } else { - log.Printf("[DEBUG] Ran init for worker to set up cache system. Docker version: %s", dockerApiVersion) + if isKubernetes != "true" { + log.Printf("[DEBUG] Ran init for worker to set up cache system. Docker version: %s", dockerApiVersion) + } else { + log.Printf("[DEBUG] Ran init for worker to set up cache system on Kubernetes") + } } log.Printf("[INFO] Setting up worker environment") @@ -3008,6 +3729,23 @@ func main() { swarmConfig := os.Getenv("SHUFFLE_SWARM_CONFIG") log.Printf("[INFO] Running with timezone %s and swarm config %#v", timezone, swarmConfig) + /*** STARTREMOVE ***/ + if swarmConfig == "run" || swarmConfig == "swarm" { + // Forcing download just in case on the first iteration. + log.Printf("[INFO] Running in swarm mode - forcing download of apps") + workflowExecution := shuffle.WorkflowExecution{} + + go baseDeploy() + + listener := webserverSetup(workflowExecution) + runWebserver(listener) + + // Should never get down here + log.Printf("[ERROR] Stopped listener %#v - exiting.", listener) + os.Exit(3) + } + /*** ENDREMOVE ***/ + authorization := "" executionId := "" @@ -3072,9 +3810,6 @@ func checkUnfinished(resp http.ResponseWriter, request *http.Request, execReques ctx := context.Background() exec, err := shuffle.GetWorkflowExecution(ctx, execRequest.ExecutionId) log.Printf("[DEBUG][%s] Rechecking execution and it's status to send to backend IF the status is EXECUTING (%s - %d/%d finished)", execRequest.ExecutionId, exec.Status, len(exec.Results), len(exec.Workflow.Actions)) - if err != nil { - return - } // FIXMe: Does this create issue with infinite loops? // Usually caused by issue during startup @@ -3101,7 +3836,7 @@ func handleRunExecution(resp http.ResponseWriter, request *http.Request) { body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("[WARNING] Failed reading body for stream result queue") - resp.WriteHeader(401) + resp.WriteHeader(400) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } @@ -3111,7 +3846,7 @@ func handleRunExecution(resp http.ResponseWriter, request *http.Request) { err = json.Unmarshal(body, &execRequest) if err != nil { log.Printf("[WARNING] Failed shuffle.WorkflowExecution unmarshaling: %s", err) - resp.WriteHeader(401) + resp.WriteHeader(400) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } @@ -3121,6 +3856,9 @@ func handleRunExecution(resp http.ResponseWriter, request *http.Request) { time.Sleep(time.Duration(30) * time.Second) checkUnfinished(resp, request, execRequest) }() + window.AddEvent(time.Now()) + + ctx := context.Background() // FIXME: This should be PER EXECUTION //if strings.ToLower(os.Getenv("SHUFFLE_PASS_APP_PROXY")) == "true" { @@ -3161,19 +3899,25 @@ func handleRunExecution(resp http.ResponseWriter, request *http.Request) { } var workflowExecution shuffle.WorkflowExecution - data = fmt.Sprintf(`{"execution_id": "%s", "authorization": "%s"}`, execRequest.ExecutionId, execRequest.Authorization) streamResultUrl := fmt.Sprintf("%s/api/v1/streams/results", baseUrl) req, err := http.NewRequest( "POST", streamResultUrl, - bytes.NewBuffer([]byte(data)), + bytes.NewBuffer([]byte(fmt.Sprintf(`{"execution_id": "%s", "authorization": "%s"}`, execRequest.ExecutionId, execRequest.Authorization))), ) + if err != nil { + log.Printf("[ERROR][%s] Failed to create a new request", execRequest.ExecutionId) + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + client := shuffle.GetExternalClient(streamResultUrl) newresp, err := client.Do(req) if err != nil { log.Printf("[ERROR] Failed making request (2): %s", err) - resp.WriteHeader(401) + resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } @@ -3181,21 +3925,21 @@ func handleRunExecution(resp http.ResponseWriter, request *http.Request) { defer newresp.Body.Close() body, err = ioutil.ReadAll(newresp.Body) if err != nil { - log.Printf("[ERROR] Failed reading body (2): %s", err) - resp.WriteHeader(401) + log.Printf("[ERROR][%s] Failed reading body (2): %s", execRequest.ExecutionId, err) + resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } if newresp.StatusCode != 200 { - log.Printf("[ERROR] Bad statuscode: %d, %s", newresp.StatusCode, string(body)) + log.Printf("[ERROR][%s] Bad statuscode: %d, %s", execRequest.ExecutionId, newresp.StatusCode, string(body)) if strings.Contains(string(body), "Workflowexecution is already finished") { log.Printf("[DEBUG] Shutting down (19)") //shutdown(workflowExecution, "", "", true) } - resp.WriteHeader(401) + resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad statuscode: %d"}`, newresp.StatusCode))) return } @@ -3203,12 +3947,11 @@ func handleRunExecution(resp http.ResponseWriter, request *http.Request) { err = json.Unmarshal(body, &workflowExecution) if err != nil { log.Printf("[ERROR] Failed workflowExecution unmarshal: %s", err) - resp.WriteHeader(401) + resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } - ctx := context.Background() //err = shuffle.SetWorkflowExecution(ctx, workflowExecution, true) err = setWorkflowExecution(ctx, workflowExecution, true) if err != nil { @@ -3239,7 +3982,7 @@ func handleRunExecution(resp http.ResponseWriter, request *http.Request) { if workflowExecution.Status != "EXECUTING" { log.Printf("[WARNING] Exiting as worker execution has status %s!", workflowExecution.Status) log.Printf("[DEBUG] Shutting down (38)") - resp.WriteHeader(401) + resp.WriteHeader(400) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad status %s for the workflow execution %s"}`, workflowExecution.Status, workflowExecution.ExecutionId))) return } @@ -3259,8 +4002,8 @@ func handleRunExecution(resp http.ResponseWriter, request *http.Request) { err = executionInit(workflowExecution) if err != nil { - log.Printf("[DEBUG][%s] Shutting down (30) - Workflow setup failed: %s", workflowExecution.ExecutionId, workflowExecution.ExecutionId, err) - resp.WriteHeader(401) + log.Printf("[DEBUG][%s] Shutting down (30) - Workflow setup failed: %s", workflowExecution.ExecutionId, err) + resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error in execution init: %s"}`, err))) return //shutdown(workflowExecution, "", "", true) @@ -3334,7 +4077,7 @@ func handleDownloadImage(resp http.ResponseWriter, request *http.Request) { } log.Printf("[INFO] Downloading image %s", image.Image) - downloadDockerImageBackend(&http.Client{Timeout: 60 * time.Second}, image.Image) + shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image.Image) // return success resp.WriteHeader(200) @@ -3345,10 +4088,50 @@ func runWebserver(listener net.Listener) { r := mux.NewRouter() r.HandleFunc("/api/v1/streams", handleWorkflowQueue).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/streams/results", handleGetStreamResults).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/execute", handleRunExecution).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/run", handleRunExecution).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/download", handleDownloadImage).Methods("POST", "OPTIONS") + // Synonyms. Require an execution ID + auth + shuffle backend + r.HandleFunc("/api/v1/execute", handleRunExecution).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/run", handleRunExecution).Methods("POST", "OPTIONS") + + // What would be require to run a workflow otherwise? + // Maybe directly /workflow/run + + /*** STARTREMOVE ***/ + if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || os.Getenv("SHUFFLE_SWARM_CONFIG") == "swarm" { + log.Printf("[DEBUG] Running webserver config for SWARM and K8s") + } + /*** ENDREMOVE ***/ + var dockercli *dockerclient.Client + ctx := context.Background() + scaleReplicas := os.Getenv("SHUFFLE_APP_REPLICAS") + if len(scaleReplicas) > 0 { + tmpInt, err := strconv.Atoi(scaleReplicas) + if err != nil { + log.Printf("[ERROR] %s is not a valid number for replication", scaleReplicas) + } else { + maxReplicas = uint64(tmpInt) + _ = tmpInt + } + + log.Printf("[DEBUG] SHUFFLE_APP_REPLICAS set to value %#v. Trying to overwrite default (%d/node)", scaleReplicas, maxReplicas) + } + + maxExecutionsPerMinute := 10 + if os.Getenv("SHUFFLE_APP_EXECUTIONS_PER_MINUTE") != "" { + tmpInt, err := strconv.Atoi(os.Getenv("SHUFFLE_APP_EXECUTIONS_PER_MINUTE")) + if err != nil { + log.Printf("[ERROR] %s is not a valid number for executions per minute", os.Getenv("SHUFFLE_APP_EXECUTIONS_PER_MINUTE")) + } else { + maxExecutionsPerMinute = tmpInt + } + + log.Printf("[DEBUG] SHUFFLE_APP_EXECUTIONS_PER_MINUTE set to value %s. Trying to overwrite default (%d)", os.Getenv("SHUFFLE_APP_EXECUTIONS_PER_MINUTE"), maxExecutionsPerMinute) + } + + if strings.ToLower(os.Getenv("SHUFFLE_SWARM_CONFIG")) == "run" || strings.ToLower(os.Getenv("SHUFFLE_APP_REPLICAS")) == "" { + go AutoScaleApps(ctx, dockercli, maxExecutionsPerMinute) + } if strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" { r.HandleFunc("/debug/pprof/", pprof.Index) r.HandleFunc("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP) @@ -3382,3 +4165,203 @@ func runWebserver(listener net.Listener) { log.Printf("[ERROR] Serve issue in worker: %#v", err) } } + +func AutoScaleApps(ctx context.Context, client *dockerclient.Client, maxExecutionsPerMinute int) { + ticker := time.NewTicker(1 * time.Second) + + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + + case <-ticker.C: + count := window.CountEvents(time.Now()) + j := numberOfApps(ctx, client) + workers := numberOfWorkers(ctx, client) + execPerMin := maxExecutionsPerMinute / workers + if count >= execPerMin { + log.Printf("[DEBUG] Too many executions per minute (%d). Scaling down to %d", count, execPerMin) + scaleApps(ctx, client, uint64(j+1)) + } + } + } +} + +func scaleApps(ctx context.Context, client *dockerclient.Client, replicas uint64) error { + client, err := dockerclient.NewEnvClient() + services, err := client.ServiceList(ctx, types.ServiceListOptions{}) + if err != nil { + log.Printf("[ERROR] Failed to find services in the swarm: %s", err) + } + + networkId, err := getNetworkId(ctx, client) + if err != nil { + log.Printf("[ERROR] Failed to get network Id in the swarm service: %s", err) + } + + workers := numberOfWorkers(ctx, client) + if replicas > uint64(workers) { + return nil + } + + for _, service := range services { + if service.Spec.Name == "shuffle-workers" { + continue + } + + inNetwork := false + for _, vip := range service.Endpoint.VirtualIPs { + if vip.NetworkID == networkId { + inNetwork = true + break + } + } + if !inNetwork { + continue // skip services not in the target network + } + + if service.Spec.Mode.Replicated == nil { + return errors.New("Service is not replicated") + } + + if *service.Spec.Mode.Replicated.Replicas >= replicas { + continue + } + + service.Spec.Mode.Replicated.Replicas = &replicas + _, err = client.ServiceUpdate(ctx, service.ID, service.Version, service.Spec, types.ServiceUpdateOptions{}) + if err != nil { + return err + } + + } + + log.Printf("[DEBUG] Scaled all services to %d replicas", replicas) + return nil +} + +func getNetworkId(ctx context.Context, dockercli *dockerclient.Client) (string, error) { + networkFilter := filters.NewArgs() + networkFilter.Add("name", swarmNetworkName) + + networks, err := dockercli.NetworkList(ctx, types.NetworkListOptions{ + Filters: networkFilter, + }) + + if err != nil || len(networks) == 0 { + return "", err + } + networkId := networks[0].ID + + return networkId, nil +} + +func numberOfApps(ctx context.Context, dockercli *dockerclient.Client) int { + // swarmNetworkName + + var err error + if swarmNetworkName == "" { + swarmNetworkName = "shuffle_swarm_executions" + } + + if dockercli == nil { + dockercli, err = dockerclient.NewEnvClient() + if err != nil { + log.Printf("[ERROR] Unable to create docker client (5): %s", err) + return 0 + } + } + + networkFilter := filters.NewArgs() + networkFilter.Add("name", swarmNetworkName) + + networks, err := dockercli.NetworkList(ctx, types.NetworkListOptions{ + Filters: networkFilter, + }) + + if err != nil || len(networks) == 0 { + return 0 + } + + networkId, err := getNetworkId(ctx, dockercli) + if err != nil { + log.Printf("[WARNING] Failed to get networkID is worker running in swarm: %s", err) + return 0 + } + + services, err := dockercli.ServiceList(ctx, types.ServiceListOptions{}) + if err != nil { + log.Printf("[WARNING] Can't found any services. %s", err) + return 0 + } + + runningReplicas := 0 + + for _, service := range services { + if service.Spec.Name == "shuffle-workers" { + continue + } + + inNetwork := false + for _, vip := range service.Endpoint.VirtualIPs { + if vip.NetworkID == networkId { + inNetwork = true + break + } + } + if !inNetwork { + continue // skip services not in the target network + } + + filterArgs := filters.NewArgs() + filterArgs.Add("service", service.Spec.Name) + filterArgs.Add("desired-state", "running") + + task, err := dockercli.TaskList(ctx, types.TaskListOptions{ + Filters: filterArgs, + }) + if err != nil { + log.Printf("[WARNING] Failed to get the list of running services %s: %s", service.Spec.Name, err) + continue + } + + runningReplicas = len(task) + break + } + + return runningReplicas +} + +func IsServiceRunning(ctx context.Context, cli *dockerclient.Client) bool { + serviceName := "shuffle-tools_1-2-0" + filterArgs := filters.NewArgs() + filterArgs.Add("name", serviceName) + + services, err := cli.ServiceList(ctx, types.ServiceListOptions{Filters: filterArgs}) + + if err != nil { + log.Printf("[ERROR] Couldn't find %s service running got error: %s", serviceName, err) + return false + } + if len(services) > 0 { + return true + } + + return false +} + +func numberOfWorkers(ctx context.Context, cli *dockerclient.Client) int { + cli, err := dockerclient.NewEnvClient() + service, _, err := cli.ServiceInspectWithRaw(ctx, "shuffle-workers", types.ServiceInspectOptions{}) + if err != nil { + return 0 + } + + if service.Spec.Mode.Replicated == nil { + return 0 + } + + replics := *service.Spec.Mode.Replicated.Replicas + return int(replics) +} diff --git a/shuffle-database/README.md b/shuffle-database/README.md new file mode 100644 index 00000000..614e8b6b --- /dev/null +++ b/shuffle-database/README.md @@ -0,0 +1 @@ +TMP diff --git a/template.yaml b/template.yaml new file mode 100644 index 00000000..2778b38b --- /dev/null +++ b/template.yaml @@ -0,0 +1,81 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: 'Create an EC2 instance with port 3001 open and auto-generate network resources' + +Resources: + VPC: + Type: AWS::EC2::VPC + Properties: + CidrBlock: 10.0.0.0/16 + EnableDnsHostnames: true + EnableDnsSupport: true + + PublicSubnet: + Type: AWS::EC2::Subnet + Properties: + VpcId: !Ref VPC + AvailabilityZone: !Select + - 0 + - !GetAZs + Ref: 'AWS::Region' + CidrBlock: 10.0.1.0/24 + MapPublicIpOnLaunch: true + + InternetGateway: + Type: AWS::EC2::InternetGateway + + AttachGateway: + Type: AWS::EC2::VPCGatewayAttachment + Properties: + VpcId: !Ref VPC + InternetGatewayId: !Ref InternetGateway + + PublicRouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: !Ref VPC + + PublicRoute: + Type: AWS::EC2::Route + DependsOn: AttachGateway + Properties: + RouteTableId: !Ref PublicRouteTable + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: !Ref InternetGateway + + SubnetRouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + SubnetId: !Ref PublicSubnet + RouteTableId: !Ref PublicRouteTable + + SecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Allow port 3001 access + VpcId: !Ref VPC + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 3001 + ToPort: 3001 + CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: 0.0.0.0/0 + + EC2Instance: + Type: AWS::EC2::Instance + Properties: + InstanceType: t3.large + SecurityGroupIds: + - !Ref SecurityGroup + SubnetId: !Ref PublicSubnet + ImageId: ami-04996ef73316c465c + +Outputs: + InstanceId: + Description: InstanceId of the newly created EC2 instance + Value: !Ref EC2Instance + PublicDNS: + Description: Public DNSName of the newly created EC2 instance + Value: !GetAtt EC2Instance.PublicDnsName \ No newline at end of file