diff --git a/.env b/.env index 7e073bd9..07deaf4a 100755 --- a/.env +++ b/.env @@ -2,13 +2,17 @@ ORG_ID=Shuffle ENVIRONMENT_NAME=Shuffle +# Sanitize liquid.py input +LIQUID_SANITIZE_INPUT=true + + # Remote github config for first load SHUFFLE_DOWNLOAD_WORKFLOW_LOCATION= SHUFFLE_DOWNLOAD_WORKFLOW_USERNAME= SHUFFLE_DOWNLOAD_WORKFLOW_PASSWORD= SHUFFLE_DOWNLOAD_WORKFLOW_BRANCH= -SHUFFLE_APP_DOWNLOAD_LOCATION=https://github.com/frikky/shuffle-apps +SHUFFLE_APP_DOWNLOAD_LOCATION=https://github.com/shuffle/python-apps SHUFFLE_DOWNLOAD_AUTH_USERNAME= SHUFFLE_DOWNLOAD_AUTH_PASSWORD= SHUFFLE_DOWNLOAD_AUTH_BRANCH= @@ -30,6 +34,8 @@ SHUFFLE_FILE_LOCATION=./shuffle-files SHUFFLE_ENCRYPTION_MODIFIER= # Other configs +BASE_URL=http://shuffle-backend:5001 +SSO_REDIRECT_URL=http://localhost:3001 BACKEND_HOSTNAME=shuffle-backend BACKEND_PORT=5001 FRONTEND_PORT=3001 @@ -48,21 +54,39 @@ SHUFFLE_PASS_WORKER_PROXY=TRUE SHUFFLE_PASS_APP_PROXY=FALSE TZ=Europe/Amsterdam # Timezone-handler in Orborus, Worker and Apps ORBORUS_CONTAINER_NAME= # Used to FIND the containername. cgroup v2: issue 501 +SHUFFLE_ORBORUS_STARTUP_DELAY= # Used for setting up a startup delay for Orborus +SHUFFLE_BASE_IMAGE_NAME=shuffle SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io -SHUFFLE_BASE_IMAGE_NAME=frikky -SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.80" +SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-1.1.0" + +## shuffle_memcached (for distributed caching) +## shuffle_SWARM_CONFIG (run vs not run) +## shuffle_Scale_Replicas (workers/node) +## shuffle_App_Replicas (apps/node) + +SHUFFLE_SWARM_BRIDGE_DEFAULT_MTU=1500 # 1500 by default +# The eth0 interface inside a container corresponds +# to the virtual Ethernet interface that connects +# the container to the docker0 +SHUFFLE_SWARM_BRIDGE_DEFAULT_INTERFACE=eth0 # Used for auto-cleanup of containers. REALLY important at scale. SHUFFLE_CONTAINER_AUTO_CLEANUP=false SHUFFLE_ELASTIC=true +SHUFFLE_LOGS_DISABLED=false +SHUFFLE_CHAT_DISABLED=false # Controls support chat +SHUFFLE_RERUN_SCHEDULE=300 +SHUFFLE_DISABLE_RERUN_AND_ABORT=false +SHUFFLE_WORKER_SERVER_URL= # Definition in case Worker & Orborus is talking to the wrong server +SHUFFLE_ORBORUS_PULL_TIME= # Definition in case Orborus is pulling too often/not often enough # DATABASE CONFIGURATIONS DATASTORE_EMULATOR_HOST=shuffle-database:8000 -#SHUFFLE_OPENSEARCH_URL=https://shuffle-opensearch:9200 -SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200 -SHUFFLE_OPENSEARCH_USERNAME= -SHUFFLE_OPENSEARCH_PASSWORD= +#SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200 +SHUFFLE_OPENSEARCH_URL=https://shuffle-opensearch:9200 +SHUFFLE_OPENSEARCH_USERNAME=admin +SHUFFLE_OPENSEARCH_PASSWORD=admin SHUFFLE_OPENSEARCH_CERTIFICATE_FILE= SHUFFLE_OPENSEARCH_APIKEY= SHUFFLE_OPENSEARCH_CLOUDID= diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 147232b4..ba788f87 100755 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -23,7 +23,7 @@ A description of what you expected to happen. **Screenshots** If applicable, add screenshots to help explain your problem. -** Debug logs ** +** Debug logs (NOT APPLICABLE FOR CLOUD)** Run the following commands and paste them ``` docker logs shuffle-backend diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index bbcbbe7d..1ba91a7a 100755 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -18,3 +18,5 @@ A clear and concise description of any alternative solutions or features you've **Additional context** Add any other context or screenshots about the feature request here. + +**Screenshots of where and how** diff --git a/.github/install-guide.md b/.github/install-guide.md index d1dfcf76..53e39401 100755 --- a/.github/install-guide.md +++ b/.github/install-guide.md @@ -1,7 +1,7 @@ # Installation guide Installation of Shuffle is currently only available in docker. Looking for how to update Shuffle? Check the [updating guide](https://shuffler.io/docs/configuration#updating_shuffle) -This document outlines a an introduction environment which is not scalable. [Read here](https://shuffler.io/docs/configuration#production_readiness) for information on production readiness. +This document outlines a an introduction environment which is not scalable. [Read here](https://shuffler.io/docs/configuration#production_readiness) for information on production readiness. This also includes system requirements and configurations for Swarm or K8s. # Docker - *nix The Docker setup is done with docker-compose @@ -10,18 +10,19 @@ The Docker setup is done with docker-compose 1. Make sure you have [Docker](https://docs.docker.com/get-docker/) and [docker-compose](https://docs.docker.com/compose/install/) installed. 2. Download Shuffle -``` -git clone https://github.com/frikky/Shuffle +```bash +git clone https://github.com/Shuffle/Shuffle cd Shuffle ``` 3. Fix prerequisites for the Opensearch database (Elasticsearch): -``` -sudo chown 1000:1000 -R shuffle-database # Required for Opensearch +```bash +mkdir shuffle-database +sudo chown -R 1000:1000 shuffle-database ``` 4. Run docker-compose. -``` +```bash docker-compose up -d ``` @@ -38,20 +39,20 @@ This step is for setting up with Docker on windows from scratch. 4. Open the .env file and change the line with "OUTER_HOSTNAME" to contain your IP: -``` +```bash OUTER_HOSTNAME=YOUR.IP.HERE ``` 6. Run docker-compose -``` -docker compose up -d +```bash +docker-compose up -d ``` ### Configurations (proxies, default users etc.) https://shuffler.io/docs/configuration ### After installation -1. After installation, go to http://localhost:3001/adminsetup (or your servername - https is on port 3443) +1. After installation, go to http://localhost:3001 (or your servername - https is on port 3443) 2. Now set up your admin account (username & password). Shuffle doesn't have a default username and password. 3. Sign in with the same Username & Password! Go to /apps and see if you have any apps yet. If not - you may need to [configure proxies](https://shuffler.io/docs/configuration#production_readiness) 4. Check out https://shuffler.io/docs/configuration as it has a lot of useful information to get started @@ -84,11 +85,15 @@ npm start ## Backend - Golang http://localhost:5001 - REST API - requires [>=go1.13](https://golang.org/dl/) ```bash -export SHUFFLE_OPENSEARCH_URL="http://localhost:9200" +export SHUFFLE_OPENSEARCH_URL="https://localhost:9200" export SHUFFLE_ELASTIC=true +export SHUFFLE_OPENSEARCH_USERNAME=admin +export SHUFFLE_OPENSEARCH_PASSWORD=admin +export SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY=true cd backend/go-app -go run *.go +go run main.go walkoff.go docker.go ``` +**WINDOWS USERS:** Follow [this guide](https://www.wikihow.com/Create-an-Environment-Variable-in-Windows-10) to add environment variables in your machine. Large portions of the backend is written in another repository - [shuffle-shared](https://github.com/frikky/shuffle-shared). If you want to update any of this code and test in realtime, we recommend following these steps: 1. Clone shuffle-shared to a local repository @@ -100,8 +105,6 @@ Large portions of the backend is written in another repository - [shuffle-shared 4. Make the changes you want, then restart the backend server! 5. With your changes made, make a pull request :fire: -**WINDOWS USERS:** You'll have to to add the "export" part as an environment variable. - ## Database - Opensearch Make sure this is running through the docker-compose, and that the backend points to it with SHUFFLE_OPENSEARCH_URL defined @@ -121,6 +124,4 @@ export BASE_URL=http://YOUR-IP:5001 export DOCKER_API_VERSION=1.40 ``` -**WINDOWS USERS:** You'll have to to add the "export" part as an environment variable. - AND THAT's it - hopefully it worked. If it didn't please email [frikky@shuffler.io](mailto:frikky@shuffler.io) diff --git a/.github/push_nightly.sh b/.github/push_nightly.sh new file mode 100644 index 00000000..b3660193 --- /dev/null +++ b/.github/push_nightly.sh @@ -0,0 +1,86 @@ +# This can be done in the dockerpush workflow itself +# Done manually for now since GHCR isn't being pushed to easily with the current Github action CI. Nightly = Latest IF we run hotfixes on latest + +### Pull latest from ghcr CI/CD +docker pull ghcr.io/shuffle/shuffle-app_sdk:nightly +docker pull ghcr.io/shuffle/shuffle-worker:nightly +docker pull ghcr.io/shuffle/shuffle-orborus:nightly +docker pull ghcr.io/shuffle/shuffle-frontend:nightly +#docker pull ghcr.io/shuffle/shuffle-backend:nightly +# +### NIGHTLY releases +docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly ghcr.io/frikky/shuffle-app_sdk:nightly +docker tag ghcr.io/shuffle/shuffle-worker:nightly ghcr.io/frikky/shuffle-worker:nightly +docker tag ghcr.io/shuffle/shuffle-orborus:nightly ghcr.io/frikky/shuffle-orborus:nightly +docker tag ghcr.io/shuffle/shuffle-frontend:nightly ghcr.io/frikky/shuffle-frontend:nightly +docker tag ghcr.io/shuffle/shuffle-backend:nightly ghcr.io/frikky/shuffle-backend:nightly + +docker push ghcr.io/frikky/shuffle-app_sdk:nightly +docker push ghcr.io/frikky/shuffle-worker:nightly +docker push ghcr.io/frikky/shuffle-orborus:nightly +docker push ghcr.io/frikky/shuffle-frontend:nightly +docker push ghcr.io/frikky/shuffle-backend:nightly + +### LATEST releases: +## shuffle/shuffle +docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly ghcr.io/shuffle/shuffle-app_sdk:latest +docker tag ghcr.io/shuffle/shuffle-worker:nightly ghcr.io/shuffle/shuffle-worker:latest +docker tag ghcr.io/shuffle/shuffle-orborus:nightly ghcr.io/shuffle/shuffle-orborus:latest +docker tag ghcr.io/shuffle/shuffle-frontend:nightly ghcr.io/shuffle/shuffle-frontend:latest +docker tag ghcr.io/shuffle/shuffle-backend:nightly ghcr.io/shuffle/shuffle-backend:latest + +docker push ghcr.io/shuffle/shuffle-app_sdk:latest +docker push ghcr.io/shuffle/shuffle-worker:latest +docker push ghcr.io/shuffle/shuffle-orborus:latest +docker push ghcr.io/shuffle/shuffle-frontend:latest +docker push ghcr.io/shuffle/shuffle-backend:latest + +## frikky/shuffle +docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly ghcr.io/frikky/shuffle-app_sdk:latest +docker tag ghcr.io/shuffle/shuffle-worker:nightly ghcr.io/frikky/shuffle-worker:latest +docker tag ghcr.io/shuffle/shuffle-orborus:nightly ghcr.io/frikky/shuffle-orborus:latest +docker tag ghcr.io/shuffle/shuffle-frontend:nightly ghcr.io/frikky/shuffle-frontend:latest +docker tag ghcr.io/shuffle/shuffle-backend:nightly ghcr.io/frikky/shuffle-backend:latest + +docker push ghcr.io/frikky/shuffle-app_sdk:latest +docker push ghcr.io/frikky/shuffle-worker:latest +docker push ghcr.io/frikky/shuffle-orborus:latest +docker push ghcr.io/frikky/shuffle-frontend:latest +docker push ghcr.io/frikky/shuffle-backend:latest + + +### 1.1.0 releases: +## shuffle/shuffle +docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly ghcr.io/shuffle/shuffle-app_sdk:1.1.0 +docker tag ghcr.io/shuffle/shuffle-worker:nightly ghcr.io/shuffle/shuffle-worker:1.1.0 +docker tag ghcr.io/shuffle/shuffle-orborus:nightly ghcr.io/shuffle/shuffle-orborus:1.1.0 +docker tag ghcr.io/shuffle/shuffle-frontend:nightly ghcr.io/shuffle/shuffle-frontend:1.1.0 +docker tag ghcr.io/shuffle/shuffle-backend:nightly ghcr.io/shuffle/shuffle-backend:1.1.0 + +docker push ghcr.io/shuffle/shuffle-app_sdk:1.1.0 +docker push ghcr.io/shuffle/shuffle-worker:1.1.0 +docker push ghcr.io/shuffle/shuffle-orborus:1.1.0 +docker push ghcr.io/shuffle/shuffle-frontend:1.1.0 +docker push ghcr.io/shuffle/shuffle-backend:1.1.0 + +## frikky/shuffle +docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly ghcr.io/frikky/shuffle-app_sdk:1.1.0 +docker tag ghcr.io/shuffle/shuffle-worker:nightly ghcr.io/frikky/shuffle-worker:1.1.0 +docker tag ghcr.io/shuffle/shuffle-orborus:nightly ghcr.io/frikky/shuffle-orborus:1.1.0 +docker tag ghcr.io/shuffle/shuffle-frontend:nightly ghcr.io/frikky/shuffle-frontend:1.1.0 +docker tag ghcr.io/shuffle/shuffle-backend:nightly ghcr.io/frikky/shuffle-backend:1.1.0 + +docker push ghcr.io/frikky/shuffle-app_sdk:1.1.0 +docker push ghcr.io/frikky/shuffle-worker:1.1.0 +docker push ghcr.io/frikky/shuffle-orborus:1.1.0 +docker push ghcr.io/frikky/shuffle-frontend:1.1.0 +docker push ghcr.io/frikky/shuffle-backend:1.1.0 + +### Manage worker-scale upload (Requires auth) +# This is supposed to be unavailable, and only be downloadable by customers +docker pull ghcr.io/shuffle/shuffle-worker-scale:latest +docker save ghcr.io/shuffle/shuffle-worker-scale:latest -o shuffle-worker.zip +echo "1. Upload shuffle-worker.zip to the shuffler.io public repo. If in Github Dev env, download the file, and upload manually." +echo "2. Have customers download it with: $ wget URL" +echo "3. Have customers use with with: docker load shuffle-worker.zip" + diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index b82be082..00006b66 100755 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,11 +39,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -54,7 +54,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v2 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -68,4 +68,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/ci.yaml b/.github/workflows/docker-build.yaml similarity index 89% rename from .github/workflows/ci.yaml rename to .github/workflows/docker-build.yaml index 6b235f6b..716b42db 100755 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/docker-build.yaml @@ -1,8 +1,8 @@ -name: ci +name: docker-build on: push: - branches: master + branches: launch jobs: main: runs-on: ubuntu-latest @@ -13,19 +13,19 @@ jobs: include: - app: frontend path: frontend - version: 0.8.3 + version: 1.0.0 experimental: true - app: backend path: backend - version: 0.8.3 + version: 1.0.0 experimental: false - app: orborus path: functions/onprem/orborus - version: 0.8.0 + version: 1.0.0 experimental: false - app: database path: backend/database - version: 0.8.0 + version: 1.0.0 experimental: false steps: - @@ -63,4 +63,4 @@ jobs: tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.app }}:${{ matrix.version }} - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/dockerbuild.yaml b/.github/workflows/dockerbuild.yaml new file mode 100644 index 00000000..0594f24b --- /dev/null +++ b/.github/workflows/dockerbuild.yaml @@ -0,0 +1,75 @@ +name: dockerbuild + +on: + push: + branches: 1.2.0 +jobs: + main: + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + include: + - app: frontend + path: frontend + version: nightly + experimental: true + - app: backend + path: backend + version: nightly + experimental: true + - app: app_sdk + path: backend/app_sdk + version: nightly + experimental: true + - app: orborus + path: functions/onprem/orborus + version: nightly + experimental: true + - app: worker + path: functions/onprem/worker + version: nightly + experimental: true + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to Ghcr + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Ghcr Build and push + id: docker_build + uses: docker/build-push-action@v3 + env: + BUILDX_NO_DEFAULT_LOAD: true + with: + logout: false + context: ${{ matrix.path }}/ + file: ${{ matrix.path }}/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + tags: | + ghcr.io/shuffle/shuffle-${{ matrix.app }}:nightly + ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:nightly + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml new file mode 100644 index 00000000..b3d1927e --- /dev/null +++ b/.github/workflows/release-please.yml @@ -0,0 +1,13 @@ +on: + push: + branches: + - launch +name: release-please +jobs: + release-please: + runs-on: ubuntu-latest + steps: + - uses: google-github-actions/release-please-action@v3 + with: + release-type: node + package-name: release-please-action diff --git a/.github/workflows/snyk-container-analysis.yml b/.github/workflows/snyk-container-analysis.yml index b3a78761..f9a406b2 100755 --- a/.github/workflows/snyk-container-analysis.yml +++ b/.github/workflows/snyk-container-analysis.yml @@ -11,7 +11,6 @@ name: Snyk Container on: push: branches: - - master - launch pull_request: # The branches below must be a subset of the branches above @@ -25,9 +24,12 @@ jobs: snyk: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v2 + - name: Build a Docker image - run: docker build -t your/image-to-test . + run: docker build -t frontend . + - name: Run Snyk to check Docker image for vulnerabilities # Snyk can be used to break the build when it detects vulnerabilities. # In this case we want to upload the issues to GitHub Code Scanning @@ -41,6 +43,7 @@ jobs: with: image: your/image-to-test args: --file=Dockerfile + - name: Upload result to GitHub Code Scanning uses: github/codeql-action/upload-sarif@v1 with: diff --git a/.github/workflows/upload_sdk.yml b/.github/workflows/upload_sdk.yml new file mode 100644 index 00000000..9faa33e4 --- /dev/null +++ b/.github/workflows/upload_sdk.yml @@ -0,0 +1,51 @@ +# This is a basic workflow to help you get started with Actions + +name: App SDK upload + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the main branch + push: + branches: [ master, launch ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v3 + + - id: 'auth' + name: 'Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v0' + with: + credentials_json: '${{ secrets.SANDBOX_CREDENTIALS }}' + + - id: 'upload_sdk' + name: Cloud Storage Uploader + uses: google-github-actions/upload-cloud-storage@v0.9.0 + with: + path: 'backend/app_sdk/app_base.py' + destination: 'shuffle-sandbox-337810.appspot.com/generated_apps/baseline' + + - id: 'upload_requirement' + name: Cloud Storage Uploader + uses: google-github-actions/upload-cloud-storage@v0.9.0 + with: + path: 'backend/app_sdk/requirements.txt' + destination: 'shuffle-sandbox-337810.appspot.com/generated_apps/baseline' + + - id: 'upload_Dockerfile' + name: Cloud Storage Uploader + uses: google-github-actions/upload-cloud-storage@v0.9.0 + with: + path: 'backend/app_sdk/Dockerfile' + destination: 'shuffle-sandbox-337810.appspot.com/generated_apps/baseline' diff --git a/.gitignore b/.gitignore index 4d3f2d80..3ad770b6 100755 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,5 @@ shuffle-database/logging_enabled.conf shuffle-database/nodes shuffle-database/performance_analyzer_enabled.conf shuffle-database/rca_enabled.conf + +*/package-lock.json diff --git a/README.md b/README.md index c09bd090..e937f939 100755 --- a/README.md +++ b/README.md @@ -1,7 +1,25 @@ -# Shuffle -[Shuffle](https://shuffler.io) is an automation platform focused on accessibility. We believe everyone should have access to efficient processes, and are striving to make that a possibility by making integrations for YOUR tools. Security Operations is complex, but it doesn't have to be. +

-[![Discord](https://img.shields.io/discord/463752820026376202.svg?label=&logo=discord&logoColor=ffffff&color=7389D8&labelColor=6A7EC2)](https://discord.gg/B2CBzUm) +[![Shuffle Logo](https://github.com/frikky/Shuffle/blob/launch/frontend/public/images/Shuffle_logo_new.png)](https://shuffler.io) + +Shuffle Automation + +[![CodeQL](https://github.com/Shuffle/Shuffle/actions/workflows/codeql-analysis.yml/badge.svg?branch=launch)](https://github.com/Shuffle/Shuffle/actions/workflows/codeql-analysis.yml) +[![Autobuild](https://github.com/Shuffle/Shuffle/actions/workflows/dockerbuild.yaml/badge.svg?branch=launch)](https://github.com/Shuffle/Shuffle/actions/workflows/dockerbuild.yaml) + +

+ +[Shuffle](https://shuffler.io) is an automation platform for and by the community, focusing on accessibility for anyone to automate. Security operations is complex, but it doesn't have to be. + +[_Key Features_](https://shuffler.io/docs/features) — +[_Community & Support_](https://discord.gg/B2CBzUm) — +[_Documentation_](https://shuffler.io/docs) — +[_Getting Started_](https://shuffler.io/docs/getting_started) — +[_Development_](https://github.com/frikky/Shuffle/blob/master/.github/CONTRIBUTING.md) + +Follow us on Twitter at [@shuffleio](https://twitter.com/shuffleio). + +

![Example Shuffle webhook integration](https://github.com/frikky/Shuffle/blob/master/frontend/src/assets/img/github_shuffle_img.png) @@ -85,15 +103,14 @@ Below is the folder structure with a short explanation ```bash ├── README.md # What you're reading right now ├── backend # Contains backend related code. -│   ├── go-app # The backend golang webserver +│ ├── go-app # The backend golang webserver │ └── app_sdk # The SDK used for apps ├── frontend # Contains frontend code. ReactJS, Material UI and cytoscape ├── functions # Has execution and extension resources, such as the Wazuh integration -│   ├── onprem # Code for onprem solutions -│  │   ├── Orborus # Distributes execution locations -│  │   ├── Worker # Runs a workflow +│ ├── onprem # Code for onprem solutions +│ │ ├── Orborus # Distributes execution locations +│ │ ├── Worker # Runs a workflow └ docker-compose.yml # Used for deployments ``` -**It's in BETA (0.8.60)** - [Get in touch](https://shuffler.io/contact), send a mail to [frikky@shuffler.io](mailto:frikky@shuffler.io) or poke me on twitter [@frikkylikeme](https://twitter.com/frikkylikeme) - +**It's in BETA** - [Get in touch](https://shuffler.io/contact), send a mail to [frikky@shuffler.io](mailto:frikky@shuffler.io) or poke me on twitter [@frikkylikeme](https://twitter.com/frikkylikeme) diff --git a/SECURITY.md b/SECURITY.md index 3c372911..51662a7d 100755 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,7 +2,7 @@ ## Supported Versions -Shuffle is currently still in beta, but we aim to support older version with critical severity issues, but do advise you to stay up to date with Major versions. +Shuffle is now live in version 1.0.0, but we aim to support older version with critical severity security issues, but do advise you to stay up to date with Major versions. | Version | Supported | | ------- | ------------------ | @@ -11,9 +11,9 @@ Shuffle is currently still in beta, but we aim to support older version with cri ## Reporting a Vulnerability -Reporting a vulnerability can either be done to (frikky@shuffler.io)[mailto:frikky@shuffler.io] or [through the contact page on our website](https://shuffler.io/contact) +Reporting a vulnerability can either be done to [support@shuffler.io](mailto:support@shuffler.io) or [through the contact page on our website](https://shuffler.io/contact) -Security.txt: https://shuffler.io/.well_known/security.txt +Security.txt: https://shuffler.io/.well-known/security.txt -When a >medium severity vulnerability is discovered, expect it to be fixed ASAP - please nag us until it is. Security is a top priority, and we expect you to keep us accountable. +When a >medium severity vulnerability is discovered, expect it to be fixed ASAP - please nag us until it is otherwise. Security is a top priority, and we expect and hope you hold us accountable. In the case it makes sense, we'll further create a security advisory, and publish a new CVE for your new glorious finding. diff --git a/backend/Dockerfile b/backend/Dockerfile index f09fe3b3..5bc61f64 100755 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.2-buster as builder +FROM golang:1.19.3-buster as builder # Add files RUN mkdir /app @@ -15,14 +15,24 @@ ADD ./app_sdk/app_base.py /app_sdk ADD ./app_gen /app_gen RUN go get -v +RUN go mod tidy +RUN go clean -modcache -RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o webapp . +# From November 2022, CGO is enabled due to packages +# that we use requiring it. This is a temporary fix +# and makes us HAVE to install libc compatibility packages farther down. +RUN CGO_ENABLED=1 GOOS=linux go build -a -installsuffix cgo -o webapp . # Certificate build - gets required certs FROM alpine:latest as certs RUN apk --update add ca-certificates -FROM alpine:3.14.2 +# Sets up the final image +FROM alpine:3.17.0 + +# FIXME: Install cgo because CGO_ENABLED=1 during build +RUN apk add --no-cache libc6-compat +RUN apk add --no-cache libstdc++ COPY --from=builder /app/ /app COPY --from=builder /app_sdk/ /app_sdk diff --git a/backend/app_sdk/Dockerfile b/backend/app_sdk/Dockerfile index 25c9aa40..d3709032 100755 --- a/backend/app_sdk/Dockerfile +++ b/backend/app_sdk/Dockerfile @@ -1,5 +1,6 @@ #FROM python:3.9.1-alpine as base FROM python:3.10.0-alpine as base +#FROM python:3.11.3-alpine as base FROM base as builder RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev tzdata coreutils diff --git a/backend/app_sdk/Dockerfile_alpine_grpc b/backend/app_sdk/Dockerfile_alpine_grpc new file mode 100644 index 00000000..dedc95b1 --- /dev/null +++ b/backend/app_sdk/Dockerfile_alpine_grpc @@ -0,0 +1,42 @@ +FROM python:3.10.0-alpine as base + +FROM base as builder +RUN apk --no-cache add --update \ + alpine-sdk \ + build-base \ + g++ \ + gcc \ + libffi \ + libffi-dev \ + libstdc++ \ + linux-headers \ + musl-dev \ + openssl-dev \ + tzdata \ + coreutils + +RUN pip install --upgrade pip && \ + pip install --prefix="/install" --no-cache-dir grpcio grpcio-tools && \ + apk del --purge \ + g++ \ + gcc \ + musl-dev \ + libffi-dev \ + libstdc++ \ + build-base \ + linux-headers + +RUN mkdir -p /install +WORKDIR /install + +FROM base + +#--no-cache +RUN apk update && apk add --update tzdata libmagic alpine-sdk libffi libffi-dev musl-dev openssl-dev coreutils + +COPY --from=builder /install /usr/local +COPY requirements.txt /requirements.txt +RUN pip3 install -r /requirements.txt + +COPY __init__.py /app/walkoff_app_sdk/__init__.py +COPY app_base.py /app/walkoff_app_sdk/app_base.py diff --git a/backend/app_sdk/Dockerfile_blackarch b/backend/app_sdk/Dockerfile_blackarch index 02480aad..e7469166 100755 --- a/backend/app_sdk/Dockerfile_blackarch +++ b/backend/app_sdk/Dockerfile_blackarch @@ -1,4 +1,4 @@ -FROM peterclemenko/blackarch as base +FROM blackarchlinux/blackarch as base FROM base as builder diff --git a/backend/app_sdk/Dockerfile_ubuntu b/backend/app_sdk/Dockerfile_ubuntu new file mode 100644 index 00000000..3f34d4bd --- /dev/null +++ b/backend/app_sdk/Dockerfile_ubuntu @@ -0,0 +1,22 @@ +FROM ubuntu as base + +FROM base as builder + +RUN apt-get update +RUN apt-get dist-upgrade -y +RUN apt install build-essential libffi-dev musl-dev openssl python3 python3-pip -y + +RUN mkdir /install +WORKDIR /install + +COPY requirements.txt /requirements.txt +RUN pip install --prefix="/install" -r /requirements.txt + +FROM base +RUN apt-get update +RUN apt-get dist-upgrade -y +RUN apt install build-essential libffi-dev musl-dev openssl python3 python3-pip -y + +COPY --from=builder /install /usr/local +COPY __init__.py /app/walkoff_app_sdk/__init__.py +COPY app_base.py /app/walkoff_app_sdk/app_base.py diff --git a/backend/app_sdk/README.md b/backend/app_sdk/README.md index 170781f3..478fb394 100755 --- a/backend/app_sdk/README.md +++ b/backend/app_sdk/README.md @@ -8,5 +8,15 @@ This is the SDK used for apps to behave like they should. 4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...) 5. Rebuild the Docker image (click load in GUI?) +## Cloud updates +1. Go to shuffle cloud on GCP +2. Go to Cloud Storage +3. Find shuffler.appspot.com +4. Navigate to generated_apps/baseline +5. Update SDK there. This will make all new apps run with the new SDK + +## Cloud app force-updates +1. Run the "stitcher.go" program in the public shuffle-shared repository. + # LICENSE Everything in here is MIT, not AGPLv3 as indicated by the license. diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index b725ae9f..b0ffee23 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1,19 +1,276 @@ import os +import ast import copy import sys import re import time +import base64 import json +import liquid import logging -import requests -import urllib.parse -import http.client import urllib3 import hashlib -from liquid import Liquid -import liquid import zipfile +import asyncio +import requests +import http.client +import urllib.parse +import jinja2 +import datetime +import dateutil + +import threading +import concurrent.futures + +from io import StringIO as StringBuffer from io import BytesIO +from liquid import Liquid, defaults + +runtime = os.getenv("SHUFFLE_SWARM_CONFIG", "") + +### +### +### +#### Filters for liquidpy +### +### +### + +defaults.MODE = 'wild' +defaults.FROM_FILE = False +from liquid.filters.manager import FilterManager +from liquid.filters.standard import standard_filter_manager + +shuffle_filters = FilterManager() +for key, value in standard_filter_manager.filters.items(): + shuffle_filters.filters[key] = value + +#@shuffle_filters.register +#def plus(a, b): +# try: +# a = int(a) +# except: +# a = 0 +# +# try: +# b = int(b) +# except: +# b = 0 +# +# return standard_filter_manager.filters["plus"](a, b) +# +#@shuffle_filters.register +#def minus(a, b): +# a = int(a) +# b = int(b) +# return standard_filter_manager.filters["minus"](a, b) +# +#@shuffle_filters.register +#def multiply(a, b): +# a = int(a) +# b = int(b) +# return standard_filter_manager.filters["multiply"](a, b) +# +#@shuffle_filters.register +#def divide(a, b): +# a = int(a) +# b = int(b) +# return standard_filter_manager.filters["divide"](a, b) + +@shuffle_filters.register +def md5(a): + a = str(a) + return hashlib.md5(a.encode('utf-8')).hexdigest() + +@shuffle_filters.register +def sha256(a): + a = str(a) + return hashlib.sha256(str(a).encode("utf-8")).hexdigest() + +@shuffle_filters.register +def md5_base64(a): + a = str(a) + foundhash = hashlib.md5(a.encode('utf-8')).hexdigest() + return base64.b64encode(foundhash.encode('utf-8')) + +@shuffle_filters.register +def base64_encode(a): + a = str(a) + try: + return base64.b64encode(a.encode('utf-8')).decode() + except: + return base64.b64encode(a).decode() + +@shuffle_filters.register +def base64_decode(a): + a = str(a) + try: + return base64.b64decode(a).decode() + except: + return base64.b64decode(a) + +@shuffle_filters.register +def json_parse(a): + return json.loads(str(a)) + +@shuffle_filters.register +def as_object(a): + return json.loads(str(a)) + +@shuffle_filters.register +def ast(a): + return ast.literal_eval(str(a)) + +@shuffle_filters.register +def escape_string(a): + a = str(a) + return a.replace("\\\'", "\'", -1).replace("\\\"", "\"", -1).replace("'", "\\\'", -1).replace("\"", "\\\"", -1) + +@shuffle_filters.register +def json_escape(a): + a = str(a) + return a.replace("\\\'", "\'", -1).replace("\\\"", "\"", -1).replace("'", "\\\\\'", -1).replace("\"", "\\\\\"", -1) + +@shuffle_filters.register +def escape_json(a): + a = str(a) + return a.replace("\\\'", "\'", -1).replace("\\\"", "\"", -1).replace("'", "\\\\\'", -1).replace("\"", "\\\\\"", -1) + +# By default using json escape to add all backslashes +@shuffle_filters.register +def escape(a): + a = str(a) + return json_escape(a) + +@shuffle_filters.register +def neat_json(a): + try: + a = json.loads(a) + except: + pass + + return json.dumps(a, indent=4, sort_keys=True) + +@shuffle_filters.register +def flatten(a): + a = list(a) + + flat_list = [a for xs in a for a in xs] + return flat_list + +@shuffle_filters.register +def last(a): + try: + a = json.loads(a) + except: + pass + + if len(a) == 0: + return "" + + return a[-1] + +@shuffle_filters.register +def first(a): + try: + a = json.loads(a) + except: + pass + + if len(a) == 0: + return "" + + return a[0] + + +@shuffle_filters.register +def csv_parse(a): + a = str(a) + splitdata = a.split("\n") + columns = [] + if len(splitdata) > 1: + columns = splitdata[0].split(",") + else: + return a.split("\n") + + allitems = [] + cnt = -1 + for item in splitdata[1:]: + cnt += 1 + commasplit = item.split(",") + + fullitem = {} + fullitem["unparsed"] = item + fullitem["index"] = cnt + fullitem["parsed"] = {} + if len(columns) != len(commasplit): + + if len(commasplit) > len(columns): + diff = len(commasplit)-len(columns) + + try: + commasplit = commasplit[0:len(commasplit)-diff] + except: + pass + else: + for item in range(0, len(columns)-len(commasplit)): + commasplit.append("") + + for key in range(len(columns)): + try: + fullitem["parsed"][columns[key]] = commasplit[key] + except: + continue + + allitems.append(fullitem) + + try: + return json.dumps(allitems) + except: + print("[ERROR] Failed dumping from JSON in csv parse") + return allitems + +@shuffle_filters.register +def parse_csv(a): + return csv_parse(a) + +@shuffle_filters.register +def format_csv(a): + return csv_parse(a) + +@shuffle_filters.register +def csv_format(a): + return csv_parse(a)@standard_filter_manager.register + +@shuffle_filters.register +def split(base, sep): + if not sep: + try: + return json.dumps(list(base)) + except: + return list(base) + + try: + return json.dumps(base.split(sep)) + except: + return base.split(sep) + +#print(shuffle_filters.filters) +#print(Liquid("{{ '10' | plus: 1}}", filters=shuffle_filters.filters).render()) +#print(Liquid("{{ '10' | minus: 1}}", filters=shuffle_filters.filters).render()) +#print(Liquid("{{ asd | size }}", filters=shuffle_filters.filters).render()) +#print(Liquid("{{ 'asd' | md5 }}", filters=shuffle_filters.filters).render()) +#print(Liquid("{{ 'asd' | sha256 }}", filters=shuffle_filters.filters).render()) +#print(Liquid("{{ 'asd' | md5_base64 | base64_decode }}", filters=shuffle_filters.filters).render()) + +### +### +### +### +### +### +### + class AppBase: __version__ = None @@ -21,13 +278,21 @@ class AppBase: def __init__(self, redis=None, logger=None, console_logger=None):#, docker_client=None): self.logger = logger if logger is not None else logging.getLogger("AppBaseLogger") + + if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true": + self.log_capture_string = StringBuffer() + ch = logging.StreamHandler(self.log_capture_string) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + ch.setFormatter(formatter) + logger.addHandler(ch) + self.redis=redis self.console_logger = logger if logger is not None else logging.getLogger("AppBaseLogger") # apikey is for the user / org # authorization is for the specific workflow - self.url = os.getenv("CALLBACK_URL", "https://shuffler.io") + self.url = os.getenv("CALLBACK_URL", "https://shuffler.io") self.base_url = os.getenv("BASE_URL", "https://shuffler.io") self.action = os.getenv("ACTION", "") self.original_action = os.getenv("ACTION", "") @@ -51,52 +316,255 @@ class AppBase: try: self.action = json.loads(self.action) self.original_action = json.loads(self.action) - except: - self.logger.info("[WARNING] Failed parsing action as JSON") + except Exception as e: + self.logger.info(f"[DEBUG] Failed parsing action as JSON (init): {e}. NOT important if running apps with webserver. This is NOT critical.") + + #print(f"ACTION: {self.action}") if len(self.base_url) == 0: self.base_url = self.url + # Checks output for whether it should be automatically parsed or not + def run_magic_parser(self, input_data): + if not isinstance(input_data, str): + self.logger.info("[DEBUG] Not string. Returning from magic") + return input_data + + # Don't touch existing JSON/lists + if (input_data.startswith("[") and input_data.endswith("]")) or (input_data.startswith("{") and input_data.endswith("}")): + self.logger.info("[DEBUG] Already JSON-like. Returning from magic") + return input_data + + if len(input_data) < 3: + self.logger.info("[DEBUG] Too short input data") + return input_data + + # Don't touch large data. + if len(input_data) > 100000: + self.logger.info("[DEBUG] Value too large. Returning from magic") + return input_data + + if not "\n" in input_data and not "," in input_data: + self.logger.info("[DEBUG] No data to autoparse - requires newline or comma") + return input_data + + new_input = input_data + try: + #new_input.strip() + new_input = input_data.split() + new_return = [] + + index = 0 + for item in new_input: + splititem = "," + if ", " in item: + splititem = ", " + elif "," in item: + splititem = "," + else: + new_return.append(item) + + index += 1 + continue + + #print("FIX ITEM %s" % item) + for subitem in item.split(splititem): + new_return.insert(index, subitem) + + index += 1 + + # Prevent large data or infinite loops + if index > 10000: + self.logger.info(f"[DEBUG] Infinite loop. Returning default data.") + return input_data + + fixed_return = [] + for item in new_return: + if not item: + continue + + if not isinstance(item, str): + fixed_return.append(item) + continue + + if item.endswith(","): + item = item[0:-1] + + fixed_return.append(item) + + new_input = fixed_return + except Exception as e: + # Not used anymore + #self.logger.info(f"[ERROR] Failed to run magic parser (2): {e}") + return input_data + + try: + new_input = input_data.split() + except Exception as e: + self.logger.info(f"[ERROR] Failed to run parser during split (1): {e}") + return input_data + + # Won't ever touch this one? + if isinstance(new_input, list) or isinstance(new_input, object): + try: + return json.dumps(new_input) + except Exception as e: + self.logger.info(f"[ERROR] Failed to run magic parser (3): {e}") + + return new_input + + def prepare_response(self, request): + try: + parsedheaders = {} + for key, value in request.headers.items(): + parsedheaders[key] = value + + cookies = {} + if request.cookies: + for key, value in request.cookies.items(): + cookies[key] = value + + + jsondata = request.text + try: + jsondata = json.loads(jsondata) + except: + pass + + return json.dumps({ + "success": True, + "status": request.status_code, + "url": request.url, + "headers": parsedheaders, + "body": jsondata, + "cookies":cookies, + }) + except Exception as e: + print(f"[WARNING] Failed in request: {e}") + return request.text + # FIXME: Add more info like logs in here. # Docker logs: https://forums.docker.com/t/docker-logs-inside-the-docker-container/68190/2 def send_result(self, action_result, headers, stream_path): if action_result["status"] == "EXECUTING": action_result["status"] = "FAILURE" + try: + #self.logger.info(f"[DEBUG] ACTION: {self.action}") + if self.action["run_magic_output"] == True: + self.logger.warning(f"[INFO] Action result ran with Magic parser output.") + action_result["result"] = self.run_magic_parser(action_result["result"]) + else: + self.logger.warning(f"[WARNING] Magic output not defined.") + except KeyError as e: + #self.logger.warning(f"[DEBUG] Failed to run magic autoparser (send result) - keyerror: {e}") + pass + except Exception as e: + #self.logger.warning(f"[DEBUG] Failed to run magic autoparser (send result): {e}") + pass + + # Try it with some magic + + action_result["completed_at"] = int(time.time()) self.logger.info(f"""[DEBUG] Inside Send result with status {action_result["status"]}""") + #if isinstance(action_result, # FIXME: Add cleanup of parameters to not send to frontend here params = {} - #action = action_result["action"] - #try: - # for item in action["authentication"]: - # for action["parameters"] - # self.logger.info("AUTH: ", key, value) - # params[item["key"]] = item["value"] - #except KeyError: - # self.logger.info("No authentication specified!") - # pass # I wonder if this actually works self.logger.info(f"[DEBUG] Before last stream result") url = "%s%s" % (self.base_url, stream_path) - #self.logger.info("[INFO] URL (URL): %s" % url) + self.logger.info(f"[INFO] URL FOR RESULT (URL): {url}") + try: - ret = requests.post(url, headers=headers, json=action_result) - #self.logger.info(f"[DEBUG] Result: {ret.status_code}") - #if ret.status_code != 200: - # self.logger.info(f"[DEBUG] Shuffle Response: {ret.text}") + log_contents = "disabled: add env SHUFFLE_LOGS_DISABLED=true to Orborus to re-enable logs for apps. Can not be enabled natively in Cloud except in Hybrid mode." + if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true": + log_contents = self.log_capture_string.getvalue() + + #print("RESULTS: %s" % log_contents) + self.logger.info(f"[WARNING] Got logs of length {len(log_contents)}") + if len(action_result["action"]["parameters"]) == 0: + action_result["action"]["parameters"] = [] + + param_found = False + for param in action_result["action"]["parameters"]: + if param["name"] == "shuffle_action_logs": + param_found = True + break + + if not param_found: + action_result["action"]["parameters"].append({ + "name": "shuffle_action_logs", + "value": log_contents, + }) + + except Exception as e: + print(f"[WARNING] Failed adding parameter for logs: {e}") + + # FIXME: Adding retries here. + try: + finished = False + for i in range (0, 10): + try: + ret = requests.post(url, headers=headers, json=action_result, timeout=10, verify=False) + + self.logger.info(f"[DEBUG] Result: {ret.status_code} (break on 200 or 201)") + if ret.status_code == 200 or ret.status_code == 201: + finished = True + break + else: + self.logger.info(f"[ERROR] RESP: {ret.text}") + + except requests.exceptions.RequestException as e: + self.logger.info(f"[DEBUG] Request problem: {e}") + time.sleep(0.1) + + #time.sleep(5) + continue + except TimeoutError as e: + self.logger.info(f"[DEBUG] Timeout or request: {e}") + time.sleep(0.1) + + #time.sleep(5) + continue + except requests.exceptions.ConnectionError as e: + self.logger.info(f"[DEBUG] Connectionerror: {e}") + time.sleep(0.1) + + #time.sleep(5) + continue + except http.client.RemoteDisconnected as e: + self.logger.info(f"[DEBUG] Remote: {e}") + time.sleep(0.1) + + #time.sleep(5) + continue + except urllib3.exceptions.ProtocolError as e: + self.logger.info(f"[DEBUG] Protocol err: {e}") + time.sleep(0.1) + + #time.sleep(5) + continue + + #time.sleep(5) + + if not finished: + # Not sure why this would work tho :) + action_result["status"] = "FAILURE" + action_result["result"] = json.dumps({"success": False, "reason": "POST error: Failed connecting to %s over 10 retries to the backend" % url}) + self.logger.info(f"[DEBUG] Before typeerror stream result - NOT finished after 10 requests") + ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False) - self.logger.info(f"[DEBUG] Successful request: Status= {ret.status_code} & Response= {ret.text}") + self.logger.info(f"""[DEBUG] Successful request result request: Status= {ret.status_code} & Response= {ret.text}. Action status: {action_result["status"]}""") except requests.exceptions.ConnectionError as e: self.logger.info(f"[DEBUG] Unexpected ConnectionError happened: {e}") - return except TypeError as e: - #self.logger.exception(e) action_result["status"] = "FAILURE" - action_result["result"] = f"POST error: {e}" + action_result["result"] = json.dumps({"success": False, "reason": "Typeerror when sending to backend URL %s" % url}) + self.logger.info(f"[DEBUG] Before typeerror stream result: {e}") - ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) + ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False) #self.logger.info(f"[DEBUG] Result: {ret.status_code}") #if ret.status_code != 200: # pr @@ -104,14 +572,26 @@ class AppBase: self.logger.info(f"[DEBUG] TypeError request: Status= {ret.status_code} & Response= {ret.text}") except http.client.RemoteDisconnected as e: self.logger.info(f"[DEBUG] Expected Remotedisconnect happened: {e}") - return except urllib3.exceptions.ProtocolError as e: self.logger.info(f"[DEBUG] Expected ProtocolError happened: {e}") - return - async def cartesian_product(self, L): + + # FIXME: Re-enable data flushing otherwise we'll overload it all + # Or nah? + if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true": + try: + self.log_capture_string.flush() + #self.log_capture_string.close() + #pass + except Exception as e: + print(f"[WARNING] Failed to flush logs: {e}") + pass + + #async def cartesian_product(self, L): + def cartesian_product(self, L): if L: - return {(a, ) + b for a in L[0] for b in await self.cartesian_product(L[1:])} + #return {(a, ) + b for a in L[0] for b in await self.cartesian_product(L[1:])} + return {(a, ) + b for a in L[0] for b in self.cartesian_product(L[1:])} else: return {()} @@ -213,7 +693,7 @@ class AppBase: #self.logger.info(f"RET: {ret.text}") #self.logger.info(f"ID: {ret.status_code}") url = f"{self.url}/api/v1/orgs/{org_id}/validate_app_values" - ret = requests.post(url, json=data) + ret = requests.post(url, json=data, verify=False) if ret.status_code == 200: json_value = ret.json() if len(json_value["found"]) > 0: @@ -247,7 +727,8 @@ class AppBase: # Returns a list of all the executions to be done in the inner loop # FIXME: Doesn't take into account whether you actually WANT to loop or not # Check if the last part of the value is #? - async def get_param_multipliers(self, baseparams): + #async def get_param_multipliers(self, baseparams): + def get_param_multipliers(self, baseparams): # Example: # {'call': ['hello', 'hello4'], 'call2': ['hello2', 'hello3'], 'call3': '1'} # @@ -425,7 +906,8 @@ class AppBase: self.logger.info("[DEBUG] Newlength of array: %d. Lists: %s" % (newlength, all_lists)) # Get the cartesian product of the arrays - cartesian = await self.cartesian_product(all_lists) + #cartesian = await self.cartesian_product(all_lists) + cartesian = self.cartesian_product(all_lists) newlist = [] for item in cartesian: newlist.append(list(item)) @@ -454,7 +936,8 @@ class AppBase: # Runs recursed versions with inner loops and such - async def run_recursed_items(self, func, baseparams, loop_wrapper): + #async def run_recursed_items(self, func, baseparams, loop_wrapper): + def run_recursed_items(self, func, baseparams, loop_wrapper): #self.logger.info(f"RECURSED ITEMS: {baseparams}") has_loop = False @@ -495,14 +978,18 @@ class AppBase: results = [] if has_loop: - self.logger.info(f"[DEBUG] Should run inner loop: {newparams}") - ret = await self.run_recursed_items(func, newparams, loop_wrapper) + #self.logger.info(f"[DEBUG] Should run inner loop: {newparams}") + self.logger.info(f"[DEBUG] Should run inner loop") + #ret = await self.run_recursed_items(func, newparams, loop_wrapper) + ret = self.run_recursed_items(func, newparams, loop_wrapper) else: - self.logger.info(f"[DEBUG] Should run multiplier check with params (inner): {newparams}") + #self.logger.info(f"[DEBUG] Should run multiplier check with params (inner): {newparams}") + self.logger.info(f"[DEBUG] Should run multiplier check with params (inner)") # 1. Find the loops that are required and create new multipliers # If here: check for multipliers within this scope. ret = [] - param_multiplier = await self.get_param_multipliers(newparams) + #param_multiplier = await self.get_param_multipliers(newparams) + param_multiplier = self.get_param_multipliers(newparams) # FIXME: This does a deduplication of the data new_params = self.validate_unique_fields(param_multiplier) @@ -520,8 +1007,10 @@ class AppBase: } self.send_result(self.action_result, {"Content-Type": "application/json", "Authorization": "Bearer %s" % self.authorization}, "/api/v1/streams") - exit() - #return + if runtime != "run": + exit() + else: + return else: #subparams = new_params #self.logger.info(f"NEW PARAMS: {new_params}") @@ -543,11 +1032,10 @@ class AppBase: for subparams in param_multiplier: #self.logger.info(f"SUBPARAMS IN MULTI: {subparams}") try: - #tmp = await func(**subparams) while True: try: - tmp = await func(**subparams) + tmp = func(**subparams) break except TypeError as e: self.logger.info("BASE TYPEERROR: %s" % e) @@ -559,21 +1047,51 @@ class AppBase: try: del subparams[field] - self.logger.info("Removed field invalid field %s" % field) + self.logger.info("Removed invalid field %s (1)" % field) except KeyError: break else: - raise e + raise Exception(json.dumps({ + "success": False, + "reason": "You may be running an old version of this action. Please delete and remake the node.", + "exception": f"TypeError: {e}", + })) + break + except: e = "" try: e = sys.exc_info()[1] except: - self.logger.info("Exc check fail: %s" % e) + self.logger.info("Exec check fail: %s" % e) pass - tmp = "An error occured during execution: %s" % e + tmp = json.dumps({ + "success": False, + "reason": f"An error occured during execution: {e}", + }) + + + # An attempt at decomposing coroutine results + # Backwards compatibility + try: + if asyncio.iscoroutine(tmp): + self.logger.info("[DEBUG] In coroutine (2)") + async def parse_value(tmp): + value = await asyncio.gather( + tmp + ) + + return value[0] + + + tmp = asyncio.run(parse_value(tmp)) + else: + #self.logger.info("[DEBUG] Not in coroutine (2)") + pass + except Exception as e: + self.logger.warning("[ERROR] Failed to parse coroutine value for old app: {e}") #self.logger.info("RET from execution: %s" % ret) new_value = tmp @@ -638,6 +1156,32 @@ class AppBase: self.logger.info("\nLOOP: %s\nRESULTS: %s" % (loop_wrapper, results)) return results + # Downloads all files from a namespace + # Currently only working on local version of Shuffle + def get_file_category_ids(self, category): + org_id = self.full_execution["workflow"]["execution_org"]["id"] + + get_path = "/api/v1/files/namespaces/%s?execution_id=%s&ids=true" % (category, self.full_execution["execution_id"]) + headers = { + "Authorization": "Bearer %s" % self.authorization, + "User-Agent": "Shuffle 1.1.0", + } + + ret = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False) + return ret.json() + #if ret1.status_code != 200: + # return { + # "success": False, + # "reason": "Status code is %d from backend for category %s" % category, + # "list": [], + # } + + #return { + # "success": True, + # "ids": ret1.json(), + #} + + # Downloads all files from a namespace # Currently only working on local version of Shuffle def get_file_namespace(self, namespace): @@ -645,17 +1189,38 @@ class AppBase: get_path = "/api/v1/files/namespaces/%s?execution_id=%s" % (namespace, self.full_execution["execution_id"]) headers = { - "Authorization": "Bearer %s" % self.authorization + "Authorization": "Bearer %s" % self.authorization, + "User-Agent": "Shuffle 1.1.0", } - ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers) + ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False) if ret1.status_code != 200: return None filebytes = BytesIO(ret1.content) myzipfile = zipfile.ZipFile(filebytes) + + # Unzip and build here! + #for member in files.namelist(): + # filename = os.path.basename(member) + # if not filename: + # continue + + # self.logger.info("File: %s" % member) + # source = files.open(member) + # with open("%s/%s" % (basedir, source.name), "wb+") as tmp: + # filedata = source.read() + # self.logger.info("Filedata (%s): %s" % (source.name, filedata)) + # tmp.write(filedata) + return myzipfile + def get_file_namespace_ids(self, namespace): + return self.get_file_category_ids(self, namespace) + + def get_file_category(self, category): + return self.get_file_namespace(self, category) + # Things to consider for files: # - How can you download / stream a file? # - Can you decide if you want a stream or the files directly? @@ -675,7 +1240,7 @@ class AppBase: returns = [] for item in value: self.logger.info("VALUE: %s" % item) - if len(item) != 36: + if len(item) != 36 and not item.startswith("file_"): self.logger.info("Bad length for file value %s" % item) continue #return { @@ -687,10 +1252,11 @@ class AppBase: get_path = "/api/v1/files/%s?execution_id=%s" % (item, full_execution["execution_id"]) headers = { "Content-Type": "application/json", - "Authorization": "Bearer %s" % self.authorization + "Authorization": "Bearer %s" % self.authorization, + "User-Agent": "Shuffle 1.1.0", } - ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers) + ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False) self.logger.info("RET1 (file get): %s" % ret1.text) if ret1.status_code != 200: returns.append({ @@ -701,7 +1267,7 @@ class AppBase: continue content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"]) - ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers) + ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers, verify=False) self.logger.info("RET2 (file get) done") if ret2.status_code == 200: tmpdata = ret1.json() @@ -737,7 +1303,7 @@ class AppBase: "value": str(value), } - response = requests.post(url, json=data) + response = requests.post(url, json=data, verify=False) try: allvalues = response.json() allvalues["key"] = key @@ -759,7 +1325,7 @@ class AppBase: "key": key, } - value = requests.post(url, json=data) + value = requests.post(url, json=data, verify=False) try: allvalues = value.json() self.logger.info("VAL1: ", allvalues) @@ -778,6 +1344,7 @@ class AppBase: #return value.json() return {"success": False} + # Wrapper for set_files def set_file(self, infiles): return self.set_files(infiles) @@ -788,7 +1355,8 @@ class AppBase: org_id = full_execution["workflow"]["execution_org"]["id"] headers = { "Content-Type": "application/json", - "Authorization": "Bearer %s" % self.authorization + "Authorization": "Bearer %s" % self.authorization, + "User-Agent": "Shuffle 1.1.0", } if not isinstance(infiles, list): @@ -811,11 +1379,11 @@ class AppBase: self.logger.info(f"KeyError in file setup: {e}") pass - ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data) - self.logger.info(f"Ret CREATE: {ret.text}") + ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data, verify=False) + #self.logger.info(f"Ret CREATE: {ret.text}") cur_id = "" if ret.status_code == 200: - self.logger.info("RET: %s" % ret.text) + #self.logger.info("RET: %s" % ret.text) ret_json = ret.json() if not ret_json["success"]: self.logger.info("Not success in file upload creation.") @@ -834,6 +1402,7 @@ class AppBase: new_headers = { "Authorization": f"Bearer {self.authorization}", + "User-Agent": "Shuffle 1.1.0", } upload_path = "/api/v1/files/%s/upload?execution_id=%s" % (cur_id, full_execution["execution_id"]) @@ -842,15 +1411,15 @@ class AppBase: files={"shuffle_file": (filename, curfile["data"])} #open(filename,'rb')} - ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers) + ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers, verify=False) self.logger.info("Ret UPLOAD: %s" % ret.text) self.logger.info("Ret2 UPLOAD: %d" % ret.status_code) self.logger.info("IDS TO RETURN: %s" % file_ids) return file_ids - async def execute_action(self, action): - + #async def execute_action(self, action): + def execute_action(self, action): # !!! Let this line stay - its used for some horrible codegeneration / stitching !!! # #STARTCOPY stream_path = "/api/v1/streams" @@ -864,35 +1433,40 @@ class AppBase: } # Simple validation of parameters in general + replace_params = False try: tmp_parameters = action["parameters"] + for param in tmp_parameters: + if param["value"] == "SHUFFLE_AUTO_REMOVED": + replace_params = True except KeyError: action["parameters"] = [] except TypeError: pass self.action = copy.deepcopy(action) - self.logger.info("[DEBUG] Sending starting action result (EXECUTING)") + self.logger.info(f"[DEBUG] Sending starting action result (EXECUTING). Param replace: {replace_params}") headers = { "Content-Type": "application/json", - "Authorization": "Bearer %s" % self.authorization + "Authorization": f"Bearer {self.authorization}", + "User-Agent": "Shuffle 1.1.0", } if len(self.action) == 0: - self.logger.info("ACTION env not defined") + self.logger.info("[WARNING] ACTION env not defined") self.action_result["result"] = "Error in setup ENV: ACTION not defined" self.send_result(self.action_result, headers, stream_path) return if len(self.authorization) == 0: - self.logger.info("AUTHORIZATION env not defined") + self.logger.info("[WARING] AUTHORIZATION env not defined") self.action_result["result"] = "Error in setup ENV: AUTHORIZATION not defined" self.send_result(self.action_result, headers, stream_path) return if len(self.current_execution_id) == 0: - self.logger.info("EXECUTIONID env not defined") + self.logger.info("[WARNING] EXECUTIONID env not defined") self.action_result["result"] = "Error in setup ENV: EXECUTIONID not defined" self.send_result(self.action_result, headers, stream_path) return @@ -904,7 +1478,7 @@ class AppBase: # FIXME: Shouldn't skip this, but it's good for minimzing API calls #try: - # ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) + # ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False) # self.logger.info("Workflow: %d" % ret.status_code) # if ret.status_code != 200: # self.logger.info(ret.text) @@ -918,43 +1492,72 @@ class AppBase: # Verify whether there are any parameters with ACTION_RESULT required # If found, we get the full results list from backend fullexecution = {} - if len(self.full_execution) == 0: - self.logger.info("[DEBUG] NO EXECUTION - LOADING!") + if isinstance(self.full_execution, str) and len(self.full_execution) == 0: + #self.logger.info("[DEBUG] NO EXECUTION - LOADING!") try: - tmpdata = { - "authorization": self.authorization, - "execution_id": self.current_execution_id - } + failed = False + rettext = "" + for i in range(0, 5): + tmpdata = { + "authorization": self.authorization, + "execution_id": self.current_execution_id + } - self.logger.info("[DEBUG] Before FULLEXEC stream result") - ret = requests.post( - "%s/api/v1/streams/results" % (self.base_url), - headers=headers, - json=tmpdata - ) + self.logger.info("[ERROR] Before FULLEXEC stream result") + ret = requests.post( + "%s/api/v1/streams/results" % (self.base_url), + headers=headers, + json=tmpdata, + verify=False + ) - if ret.status_code == 200: - fullexecution = ret.json() - else: - try: - self.logger.info("Error: Data: ", ret.json()) - self.logger.info("Error with status code for results. Crashing because ACTION_RESULTS or WORKFLOW_VARIABLE can't be handled. Status: %d" % ret.status_code) - except json.decoder.JSONDecodeError: - pass + if ret.status_code == 200: + fullexecution = ret.json() + failed = False + break + + #elif ret.status_code == 500 or ret.status_code == 400: + elif ret.status_code >= 400: + self.logger.info("[ERROR] (fails: %d) Error in app with status code %d for results (1). RETRYING because results can't be handled" % (i+1, ret.status_code)) + + rettext = ret.text + failed = True + time.sleep(8) + continue + + else: + self.logger.info("[ERROR] Error in app with status code %d for results (2). Crashing because results can't be handled" % ret.status_code) + + rettext = ret.text + failed = True + time.sleep(8) + break + + if failed: + self.action_result["result"] = json.dumps({ + "success": False, + "reason": f"Bad result from backend during startup of app: {ret.status_code}", + "extended_reason": f"{rettext}" + }) - self.action_result["result"] = "Bad result from backend: %d" % ret.status_code self.send_result(self.action_result, headers, stream_path) return + except requests.exceptions.ConnectionError as e: - self.logger.info("Connectionerror: %s" % e) - self.action_result["result"] = "Connection error during startup: %s" % e + self.logger.info("[ERROR] FullExec Connectionerror: %s" % e) + self.action_result["result"] = json.dumps({ + "success": False, + "reason": f"Connection error during startup: {e}" + }) + self.send_result(self.action_result, headers, stream_path) return else: + self.logger.info(f"[DEBUG] Setting execution to default value with type {type(self.full_execution)}") try: fullexecution = json.loads(self.full_execution) except json.decoder.JSONDecodeError as e: - self.logger.info("[WARNING] Json decode execution error: %s" % e) + self.logger.info("[ERROR] Json decode execution error: %s" % e) self.action_result["result"] = "Json error during startup: %s" % e self.send_result(self.action_result, headers, stream_path) return @@ -963,7 +1566,40 @@ class AppBase: self.full_execution = fullexecution - self.logger.info("[DEBUG] AFTER FULLEXEC stream result (init)") + + #try: + # if "backend_url" in self.full_execution: + # self.url = self.full_execution["backend_url"] + # self.base_url = self.full_execution["backend_url"] + #except KeyError: + # pass + + try: + if replace_params == True: + for inner_action in self.full_execution["workflow"]["actions"]: + self.logger.info("[DEBUG] ID: %s vs %s" % (inner_action["id"], self.action["id"])) + + # In case of some kind of magic, we're just doing params + if inner_action["id"] == self.action["id"]: + self.logger.info("FOUND!") + + if isinstance(self.action, str): + self.logger.info("Params is in string object for self.action?") + else: + self.action["parameters"] = inner_action["parameters"] + self.action_result["action"]["parameters"] = inner_action["parameters"] + + if isinstance(self.original_action, str): + self.logger.info("Params for original actions is in string object?") + else: + self.original_action["parameters"] = inner_action["parameters"] + + break + + except Exception as e: + self.logger.info(f"[WARNING] Failed in replace params action parsing: {e}") + + self.logger.info(f"[DEBUG] AFTER FULLEXEC stream result (init): {self.current_execution_id}") # Gets the value at the parenthesis level you want def parse_nested_param(string, level): @@ -1147,6 +1783,11 @@ class AppBase: except TypeError: return data, False + # Because liquid can handle ALL of this now. + # Implemented for >0.9.25 + #self.logger.info("[DEBUG] Skipping parser because use of its been deprecated >0.9.25 due to Liquid implementation") + return data, False + wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length", "lenght", "join", "replace"] if not any(wrapper in data for wrapper in wrappers): @@ -1197,7 +1838,7 @@ class AppBase: else: parse_string = inner_result - print("PARSE STRING: %s" % parse_string) + #print("PARSE STRING: %s" % parse_string) return parse_string, True # Looks for parantheses to grab special cases within a string, e.g: @@ -1219,7 +1860,7 @@ class AppBase: if isinstance(data, str) and len(data) > 4: if (data[0] == "{" or data[0] == "[") and (data[len(data)-1] == "]" or data[len(data)-1] == "}"): - self.logger.info("Skipping parser because use of {[ and ]}") + self.logger.info("[DEBUG] Skipping parser because use of {[ and ]}") return data newdata = [] @@ -1305,6 +1946,7 @@ class AppBase: # value = value.replace(" ", "_", -1) actualitem = re.findall(match, value, re.MULTILINE) + # Goes here if loop if value == "#": newvalue = [] for innervalue in basejson: @@ -1323,18 +1965,24 @@ class AppBase: # it as multi execution return newvalue, True + # Checks specific regex like #1-2 for index 1-2 in a loop elif len(actualitem) > 0: is_loop = True newvalue = [] firstitem = actualitem[0][0] seconditem = actualitem[0][1] - print("[DEBUG] ACTUAL PARSED: %s" % actualitem) + if isinstance(firstitem, int): + firstitem = str(firstitem) + if isinstance(seconditem, int): + seconditem = str(seconditem) + + #print("[DEBUG] ACTUAL PARSED: %s" % actualitem) # Means it's a single item -> continue if seconditem == "": print("[INFO] In first - handling %s. Len: %d" % (firstitem, len(basejson))) - if firstitem.lower() == "max" or firstitem.lower() == "last": + if firstitem.lower() == "max" or firstitem.lower() == "last" or firstitem.lower() == "end": firstitem = len(basejson)-1 elif firstitem.lower() == "min" or firstitem.lower() == "first": firstitem = 0 @@ -1349,17 +1997,23 @@ class AppBase: newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:]) else: print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem)) - if firstitem.lower() == "max" or firstitem.lower() == "last": - firstitem = len(basejson)-1 - elif firstitem.lower() == "min" or firstitem.lower() == "first": - firstitem = 0 + if isinstance(firstitem, str): + if firstitem.lower() == "max" or firstitem.lower() == "last" or firstitem.lower() == "end": + firstitem = len(basejson)-1 + elif firstitem.lower() == "min" or firstitem.lower() == "first": + firstitem = 0 + else: + firstitem = int(firstitem) else: firstitem = int(firstitem) - if seconditem.lower() == "max" or seconditem.lower() == "last": - seconditem = len(basejson)-1 - elif seconditem.lower() == "min" or seconditem.lower() == "first": - seconditem = 0 + if isinstance(seconditem, str): + if seconditem.lower() == "max" or seconditem.lower() == "last" or firstitem.lower() == "end": + seconditem = len(basejson)-1 + elif seconditem.lower() == "min" or seconditem.lower() == "first": + seconditem = 0 + else: + seconditem = int(seconditem) else: seconditem = int(seconditem) @@ -1394,9 +2048,12 @@ class AppBase: return basejson, False elif isinstance(basejson[value], str): try: - basejson = json.loads(basejson[value]) + if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")): + basejson = json.loads(basejson[value]) + else: + return str(basejson[value]), False except json.decoder.JSONDecodeError as e: - return basejson[value], False + return str(basejson[value]), False else: basejson = basejson[value] except KeyError as e: @@ -1412,11 +2069,14 @@ class AppBase: elif isinstance(basejson[value], str): print(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value]) try: - basejson = json.loads(basejson[value]) print("[DEBUG] BASEJSON: %s" % basejson) + if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")): + basejson = json.loads(basejson[value]) + else: + return str(basejson[value]), False except json.decoder.JSONDecodeError as e: print("[DEBUG] RETURNING BECAUSE '%s' IS A NORMAL STRING (1)" % basejson[value]) - return basejson[value], False + return str(basejson[value]), False else: basejson = basejson[value] @@ -1439,13 +2099,13 @@ class AppBase: actionname_lower = parsersplit[0][1:].lower() #Actionname: Start_node - print(f"\n[INFO] Actionname: {actionname_lower}") + #print(f"\n[INFO] Actionname: {actionname_lower}") # 1. Find the action baseresult = "" appendresult = "" - print("[INFO] Parsersplit length: %d" % len(parsersplit)) + #print("[INFO] Parsersplit length: %d" % len(parsersplit)) if (actionname_lower.startswith("exec ") or actionname_lower.startswith("webhook ") or actionname_lower.startswith("schedule ") or actionname_lower.startswith("userinput ") or actionname_lower.startswith("email_trigger ") or actionname_lower.startswith("trigger ")) and len(parsersplit) == 1: record = False for char in actionname_lower: @@ -1456,7 +2116,7 @@ class AppBase: appendresult += char actionname_lower = "exec" - elif actionname_lower.startswith("shuffle_cache "): + elif actionname_lower.startswith("shuffle_cache ") or actionname_lower.startswith("shuffle_db "): actionname_lower = "shuffle_cache" actionname_lower = actionname_lower.replace(" ", "_", -1) @@ -1488,7 +2148,6 @@ class AppBase: print("[DEBUG] No results to get values from.") baseresult = "$" + parsersplit[0][1:] - print("[DEBUG] BEFORE VARIABLES!") if len(baseresult) == 0: try: for variable in execution_data["workflow"]["workflow_variables"]: @@ -1499,13 +2158,12 @@ class AppBase: break except KeyError as e: - print("[INFO] KeyError wf variables: %s" % e) + #print("[INFO] KeyError wf variables: %s" % e) pass except TypeError as e: - print("[INFO] TypeError wf variables: %s" % e) + #print("[INFO] TypeError wf variables: %s" % e) pass - print("[DEBUG] BEFORE EXECUTION VAR") if len(baseresult) == 0: try: for variable in execution_data["execution_variables"]: @@ -1514,22 +2172,24 @@ class AppBase: baseresult = variable["value"] break except KeyError as e: - print("[INFO] KeyError exec variables: %s" % e) + #print("[INFO] KeyError exec variables: %s" % e) pass except TypeError as e: - print("[INFO] TypeError exec variables: %s" % e) + #print("[INFO] TypeError exec variables: %s" % e) pass except KeyError as error: print(f"[DEBUG] KeyError in JSON: {error}") - print(f"[INFO] After first trycatch. Baseresult")#, baseresult) + #print(f"[INFO] After first trycatch. Baseresult")#, baseresult) # 2. Find the JSON data + # Returns if there isn't any JSON in the base ($nodename) if len(baseresult) == 0: return ""+appendresult, False - print("[INFO] After second return") + #print("[INFO] After second return") + # Returns if the result is JUST something like $nodename, not $nodename.value if len(parsersplit) == 1: returndata = str(baseresult)+str(appendresult) print("[DEBUG] RETURNING!")#: %s" % returndata) @@ -1538,7 +2198,8 @@ class AppBase: baseresult = baseresult.replace(" True,", " true,") baseresult = baseresult.replace(" False", " false,") - print("[INFO] After third parser return - Formatted")#, baseresult) + # Tries to actually read it as JSON with some stupid formatting + #print("[INFO] After third parser return - Formatted")#, baseresult) basejson = {} try: basejson = json.loads(baseresult) @@ -1547,10 +2208,11 @@ class AppBase: baseresult = baseresult.replace("\'", "\"") basejson = json.loads(baseresult) except json.decoder.JSONDecodeError as e: - print("Parser issue with JSON: %s" % e) + print(f"[ERROR] Parser issue with JSON for {baseresult}: {e}") return str(baseresult)+str(appendresult), False print("[INFO] After fourth parser return as JSON") + # Finds the ACTUAL value which is in the $nodename.value.test - focusing on value.test data, is_loop = recurse_json(basejson, parsersplit[1:]) parseditem = data @@ -1561,7 +2223,6 @@ class AppBase: print("[WARNING] Parseditem issue: %s" % e) pass - print("[DEBUG] DATA: (%s) %s" % (type(data), data)) if is_loop: print("[DEBUG] DATA IS A LOOP - SHOULD WRAP") if parsersplit[-1] == "#": @@ -1573,27 +2234,37 @@ class AppBase: parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data)) - print("[DEBUG] Before last return with %s" % appendresult) returndata = str(parseditem)+str(appendresult) # New in 0.8.97: Don't return items without lists #self.logger.info("RETURNDATA: %s" % returndata) #return returndata, is_loop + + # 0.9.70: + # The {} and [] checks are required because e.g. 7e7 is valid JSON for some reason... + # This breaks EVERYTHING try: - return json.dumps(json.loads(returndata)), is_loop + if (returndata.endswith("}") and returndata.endswith("}")) or (returndata.startswith("[") and returndata.endswith("]")): + return json.dumps(json.loads(returndata)), is_loop + else: + return returndata, is_loop except json.decoder.JSONDecodeError as e: - print("Error in decoder: %s" % e) return returndata, is_loop # Sending self as it's not a normal function def parse_liquid(template, self): - - #self.logger.info("Inside liquid with glob: %s" % globals()) + + errors = False + error_msg = "" try: - if len(template) > 5000000: + if len(template) > 10000000: self.logger.info("[DEBUG] Skipping liquid - size too big (%d)" % len(template)) return template + if "${" in template and "}$" in template: + self.logger.info("[DEBUG] Shuffle loop shouldn't run in liquid. Data length: %d" % len(template)) + return template + #if not "{{" in template or not "}}" in template: # if not "{%" in template or not "%}" in template: # self.logger.info("Skipping liquid - missing {{ }} and {% %}") @@ -1603,27 +2274,175 @@ class AppBase: # return template #self.logger.info(globals()) - self.logger.info("[DEBUG] Running liquid with data of length %d" % len(template)) - run = Liquid(template, mode="wild", from_file=False) + #if len(template) > 100: + # self.logger.info("[DEBUG] Running liquid with data of length %d" % len(template)) + #self.logger.info(f"[DEBUG] Data: {template}") + run = Liquid(template, mode="wild", from_file=False, filters=shuffle_filters.filters) # Can't handle self yet (?) ret = run.render(**globals()) return ret - #try: - #run = Liquid(template) - #return ret - #except liquid.exceptions.LiquidSyntaxError as e: - # run = Liquid(template, {'mode': 'python'}) - # ret = run.render(**globals()) - # return ret - #except liquid.exceptions.LiquidRenderError as e: - # self.logger.info("Render error: %s" % e) except jinja2.exceptions.TemplateNotFound as e: - self.logger.info("[ERROR] Template error: %s" % e) + self.logger.info(f"[ERROR] Liquid Template error: {e}") + error = True + error_msg = e + + self.action["parameters"].append({ + "name": "liquid_template_error", + "value": f"There was a Liquid input error (1). Details: {e}", + }) + + self.action_result["action"] = self.action + except SyntaxError as e: + self.logger.info(f"[ERROR] Liquid Syntax error: {e}") + error = True + error_msg = e + + self.action["parameters"].append({ + "name": "liquid_python_syntax_error", + "value": f"There was a syntax error in your Liquid input (2). Details: {e}", + }) + + self.action_result["action"] = self.action + except IndentationError as e: + self.logger.info(f"[ERROR] Liquid IndentationError: {e}") + error = True + error_msg = e + + self.action["parameters"].append({ + "name": "liquid_indentiation_error", + "value": f"There was an indentation error in your Liquid input (2). Details: {e}", + }) + + self.action_result["action"] = self.action except jinja2.exceptions.TemplateSyntaxError as e: - self.logger.info("[ERROR] Syntax error: %s" % e) - except: - self.logger.info("[ERROR] General exception for liquid") + self.logger.info(f"[ERROR] Liquid Syntax error: {e}") + error = True + error_msg = e + + self.action["parameters"].append({ + "name": "liquid_syntax_error", + "value": f"There was a syntax error in your Liquid input (2). Details: {e}", + }) + + self.action_result["action"] = self.action + except json.decoder.JSONDecodeError as e: + self.logger.info(f"[ERROR] Liquid JSON Syntax error: {e}") + + replace = False + skip_next = False + newlines = [] + thisline = [] + for line in template.split("\n"): + #print("LINE: %s" % repr(line)) + if "\"\"\"" in line or "\'\'\'" in line: + if replace: + skip_next = True + else: + replace = not replace + + if replace == True: + thisline.append(line) + if skip_next == True: + if len(thisline) > 0: + #print(thisline) + newlines.append(" ".join(thisline)) + thisline = [] + + replace = False + else: + newlines.append(line) + + new_template = "\n".join(newlines) + if new_template != template: + #check_template(new_template) + return parse_liquid(new_template, self) + else: + error = True + error_msg = e + + self.action["parameters"].append({ + "name": "liquid_json_error", + "value": f"There was a syntax error in your input JSON(2). This is typically an issue with escaping newlines. Details: {e}", + }) + + self.action_result["action"] = self.action + except TypeError as e: + try: + if "string as left operand" in f"{e}": + #print(f"HANDLE REPLACE: {template}") + split_left = template.split("|") + if len(split_left) < 2: + return template + + splititem = split_left[0] + additem = "{{" + if "{{" in splititem: + splititem = splititem.replace("{{", "", -1) + + if "{%" in splititem: + splititem = splititem.replace("{%", "", -1) + additem = "{%" + + splititem = "%s \"%s\"" % (additem, splititem.strip()) + parsed_template = template.replace(split_left[0], splititem) + run = Liquid(parsed_template, mode="wild", from_file=False) + return run.render(**globals()) + + except Exception as e: + print(f"SubError in Liquid: {e}") + + self.action["parameters"].append({ + "name": "liquid_general_error", + "value": f"There was general error Liquid input (2). Details: {e}", + }) + + self.action_result["action"] = self.action + #return template + + self.logger.info(f"[ERROR] Liquid TypeError error: {e}") + error = True + error_msg = e + + except Exception as e: + self.logger.info(f"[ERROR] General exception for liquid: {e}") + error = True + error_msg = e + + self.action["parameters"].append({ + "name": "liquid_general_exception", + "value": f"There was general exception Liquid input (2). Details: {e}", + }) + + self.action_result["action"] = self.action + + if "fmt" in error_msg and "liquid_date" in error_msg: + return template + + self.logger.info("Done in liquid") + if error == True: + self.action_result["status"] = "FAILURE" + data = { + "success": False, + "reason": f"Failed to parse LiquidPy: {error_msg}", + "input": template, + } + + try: + self.action_result["result"] = json.dumps(data) + except Exception as e: + self.action_result["result"] = f"Failed to parse LiquidPy: {error_msg}" + print("[WARNING] Failed to set LiquidPy result") + + self.action_result["completed_at"] = int(time.time()) + self.send_result(self.action_result, headers, stream_path) + + self.logger.info(f"[ERROR] Sent FAILURE response to backend due to : {e}") + + if runtime == "run": + return template + else: + os.exit() return template @@ -1663,9 +2482,11 @@ class AppBase: newvalue[key] = recurse_cleanup_script(value) except json.decoder.JSONDecodeError as e: - print(f"[WARNING] Failed JSON replacement for OpenAPI keys (3) {e}") + # Since here the data isn't at all JSON compatible..? + # Seems to happen with newlines in variables being parsed in as strings? + print(f"[ERROR] Failed JSON replacement for OpenAPI keys (3) {e}. Value: {data}") except Exception as e: - print(f"[WARNING] Failed as an exception (1): {e}") + print(f"[ERROR] Failed as an exception (1): {e}") try: for deletekey in deletekeys: @@ -1716,7 +2537,8 @@ class AppBase: #match = ".*?([$]{1}([a-zA-Z0-9 _-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})[$/, ]?" #match = ".*?([$]{1}([a-zA-Z0-9 _-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})" - match = ".*?([$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})" # Removed space - no longer ok. Force underscore. + #match = ".*?([$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})" # Removed space - no longer ok. Force underscore. + match = "([$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})" # Removed .*? to make it work with large amounts of data # Extra replacements for certain scenarios escaped_dollar = "\\$" @@ -1729,16 +2551,31 @@ class AppBase: except: self.logger.info("Error in initial replacement of escaped dollar!") - #self.logger.info("POST input value: %s" % parameter["value"]) + paramname = "" + try: + paramname = parameter["name"] + except: + pass + + # Basic fix in case variant isn't set + # Variant is ALWAYS STATIC_VALUE from mid 2021~ + try: + #self.logger.info(f"[DEBUG] Parameter '{paramname}' of length {len(parameter['value'])}") + parameter["variant"] = parameter["variant"] + except: + parameter["variant"] = "STATIC_VALUE" # Regex to find all the things + # Should just go in here if data is ... not so big + #if parameter["variant"] == "STATIC_VALUE" and len(parameter["value"]) < 1000000: + #if parameter["variant"] == "STATIC_VALUE" and len(parameter["value"]) < 5000000: if parameter["variant"] == "STATIC_VALUE": data = parameter["value"] actualitem = re.findall(match, data, re.MULTILINE) #self.logger.debug(f"\n\nHandle static data with JSON: {data}\n\n") #self.logger.info("STATIC PARSED: %s" % actualitem) + #self.logger.info("[INFO] Done with regex matching") if len(actualitem) > 0: - #self.logger.info("[DEACTUAL: ", actualitem) for replace in actualitem: try: to_be_replaced = replace[0] @@ -1750,8 +2587,7 @@ class AppBase: # Trying without string dumping. value, is_loop = get_json_value(fullexecution, to_be_replaced) - #self.logger.info("\n\nType of value: %s. Value: %s" % (type(value), value)) - self.logger.info("\n\nType of value: %s" % type(value)) + #self.logger.info(f"\n\nType of value: {type(value)}") if isinstance(value, str): parameter["value"] = parameter["value"].replace(to_be_replaced, value) elif isinstance(value, dict) or isinstance(value, list): @@ -1765,13 +2601,15 @@ class AppBase: # parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) # self.logger.info("Failed parsing value as string?") else: - self.logger.info("Unknown type %s" % type(value)) + self.logger.info("[WARNING] Unknown type %s" % type(value)) try: parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) except json.decoder.JSONDecodeError as e: parameter["value"] = parameter["value"].replace(to_be_replaced, value) #self.logger.info("VALUE: %s" % parameter["value"]) + else: + self.logger.info(f"[ERROR] Not running static variant regex parsing (slow) on value with length {len(parameter['value'])}. Max is 5Mb~.") if parameter["variant"] == "WORKFLOW_VARIABLE": self.logger.info("[DEBUG] Handling workflow variable") @@ -1857,7 +2695,7 @@ class AppBase: parameter["value"] = parameter["value"].replace(end_variable, "", -1) parameter["value"] = parameter["value"].replace(escape_replacement, "$", -1) except: - self.logger.info("Error in datareplacement") + self.logger.info(f"[ERROR] Problem in datareplacement: {e}") # Just here in case it breaks # Implemented 02.08.2021 @@ -1867,9 +2705,6 @@ class AppBase: except: pass - #self.logger.info("Replaced data: %s" % parameter["value"]) - - #self.logger.info("POST liquid: %s" % parameter["value"]) return "", parameter["value"], is_loop def run_validation(sourcevalue, check, destinationvalue): @@ -1899,19 +2734,6 @@ class AppBase: return True return False - #if tmp == "[]": - # tmp = [] - - #if type(tmp) == list and len(tmp) == 0 and not flip: - # new_list.append(item) - #elif type(tmp) == list and len(tmp) > 0 and flip: - # new_list.append(item) - #elif type(tmp) == str and not tmp and not flip: - # new_list.append(item) - #elif type(tmp) == str and tmp and flip: - # new_list.append(item) - #else: - # failed_list.append(item) elif check.lower() == "contains_any_of": newvalue = [destinationvalue.lower()] @@ -1960,7 +2782,7 @@ class AppBase: return True else: print("[DEBUG] Condition: can't handle %s yet. Setting to true" % check) - + return False def check_branch_conditions(action, fullexecution, self): @@ -1971,66 +2793,93 @@ class AppBase: except KeyError: return True, "" + # Startnode should always run - no need to check incoming + try: + if action["id"] == fullexecution["start"]: + return True, "" + except Exception as error: + self.logger.info(f"[WARNING] Failed checking startnode: {error}") + return True, "" + + available_checks = [ + "=", + "equals", + "!=", + "does not equal", + ">", + "larger than", + "<", + "less than", + ">=", + "<=", + "startswith", + "endswith", + "contains", + "contains_any_of", + "re", + "matches regex", + ] + relevantbranches = [] + correct_branches = 0 + matching_branches = 0 for branch in fullexecution["workflow"]["branches"]: if branch["destination_id"] != action["id"]: continue + matching_branches += 1 + + # Find if previous is skipped or failed. Skipped != correct branch + try: + should_skip = False + for res in fullexecution["results"]: + if res["action"]["id"] == branch["source_id"]: + if res["status"] == "FAILURE" or res["status"] == "SKIPPED": + should_skip = True + + break + + if should_skip: + continue + except Exception as e: + self.logger.info("[WARNING] Failed handling check of if parent is skipped") + + # Remove anything without a condition try: if (branch["conditions"]) == 0 or branch["conditions"] == None: + correct_branches += 1 continue except KeyError: + correct_branches += 1 continue - self.logger.info("Relevant conditions: %s" % branch["conditions"]) successful_conditions = [] failed_conditions = [] + successful_conditions = 0 + total_conditions = len(branch["conditions"]) for condition in branch["conditions"]: - self.logger.info("Getting condition value of %s" % condition) + self.logger.info("[DEBUG] Getting condition value of %s" % condition) # Parse all values first here sourcevalue = condition["source"]["value"] check, sourcevalue, is_loop = parse_params(action, fullexecution, condition["source"], self) if check: - return False, {"success": False, "reason": "Failed condition (1): %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)} + continue - #sourcevalue = sourcevalue.encode("utf-8") sourcevalue = parse_wrapper_start(sourcevalue, self) destinationvalue = condition["destination"]["value"] check, destinationvalue, is_loop = parse_params(action, fullexecution, condition["destination"], self) if check: - return False, {"success": False, "reason": "Failed condition (2): %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)} + continue - #destinationvalue = destinationvalue.encode("utf-8") destinationvalue = parse_wrapper_start(destinationvalue, self) - available_checks = [ - "=", - "equals", - "!=", - "does not equal", - ">", - "larger than", - "<", - "less than", - ">=", - "<=", - "startswith", - "endswith", - "contains", - "contains_any_of", - "re", - "matches regex", - ] if not condition["condition"]["value"] in available_checks: self.logger.warning("Skipping %s %s %s because %s is invalid." % (sourcevalue, condition["condition"]["value"], destinationvalue, condition["condition"]["value"])) continue - #self.logger.info(destinationvalue) - # NEGATE - # Configuration = negated because of WorkflowAppActionParam.. validation = run_validation(sourcevalue, condition["condition"]["value"], destinationvalue) try: @@ -2039,42 +2888,63 @@ class AppBase: except KeyError: pass - if not validation: - self.logger.info("Failed condition check for %s %s %s." % (sourcevalue, condition["condition"]["value"], destinationvalue)) - return False, {"success": False, "reason": "Failed condition (3): %s %s %s" % (sourcevalue, condition["condition"]["value"], destinationvalue)} + if validation == True: + successful_conditions += 1 - - # Make a general parser here, at least to get param["name"] = param["value"] in maparameter[string]string - #for condition in branch.conditons: + if total_conditions == successful_conditions: + correct_branches += 1 - return True, "" + if matching_branches == 0: + return True, "" + + if matching_branches > 0 and correct_branches > 0: + return True, "" + + self.logger.info("[DEBUG] Correct branches vs matching branches: %d vs %d" % (correct_branches, matching_branches)) + return False, {"success": False, "reason": "Minimum of one branch's conditions must be correct to continue. Total: %d of %d" % (correct_branches, matching_branches)} + # + # + # + # + # CONT + # CONT + # CONT + # CONT + # CONT + # CONT + # CONT + # CONT + # CONT + # CONT + # CONT + # CONT + # CONT + # + # + # + # # THE START IS ACTUALLY RIGHT HERE :O # Checks whether conditions are met, otherwise set branchcheck, tmpresult = check_branch_conditions(action, fullexecution, self) - if isinstance(tmpresult, object) or isinstance(tmpresult, list): - self.logger.info("[DEBUG] Fixing branch return as object -> string") + if isinstance(tmpresult, object) or isinstance(tmpresult, list) or isinstance(tmpresult, dict): + #self.logger.info("[DEBUG] Fixing branch return as object -> string") try: #tmpresult = tmpresult.replace("'", "\"") tmpresult = json.dumps(tmpresult) except json.decoder.JSONDecodeError as e: self.logger.info(f"[WARNING] Failed condition parsing {tmpresult} to string") + # IF branches fail: Exit! if not branchcheck: self.logger.info("Failed one or more branch conditions.") self.action_result["result"] = tmpresult self.action_result["status"] = "SKIPPED" - try: - ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=self.action_result) - self.logger.info("Result: %d" % ret.status_code) - if ret.status_code != 200: - self.logger.info(ret.text) - except requests.exceptions.ConnectionError as e: - self.logger.exception(e) + self.action_result["completed_at"] = int(time.time()) - self.logger.info("\n\n[DEBUG] RETURNING BECAUSE A BRANCH FAILED: %s\n\n" % tmpresult) + self.send_result(self.action_result, headers, stream_path) return # Replace name cus there might be issues @@ -2083,7 +2953,8 @@ class AppBase: if " " in actionname: actionname.replace(" ", "_", -1) - + #print("ACTION: ", action) + #print("exec: ", self.full_execution) #if action.generated: # actionname = actionname.lower() @@ -2091,13 +2962,18 @@ class AppBase: try: func = getattr(self, actionname, None) if func == None: - self.logger.debug(f"Failed executing {actionname} because func is None.") + self.logger.debug(f"[DEBUG] Failed executing {actionname} because func is None (no function specified).") self.action_result["status"] = "FAILURE" - self.action_result["result"] = "Function %s doesn't exist." % actionname + self.action_result["result"] = json.dumps({ + "success": False, + "reason": f"Function {actionname} doesn't exist, or the App is out of date.", + "details": "If this persists, please restart delete the Docker image locally, restart your Orborus instance and then try again to force-download the latest version. Contact support@shuffler.io with this data if the issue persists.", + }) elif callable(func): try: if len(action["parameters"]) < 1: - result = await func() + #result = await func() + result = func() else: # Potentially parse JSON here # FIXME - add potential authentication as first parameter(s) here @@ -2106,14 +2982,14 @@ class AppBase: # What variables are necessary here tho hmm params = {} - try: - for item in action["authentication"]: - #self.logger.info("AUTH: ", key, value) - params[item["key"]] = item["value"] - except KeyError: - self.logger.info("[DEBUG] No authentication specified!") - pass - #action["authentication"] + # This replacement should happen in backend as part of params + # error log is useless + #try: + # for item in action["authentication"]: + # self.logger.info("AUTH PARAM: ", key, value) + # #params[item["key"]] = item["value"] + #except KeyError as e: + # self.logger.info(f"[WARNING] No authentication specified! Is this correct? err: {e}") # Fixes OpenAPI body parameters for later. newparams = [] @@ -2143,6 +3019,8 @@ class AppBase: if parameter["name"] == "body": bodyindex = counter #self.logger.info("PARAM: %s" % parameter) + + # FIXMe: This should also happen after liquid & param parsing.. try: values = parameter["value_replace"] if values != None: @@ -2150,16 +3028,24 @@ class AppBase: for val in values: replace_value = val["value"] replace_key = val["key"] + if (val["value"].startswith("{") and val["value"].endswith("}")) or (val["value"].startswith("[") and val["value"].endswith("]")): self.logger.info(f"""Trying to parse as JSON: {val["value"]}""") try: - value_replace = json.loads(val["value"]) - # If it gets here, remove the "" infront and behind the key as well since this is preventing the JSON from being loaded + newval = val["value"] + + # If it gets here, remove the "" infront and behind the key as well + # since this is preventing the JSON from being loaded + tmpvalue = json.loads(newval) replace_key = f"\"{replace_key}\"" except json.decoder.JSONDecodeError as e: - self.logger.info("Failed JSON replacement for OpenAPI %s", val["key"]) + self.logger.info("[WARNING] Failed JSON replacement for OpenAPI %s", val["key"]) + elif val["value"].lower() == "true" or val["value"].lower() == "false": replace_key = f"\"{replace_key}\"" + else: + if "\"" in replace_value and not "\\\"" in replace_value: + replace_value = replace_value.replace("\"", "\\\"", -1) action["parameters"][counter]["value"] = action["parameters"][counter]["value"].replace(replace_key, replace_value, 1) @@ -2174,13 +3060,12 @@ class AppBase: pass - self.logger.info(f"""HANDLING {action["parameters"][counter]["value"]}""") + self.logger.info(f"""HANDLING BODY: {action["parameters"][counter]["value"]}""") action["parameters"][counter]["value"] = recurse_cleanup_script(action["parameters"][counter]["value"]) #self.logger.info(action["parameters"]) # This seems redundant now - self.logger.info("[DEBUG] Pre parameters") for parameter in newparams: action["parameters"].append(parameter) @@ -2202,7 +3087,6 @@ class AppBase: # Multi_parameter has the data for each. variable minlength = 0 - self.logger.info("[DEBUG] Pre-loading parameters") multi_parameters = json.loads(json.dumps(params)) multiexecution = False multi_execution_lists = [] @@ -2210,7 +3094,14 @@ class AppBase: for parameter in action["parameters"]: check, value, is_loop = parse_params(action, fullexecution, parameter, self) if check: - raise "Value check error: %s" % Exception(check) + raise Exception(json.dumps({ + "success": False, + "reason": "Parameter {parameter} has an issue", + "exception": f"Value Error: {check}", + })) + + if parameter["name"] == "body": + self.logger.info(f"[INFO] Should debug field with liquid and other checks as it's BODY: {value}") # Custom format for ${name[0,1,2,...]}$ #submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" @@ -2279,6 +3170,11 @@ class AppBase: else: newvalue = tmpitem.replace(str(actualitem[index][0]), str(json_replacement[i]), 1) + try: + newvalue = parse_liquid(newvalue, self) + except Exception as e: + self.logger.info(f"[WARNING] Failed liquid parsing in loop (2): {e}") + try: newvalue = json.loads(newvalue) except json.decoder.JSONDecodeError as e: @@ -2289,7 +3185,7 @@ class AppBase: self.logger.info("New replacement: %s" % new_replacement) - # New + # FIXME: Should this use new_replacement? tmpitem = tmpitem.replace(actualitem[index][0], replacement, 1) # This code handles files. @@ -2383,7 +3279,13 @@ class AppBase: for i in range(0, curminlength): tmpitem = json.loads(json.dumps(parameter["value"])) for key, value in replacements.items(): - replacement = json.dumps(json.loads(value)[i]) + replacement = value + try: + replacement = json.dumps(json.loads(value)[i]) + except IndexError as e: + self.logger.info(f"[ERROR] Failed handling value parsing with index: {e}") + pass + if replacement.startswith("\"") and replacement.endswith("\""): replacement = replacement[1:len(replacement)-1] #except json.decoder.JSONDecodeError as e: @@ -2391,6 +3293,10 @@ class AppBase: #self.logger.info("REPLACING %s with %s" % (key, replacement)) #replacement = parse_wrapper_start(replacement) tmpitem = tmpitem.replace(key, replacement, -1) + try: + tmpitem = parse_liquid(tmpitem, self) + except Exception as e: + self.logger.info(f"[WARNING] Failed liquid parsing in loop (2): {e}") # This code handles files. @@ -2434,13 +3340,17 @@ class AppBase: multi_parameters[parameter["name"]] = resultarray else: # Parses things like int(value) - self.logger.info("[DEBUG] Normal parsing (not looping)")#with data %s" % value) + #self.logger.info("[DEBUG] Normal parsing (not looping)")#with data %s" % value) # This part has fucked over so many random JSON usages because of weird paranthesis parsing value = parse_wrapper_start(value, self) - self.logger.info("[DEBUG] Post return: %s" % value) - #self.logger.info("POST data value: %s" % value) + try: + if str(value).startswith("b'") and str(value).endswith("'"): + value = value[2:-1] + except Exception as e: + print(f"Value rawbytes Exception: {e}") + params[parameter["name"]] = value multi_parameters[parameter["name"]] = value @@ -2522,37 +3432,195 @@ class AppBase: self.send_result(self.action_result, headers, stream_path) return - self.logger.info("[INFO] Running normal execution (not loop)\n") + self.logger.info("[INFO] Running normal execution (not loop)\n\n") + + # Added literal evaluation of anything resembling a string + # The goal is to parse objects that e.g. use single quotes and the like + # FIXME: add this to Multi exec as well. + try: + for key, value in params.items(): + if "-" in key: + try: + newkey = key.replace("-", "_", -1).lower() + params[newkey] = params[key] + except Exception as e: + self.logger.info("[DEBUG] Failed updating key with dash in it: %s" % e) + + try: + if isinstance(value, str) and ((value.startswith("{") and value.endswith("}")) or (value.startswith("[") and value.endswith("]"))): + params[key] = json.loads(value) + except Exception as e: + try: + if isinstance(value, str) and ((value.startswith("{") and value.endswith("}")) or (value.startswith("[") and value.endswith("]"))): + params[key] = ast.literal_eval(value) + except Exception as e: + self.logger.info(f"[DEBUG] Failed parsing value with ast and json.loads - noncritical. Trying next: {e}") + continue + except Exception as e: + self.logger.info("[DEBUG] Failed looping objects. Non critical: {e}") + + # Uncomment below to get the param input + # self.logger.info(f"[DEBUG] PARAMS: {params}") - #newres = await func(**params) - #self.logger.info("PARAMS: %s" % params) #newres = "" + iteration_count = 0 + found_error = "" while True: - try: - newres = await func(**params) + iteration_count += 1 + if iteration_count >= 10: + newres = { + "success": False, + "reason": "Iteration count more than 10. This happens if the input to the action is wrong. Try remaking the action, and contact support@shuffler.io if this persists.", + "details": found_error, + } break + + try: + #try: + # Individual functions shouldn't take longer than this + # This is an attempt to make timeouts occur less, incentivizing users to make use efficient API's + # PS: Not implemented for lists - only single actions as of May 2023 + timeout = 30 + + # Check if current app is Shuffle Tools, then set to 55 due to certain actions being slow (ioc parser..) + #uu In general, this should be disabled for onprem + if self.action["app_name"].lower() == "shuffle tools": + timeout = 55 + + timeout = 30 + + try: + executor = concurrent.futures.ThreadPoolExecutor() + future = executor.submit(func, **params) + newres = future.result(timeout) + + if not future.done(): + # The future is still running, so we need to cancel it + future.cancel() + newres = json.dumps({ + "success": False, + "reason": "Timeout error within %d seconds. This happens if we can't reach or use the API you're trying to use within the time limit." % timeout, + "exception": str(e), + }) + + else: + # The future is done, so we can just get the result from newres :) + #newres = future.result() + #print("Future is done!") + pass + + except concurrent.futures.TimeoutError as e: + newres = json.dumps({ + "success": False, + "reason": "Timeout error within %d seconds (2). This happens if we can't reach or use the API you're trying to use within the time limit" % timeout + }) + + break + + + + #thread = threading.Thread(target=func, args=(**params,)) + #thread.start() + + #thread.join(timeout) + + #if thread.is_alive(): + # # The thread is still running, so we need to stop it + # # You can handle this as needed, such as raising an exception + # timeout_handler() + + + #with Timeout(timeout): + # newres = func(**params) + # break + #except Timeout.Timeout as e: + # self.logger.info(f"[DEBUG] Timeout error: {e}") + # newres = json.dumps({ + # "success": False, + # "reason": "Timeout error within %d seconds. This typically happens if we can't reach the API you're trying to reach." % timeout, + # "exception": str(e), + # }) + + # break + except TypeError as e: newres = "" - self.logger.info(f"[DEBUG] Got exec error: {errorstring}") + self.logger.info(f"[DEBUG] Got exec type error: {e}") + try: + e = json.loads(f"{e}") + except: + e = f"{e}" + + found_error = e errorstring = f"{e}" - if "got an unexpected keyword argument" in errorstring: + + if "the JSON object must be" in errorstring: + self.logger.info("[ERROR] Something is wrong with the input for this function. Are lists and JSON data handled parsed properly (0)? the JSON object must be in...") + try: + e = json.loads(f"{e}") + except: + e = f"{e}" + + newres = json.dumps({ + "success": False, + "reason": "An exception occurred while running this function (1). See exception for more details and contact support if this persists (support@shuffler.io)", + "exception": e, + }) + break + elif "got an unexpected keyword argument" in errorstring: fieldsplit = errorstring.split("'") if len(fieldsplit) > 1: field = fieldsplit[1] try: del params[field] - self.logger.info("[WARNING] Removed field invalid field %s" % field) + self.logger.info("[WARNING] Removed invalid field %s (2)" % field) except KeyError: break else: - raise e - #break + newres = json.dumps({ + "success": False, + "reason": "You may be running an old version of this action. Try remaking the node, then contact us at support@shuffler.io if it doesn't work with all these details.", + "exception": f"TypeError: {e}", + }) + break + except Exception as e: + self.logger.info(f"[ERROR] Something is wrong with the input for this function. Are lists and JSON data handled parsed properly (1)? err: {e}") - self.logger.info("\n[INFO] Returned from execution with types %s" % type(newres)) + try: + e = json.loads(f"{e}") + except: + e = f"{e}" + + newres = json.dumps({ + "success": False, + "reason": "An exception occurred while running this function (2). See exception for more details and contact support if this persists (support@shuffler.io)", + "exception": e, + }) + break + + # Forcing async wait in case of old apps that use async (backwards compatibility) + try: + if asyncio.iscoroutine(newres): + self.logger.info("[DEBUG] In coroutine (1)") + async def parse_value(newres): + value = await asyncio.gather( + newres + ) + + return value[0] + + newres = asyncio.run(parse_value(newres)) + else: + #self.logger.info("[DEBUG] Not in coroutine (1)") + pass + except Exception as e: + self.logger.warning("[ERROR] Failed to parse coroutine value for old app: {e}") + + self.logger.info("\n\n\n[INFO] Returned from execution with type(s) %s" % type(newres)) #self.logger.info("\n[INFO] Returned from execution with %s of types %s" % (newres, type(newres)))#, newres) if isinstance(newres, tuple): - self.logger.info("[INFO] Handling return as tuple") + self.logger.info(f"[INFO] Handling return as tuple: {newres}") # Handles files. filedata = "" file_ids = [] @@ -2570,6 +3638,7 @@ class AppBase: self.logger.info("[INFO] NO FILES TO HANDLE") tmp_result = { + "success": True, "result": newres[0], "file_ids": file_ids } @@ -2582,13 +3651,15 @@ class AppBase: try: result += json.dumps(newres, indent=4) except json.JSONDecodeError as e: - self.logger.info("Failed decoding result: %s" % e) - + self.logger.info("[WARNING] Failed decoding result: %s" % e) try: result += str(newres) except ValueError: result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) self.logger.info("Can't handle type %s value from function" % (type(newres))) + except Exception as e: + self.logger.info("[ERROR] Failed to json dump. Returning as string.") + result += str(newres) else: try: result += str(newres) @@ -2596,7 +3667,7 @@ class AppBase: result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) self.logger.info("Can't handle type %s value from function" % (type(newres))) - self.logger.info("[INFO] POST NEWRES RESULT!")#, result) + #self.logger.info("[INFO] POST NEWRES RESULT!")#, result) else: #self.logger.info("[INFO] APP_SDK DONE: Starting MULTI execution (length: %d) with values %s" % (minlength, multi_parameters)) # 1. Use number of executions based on the arrays being similar @@ -2604,119 +3675,15 @@ class AppBase: self.logger.info("[INFO] Running WITHOUT outer loop (looping)") json_object = False - results = await self.run_recursed_items(func, multi_parameters, {}) + #results = await self.run_recursed_items(func, multi_parameters, {}) + results = self.run_recursed_items(func, multi_parameters, {}) if isinstance(results, dict) or isinstance(results, list): json_object = True - #for i in range(0, minlength): - # # To be able to use the results as a list: - # self.logger.info("1: %s" % multi_parameters) - # #baseparams = json.loads(json.dumps(multi_parameters)) - # baseparams = copy.deepcopy(multi_parameters) - - # self.logger.info("2: %s: %s" % (type(baseparams), baseparams)) - - # self.logger.info("4") - # self.logger.info("Running with params (1): %s" % baseparams) - - # results = await self.run_recursed_items(func, baseparams, {}) - # if isinstance(results, dict) or isinstance(results, list): - # json_object = True - - # {'call': ['GoogleSafebrowsing_2_0', 'VirusTotal_GetReport_3_0']} - # 1. Check if list length is same as minlength - # 2. If NOT same length, duplicate based on length of array - # arraylength = 3 ["1", "2", "3"] - # arraylength = 4 ["1", "2", "3", "4"] - # minlength = 12 - 12/3 = 4 per item = ["1", "1", "1", "1", "2", "2", ...] - - #try: - # firstlist = True - # for key, value in baseparams.items(): - # self.logger.info("Itemtype: %s" % type(value)) - # if isinstance(value, list): - # try: - # newvalue = value[i] - # except IndexError: - # pass - - # if len(value) != minlength and len(value) > 0: - # newarray = [] - # self.logger.info("VALUE: ", value) - # additiontime = minlength/len(value) - # self.logger.info("Bad length for value: %d - should be %d. Additiontime: %d" % (len(value), minlength, additiontime)) - # if firstlist: - # self.logger.info("Running normal list (FIRST)") - # for subvalue in value: - # for number in range(int(additiontime)): - # newarray.append(subvalue) - # else: - # #self.logger.info("Running secondary lists") - # ## 1. Set up length of array - # ## 2. Put values spread out - # # FIXME: This works well, except if lists are same length - # newarray = [""] * minlength - - # cnt = 0 - # for number in range(int(additiontime)): - # for subvaluerange in range(len(value)): - # # newlocation = number+(additiontime*subvaluerange) - # # self.logger.info("%d+(%d*%d) = %d. VAL: %s" % (number, additiontime, subvaluerange, newlocation, value[subvaluerange])) - # # Reverse if same length? - # if int(minlength/len(value)) == len(value): - # tmp = int(len(value)-subvaluerange-1) - # self.logger.info("NEW: %d" % tmp) - # newarray[cnt] = value[tmp] - # else: - # newarray[cnt] = value[subvaluerange] - # cnt += 1 - - # #self.logger.info("Newarray =", newarray) - # newvalue = newarray[i] - # firstlist = False - - # baseparams[key] = newvalue - - # self.logger.info("3") - #except IndexError as e: - # self.logger.info("IndexError: %s" % e) - # baseparams[key] = "IndexError: %s" % e - #except KeyError as e: - # self.logger.info("KeyError: %s" % e) - # baseparams[key] = "KeyError: %s" % e - #self.logger.info("4") - #self.logger.info("Running with params (1): %s" % baseparams) - - #results = await self.run_recursed_items(func, baseparams, {}) - #if isinstance(results, dict) or isinstance(results, list): - # json_object = True - - # Check the structure here. If "isloop", try to recurse? - # ret, is_loop = recurse_json(innervalue, parsersplit[outercnt+1:]) - #ret = await func(**baseparams) - #self.logger.info("Return from execution: %s" % ret) - #if ret == None: - # results.append("") - # json_object = False - #elif isinstance(ret, dict) or isinstance(ret, list): - # results.append(ret) - # json_object = True - #else: - # ret = ret.replace("\"", "\\\"", -1) - - # try: - # results.append(json.loads(ret)) - # json_object = True - # except json.decoder.JSONDecodeError as e: - # #self.logger.info("Json: %s" % e) - # results.append(ret) - - #self.logger.info("Inner ret parsed: %s" % ret) - # Dump the result as a string of a list #self.logger.info("RESULTS: %s" % results) if isinstance(results, list) or isinstance(results, dict): - self.logger.info("JSON OBJECT? ", json_object) + self.logger.info(f"JSON OBJECT? {json_object}") # This part is weird lol if json_object: @@ -2758,75 +3725,234 @@ class AppBase: self.logger.debug(f"[DEBUG] Executed {action['label']}-{action['id']}")#with result: {result}") #self.logger.debug(f"Data: %s" % action_result) except TypeError as e: - self.logger.info("TypeError issue: %s" % e) + self.logger.info("[ERROR] TypeError issue: %s" % e) self.action_result["status"] = "FAILURE" - self.action_result["result"] = "TypeError: %s" % str(e) + self.action_result["result"] = json.dumps({ + "success": False, + "reason": f"Typeerror. Most likely due to a list that should've been a string. See details for more info.", + "details": e, + }) + #self.action_result["result"] = "TypeError: %s" % str(e) else: self.logger.info("[DEBUG] Function %s doesn't exist?" % action["name"]) self.logger.error(f"[ERROR] App {self.__class__.__name__}.{action['name']} is not callable") self.action_result["status"] = "FAILURE" - self.action_result["result"] = "Function %s is not callable." % actionname + #self.action_result["result"] = "Function %s is not callable." % actionname + + self.action_result["result"] = json.dumps({ + "success": False, + "reason": f"Function %s doesn't exist." % actionname, + }) # https://ptb.discord.com/channels/747075026288902237/882017498550112286/882043773138382890 except (requests.exceptions.RequestException, TimeoutError) as e: - self.logger.info(f"Failed to execute request: {e}") - self.logger.exception(f"Failed to execute {e}-{action['id']}") + self.logger.info(f"[ERROR] Failed to execute request (requests): {e}") + self.logger.exception(f"[ERROR] Failed to execute {e}-{action['id']}") self.action_result["status"] = "SUCCESS" + try: + e = json.loads(f"{e}") + except: + e = f"{e}" + try: self.action_result["result"] = json.dumps({ "success": False, "reason": f"Request error - failing silently. Details in detail section", - "details": f"{e}", + "details": e, }) except json.decoder.JSONDecodeError as e: self.action_result["result"] = f"Request error: {e}" except Exception as e: - self.logger.info(f"Failed to execute: {e}") - self.logger.exception(f"Failed to execute {e}-{action['id']}") + self.logger.info(f"[ERROR] Failed to execute: {e}") + self.logger.exception(f"[ERROR] Failed to execute {e}-{action['id']}") self.action_result["status"] = "FAILURE" - self.action_result["result"] = f"General exception: {e}" + try: + e = json.loads(f"{e}") + except: + e = f"{e}" - self.action_result["completed_at"] = int(time.time()) + self.action_result["result"] = json.dumps({ + "success": False, + "reason": f"General exception in the app. See shuffle action logs for more details.", + "details": e, + }) # Send the result :) + self.action_result["completed_at"] = int(time.time()) self.send_result(self.action_result, headers, stream_path) + + #try: + # try: + # self.log_capture_string.flush() + # except Exception as e: + # print(f"[WARNING] Failed to flush logs (2): {e}") + # pass + + # self.log_capture_string.close() + #except: + # print(f"[WARNING] Failed to close logs (2): {e}") + return @classmethod - async def run(cls, action=""): + def run(cls, action=""): logging.basicConfig(format="{asctime} - {name} - {levelname}:{message}", style='{') logger = logging.getLogger(f"{cls.__name__}") logger.setLevel(logging.DEBUG) + + logger.info("[DEBUG] Normal execution.") - #self.logger.info("Started execution: %s!!" % cls) - #self.logger.info("Action: %s" % action) - #if isinstance(cls, object): - # self.action = cls + ############################################## - app = cls(redis=None, logger=logger, console_logger=logger) - if isinstance(action, str): - print("[DEBUG] Normal execution. Action is a string.") - elif isinstance(action, object): - print("[DEBUG] OBJECT execution. Action is NOT a string.") - app.action = action + exposed_port = os.getenv("SHUFFLE_APP_EXPOSED_PORT", "") + logger.info(f"[DEBUG] \"{runtime}\" - run indicates microservices. Port: \"{exposed_port}\"") + if runtime == "run" and exposed_port != "": + # Base port is 33334. Exposed port may differ based on discovery from Worker + port = int(exposed_port) + logger.info(f"[DEBUG] Starting webserver on port {port} (same as exposed port)") + from flask import Flask, request + from waitress import serve + + flask_app = Flask(__name__) + #flask_app.config['PERMANENT_SESSION_LIFETIME'] = datetime.timedelta(minutes=5) + + #async def execute(): + @flask_app.route("/api/v1/health", methods=["GET", "POST"]) + def check_health(): + return "OK" - try: - app.authorization = action["authorization"] - app.current_execution_id = action["execution_id"] - except: - pass + @flask_app.route("/api/v1/run", methods=["POST"]) + def execute(): + if request.method == "POST": + #print(request.get_json(force=True)) + requestdata = {} + try: + requestdata = json.loads(request.data) + except Exception as e: + return { + "success": False, + "reason": f"Invalid Action data {e}", + } + + #logger.info(f"[DEBUG] Datatype: {type(requestdata)}: {requestdata}") - try: - app.url = action["url"] - except: - pass + # Remaking class for each request + + app = cls(redis=None, logger=logger, console_logger=logger) + extra_info = "" + try: + #asyncio.run(AppBase.run(action=requestdata), debug=True) + #value = json.dumps(value) + try: + app.full_execution = json.dumps(requestdata["workflow_execution"]) + except Exception as e: + logger.info(f"[ERROR] Failed parsing full execution from workflow_execution: {e}") + extra_info += f"\n{e}" - try: - app.base_url = action["base_url"] - except: - pass + try: + app.action = requestdata["action"] + except Exception as e: + logger.info(f"[ERROR] Failed parsing action: {e}") + extra_info += f"\n{e}" + + try: + app.authorization = requestdata["authorization"] + app.current_execution_id = requestdata["execution_id"] + except Exception as e: + logger.info(f"[ERROR] Failed parsing auth and exec id: {e}") + extra_info += f"\n{e}" + + # BASE URL (backend) + try: + app.url = requestdata["url"] + logger.info(f"BACKEND URL (url): {app.url}") + except Exception as e: + logger.info(f"[ERROR] Failed parsing url (backend): {e}") + extra_info += f"\n{e}" + + # URL (worker) + try: + app.base_url = requestdata["base_url"] + logger.info(f"WORKER URL (base url): {app.base_url}") + except Exception as e: + logger.info(f"[ERROR] Failed parsing base url (worker): {e}") + extra_info += f"\n{e}" + + #await + app.execute_action(app.action) + logger.info("[DEBUG] Done awaiting app action running") + except Exception as e: + return { + "success": False, + "reason": f"Problem in execution {e}", + "execution_issues": extra_info, + } + + return { + "success": True, + "reason": "App successfully finished", + "execution_issues": extra_info, + } + else: + return { + "success": False, + "reason": f"HTTP method {request.method} not allowed", + } + + logger.info(f"[DEBUG] Serving on port {port}") + + #flask_app.run( + # host="0.0.0.0", + # port=port, + # threaded=True, + # processes=1, + # debug=False, + #) + + serve( + flask_app, + host="0.0.0.0", + port=port, + threads=8, + channel_timeout=30, + expose_tracebacks=True, + asyncore_use_poll=True, + ) + ####################### else: - self.logger.info("ACTION TYPE (unhandled): %s" % type(action)) + # Has to start like this due to imports in other apps + # Move it outside everything? + app = cls(redis=None, logger=logger, console_logger=logger) + #logger.info(f"[DEBUG] Action: {action}") + + if isinstance(action, str): + logger.info("[DEBUG] Normal execution (env var). Action is a string.") + elif isinstance(action, object): + logger.info("[DEBUG] OBJECT execution (cloud). Action is NOT a string.") + app.action = action - await app.execute_action(app.action) + try: + app.authorization = action["authorization"] + app.current_execution_id = action["execution_id"] + except: + pass + + # BASE URL (worker) + try: + app.url = action["url"] + except: + pass + + # Callback URL (backend) + try: + app.base_url = action["base_url"] + except: + pass + else: + self.logger.info("ACTION TYPE (unhandled): %s" % type(action)) + + app.execute_action(app.action) + +if __name__ == "__main__": + AppBase.run() diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index ff991f1f..dd540b94 100755 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,36 +1,51 @@ #!/bin/bash - ### DEFAULT NAME=shuffle-app_sdk -VERSION=0.9.25 +VERSION=1.2.0 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force -docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly - -#docker push frikky/$NAME:$VERSION -#docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -#docker push ghcr.io/frikky/$NAME:$VERSION -#docker tag ghcr.io/frikky/$NAME:$VERSION frikky/shuffle:app_sdk +docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly -t shuffle/shuffle:app_sdk -t shuffle/$NAME:$VERSION -t docker.pkg.github.com/shuffle/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:nightly docker push frikky/shuffle:app_sdk docker push ghcr.io/frikky/$NAME:$VERSION docker push ghcr.io/frikky/$NAME:nightly docker push ghcr.io/frikky/$NAME:latest -#### KALI ### -NAME=shuffle-app_sdk_kali -docker build . -f Dockerfile_kali -t frikky/shuffle:app_sdk_kali -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION +docker push shuffle/shuffle:app_sdk +docker push ghcr.io/shuffle/$NAME:$VERSION +docker push ghcr.io/shuffle/$NAME:nightly +docker push ghcr.io/shuffle/$NAME:latest -docker push frikky/shuffle:app_sdk_kali + + + +#### UBUNTU +NAME=shuffle-app_sdk_ubuntu +docker build . -f Dockerfile_ubuntu -t frikky/shuffle:app_sdk_ubuntu -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION +docker push frikky/shuffle:app_sdk_ubuntu docker push ghcr.io/frikky/$NAME:$VERSION -docker push ghcr.io/frikky/$NAME:nightly + +#### Alpine GRPC +NAME=shuffle-app_sdk_grpc +docker build . -f Dockerfile_alpine_grpc -t frikky/shuffle:app_sdk_grpc -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION +docker push frikky/shuffle:app_sdk_grpc +docker push ghcr.io/frikky/$NAME:$VERSION + + + +#### KALI ### +#NAME=shuffle-app_sdk_kali +#docker build . -f Dockerfile_kali -t frikky/shuffle:app_sdk_kali -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION +# +#docker push frikky/shuffle:app_sdk_kali +#docker push ghcr.io/frikky/$NAME:$VERSION +#docker push ghcr.io/frikky/$NAME:nightly ### BLACKARCH ### -NAME=shuffle-app_sdk_blackarch -docker build . -f Dockerfile_blackarch -t frikky/shuffle:app_sdk_blackarch -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION - -docker push frikky/shuffle:app_sdk_blackarch -docker push ghcr.io/frikky/$NAME:$VERSION -docker push ghcr.io/frikky/$NAME:nightly - +#NAME=shuffle-app_sdk_blackarch +#docker build . -f Dockerfile_blackarch -t frikky/shuffle:app_sdk_blackarch -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION +# +#docker push frikky/shuffle:app_sdk_blackarch +#docker push ghcr.io/frikky/$NAME:$VERSION +#docker push ghcr.io/frikky/$NAME:nightly diff --git a/backend/app_sdk/requirements.txt b/backend/app_sdk/requirements.txt index 79cf5ea2..bacb3bc8 100755 --- a/backend/app_sdk/requirements.txt +++ b/backend/app_sdk/requirements.txt @@ -1,4 +1,8 @@ urllib3==1.26.5 requests==2.25.1 MarkupSafe==2.0.1 -liquidpy==0.7.1 +liquidpy==0.7.6 +flask[async]==2.0.2 +waitress==2.1.0 +#flask==1.1.2 +python-dateutil==2.8.1 diff --git a/backend/build.sh b/backend/build.sh index bb1f4a28..17c372c7 100755 --- a/backend/build.sh +++ b/backend/build.sh @@ -1,10 +1,10 @@ #!/bin/sh docker stop shuffle-backend docker rm shuffle-backend -docker rmi frikky/shuffle:backend +docker rmi ghcr.io/shuffle/shuffle-backend:nightly -docker build . -t frikky/shuffle:backend -docker push frikky/shuffle:backend +docker build . -t ghcr.io/shuffle/shuffle-backend:nightly +docker push ghcr.io/shuffle/shuffle-backend:nightly echo "Starting server" #docker run -it \ diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 391386d5..a6324faf 100755 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -2,18 +2,21 @@ package main // Docker import ( + "archive/tar" + "github.com/shuffle/shuffle-shared" - "archive/tar" //"bufio" "path/filepath" //"strconv" "bytes" "context" + "encoding/base64" "encoding/json" "errors" "fmt" + //"github.com/docker/docker" "github.com/docker/docker/api/types" //"github.com/docker/docker/api/types/container" @@ -238,16 +241,15 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin BuildArgs: map[string]*string{}, Labels: labels, } - // NetworkMode: "host", httpProxy := os.Getenv("HTTP_PROXY") if len(httpProxy) > 0 { - buildOptions.BuildArgs["http_proxy"] = &httpProxy + buildOptions.BuildArgs["HTTP_PROXY"] = &httpProxy } httpsProxy := os.Getenv("HTTPS_PROXY") if len(httpProxy) > 0 { - buildOptions.BuildArgs["https_proxy"] = &httpsProxy + buildOptions.BuildArgs["HTTPS_PROXY"] = &httpsProxy } // Build the actual image @@ -260,7 +262,7 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin //log.Printf("RESPONSE: %#v", imageBuildResponse) //log.Printf("Response: %#v", imageBuildResponse.Body) - log.Printf("[DEBUG] IMAGERESPONSE: %#v", imageBuildResponse.Body) + //log.Printf("[DEBUG] IMAGERESPONSE: %#v", imageBuildResponse.Body) if imageBuildResponse.Body != nil { defer imageBuildResponse.Body.Close() @@ -299,6 +301,7 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin } if !downloaded { + return errors.New(fmt.Sprintf("Failed to build / download images %s", strings.Join(tags, ","))) } //baseDockerName @@ -348,7 +351,7 @@ func buildImage(tags []string, dockerfileFolder string) error { httpProxy := os.Getenv("HTTP_PROXY") if len(httpProxy) > 0 { - buildOptions.BuildArgs["http_proxy"] = &httpProxy + buildOptions.BuildArgs["HTTP_PROXY"] = &httpProxy } httpsProxy := os.Getenv("HTTPS_PROXY") if len(httpProxy) > 0 { @@ -382,170 +385,6 @@ func buildImage(tags []string, dockerfileFolder string) error { return nil } -// FIXME - very specific for webhooks. Make it easier? -func stopWebhook(image string, identifier string) error { - ctx := context.Background() - - containername := fmt.Sprintf("%s-%s", image, identifier) - - cli, err := client.NewEnvClient() - if err != nil { - log.Println("Unable to create docker client") - return err - } - - // containers, err := cli.ContainerList(ctx, types.ContainerListOptions{ - // All: true, - // }) - - if err := cli.ContainerStop(ctx, containername, nil); err != nil { - log.Printf("Unable to stop container %s - running removal anyway, just in case: %s", containername, err) - } - - removeOptions := types.ContainerRemoveOptions{ - RemoveVolumes: true, - Force: true, - } - - if err := cli.ContainerRemove(ctx, containername, removeOptions); err != nil { - log.Printf("Unable to remove container: %s", err) - } - - return nil -} - -// Starts a new webhook -func handleStopHookDocker(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - location := strings.Split(request.URL.String(), "/") - - var fileId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if len(fileId) != 32 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) - return - } - - ctx := context.Background() - hook, err := shuffle.GetHook(ctx, fileId) - if err != nil { - log.Printf("Failed getting hook %s (stop docker): %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Printf("Status: %s", hook.Status) - log.Printf("Running: %t", hook.Running) - if !hook.Running { - message := fmt.Sprintf("Error: %s isn't running", hook.Id) - log.Println(message) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "%s"}`, message))) - return - } - - hook.Status = "stopped" - hook.Running = false - hook.Actions = []shuffle.HookAction{} - err = shuffle.SetHook(ctx, *hook) - if err != nil { - log.Printf("Failed setting hook: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - image := "webhook" - - // This is here to force stop and remove the old webhook - err = stopWebhook(image, fileId) - if err != nil { - log.Printf("Container stop issue for %s-%s: %s", image, fileId, err) - } - - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true, "message": "Stopped webhook"}`)) -} - -// THis is an example -// Can also be used as base data? -var webhook = `{ - "id": "d6ef8912e8bd37776e654cbc14c2629c", - "info": { - "url": "http://localhost:5001", - "name": "TheHive", - "description": "Webhook for TheHive" - }, - "transforms": {}, - "actions": {}, - "type": "webhook", - "running": false, - "status": "stopped" -}` - -// Starts a new webhook -func handleDeleteHookDocker(resp http.ResponseWriter, request *http.Request) { - ctx := context.Background() - cors := handleCors(resp, request) - if cors { - return - } - - location := strings.Split(request.URL.String(), "/") - - var fileId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if len(fileId) != 32 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) - return - } - - err := shuffle.DeleteKey(ctx, "hooks", fileId) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "message": "Can't delete"}`)) - return - } - - image := "webhook" - - // This is here to force stop and remove the old webhook - err = stopWebhook(image, fileId) - if err != nil { - log.Printf("Container stop issue for %s-%s: %s", image, fileId, err) - resp.Write([]byte(`{"success": false, "message": "Couldn't stop webhook"}`)) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true, "message": "Deleted webhook"}`)) -} - // Checks if an image exists func imageCheckBuilder(images []string) error { //log.Printf("[FIXME] ImageNames to check: %#v", images) @@ -591,34 +430,9 @@ func imageCheckBuilder(images []string) error { return nil } -func hookTest() { - var hook shuffle.Hook - err := json.Unmarshal([]byte(webhook), &hook) - log.Println(webhook) - if err != nil { - log.Printf("Failed hook unmarshaling: %s", err) - return - } - - ctx := context.Background() - err = shuffle.SetHook(ctx, hook) - if err != nil { - log.Printf("Failed setting hook: %s", err) - } - - returnHook, err := shuffle.GetHook(ctx, hook.Id) - if err != nil { - log.Printf("Failed getting hook %s (test): %s", hook.Id, err) - } - - if len(returnHook.Id) > 0 { - log.Printf("Success! - %s", returnHook.Id) - } -} - -//https://stackoverflow.com/questions/23935141/how-to-copy-docker-images-from-one-host-to-another-without-using-a-repository +// https://stackoverflow.com/questions/23935141/how-to-copy-docker-images-from-one-host-to-another-without-using-a-repository func getDockerImage(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -639,14 +453,9 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { return } - type requestCheck struct { - Name string `datastore:"name" json:"name" yaml:"name"` - } - // This has to be done in a weird way because Datastore doesn't // support map[string]interface and similar (openapi3.Swagger) - var version requestCheck - + var version shuffle.DockerRequestCheck err = json.Unmarshal(body, &version) if err != nil { resp.WriteHeader(422) @@ -654,7 +463,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("[DEBUG] Image to load: %s", version.Name) + //log.Printf("[DEBUG] Image to load: %s", version.Name) dockercli, err := client.NewEnvClient() if err != nil { log.Printf("[WARNING] Unable to create docker client: %s", err) @@ -680,8 +489,11 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { alternativeName = strings.Join(alternativeNameSplit[1:3], "/") } + log.Printf("[INFO] Trying to download image: %s. Alt: %s", version.Name, alternativeName) + for _, image := range images { for _, tag := range image.RepoTags { + //log.Printf("[DEBUG] Tag: %s", tag) if strings.ToLower(tag) == strings.ToLower(version.Name) { img = image tagFound = tag @@ -695,6 +507,29 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { } } + pullOptions := types.ImagePullOptions{} + if len(img.ID) == 0 { + _, err := dockercli.ImagePull(context.Background(), version.Name, pullOptions) + if err == nil { + tagFound = version.Name + img.ID = version.Name + img2.ID = version.Name + + dockercli.ImageTag(ctx, version.Name, alternativeName) + } + } + + if len(img2.ID) == 0 { + _, err := dockercli.ImagePull(context.Background(), alternativeName, pullOptions) + if err == nil { + tagFound = alternativeName + img.ID = alternativeName + img2.ID = alternativeName + + dockercli.ImageTag(ctx, alternativeName, version.Name) + } + } + // REBUILDS THE APP if len(img.ID) == 0 { if len(img2.ID) == 0 { @@ -705,10 +540,10 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { imageVersion := "" newNameSplit := strings.Split(version.Name, ":") if len(newNameSplit) == 2 { - log.Printf("[DEBUG] Found name %#v", newNameSplit) + //log.Printf("[DEBUG] Found name %#v", newNameSplit) findVersionSplit := strings.Split(newNameSplit[1], "_") - log.Printf("[DEBUG] Found another split %#v", findVersionSplit) + //log.Printf("[DEBUG] Found another split %#v", findVersionSplit) if len(findVersionSplit) == 2 { imageVersion = findVersionSplit[len(findVersionSplit)-1] imageName = findVersionSplit[0] @@ -724,7 +559,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { foundApp := shuffle.WorkflowApp{} imageName = strings.ToLower(imageName) imageVersion = strings.ToLower(imageVersion) - log.Printf("[DEBUG] Looking for appname %s with version %s", imageName, imageVersion) + log.Printf("[DEBUG] Docker Looking for appname %s with version %s", imageName, imageVersion) for _, app := range workflowapps { if strings.ToLower(strings.Replace(app.Name, " ", "_", -1)) == imageName && app.AppVersion == imageVersion { @@ -755,7 +590,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { tagFound = version.Name } - buildSwaggerApp(resp, []byte(openApiApp.Body), user) + buildSwaggerApp(resp, []byte(openApiApp.Body), user, false) } } } @@ -774,7 +609,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { } //log.Printf("[INFO] Img found (%s): %#v", tagFound, img) - log.Printf("[INFO] Img found to be downloaded by client: %s", tagFound) + //log.Printf("[INFO] Img found to be downloaded by client: %s", tagFound) newClient, err := newdockerclient.NewClientFromEnv() if err != nil { @@ -797,4 +632,194 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't export image"}`))) return } + + //resp.WriteHeader(200) +} + +// Downloads and activates an app from shuffler.io if possible +func handleRemoteDownloadApp(resp http.ResponseWriter, ctx context.Context, user shuffle.User, appId string) { + url := fmt.Sprintf("https://shuffler.io/api/v1/apps/%s/config", appId) + log.Printf("Downloading API from %s", url) + req, err := http.NewRequest( + "GET", + url, + nil, + ) + + if err != nil { + log.Printf("[ERROR] Failed auto-downloading app %s: %s", appId, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + return + } + + httpClient := &http.Client{} + newresp, err := httpClient.Do(req) + if err != nil { + log.Printf("[ERROR] Failed running auto-download request for %s: %s", appId, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + return + } + + respBody, err := ioutil.ReadAll(newresp.Body) + if err != nil { + log.Printf("[ERROR] Failed setting respbody for workflow download: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + return + } + + if len(respBody) > 0 { + type tmpapp struct { + Success bool `json:"success"` + OpenAPI string `json:"openapi"` + } + + app := tmpapp{} + err := json.Unmarshal(respBody, &app) + if err != nil || app.Success == false || len(app.OpenAPI) == 0 { + log.Printf("[ERROR] Failed app unmarshal during auto-download. Success: %#v. Applength: %d: %s", app.Success, len(app.OpenAPI), err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + return + } + + key, err := base64.StdEncoding.DecodeString(app.OpenAPI) + if err != nil { + log.Printf("[ERROR] Failed auto-setting OpenAPI app: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + return + } + + cacheKey := fmt.Sprintf("workflowapps-sorted-100") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-1000") + shuffle.DeleteCache(ctx, cacheKey) + + newapp := shuffle.ParsedOpenApi{} + err = json.Unmarshal(key, &newapp) + if err != nil { + log.Printf("[ERROR] Failed openapi unmarshal during auto-download: %s", app.Success, len(app.OpenAPI), err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + return + } + + err = json.Unmarshal(key, &newapp) + if err != nil { + log.Printf("[ERROR] Failed openapi unmarshal during auto-download: %s", app.Success, len(app.OpenAPI), err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + return + } + + buildSwaggerApp(resp, []byte(newapp.Body), user, true) + return + } +} + +func activateWorkflowAppDocker(resp http.ResponseWriter, request *http.Request) { + cors := shuffle.HandleCors(resp, request) + if cors { + return + } + + user, err := shuffle.HandleApiAuthentication(resp, request) + if err != nil { + log.Printf("[WARNING] Api authentication failed in get active apps: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if user.Role == "org-reader" { + log.Printf("[WARNING] Org-reader doesn't have access to activate workflow app (shared): %s (%s)", user.Username, user.Id) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Read only user"}`)) + return + } + + ctx := context.Background() + location := strings.Split(request.URL.String(), "/") + var fileId string + if location[1] == "api" { + if len(location) <= 4 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fileId = location[4] + } + + app, err := shuffle.GetApp(ctx, fileId, user, false) + if err != nil { + appName := request.URL.Query().Get("app_name") + appVersion := request.URL.Query().Get("app_version") + + if len(appName) > 0 && len(appVersion) > 0 { + apps, err := shuffle.FindWorkflowAppByName(ctx, appName) + //log.Printf("[INFO] Found %d apps for %s", len(apps), appName) + if err != nil || len(apps) == 0 { + log.Printf("[WARNING] Error getting app %s (app config). Starting remote download.: %s", appName, err) + + handleRemoteDownloadApp(resp, ctx, user, fileId) + return + } + + selectedApp := shuffle.WorkflowApp{} + for _, app := range apps { + if !app.Sharing && !app.Public { + continue + } + + if app.Name == appName { + selectedApp = app + } + + if app.Name == appName && app.AppVersion == appVersion { + selectedApp = app + } + } + + app = &selectedApp + } else { + log.Printf("[WARNING] Error getting app with ID %s (app config): %s. Starting remote download(2)", fileId, err) + handleRemoteDownloadApp(resp, ctx, user, fileId) + return + //resp.WriteHeader(401) + //resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + //return + } + } + + // Just making sure it's being built properly + if app == nil { + log.Printf("[WARNING] App is nil. This shouldn't happen. Starting remote download(3)") + handleRemoteDownloadApp(resp, ctx, user, fileId) + return + } + + // Check the app.. hmm + openApiApp, err := shuffle.GetOpenApiDatastore(ctx, app.ID) + if err != nil { + log.Printf("[WARNING] Error getting app %s (openapi config): %s", app.ID, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Couldn't find app OpenAPI"}`)) + return + } + + log.Printf("[INFO] User %s (%s) is activating %s. Public: %t, Shared: %t", user.Username, user.Id, app.Name, app.Public, app.Sharing) + buildSwaggerApp(resp, []byte(openApiApp.Body), user, true) + + //app.Active = true + //app.Generated = true + //app, err := shuffle.SetApp(ctx, app) + + //resp.WriteHeader(200) + //resp.Write([]byte(`{"success": true}`)) } diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index e89d5686..b43db79b 100755 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -1,35 +1,105 @@ -module shuffle - -go 1.13 +module shuffle-shared replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared -//replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi +go 1.19 require ( - cloud.google.com/go/datastore v1.4.0 - cloud.google.com/go/pubsub v1.3.1 - cloud.google.com/go/storage v1.12.0 + cloud.google.com/go/datastore v1.11.0 + cloud.google.com/go/pubsub v1.31.0 + cloud.google.com/go/storage v1.30.1 github.com/basgys/goxml2json v1.1.0 - github.com/bitly/go-simplejson v0.5.1 // indirect github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 - github.com/docker/distribution v2.7.1+incompatible // indirect - github.com/docker/docker v20.10.3-0.20210216175712-646072ed6524+incompatible - github.com/frikky/kin-openapi v0.41.0 - github.com/fsouza/go-dockerclient v1.7.2 + github.com/docker/docker v24.0.2+incompatible + github.com/frikky/kin-openapi v0.42.0 + github.com/fsouza/go-dockerclient v1.9.7 github.com/ghodss/yaml v1.0.0 - github.com/go-git/go-billy/v5 v5.0.0 - github.com/go-git/go-git/v5 v5.0.0 + github.com/go-git/go-billy/v5 v5.4.1 + github.com/go-git/go-git/v5 v5.7.0 github.com/gorilla/mux v1.8.0 - github.com/h2non/filetype v1.0.12 - github.com/opensearch-project/opensearch-go v1.1.0 // indirect - github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect + github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.1.15 - golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 - google.golang.org/api v0.36.0 + github.com/shuffle/shuffle-shared v0.4.19 + golang.org/x/crypto v0.9.0 + google.golang.org/api v0.125.0 google.golang.org/appengine v1.6.7 - google.golang.org/grpc v1.34.1 + google.golang.org/grpc v1.55.0 gopkg.in/src-d/go-git.v4 v4.13.1 gopkg.in/yaml.v3 v3.0.1 ) + +require ( + cloud.google.com/go v0.110.2 // indirect + cloud.google.com/go/compute v1.19.3 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v1.0.1 // indirect + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Masterminds/semver v1.5.0 // indirect + github.com/Microsoft/go-winio v0.6.0 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect + github.com/acomagu/bufpipe v1.0.4 // indirect + github.com/adrg/strutil v0.2.3 // indirect + github.com/algolia/algoliasearch-client-go/v3 v3.18.1 // indirect + github.com/bitly/go-simplejson v0.5.0 // indirect + github.com/bradfitz/gomemcache v0.0.0-20221031212613-62deef7fc822 // indirect + github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect + github.com/cloudflare/circl v1.3.3 // indirect + github.com/containerd/containerd v1.6.18 // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/swag v0.19.5 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/go-github/v28 v28.1.1 // indirect + github.com/google/go-querystring v1.0.0 // indirect + github.com/google/s2a-go v0.1.4 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/googleapis/gax-go/v2 v2.10.0 // indirect + github.com/imdario/mergo v0.3.15 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/klauspost/compress v1.11.13 // indirect + github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e // indirect + github.com/moby/patternmatcher v0.5.0 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect + github.com/opencontainers/runc v1.1.5 // indirect + github.com/opensearch-project/opensearch-go v1.1.0 // indirect + github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect + github.com/patrickmn/go-cache v2.1.0+incompatible // indirect + github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/sergi/go-diff v1.1.0 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect + github.com/skeema/knownhosts v1.1.1 // indirect + github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect + github.com/src-d/gcfg v1.4.0 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + go.opencensus.io v0.24.0 // indirect + go4.org v0.0.0-20201209231011-d4a079459e60 // indirect + golang.org/x/mod v0.8.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/oauth2 v0.8.0 // indirect + golang.org/x/sync v0.2.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect + golang.org/x/tools v0.6.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + +) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 356f16e0..b1304b4f 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1,14 +1,19 @@ package main import ( + uuid "github.com/satori/go.uuid" "github.com/shuffle/shuffle-shared" + "archive/zip" "bufio" "bytes" "context" "crypto/md5" + "strconv" + //"crypto/tls" //"crypto/x509" + //"encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -21,6 +26,7 @@ import ( "os" "os/exec" "path/filepath" + //"regexp" "strings" "time" @@ -31,9 +37,6 @@ import ( "cloud.google.com/go/storage" "google.golang.org/appengine/mail" - //"github.com/elastic/go-elasticsearch/v7" - //"github.com/elastic/go-elasticsearch/v8/esapi" - "github.com/frikky/kin-openapi/openapi2" "github.com/frikky/kin-openapi/openapi2conv" "github.com/frikky/kin-openapi/openapi3" @@ -49,12 +52,14 @@ import ( "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" + + //cv "github.com/nirasan/go-oauth-pkce-code-verifier" + //githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http" // Random xj "github.com/basgys/goxml2json" newscheduler "github.com/carlescere/scheduler" - "github.com/satori/go.uuid" "golang.org/x/crypto/bcrypt" "gopkg.in/yaml.v3" @@ -73,18 +78,14 @@ import ( var gceProject = "shuffle" var bucketName = "shuffler.appspot.com" var baseAppPath = "/home/frikky/git/shaffuru/tmp/apps" + var baseDockerName = "frikky/shuffle" var registryName = "registry.hub.docker.com" var runningEnvironment = "onprem" var syncUrl = "https://shuffler.io" - -//var syncUrl = "http://localhost:5002" var syncSubUrl = "https://shuffler.io" -//var syncUrl = "http://localhost:5002" -//var syncSubUrl = "https://050196912a9d.ngrok.io" - var dbclient *datastore.Client type Userapi struct { @@ -682,13 +683,25 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) err = shuffle.SetUser(ctx, newUser, true) if err != nil { - log.Printf("Error adding User %s: %s", username, err) + log.Printf("[ERROR] Problem adding User %s: %s", username, err) return err } neworg, err := shuffle.GetOrg(ctx, org.Id) if err == nil { //neworg.Users = append(neworg.Users, *newUser) + for tutorialIndex, tutorial := range neworg.Tutorials { + if tutorial.Name == "Invite teammates" { + neworg.Tutorials[tutorialIndex].Description = fmt.Sprintf("%d users are in your org. Org name and Image change next.", len(neworg.Users)) + if len(neworg.Users) > 1 { + neworg.Tutorials[tutorialIndex].Done = true + neworg.Tutorials[tutorialIndex].Link = "/admin?tab=users" + } + + break + } + } + err = shuffle.SetOrg(ctx, *neworg, neworg.Id) if err != nil { log.Printf("Failed updating org with user %s", newUser.Username) @@ -706,7 +719,7 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) } func handleRegister(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -720,7 +733,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { if err != nil { if (countErr == nil && count > 0) || countErr != nil { resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Can't register without being admin"}`)) + resp.Write([]byte(`{"success": false, "reason": "Users already exist. Please go to /login to log into your admin user."}`)) return } } @@ -775,7 +788,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { CloudSync: false, } - err = shuffle.SetOrg(ctx, newOrg, orgId) + err = shuffle.SetOrg(ctx, newOrg, newOrg.Id) if err != nil { log.Printf("[WARNING] Failed setting init organization: %s", err) } else { @@ -834,8 +847,10 @@ func handleCookie(request *http.Request) bool { return true } +// Returns whether the user is logged in or not etc. +// Also has more data about the user and org func handleInfo(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -925,54 +940,111 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { }) // Updating user info if there's something wrong - if (len(userInfo.ActiveOrg.Name) == 0 || len(userInfo.ActiveOrg.Id) == 0) && len(userInfo.Orgs) > 0 { - _, err := shuffle.GetOrg(ctx, userInfo.Orgs[0]) - if err != nil { + if len(userInfo.ActiveOrg.Name) == 0 || len(userInfo.ActiveOrg.Id) == 0 { + if len(userInfo.Orgs) == 0 || (len(userInfo.Orgs) > 0 && userInfo.Orgs[0] == "") { orgs, err := shuffle.GetAllOrgs(ctx) - if err == nil { - newStringOrgs := []string{} - newOrgs := []shuffle.Org{} + log.Printf("[INFO] Fixing organization for user %s (%s). Found orgs: %d", userInfo.Username, userInfo.Id, len(orgs)) + if err == nil && len(orgs) > 0 { for _, org := range orgs { - if strings.ToLower(org.Name) == strings.ToLower(userInfo.Orgs[0]) { - newOrgs = append(newOrgs, org) - newStringOrgs = append(newStringOrgs, org.Id) + if len(org.Id) == 0 { + continue } - } - if len(newOrgs) > 0 { + // Prolly some way here to jump into another org + // when you have access to the DB userInfo.ActiveOrg = shuffle.OrgMini{ - Id: newOrgs[0].Id, - Name: newOrgs[0].Name, - } - - userInfo.Orgs = newStringOrgs - - err = shuffle.SetUser(ctx, &userInfo, true) - if err != nil { - log.Printf("Error patching User for activeOrg: %s", err) - } else { - log.Printf("Updated the users' org") + Name: org.Name, + Id: org.Id, + Role: "admin", } + userInfo.Orgs = []string{org.Id} + break } - } else { - log.Printf("Failed getting orgs for user. Major issue.: %s", err) } - } else { - // 1. Check if the org exists by ID - // 2. if it does, overwrite user - userInfo.ActiveOrg = shuffle.OrgMini{ - Id: userInfo.Orgs[0], + // Make a new one in case we couldn't find one + if len(userInfo.ActiveOrg.Id) == 0 { + orgSetupName := "default" + orgId := uuid.NewV4().String() + newOrg := shuffle.Org{ + Name: orgSetupName, + Id: orgId, + Org: orgSetupName, + Users: []shuffle.User{}, + Roles: []string{"admin", "user"}, + CloudSync: false, + } + + err = shuffle.SetOrg(ctx, newOrg, newOrg.Id) + if err == nil { + userInfo.ActiveOrg = shuffle.OrgMini{ + Name: newOrg.Name, + Id: newOrg.Id, + Role: "admin", + } + userInfo.Orgs = []string{newOrg.Id} + } else { + log.Printf("[WARNING] Failed to set new org: %s", err) + } } + + // Set user err = shuffle.SetUser(ctx, &userInfo, true) if err != nil { - log.Printf("[INFO] Error patching User for activeOrg: %s", err) + log.Printf("[WARNING] Failed fixing org info for user %s (%s)", userInfo.Username, userInfo.Id) + } else { + log.Printf("[INFO] Set organization for %s (%s) to be %s (%s)", userInfo.Username, userInfo.Id, userInfo.ActiveOrg.Name, userInfo.ActiveOrg.Id) + } + } else if len(userInfo.Orgs) > 0 && userInfo.Orgs[0] != "" { + _, err := shuffle.GetOrg(ctx, userInfo.Orgs[0]) + if err != nil { + orgs, err := shuffle.GetAllOrgs(ctx) + if err == nil { + newStringOrgs := []string{} + newOrgs := []shuffle.Org{} + for _, org := range orgs { + if strings.ToLower(org.Name) == strings.ToLower(userInfo.Orgs[0]) { + newOrgs = append(newOrgs, org) + newStringOrgs = append(newStringOrgs, org.Id) + } + } + + if len(newOrgs) > 0 { + userInfo.ActiveOrg = shuffle.OrgMini{ + Id: newOrgs[0].Id, + Name: newOrgs[0].Name, + } + + userInfo.Orgs = newStringOrgs + + err = shuffle.SetUser(ctx, &userInfo, true) + if err != nil { + log.Printf("Error patching User for activeOrg: %s", err) + } else { + log.Printf("Updated the users' org") + } + } + } else { + log.Printf("Failed getting orgs for user. Major issue.: %s", err) + } + + } else { + // 1. Check if the org exists by ID + // 2. if it does, overwrite user + userInfo.ActiveOrg = shuffle.OrgMini{ + Id: userInfo.Orgs[0], + } + err = shuffle.SetUser(ctx, &userInfo, true) + if err != nil { + log.Printf("[INFO] Error patching User for activeOrg: %s", err) + } } } } org, err := shuffle.GetOrg(ctx, userInfo.ActiveOrg.Id) - if err == nil { + //if err == nil { + if len(org.Id) > 0 { userInfo.ActiveOrg = shuffle.OrgMini{ Id: org.Id, Name: org.Name, @@ -981,6 +1053,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { Image: org.Image, } } + //} userInfo.ActiveOrg.Users = []shuffle.UserMini{} userOrgs := []shuffle.OrgMini{} @@ -991,7 +1064,8 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { } org, err := shuffle.GetOrg(ctx, item) - if err == nil { + _ = err + if len(org.Id) > 0 { userOrgs = append(userOrgs, shuffle.OrgMini{ Id: org.Id, Name: org.Name, @@ -1003,6 +1077,61 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { } } + // FIXME: This is bad, but we've had a lot of bugs with edit users, and this is the quick fix. + if userInfo.Role == "" && userInfo.ActiveOrg.Role == "" && parsedAdmin == "false" { + userInfo.Role = "admin" + userInfo.ActiveOrg.Role = "admin" + parsedAdmin = "true" + + err = shuffle.SetUser(ctx, &userInfo, true) + if err != nil { + log.Printf("[WARNING] Automatically asigning user as admin to their org because they don't have a role at all failed: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false}`)) + return + } else { + log.Printf("[DEBUG] Made user %s org-admin as they didn't have any role specified", err) + + } + } + + chatDisabled := false + if os.Getenv("SHUFFLE_CHAT_DISABLED") == "true" { + chatDisabled = true + } + + userOrgs = shuffle.SortOrgList(userOrgs) + orgPriorities := org.Priorities + if len(org.Priorities) < 10 { + log.Printf("[WARNING] Should find and add priorities as length is less than 10 for org %s", userInfo.ActiveOrg.Id) + newPriorities, err := shuffle.GetPriorities(ctx, userInfo, org) + if err != nil { + log.Printf("[WARNING] Failed getting new priorities for org %s: %s", org.Id, err) + //orgPriorities = []shuffle.Priority{} + } else { + orgPriorities = newPriorities + + // A way to manage them over time + } + } + + tutorialsFinished := []shuffle.Tutorial{} + for _, tutorial := range userInfo.PersonalInfo.Tutorials { + tutorialsFinished = append(tutorialsFinished, shuffle.Tutorial{ + Name: tutorial, + }) + } + + if len(org.SecurityFramework.SIEM.Name) > 0 || len(org.SecurityFramework.Network.Name) > 0 || len(org.SecurityFramework.EDR.Name) > 0 || len(org.SecurityFramework.Cases.Name) > 0 || len(org.SecurityFramework.IAM.Name) > 0 || len(org.SecurityFramework.Assets.Name) > 0 || len(org.SecurityFramework.Intel.Name) > 0 || len(org.SecurityFramework.Communication.Name) > 0 { + tutorialsFinished = append(tutorialsFinished, shuffle.Tutorial{ + Name: "find_integrations", + }) + } + + for _, tutorial := range org.Tutorials { + tutorialsFinished = append(tutorialsFinished, tutorial) + } + returnValue := shuffle.HandleInfo{ Success: true, Username: userInfo.Username, @@ -1017,6 +1146,11 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { Expiration: expiration.Unix(), }, }, + EthInfo: userInfo.EthInfo, + ChatDisabled: chatDisabled, + Tutorials: tutorialsFinished, + + Priorities: orgPriorities, } returnData, err := json.Marshal(returnValue) @@ -1097,7 +1231,7 @@ func increaseStatisticsField(ctx context.Context, fieldname, id string, amount i // FIXME - forward this to emails or whatever CRM system in use func handleContact(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1143,7 +1277,7 @@ func handleContact(resp http.ResponseWriter, request *http.Request) { } func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1165,125 +1299,42 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { return } - //ssoUrl = org.SSOConfig.SOSOEntrypoint - redirectUri := shuffle.SSOUrl + baseSSOUrl := "" + handled := []string{} + for _, user := range users { + if shuffle.ArrayContains(handled, user.ActiveOrg.Id) { + continue + } + + handled = append(handled, user.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) + if err != nil { + log.Printf("[WARNING] Error getting org in admin check: %s", err) + continue + } + + // No childorg setup, only parent org + if len(org.ManagerOrgs) > 0 || len(org.CreatorOrg) > 0 { + continue + } + + // Should run calculations + if len(org.SSOConfig.OpenIdAuthorization) > 0 { + baseSSOUrl = shuffle.GetOpenIdUrl(request, *org) + + break + } + + if len(org.SSOConfig.SSOEntrypoint) > 0 { + log.Printf("[DEBUG] Found SAML SSO url: %s", org.SSOConfig.SSOEntrypoint) + baseSSOUrl = org.SSOConfig.SSOEntrypoint + break + } + } + + //log.Printf("[DEBUG] OpenID URL: %s", baseSSOUrl) resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "redirect", "sso_url": "%s"}`, redirectUri))) -} - -func handleLogin(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - // Gets a struct of Username, password - data, err := shuffle.ParseLoginParameters(resp, request) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - - log.Printf("[INFO] Handling login of %s", data.Username) - - err = checkUsername(data.Username) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - - ctx := context.Background() - log.Printf("[INFO] Login Username: %s", data.Username) - users, err := shuffle.FindUser(ctx, strings.ToLower(strings.TrimSpace(data.Username))) - if err != nil && len(users) == 0 { - log.Printf("[WARNING] Failed getting user %s: %s", data.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) - return - } - - if len(users) != 1 { - log.Printf(`Found multiple or no users with the same username: %s: %d`, data.Username, len(users)) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error: %d users with username %s"}`, len(users), data.Username))) - return - } - - Userdata := users[0] - - err = bcrypt.CompareHashAndPassword([]byte(Userdata.Password), []byte(data.Password)) - if err != nil { - log.Printf("Password for %s is incorrect: %s", data.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) - return - } - - if !Userdata.Active { - log.Printf("%s is not active, but tried to login. Error: %v", data.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "This user is deactivated"}`)) - return - } - - // FIXME - have timeout here - loginData := `{"success": true}` - if len(Userdata.Session) != 0 { - log.Println("[INFO] User session already exists - resetting it") - expiration := time.Now().Add(3600 * time.Second) - - http.SetCookie(resp, &http.Cookie{ - Name: "session_token", - Value: Userdata.Session, - Expires: expiration, - }) - - loginData = fmt.Sprintf(`{"success": true, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}]}`, Userdata.Session, expiration.Unix()) - //log.Printf("SESSION LENGTH MORE THAN 0 IN LOGIN: %s", Userdata.Session) - - err = shuffle.SetSession(ctx, Userdata, Userdata.Session) - if err != nil { - log.Printf("Error adding session to database: %s", err) - } - - resp.WriteHeader(200) - resp.Write([]byte(loginData)) - return - } else { - log.Printf("[INFO] User session is empty - create one!") - - sessionToken := uuid.NewV4().String() - expiration := time.Now().Add(3600 * time.Second) - http.SetCookie(resp, &http.Cookie{ - Name: "session_token", - Value: sessionToken, - Expires: expiration, - }) - - // ADD TO DATABASE - err = shuffle.SetSession(ctx, Userdata, sessionToken) - if err != nil { - log.Printf("Error adding session to database: %s", err) - } - - Userdata.Session = sessionToken - err = shuffle.SetUser(ctx, &Userdata, true) - if err != nil { - log.Printf("Failed updating user when setting session: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false}`)) - return - } - - loginData = fmt.Sprintf(`{"success": true, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}]}`, sessionToken, expiration.Unix()) - } - - log.Printf("[INFO] %s SUCCESSFULLY LOGGED IN with session %s", data.Username, Userdata.Session) - - resp.WriteHeader(200) - resp.Write([]byte(loginData)) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "redirect", "sso_url": "%s"}`, baseSSOUrl))) } func fixOrgUser(ctx context.Context, org *shuffle.Org) *shuffle.Org { @@ -1387,7 +1438,7 @@ func fixUserOrg(ctx context.Context, user *shuffle.User) *shuffle.User { org.Users = append(org.Users, *user) } - err = shuffle.SetOrg(ctx, *org, orgId) + err = shuffle.SetOrg(ctx, *org, org.Id) if err != nil { log.Printf("Failed setting org %s", orgId) } @@ -1397,8 +1448,12 @@ func fixUserOrg(ctx context.Context, user *shuffle.User) *shuffle.User { } // Used for testing only. Shouldn't impact production. -func handleCors(resp http.ResponseWriter, request *http.Request) bool { - allowedOrigins := "http://localhost:3000" +/* +func shuffle.HandleCors(resp http.ResponseWriter, request *http.Request) bool { + // Used for Codespace dev + allowedOrigins := "https://frikky-shuffle-5gvr4xx62w64-3000.githubpreview.dev" + //origin := request.Header["Origin"] + //log.Printf("Origin: %s", origin) //allowedOrigins := "http://localhost:3002" resp.Header().Set("Vary", "Origin") @@ -1415,6 +1470,7 @@ func handleCors(resp http.ResponseWriter, request *http.Request) bool { return false } +*/ func parseWorkflowParameters(resp http.ResponseWriter, request *http.Request) (map[string]interface{}, error) { body, err := ioutil.ReadAll(request.Body) @@ -1559,7 +1615,7 @@ func SearchNested(obj interface{}, key string) (interface{}, bool) { } func handleSetHook(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1747,7 +1803,7 @@ func verifyHook(hook shuffle.Hook) (bool, string) { } func setSpecificSchedule(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1807,7 +1863,7 @@ func setSpecificSchedule(resp http.ResponseWriter, request *http.Request) { } func getSpecificWebhook(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1859,7 +1915,7 @@ func getSpecificWebhook(resp http.ResponseWriter, request *http.Request) { // Starts a new webhook func handleDeleteSchedule(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1914,7 +1970,7 @@ func handleDeleteSchedule(resp http.ResponseWriter, request *http.Request) { // Starts a new webhook func handleNewSchedule(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1955,9 +2011,18 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { // 1. Get callback data // 2. Load the configuration // 3. Execute the workflow - cors := shuffle.HandleCors(resp, request) - if cors { - return + //cors := shuffle.HandleCors(resp, request) + //if cors { + // return + //} + + if request.Method != "POST" { + request.Method = "POST" + } + + if request.Body == nil { + stringReader := strings.NewReader("") + request.Body = ioutil.NopCloser(stringReader) } path := strings.Split(request.URL.String(), "/") @@ -1994,6 +2059,15 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { } } + // Find user agent header + userAgent := request.Header.Get("User-Agent") + if strings.Contains(strings.ToLower(userAgent), "microsoftpreview") || strings.Contains(strings.ToLower(userAgent), "googlebot") { + log.Printf("[AUDIT] Blocking googlebot and microsoftbot for webhooks. UA: '%s'", userAgent) + resp.WriteHeader(400) + resp.Write([]byte(`{"success": false, "reason": "Google/Microsoft preview bots not allowed. Please change the useragent."}`)) + return + } + // ID: webhook_ if len(hookId) != 44 { log.Printf("[INFO] Couldn't handle hookId. Too short in webhook: %d", len(hookId)) @@ -2079,7 +2153,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { if err == nil { for _, branch := range workflow.Branches { if branch.SourceID == hook.Id { - log.Printf("[INFO] Found ID %s for hook", hook.Id) + log.Printf("[DEBUG] Found ID %s for hook", hook.Id) if branch.DestinationID != hook.Start { newBody.Start = branch.DestinationID break @@ -2091,57 +2165,76 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { b, err := json.Marshal(newBody) if err != nil { - log.Printf("Failed newBody marshaling: %s", err) - resp.WriteHeader(401) + log.Printf("[ERROR] Failed newBody marshaling for webhook: %s", err) + resp.WriteHeader(500) resp.Write([]byte(`{"success": false}`)) return } + // Should wrap the response input Body as well? for _, item := range hook.Workflows { - //log.Printf("Running webhook for workflow %s with startnode %s", item, hook.Start) + log.Printf("[INFO] Running webhook for workflow %s with startnode %s", item, hook.Start) + + // This ID is empty to force it to get the webhook within the execution workflow := shuffle.Workflow{ ID: "", } - //parsedBody := string(body) - //parsedBody = strings.Replace(parsedBody, "\"", "\\\"", -1) - //if len(parsedBody) > 0 { - // if string(parsedBody[0]) == `"` && string(parsedBody[len(parsedBody)-1]) == "\"" { - // parsedBody = parsedBody[1 : len(parsedBody)-1] - // } - //} - - //bodyWrapper := fmt.Sprintf(`{"start": "%s", "execution_source": "webhook", "execution_argument": "%s"}`, hook.Start, string(parsedBody)) - //if len(hook.Start) == 0 { - // log.Printf("No start node for hook %s - running with workflow default.", hook.Id) - // bodyWrapper = string(parsedBody) - //} + if len(hook.Start) == 0 { + log.Printf("[WARNING] No start node for hook %s - running with workflow default.", hook.Id) + //bodyWrapper = string(parsedBody) + } newRequest := &http.Request{ URL: &url.URL{}, Method: "POST", Body: ioutil.NopCloser(bytes.NewReader(b)), } - //start, startok := request.URL.Query()["start"] // OrgId: activeOrgs[0].Id, - workflowExecution, executionResp, err := handleExecution(item, workflow, newRequest) - if err == nil { - /* - err = increaseStatisticsField(ctx, "total_webhooks_ran", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) - if err != nil { - log.Printf("Failed to increase total apps loaded stats: %s", err) - } - */ + workflowExecution, executionResp, err := handleExecution(item, workflow, newRequest, hook.OrgId) + if err == nil { + if hook.Version == "v2" { + timeout := 15 + //if hook.VersionTimeout != 0 { + // timeout = hook.VersionTimeout + //} + + log.Printf("[DEBUG] Waiting for Webhook response from %s for max %d seconds! Checking every 1 second. Hook ID: %s", workflowExecution.ExecutionId, timeout, hook.Id) + // Try every second for 15 seconds + for i := 0; i < timeout; i++ { + time.Sleep(1 * time.Second) + + newExec, err := shuffle.GetWorkflowExecution(ctx, workflowExecution.ExecutionId) + if err != nil { + log.Printf("[ERROR] Failed to get workflow execution: %s", err) + break + } + + if newExec.Status != "EXECUTING" { + log.Printf("[INFO] Got response from webhook v2 of length '%d' <- %s", len(newExec.Result), newExec.ExecutionId) + resp.WriteHeader(200) + resp.Write([]byte(newExec.Result)) + return + } + } + } + + // Fallback resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization))) + if len(hook.CustomResponse) > 0 { + resp.Write([]byte(hook.CustomResponse)) + } else { + resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s"}`, workflowExecution.ExecutionId))) + } return } resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, executionResp))) } + } func executeCloudAction(action shuffle.CloudSyncJob, apikey string) error { @@ -2151,10 +2244,11 @@ func executeCloudAction(action shuffle.CloudSyncJob, apikey string) error { return err } - transport := http.DefaultTransport.(*http.Transport).Clone() - client := &http.Client{ - Transport: transport, - } + //transport := http.DefaultTransport.(*http.Transport).Clone() + //client := &http.Client{ + // Transport: transport, + //} + client := &http.Client{} syncUrl := fmt.Sprintf("%s/api/v1/cloud/sync/handle_action", syncUrl) req, err := http.NewRequest( @@ -2190,6 +2284,8 @@ func executeCloudAction(action shuffle.CloudSyncJob, apikey string) error { return errors.New(fmt.Sprintf("Cloud error from Shuffler: %s", responseData.Reason)) } + log.Printf("[INFO] Cloud action executed successfully for '%s'", action.Action) + return nil } @@ -2199,7 +2295,7 @@ func getSpecificSchedule(resp http.ResponseWriter, request *http.Request) { return } - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -2265,7 +2361,7 @@ func loadYaml(fileLocation string) (ApiYaml, error) { // This should ALWAYS come from an OUTPUT func executeSchedule(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -2758,7 +2854,7 @@ type Result struct { // r.HandleFunc("/api/v1/docs/{key}", getDocs).Methods("GET", "OPTIONS") func getOpenapi(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -2818,75 +2914,6 @@ func getOpenapi(resp http.ResponseWriter, request *http.Request) { resp.Write(data) } -func echoOpenapiData(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - // Just here to verify that the user is logged in - _, err := shuffle.HandleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in validate swagger: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed authentication"}`)) - return - } - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Bodyreader err: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`)) - return - } - - newbody := string(body) - newbody = strings.TrimSpace(newbody) - if strings.HasPrefix(newbody, "\"") { - newbody = newbody[1:len(newbody)] - } - - if strings.HasSuffix(newbody, "\"") { - newbody = newbody[0 : len(newbody)-1] - } - - req, err := http.NewRequest("GET", newbody, nil) - if err != nil { - log.Printf("[ERROR] Requestbuilder err: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed building request"}`)) - return - } - - httpClient := &http.Client{} - newresp, err := httpClient.Do(req) - if err != nil { - log.Printf("[ERROR] Grabbing error: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed making remote request to get the data"}`))) - return - } - defer newresp.Body.Close() - - urlbody, err := ioutil.ReadAll(newresp.Body) - if err != nil { - log.Printf("[ERROR] URLbody error: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't get data from selected uri"}`))) - return - } - - if newresp.StatusCode >= 400 { - resp.WriteHeader(201) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, urlbody))) - return - } - - resp.WriteHeader(200) - resp.Write(urlbody) -} - func handleSwaggerValidation(body []byte) (shuffle.ParsedOpenApi, error) { type versionCheck struct { Swagger string `datastore:"swagger" json:"swagger" yaml:"swagger"` @@ -2924,7 +2951,7 @@ func handleSwaggerValidation(body []byte) (shuffle.ParsedOpenApi, error) { } } else { isJson = true - log.Printf("Successfully parsed JSON!") + //log.Printf("[DEBUG] Successfully parsed JSON!") } if len(version.SwaggerVersion) > 0 && len(version.Swagger) == 0 { @@ -3004,18 +3031,19 @@ func handleSwaggerValidation(body []byte) (shuffle.ParsedOpenApi, error) { return parsed, err } -func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) { +func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User, skipEdit bool) { type Test struct { - Editing bool `datastore:"editing"` - Id string `datastore:"id"` - Image string `datastore:"image"` + Editing bool `json:"editing" datastore:"editing"` + Id string `json:"id" datastore:"id"` + Image string `json:"image" datastore:"image"` + Body string `json:"body" datastore:"body"` } var test Test err := json.Unmarshal(body, &test) if err != nil { - log.Printf("[WARNING] Failed unmarshalling test: %s", err) - resp.WriteHeader(401) + log.Printf("[ERROR] Failed unmarshalling in swagger build: %s", err) + resp.WriteHeader(400) resp.Write([]byte(`{"success": false}`)) return } @@ -3025,13 +3053,13 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) { hasher.Write(body) newmd5 := hex.EncodeToString(hasher.Sum(nil)) - if test.Editing && len(user.Id) > 0 { + if test.Editing && len(user.Id) > 0 && skipEdit != true { // Quick verification test ctx := context.Background() app, err := shuffle.GetApp(ctx, test.Id, user, false) if err != nil { - log.Printf("[WARNING] Error getting app when editing: %s", app.Name) - resp.WriteHeader(401) + log.Printf("[ERROR] Error getting app when editing: %s", app.Name) + resp.WriteHeader(400) resp.Write([]byte(`{"success": false}`)) return } @@ -3039,7 +3067,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) { // FIXME: Check whether it's in use. if user.Id != app.Owner && user.Role != "admin" { log.Printf("[WARNING] Wrong user (%s) for app %s when verifying swagger", user.Username, app.Name) - resp.WriteHeader(401) + resp.WriteHeader(400) resp.Write([]byte(`{"success": false}`)) return } @@ -3063,12 +3091,13 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) { } if swagger.Info == nil { - log.Printf("[ERORR] Info is nil?: %#v", swagger) + log.Printf("[ERORR] Info is nil in swagger?") resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Info not parsed"}`)) return } + swagger.Info.Title = shuffle.FixFunctionName(swagger.Info.Title, swagger.Info.Title, false) if strings.Contains(swagger.Info.Title, " ") { swagger.Info.Title = strings.Replace(swagger.Info.Title, " ", "_", -1) } @@ -3172,7 +3201,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) { //log.Println(stitched) // 3. Zip and stream it directly in the directory - _, err = shuffle.StreamZipdata(ctx, identifier, stitched, "requests\nurllib3", "") + _, err = shuffle.StreamZipdata(ctx, identifier, stitched, shuffle.GetAppRequirements(), "") if err != nil { log.Printf("[ERROR] Zipfile error: %s", err) resp.WriteHeader(500) @@ -3193,14 +3222,6 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) { fmt.Sprintf("%s:%s", baseDockerName, versionName), } - err = buildImage(dockerTags, dockerLocation) - if err != nil { - log.Printf("[ERROR] Docker build error: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error in Docker build: %s"}`, err))) - return - } - found := false foundNumber := 0 log.Printf("[INFO] Checking for api with ID %s", newmd5) @@ -3284,6 +3305,16 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) { shuffle.DeleteCache(ctx, cacheKey) shuffle.DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id)) + // Doing this last to ensure we can copy the docker image over + // even though builds fail + err = buildImage(dockerTags, dockerLocation) + if err != nil { + log.Printf("[ERROR] Docker build error: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error in Docker build: %s"}`, err))) + return + } + log.Printf("[DEBUG] Successfully built app %s (%s)", api.Name, api.ID) if len(user.Id) > 0 { resp.WriteHeader(200) @@ -3293,7 +3324,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) { // Creates an app from the app builder func verifySwagger(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -3307,6 +3338,13 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { return } + if user.Role == "org-reader" { + log.Printf("[WARNING] Org-reader doesn't have access to check swagger doc: %s (%s)", user.Username, user.Id) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Read only user"}`)) + return + } + body, err := ioutil.ReadAll(request.Body) if err != nil { resp.WriteHeader(401) @@ -3314,11 +3352,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { return } - buildSwaggerApp(resp, body, user) -} - -func healthCheckHandler(resp http.ResponseWriter, request *http.Request) { - fmt.Fprint(resp, "OK") + buildSwaggerApp(resp, body, user, false) } // Creates osfs from folderpath with a basepath as directory base @@ -3352,11 +3386,6 @@ func createFs(basepath, pathname string) (billy.Filesystem, error) { return err } - //if strings.Contains(path, "yaml") { - // log.Printf("PATH: %s -> %s", path, fullpath) - // //log.Printf("DATA: %s", string(srcData)) - //} - dst, err := fs.Create(fullpath) if err != nil { log.Printf("Dst error: %s", err) @@ -3394,7 +3423,6 @@ func handleAppHotload(ctx context.Context, location string, forceUpdate bool) er return err } - //log.Printf("Reading app folder: %#v", dir) _, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate) if err != nil { log.Printf("[WARNING] Githubfolders error: %s", err) @@ -3467,7 +3495,7 @@ func handleCloudExecutionOnprem(workflowId, startNode, executionSource, executio Body: ioutil.NopCloser(bytes.NewReader(b)), } - _, _, err = handleExecution(workflowId, shuffle.Workflow{}, newRequest) + _, _, err = handleExecution(workflowId, shuffle.Workflow{}, newRequest, workflow.OrgId) return err } @@ -3538,7 +3566,7 @@ func handleCloudJob(job shuffle.CloudSyncJob) error { } else if job.Type == "schedule" { if job.Action == "execute" { - log.Printf("Should handle schedule for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) + log.Printf("[INFO] Should handle schedule for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) err := handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "schedule", job.ThirdItem) if err != nil { log.Printf("[INFO] Failed executing workflow from cloud schedule: %s", err) @@ -3548,7 +3576,7 @@ func handleCloudJob(job shuffle.CloudSyncJob) error { } } else if job.Type == "email_trigger" { if job.Action == "execute" { - log.Printf("Should handle email for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) + log.Printf("[INFO] Should handle email for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) err := handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "email_trigger", job.ThirdItem) if err != nil { log.Printf("Failed executing workflow from email trigger: %s", err) @@ -3559,7 +3587,7 @@ func handleCloudJob(job shuffle.CloudSyncJob) error { } else if job.Type == "user_input" { if job.Action == "continue" { - log.Printf("Should handle user_input CONTINUE for workflow %s with start node %s and execution ID %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) + log.Printf("[INFO] Should handle user_input CONTINUE for workflow %s with start node %s and execution ID %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) // FIXME: Handle authorization ctx := context.Background() workflowExecution, err := shuffle.GetWorkflowExecution(ctx, job.ThirdItem) @@ -3588,7 +3616,7 @@ func handleCloudJob(job shuffle.CloudSyncJob) error { return err } - _, _, err = handleExecution(job.PrimaryItemId, shuffle.Workflow{}, newRequest) + _, _, err = handleExecution(job.PrimaryItemId, shuffle.Workflow{}, newRequest, job.OrgId) if err != nil { log.Printf("Failed continuing workflow from cloud user_input: %s", err) return err @@ -3657,47 +3685,50 @@ func remoteOrgJobController(org shuffle.Org, body []byte) error { if !responseData.Success { log.Printf("[WARNING] Should stop org job controller because no success?") - if strings.Contains(responseData.Reason, "Bad apikey") || strings.Contains(responseData.Reason, "Error getting the organization") || strings.Contains(responseData.Reason, "Organization isn't syncing") { + if strings.Contains(strings.ToLower(responseData.Reason), "bad apikey") || strings.Contains(responseData.Reason, "Error getting the organization") || strings.Contains(responseData.Reason, "Organization isn't syncing") { log.Printf("[WARNING] Remote error; Bad apikey or org error. Stopping sync for org: %s", responseData.Reason) if value, exists := scheduledOrgs[org.Id]; exists { // Looks like this does the trick? Hurr - log.Printf("[WARNING] STOPPING ORG SCHEDULE for: %s", org.Id) - + log.Printf("[INFO] STOPPING ORG SCHEDULE for: %s", org.Id) value.Lock() - org, err := shuffle.GetOrg(ctx, org.Id) - if err != nil { - log.Printf("[WARNING] Failed finding org %s: %s", org.Id, err) - return err - } - - org.SyncConfig.Interval = 0 - org.SyncConfig.Apikey = "" - org.CloudSync = false - - // Just in case - org, err = handleStopCloudSync(syncUrl, *org) - - startDate := time.Now().Unix() - org.SyncFeatures.Webhook = shuffle.SyncData{Active: false, Type: "trigger", Name: "Webhook", StartDate: startDate} - org.SyncFeatures.UserInput = shuffle.SyncData{Active: false, Type: "trigger", Name: "User Input", StartDate: startDate} - org.SyncFeatures.EmailTrigger = shuffle.SyncData{Active: false, Type: "action", Name: "Email Trigger", StartDate: startDate} - org.SyncFeatures.Schedules = shuffle.SyncData{Active: false, Type: "trigger", Name: "Schedule", StartDate: startDate, Limit: 0} - org.SyncFeatures.SendMail = shuffle.SyncData{Active: false, Type: "action", Name: "Send Email", StartDate: startDate, Limit: 0} - org.SyncFeatures.SendSms = shuffle.SyncData{Active: false, Type: "action", Name: "Send SMS", StartDate: startDate, Limit: 0} - org.CloudSyncActive = false - - err = shuffle.SetOrg(ctx, *org, org.Id) - if err != nil { - log.Printf("[WARNING] Failed setting organization when stopping sync: %s", err) - } else { - log.Printf("[INFO] Successfully STOPPED org cloud sync for %s", org.Id) - } - - return errors.New("Stopped schedule for org locally because of bad apikey.") } else { - return errors.New(fmt.Sprintf("Failed finding the schedule for org %s", org.Id)) + log.Printf("[INFO] Failed finding the schedule for org %s (%s)", org.Name, org.Id) } + + org, err := shuffle.GetOrg(ctx, org.Id) + if err != nil { + log.Printf("[WARNING] Failed finding org %s: %s", org.Id, err) + return err + } + + // Just in case + org, err = handleStopCloudSync(syncUrl, *org) + if err != nil { + log.Printf("[ERROR] Failed stopping cloud sync remotely: %s", err) + } + + org.SyncConfig.Interval = 0 + org.CloudSync = false + org.SyncConfig.Apikey = "" + + startDate := time.Now().Unix() + org.SyncFeatures.Webhook = shuffle.SyncData{Active: false, Type: "trigger", Name: "Webhook", StartDate: startDate} + org.SyncFeatures.UserInput = shuffle.SyncData{Active: false, Type: "trigger", Name: "User Input", StartDate: startDate} + org.SyncFeatures.EmailTrigger = shuffle.SyncData{Active: false, Type: "action", Name: "Email Trigger", StartDate: startDate} + org.SyncFeatures.Schedules = shuffle.SyncData{Active: false, Type: "trigger", Name: "Schedule", StartDate: startDate, Limit: 0} + org.SyncFeatures.SendMail = shuffle.SyncData{Active: false, Type: "action", Name: "Send Email", StartDate: startDate, Limit: 0} + org.SyncFeatures.SendSms = shuffle.SyncData{Active: false, Type: "action", Name: "Send SMS", StartDate: startDate, Limit: 0} + org.CloudSyncActive = false + + err = shuffle.SetOrg(ctx, *org, org.Id) + if err != nil { + log.Printf("[WARNING] Failed setting organization when stopping sync: %s", err) + } else { + log.Printf("[INFO] Successfully STOPPED org cloud sync for %s (%s)", org.Name, org.Id) + } + + return nil } return errors.New("[ERROR] Remote job handler issues.") @@ -3766,15 +3797,15 @@ func runInitCloudSetup() { } func runInitEs(ctx context.Context) { - log.Printf("[DEBUG] Starting INIT setup (ES)") + log.Printf("[DEBUG] Starting INIT setup for Elasticsearch/Opensearch") httpProxy := os.Getenv("HTTP_PROXY") if len(httpProxy) > 0 { - log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy) + log.Printf("[INFO] Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy) } httpsProxy := os.Getenv("HTTPS_PROXY") if len(httpsProxy) > 0 { - log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy) + log.Printf("[INFO] Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy) } defaultEnv := os.Getenv("ORG_ID") @@ -3783,8 +3814,9 @@ func runInitEs(ctx context.Context) { log.Printf("[DEBUG] Setting default environment for org to %s", defaultEnv) } - log.Printf("[DEBUG] Getting organizations") + log.Printf("[DEBUG] Getting organizations for Elasticsearch/Opensearch") activeOrgs, err := shuffle.GetAllOrgs(ctx) + setUsers := false //log.Printf("ORGS: %d", len(activeOrgs)) if err != nil { @@ -3837,7 +3869,7 @@ func runInitEs(ctx context.Context) { //} } else { - log.Printf("[DEBUG] There are %d org(s).", len(activeOrgs)) + log.Printf("[DEBUG] Found %d org(s) in total.", len(activeOrgs)) if len(activeOrgs) == 1 { if len(activeOrgs[0].Users) == 0 { @@ -3872,6 +3904,12 @@ func runInitEs(ctx context.Context) { } } + if strings.Contains(os.Getenv("SHUFFLE_OPENSEARCH_URL"), "https") { + log.Printf("[INFO] Waiting during init to make sure the opensearch instance is up and running with security features properly") + time.Sleep(30 * time.Second) + } + + _ = setUsers schedules, err := shuffle.GetAllSchedules(ctx, "ALL") if err != nil { log.Printf("[WARNING] Failed getting schedules during service init: %s", err) @@ -3889,7 +3927,12 @@ func runInitEs(ctx context.Context) { Body: ioutil.NopCloser(strings.NewReader(schedule.WrappedArgument)), } - _, _, err := handleExecution(schedule.WorkflowId, shuffle.Workflow{}, request) + orgId := "" + if len(activeOrgs) > 0 { + orgId = activeOrgs[0].Id + } + + _, _, err := handleExecution(schedule.WorkflowId, shuffle.Workflow{}, request, orgId) if err != nil { log.Printf("[WARNING] Failed to execute %s: %s", schedule.WorkflowId, err) } @@ -3897,7 +3940,7 @@ func runInitEs(ctx context.Context) { } for _, schedule := range schedules { - if schedule.Environment == "cloud" { + if strings.ToLower(schedule.Environment) == "cloud" { log.Printf("Skipping cloud schedule") continue } @@ -3913,6 +3956,7 @@ func runInitEs(ctx context.Context) { } } + parsedApikey := "" users, err := shuffle.GetAllUsers(ctx) if len(users) == 0 { log.Printf("[INFO] Trying to set up user based on environments SHUFFLE_DEFAULT_USERNAME & SHUFFLE_DEFAULT_PASSWORD") @@ -3924,6 +3968,10 @@ func runInitEs(ctx context.Context) { } else { apikey := os.Getenv("SHUFFLE_DEFAULT_APIKEY") + if len(parsedApikey) == 0 { + parsedApikey = apikey + } + log.Printf("[DEBUG] Creating org for default user %s", username) orgId := uuid.NewV4().String() orgSetupName := "default" @@ -3936,7 +3984,7 @@ func runInitEs(ctx context.Context) { CloudSync: false, } - err = shuffle.SetOrg(ctx, newOrg, orgId) + err = shuffle.SetOrg(ctx, newOrg, newOrg.Id) setUsers := false if err != nil { log.Printf("[WARNING] Failed setting organization when creating original user: %s", err) @@ -3972,8 +4020,15 @@ func runInitEs(ctx context.Context) { } } } + } else { + for _, user := range users { + if user.Role == "admin" && len(user.ApiKey) > 0 { + parsedApikey = user.ApiKey + log.Printf("[DEBUG] Using apikey of %s (%s) for cleanup", user.Username, user.Id) + break + } + } } - _ = setUsers log.Printf("[INFO] Starting cloud schedules for orgs if enabled!") type requestStruct struct { @@ -3981,8 +4036,13 @@ func runInitEs(ctx context.Context) { } for _, org := range activeOrgs { + if len(org.Id) == 0 { + log.Printf("[DEBUG] No ID found for org with name '%s'. Why was it made?", org.Name) + continue + } + if !org.CloudSync { - log.Printf("[WARNING] Skipping org syncCheck for %s because sync isn't set (1).", org.Id) + log.Printf("[INFO] Skipping org syncCheck for '%s' because sync isn't set (1).", org.Id) continue } @@ -3993,7 +4053,7 @@ func runInitEs(ctx context.Context) { continue } - log.Printf("[DEBUG] Should start schedule for org %s", org.Name) + log.Printf("[DEBUG] Should start schedule for org %s (%s)", org.Name, org.Id) job := func() { err := remoteOrgJobHandler(org, interval) if err != nil { @@ -4017,6 +4077,97 @@ func runInitEs(ctx context.Context) { forceUpdate = true } + // FIXME: Have this for all envs in all orgs (loop and find). + if len(parsedApikey) > 0 { + cleanupSchedule := 300 + + if len(os.Getenv("SHUFFLE_RERUN_SCHEDULE")) > 0 { + newfrequency, err := strconv.Atoi(os.Getenv("SHUFFLE_RERUN_SCHEDULE")) + if err == nil { + cleanupSchedule = newfrequency + + if cleanupSchedule < 300 { + log.Printf("[WARNING] A Cleanupschedule of less than 300 seconds won't help.") + cleanupSchedule = 300 + } + } + } + + environments := []string{"Shuffle"} + log.Printf("[DEBUG] Starting schedule setup for execution cleanup every %d seconds. Running first immediately.", cleanupSchedule) + cleanupJob := func() func() { + return func() { + log.Printf("[INFO] Running schedule for cleaning up or re-running unfinished workflows in %d environments.", len(environments)) + + for _, environment := range environments { + httpClient := &http.Client{} + url := fmt.Sprintf("http://localhost:5001/api/v1/environments/%s/stop", environment) + req, err := http.NewRequest( + "GET", + url, + nil, + ) + + req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, parsedApikey)) + if err != nil { + log.Printf("[ERROR] Failed CREATING environment request for %s: %s", environment, err) + continue + + } + + newresp, err := httpClient.Do(req) + if err != nil { + log.Printf("[ERROR] Failed running environment request %s: %s", environment, err) + continue + } + + respBody, err := ioutil.ReadAll(newresp.Body) + if err != nil { + log.Printf("[ERROR] Failed setting respbody %s", err) + continue + } + log.Printf("[DEBUG] Successfully ran workflow cleanup request for %s. Body: %s", environment, string(respBody)) + + url = fmt.Sprintf("http://localhost:5001/api/v1/environments/%s/rerun", environment) + req, err = http.NewRequest( + "GET", + url, + nil, + ) + + req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, parsedApikey)) + if err != nil { + log.Printf("[ERROR] Failed CREATING environment request to rerun for %s: %s", environment, err) + continue + + } + + newresp, err = httpClient.Do(req) + if err != nil { + log.Printf("[ERROR] Failed running environment request to rerun for %s: %s", environment, err) + continue + } + + respBody, err = ioutil.ReadAll(newresp.Body) + if err != nil { + log.Printf("[ERROR] Failed setting respbody %s", err) + continue + } + log.Printf("[DEBUG] Successfully ran workflow RERUN request for %s. Body: %s", environment, string(respBody)) + } + } + } + + jobret, err := newscheduler.Every(cleanupSchedule).Seconds().Run(cleanupJob()) + if err != nil { + log.Printf("[ERROR] Failed to schedule Cleanup: %s", err) + } else { + _ = jobret + } + } else { + log.Printf("[DEBUG] Couldn't find a valid API-key, hence couldn't run cleanup") + } + // Getting apps to see if we should initialize a test // FIXME: Isn't this a little backwards? workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000, 0) @@ -4041,7 +4192,10 @@ func runInitEs(ctx context.Context) { url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION") if len(url) == 0 { - url = "https://github.com/frikky/shuffle-apps" + log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/shuffle/shuffle-apps") + url = "https://github.com/shuffle/shuffle-apps" + //url = "" + //return } username := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_USERNAME") @@ -4063,10 +4217,9 @@ func runInitEs(ctx context.Context) { cloneOptions.ReferenceName = plumbing.ReferenceName(branch) } - log.Printf("[DEBUG] Getting apps from %s", url) + log.Printf("[DEBUG] Getting apps from url '%s'", url) r, err := git.Clone(storer, fs, cloneOptions) - if err != nil { log.Printf("[WARNING] Failed loading repo into memory (init): %s", err) } @@ -4078,7 +4231,6 @@ func runInitEs(ctx context.Context) { _ = r //iterateAppGithubFolders(fs, dir, "", "testing") - // FIXME: Get all the apps? _, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate) if err != nil { log.Printf("[WARNING] Error from app load in init: %s", err) @@ -4093,7 +4245,7 @@ func runInitEs(ctx context.Context) { } log.Printf("[INFO] Downloading OpenAPI data for search - EXTRA APPS") - apis := "https://github.com/frikky/security-openapis" + apis := "https://github.com/shuffle/security-openapis" // THis gets memory problems hahah //apis := "https://github.com/APIs-guru/openapi-directory" @@ -4128,7 +4280,7 @@ func runInit(ctx context.Context) { //} //log.Printf("[DEBUG] Finalized init statistics update") - log.Printf("[DEBUG] Starting INIT setup") + log.Printf("[DEBUG] Starting INIT setup (NOT Opensearch/Elasticsearch!)") httpProxy := os.Getenv("HTTP_PROXY") if len(httpProxy) > 0 { log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy) @@ -4200,11 +4352,11 @@ func runInit(ctx context.Context) { CloudSync: false, } - err = shuffle.SetOrg(ctx, newOrg, orgId) + err = shuffle.SetOrg(ctx, newOrg, newOrg.Id) if err != nil { - log.Printf("Failed setting organization: %s", err) + log.Printf("[WARNING] Failed setting organization: %s", err) } else { - log.Printf("Successfully created the default org!") + log.Printf("[WARNING] Successfully created the default org!") setUsers = true } } else { @@ -4572,7 +4724,6 @@ func runInit(ctx context.Context) { continue } - log.Printf("ENV: %s", item.Environment) if item.Environment == "cloud" { log.Printf("Skipping cloud schedule") continue @@ -4604,7 +4755,7 @@ func runInit(ctx context.Context) { continue } - log.Printf("[DEBUG] Should start schedule for org %s", org.Name) + log.Printf("[DEBUG] Should start schedule for org %s (%s)", org.Name, org.Id) job := func() { err := remoteOrgJobHandler(org, interval) if err != nil { @@ -4646,7 +4797,12 @@ func runInit(ctx context.Context) { Body: ioutil.NopCloser(strings.NewReader(schedule.WrappedArgument)), } - _, _, err := handleExecution(schedule.WorkflowId, shuffle.Workflow{}, request) + orgId := "" + if len(activeOrgs) > 0 { + orgId = activeOrgs[0].Id + } + + _, _, err := handleExecution(schedule.WorkflowId, shuffle.Workflow{}, request, orgId) if err != nil { log.Printf("[WARNING] Failed to execute %s: %s", schedule.WorkflowId, err) } @@ -4671,7 +4827,7 @@ func runInit(ctx context.Context) { } // Getting apps to see if we should initialize a test - workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000,0 ) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000, 0) log.Printf("[INFO] Getting and validating workflowapps. Got %d with err %s", len(workflowapps), err) if err != nil && len(workflowapps) == 0 { log.Printf("[WARNING] Failed getting apps (runInit): %s", err) @@ -4697,7 +4853,7 @@ func runInit(ctx context.Context) { url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION") if len(url) == 0 { - url = "https://github.com/frikky/shuffle-apps" + url = "https://github.com/shuffle/shuffle-apps" } username := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_USERNAME") @@ -4718,7 +4874,7 @@ func runInit(ctx context.Context) { cloneOptions.ReferenceName = plumbing.ReferenceName(branch) } - log.Printf("[DEBUG] Getting apps from %s", url) + log.Printf("[DEBUG] Getting apps from URL '%s'", url) r, err := git.Clone(storer, fs, cloneOptions) @@ -4748,7 +4904,7 @@ func runInit(ctx context.Context) { } log.Printf("[INFO] Downloading OpenAPI data for search - EXTRA APPS") - apis := "https://github.com/frikky/security-openapis" + apis := "https://github.com/shuffle/security-openapis" // FIXME: This part gets memory problems. Fix in the future to load these apps too. //apis := "https://github.com/APIs-guru/openapi-directory" @@ -4875,7 +5031,7 @@ func handleStopCloudSync(syncUrl string, org shuffle.Org) (*shuffle.Org, error) if err != nil { return &org, err } - log.Printf("Remote disable ret: %s", string(respBody)) + log.Printf("[INFO] Remote disable ret: %s", string(respBody)) responseData := retStruct{} err = json.Unmarshal(respBody, &responseData) @@ -4943,7 +5099,7 @@ func handleStopCloudSync(syncUrl string, org shuffle.Org) (*shuffle.Org, error) This is here to both enable and disable cloud sync features for an organization */ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -5045,8 +5201,19 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { _, err = handleStopCloudSync(syncPath, *org) if err != nil { + ret := shuffle.ResultChecker{ + Success: false, + Reason: fmt.Sprintf("%s", err), + } + resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + b, err := json.Marshal(ret) + if err != nil { + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + + resp.Write(b) } else { resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Successfully disabled cloud sync for org."}`))) @@ -5482,6 +5649,13 @@ func makeWorkflowPublic(resp http.ResponseWriter, request *http.Request) { return } + if user.Role == "org-reader" { + log.Printf("[WARNING] Org-reader doesn't have access publish workflow: %s (%s)", user.Username, user.Id) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Read only user"}`)) + return + } + location := strings.Split(request.URL.String(), "/") var fileId string if location[1] == "api" { @@ -5608,16 +5782,70 @@ func makeWorkflowPublic(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } +func handleAppZipUpload(resp http.ResponseWriter, request *http.Request) { + cors := shuffle.HandleCors(resp, request) + if cors { + return + } + + //https://stackoverflow.com/questions/22964950/http-request-formfile-handle-zip-files + request.ParseMultipartForm(32 << 20) + f, _, err := request.FormFile("shuffle_file") + if err != nil { + log.Printf("[ERROR] Couldn't upload file: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Failed uploading file. Correct usage is: shuffle_file=@filepath"}`)) + return + } + + fileSize, err := f.Seek(0, 2) //2 = from end + if err != nil { + panic(err) + } + _, err = f.Seek(0, 0) + if err != nil { + panic(err) + } + + buf := new(bytes.Buffer) + fileSize, err = io.Copy(buf, f) + if err != nil { + panic(err) + } + + zipdata, err := zip.NewReader(bytes.NewReader(buf.Bytes()), fileSize) + if err != nil { + panic(err) + } + + // https://github.com/alexmullins/zip/blob/master/example_test.go + for _, item := range zipdata.File { + log.Printf("\n\nName: %s\n\n", item.FileHeader.Name) + log.Printf("item: %#v", item) + + rr, err := item.Open() + if err != nil { + log.Fatal(err) + } + + _, err = io.Copy(os.Stdout, rr) + if err != nil { + log.Fatal(err) + } + rr.Close() + + } + + resp.WriteHeader(200) + resp.Write([]byte("OK")) +} + func initHandlers() { var err error ctx := context.Background() log.Printf("[DEBUG] Starting Shuffle backend - initializing database connection") //requestCache = cache.New(5*time.Minute, 10*time.Minute) - dbclient, err = datastore.NewClient(ctx, gceProject, option.WithGRPCDialOption(grpc.WithNoProxy())) - if err != nil { - log.Fatalf("[DEBUG] Database client error during init: %s", err) - } //es := shuffle.GetEsConfig() elasticConfig := "elasticsearch" @@ -5625,6 +5853,20 @@ func initHandlers() { elasticConfig = "" } + dbclient, err = datastore.NewClient(ctx, gceProject, option.WithGRPCDialOption(grpc.WithNoProxy())) + if err != nil { + if elasticConfig == "" { + log.Printf("[ERROR] Database client error during init: %s. Env: SHUFFLE_ELASTIC=false", err) + } else { + if !strings.Contains(fmt.Sprintf("%s", err), "find default credentials") { + log.Printf("[DEBUG] Database client error info during init: %s. Here for backwards compatibility: not critical.", err) + } + dbclient = &datastore.Client{} + } + } else { + //log.Printf("Database client initiated: %s", dbclient) + } + for { _, err = shuffle.RunInit(*dbclient, storage.Client{}, gceProject, "onprem", true, elasticConfig) if err != nil { @@ -5639,13 +5881,14 @@ func initHandlers() { log.Printf("[DEBUG] Initialized Shuffle database connection. Setting up environment.") if elasticConfig == "elasticsearch" { + time.Sleep(5 * time.Second) go runInitEs(ctx) } else { go runInit(ctx) } r := mux.NewRouter() - r.HandleFunc("/api/v1/_ah/health", healthCheckHandler) + r.HandleFunc("/api/v1/_ah/health", shuffle.HealthCheckHandler) // Make user related locations // Fix user changes with org @@ -5661,11 +5904,13 @@ func initHandlers() { r.HandleFunc("/api/v1/users/updateuser", shuffle.HandleUpdateUser).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/users/{user}", shuffle.DeleteUser).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/users/passwordchange", shuffle.HandlePasswordChange).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/users/{key}/get2fa", shuffle.HandleGet2fa).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/users/{key}/set2fa", shuffle.HandleSet2fa).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/users", shuffle.HandleGetUsers).Methods("GET", "OPTIONS") // General - duplicates and old. r.HandleFunc("/api/v1/getusers", shuffle.HandleGetUsers).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/login", handleLogin).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/login", shuffle.HandleLogin).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/logout", shuffle.HandleLogout).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/register", handleRegister).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/checkusers", checkAdminLogin).Methods("GET", "OPTIONS") @@ -5688,11 +5933,18 @@ func initHandlers() { r.HandleFunc("/api/v1/streams/results", handleGetStreamResults).Methods("POST", "OPTIONS") // Used by orborus - r.HandleFunc("/api/v1/workflows/queue", handleGetWorkflowqueue).Methods("GET") + r.HandleFunc("/api/v1/workflows/queue", handleGetWorkflowqueue).Methods("GET", "POST") r.HandleFunc("/api/v1/workflows/queue/confirm", handleGetWorkflowqueueConfirm).Methods("POST") // App specific // From here down isnt checked for org specific + r.HandleFunc("/api/v1/apps/{key}/execute", executeSingleAction).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/apps/categories", shuffle.GetActiveCategories).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/apps/categories/run", shuffle.RunCategoryAction).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/apps/upload", handleAppZipUpload).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/apps/{appId}/activate", activateWorkflowAppDocker).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/apps/frameworkConfiguration", shuffle.GetFrameworkConfiguration).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/apps/frameworkConfiguration", shuffle.SetFrameworkConfiguration).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}", shuffle.UpdateWorkflowAppConfig).Methods("PATCH", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}", shuffle.DeleteWorkflowApp).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}/config", shuffle.GetWorkflowAppConfig).Methods("GET", "OPTIONS") @@ -5709,6 +5961,15 @@ func initHandlers() { r.HandleFunc("/api/v1/apps/authentication/{appauthId}/config", shuffle.SetAuthenticationConfig).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/authentication/{appauthId}", shuffle.DeleteAppAuthentication).Methods("DELETE", "OPTIONS") + // Related to NFT things + r.HandleFunc("/api/v1/workflows/collections/load", shuffle.LoadCollections).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/workflows/collections/{key}", shuffle.HandleGetCollection).Methods("GET", "OPTIONS") + + // Related to use-cases that are not directly workflows. + r.HandleFunc("/api/v1/workflows/usecases/{key}", shuffle.HandleGetUsecase).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/workflows/usecases", shuffle.LoadUsecases).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/workflows/usecases", shuffle.UpdateUsecases).Methods("POST", "OPTIONS") + // Legacy app things r.HandleFunc("/api/v1/workflows/apps/validate", validateAppInput).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/apps", getWorkflowApps).Methods("GET", "OPTIONS") @@ -5726,20 +5987,29 @@ func initHandlers() { r.HandleFunc("/api/v1/workflows/download_remote", loadSpecificWorkflows).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/execute", executeWorkflow).Methods("GET", "POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/schedule/{schedule}", stopSchedule).Methods("DELETE", "OPTIONS") + r.HandleFunc("/api/v1/workflows/{key}/stream", shuffle.HandleStreamWorkflow).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/workflows/{key}/stream", shuffle.HandleStreamWorkflowUpdate).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}", deleteWorkflow).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}", shuffle.SaveWorkflow).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}", shuffle.GetSpecificWorkflow).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/workflows/recommend", shuffle.HandleActionRecommendation).Methods("POST", "OPTIONS") + + // New for recommendations in Shuffle + r.HandleFunc("/api/v1/recommendations/get_actions", shuffle.HandleActionRecommendation).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/recommendations/modify", shuffle.HandleRecommendationAction).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/revisions", shuffle.GetWorkflowRevisions).Methods("GET", "OPTIONS") // Triggers r.HandleFunc("/api/v1/hooks/new", shuffle.HandleNewHook).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/hooks/{key}", handleWebhookCallback).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/hooks", shuffle.HandleNewHook).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/hooks/{key}", handleWebhookCallback).Methods("POST", "GET", "PATCH", "PUT", "DELETE", "OPTIONS") r.HandleFunc("/api/v1/hooks/{key}/delete", shuffle.HandleDeleteHook).Methods("DELETE", "OPTIONS") + r.HandleFunc("/api/v1/hooks/{key}", shuffle.HandleDeleteHook).Methods("DELETE", "OPTIONS") // OpenAPI configuration r.HandleFunc("/api/v1/verify_swagger", verifySwagger).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/verify_openapi", verifySwagger).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/get_openapi_uri", echoOpenapiData).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/get_openapi_uri", shuffle.EchoOpenapiData).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/validate_openapi", shuffle.ValidateSwagger).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/get_openapi/{key}", getOpenapi).Methods("GET", "OPTIONS") @@ -5776,15 +6046,22 @@ func initHandlers() { // This is a new API that validates if a key has been seen before. // Not sure what the best course of action is for it. + r.HandleFunc("/api/v1/environments/{key}/stop", shuffle.HandleStopExecutions).Methods("GET", "POST", "OPTIONS") + r.HandleFunc("/api/v1/environments/{key}/rerun", shuffle.HandleRerunExecutions).Methods("GET", "POST", "OPTIONS") + r.HandleFunc("/api/v1/orgs/{orgId}/validate_app_values", shuffle.HandleKeyValueCheck).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/orgs/{orgId}/list_cache", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/get_cache", shuffle.HandleGetCacheKey).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/set_cache", shuffle.HandleSetCacheKey).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/apps/{key}/execute", executeSingleAction).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/orgs/{orgId}/cache/{cache_key}", shuffle.HandleDeleteCacheKey).Methods("DELETE", "OPTIONS") + r.HandleFunc("/api/v1/orgs/{orgId}/stats", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/orgs/{orgId}/revisions", shuffle.GetWorkflowRevisions).Methods("GET", "OPTIONS") // Docker orborus specific - downloads an image r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/migrate_database", migrateDatabase).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/login_sso", shuffle.HandleSSO).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/login_sso", shuffle.HandleSSO).Methods("GET", "POST", "OPTIONS") + r.HandleFunc("/api/v1/login_openid", shuffle.HandleOpenId).Methods("GET", "POST", "OPTIONS") // Important for email, IDS etc. Create this by: // PS: For cloud, this has to use cloud storage. @@ -5793,7 +6070,8 @@ func initHandlers() { r.HandleFunc("/api/v1/files/namespaces/{namespace}", shuffle.HandleGetFileNamespace).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}/content", shuffle.HandleGetFileContent).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/create", shuffle.HandleCreateFile).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/files/{fileId}/upload", shuffle.HandleUploadFile).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}/upload", shuffle.HandleUploadFile).Methods("POST", "OPTIONS", "PATCH") + r.HandleFunc("/api/v1/files/{fileId}/edit", shuffle.HandleEditFile).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}", shuffle.HandleGetFileMeta).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}", shuffle.HandleDeleteFile).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/files", shuffle.HandleGetFiles).Methods("GET", "OPTIONS") @@ -5803,6 +6081,15 @@ func initHandlers() { r.HandleFunc("/api/v1/notifications/clear", shuffle.HandleClearNotifications).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS") //r.HandleFunc("/api/v1/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/users/notifications", shuffle.HandleGetNotifications).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/users/notifications/clear", shuffle.HandleClearNotifications).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/users/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS") + + r.HandleFunc("/api/v1/conversation", shuffle.RunActionAI).Methods("POST", "OPTIONS") + + //r.HandleFunc("/api/v1/users/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/dashboards/{key}/widgets", shuffle.HandleNewWidget).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/dashboards/{key}/widgets/{widget_id}", shuffle.HandleGetWidget).Methods("GET", "OPTIONS") http.Handle("/", r) } diff --git a/backend/go-app/main_test.go b/backend/go-app/main_test.go index e9d358b6..4e39ea0e 100755 --- a/backend/go-app/main_test.go +++ b/backend/go-app/main_test.go @@ -24,6 +24,7 @@ type endpoint struct { handler http.HandlerFunc path string method string + body []byte } func init() { @@ -44,7 +45,7 @@ func TestAuthenticationRequired(t *testing.T) { {handler: shuffle.HandleNewOutlookRegister, path: "/functions/outlook/register", method: "GET"}, {handler: shuffle.HandleGetOutlookFolders, path: "/functions/outlook/getFolders", method: "GET"}, {handler: shuffle.HandleApiGeneration, path: "/api/v1/users/generateapikey", method: "GET"}, - {handler: handleLogin, path: "/api/v1/users/login", method: "POST"}, // prob not this one + {handler: shuffle.HandleLogin, path: "/api/v1/users/login", method: "POST"}, // prob not this one // handleRegister generates nil pointer exception. Not necessary for this anyway. //{handler: handleRegister, path: "/api/v1/users/register", method: "POST"}, {handler: shuffle.HandleGetUsers, path: "/api/v1/users/getusers", method: "GET"}, @@ -108,8 +109,8 @@ func TestAuthenticationRequired(t *testing.T) { {handler: verifySwagger, path: "/api/v1/verify_swagger", method: "POST"}, {handler: verifySwagger, path: "/api/v1/verify_openapi", method: "POST"}, - {handler: echoOpenapiData, path: "/api/v1/get_openapi_uri", method: "POST"}, - {handler: echoOpenapiData, path: "/api/v1/validate_openapi", method: "POST"}, + {handler: shuffle.EchoOpenapiData, path: "/api/v1/get_openapi_uri", method: "POST"}, + {handler: shuffle.EchoOpenapiData, path: "/api/v1/validate_openapi", method: "POST"}, {handler: shuffle.ValidateSwagger, path: "/api/v1/validate_openapi", method: "POST"}, {handler: getOpenapi, path: "/api/v1/get_openapi", method: "GET"}, @@ -117,7 +118,7 @@ func TestAuthenticationRequired(t *testing.T) { {handler: handleCloudSetup, path: "/api/v1/cloud/setup", method: "POST"}, {handler: shuffle.HandleGetOrgs, path: "/api/v1/orgs", method: "POST"}, - {handler: shuffle.HandleGetFileContent, path: "/api/v1/files/{fileId}/content", method: "POST"}, + {handler: shuffle.HandleGetFileContent, path: "/api/v1/files/{fileId}/content", method: "POST", body: []byte("hi")}, } var err error @@ -197,10 +198,11 @@ func TestAuthenticationNotRequired(t *testing.T) { // requirements might change after the refactor. func TestCors(t *testing.T) { handlers := []endpoint{ + {handler: handleLogin, path: "/api/v1/users/login", method: "POST"}, // prob not this one + {handler: shuffle.HandleNewOutlookRegister, path: "/functions/outlook/register", method: "GET"}, {handler: shuffle.HandleGetOutlookFolders, path: "/functions/outlook/getFolders", method: "GET"}, {handler: shuffle.HandleApiGeneration, path: "/api/v1/users/generateapikey", method: "GET"}, - {handler: handleLogin, path: "/api/v1/users/login", method: "POST"}, // prob not this one // handleRegister generates nil pointer exception {handler: handleRegister, path: "/api/v1/users/register", method: "POST"}, {handler: shuffle.HandleGetUsers, path: "/api/v1/users/getusers", method: "GET"}, diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 05de34a2..2f057c8e 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -11,6 +11,7 @@ import ( "io" "io/ioutil" "log" + "math/rand" "net/http" "net/url" "os" @@ -21,6 +22,7 @@ import ( "github.com/docker/docker/api/types" dockerclient "github.com/docker/docker/client" + //gyaml "github.com/ghodss/yaml" "github.com/h2non/filetype" @@ -34,6 +36,7 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" + //"github.com/gorilla/websocket" //"google.golang.org/appengine" //"google.golang.org/appengine/memcache" @@ -50,425 +53,6 @@ var cloudname = "cloud" var scheduledJobs = map[string]*newscheduler.Job{} var scheduledOrgs = map[string]*newscheduler.Job{} -// To test out firestore before potential merge -//var upgrader = websocket.Upgrader{ -// ReadBufferSize: 1024, -// WriteBufferSize: 1024, -// CheckOrigin: func(r *http.Request) bool { -// return true -// }, -//} - -//type ExecutionRequest struct { -// ExecutionId string `json:"execution_id,omitempty"` -// ExecutionArgument string `json:"execution_argument,omitempty"` -// ExecutionSource string `json:"execution_source,omitempty"` -// WorkflowId string `json:"workflow_id,omitempty"` -// Environments []string `json:"environments,omitempty"` -// Authorization string `json:"authorization,omitempty"` -// Status string `json:"status,omitempty"` -// Start string `json:"start,omitempty"` -// Type string `json:"type,omitempty"` -//} -// -//type SyncFeatures struct { -// Webhook SyncData `json:"webhook" datastore:"webhook"` -// Schedules SyncData `json:"schedules" datastore:"schedules"` -// UserInput SyncData `json:"user_input" datastore:"user_input"` -// SendMail SyncData `json:"send_mail" datastore:"send_mail"` -// SendSms SyncData `json:"send_sms" datastore:"send_sms"` -// Updates SyncData `json:"updates" datastore:"updates"` -// Notifications SyncData `json:"notifications" datastore:"notifications"` -// EmailTrigger SyncData `json:"email_trigger" datastore:"email_trigger"` -// AppExecutions SyncData `json:"app_executions" datastore:"app_executions"` -// WorkflowExecutions SyncData `json:"workflow_executions" datastore:"workflow_executions"` -// Apps SyncData `json:"apps" datastore:"apps"` -// Workflows SyncData `json:"workflows" datastore:"workflows"` -// Autocomplete SyncData `json:"autocomplete" datastore:"autocomplete"` -// Authentication SyncData `json:"authentication" datastore:"authentication"` -// Schedule SyncData `json:"schedule" datastore:"schedule"` -//} -// -//type SyncData struct { -// Active bool `json:"active" datastore:"active"` -// Type string `json:"type,omitempty" datastore:"type"` -// Name string `json:"name,omitempty" datastore:"name"` -// Description string `json:"description,omitempty" datastore:"description"` -// Limit int64 `json:"limit,omitempty" datastore:"limit"` -// StartDate int64 `json:"start_date,omitempty" datastore:"start_date"` -// EndDate int64 `json:"end_date,omitempty" datastore:"end_date"` -// DataCollection int64 `json:"data_collection,omitempty" datastore:"data_collection"` -//} -// -//type SyncConfig struct { -// Interval int64 `json:"interval" datastore:"interval"` -// Apikey string `json:"api_key" datastore:"api_key"` -//} -// -//type PaymentSubscription struct { -// Active bool `json:"active" datastore:"active"` -// Startdate int64 `json:"startdate" datastore:"startdate"` -// CancellationDate int64 `json:"cancellationdate" datastore:"cancellationdate"` -// Enddate int64 `json:"enddate" datastore:"enddate"` -// Name string `json:"name" datastore:"name"` -// Recurrence string `json:"recurrence" datastore:"recurrence"` -// Reference string `json:"reference" datastore:"reference"` -// Level string `json:"level" datastore:"level"` -// Amount string `json:"amount" datastore:"amount"` -// Currency string `json:"currency" datastore:"currency"` -//} -// -//type Defaults struct { -// AppDownloadRepo string `json:"app_download_repo" datastore:"app_download_repo"` -// AppDownloadBranch string `json:"app_download_branch" datastore:"app_download_branch"` -// WorkflowDownloadRepo string `json:"workflow_download_repo" datastore:"workflow_download_repo"` -// WorkflowDownloadBranch string `json:"workflow_download_branch" datastore:"workflow_download_branch"` -//} -// -//type AppAuthenticationStorage struct { -// Active bool `json:"active" datastore:"active"` -// Label string `json:"label" datastore:"label"` -// Id string `json:"id" datastore:"id"` -// App WorkflowApp `json:"app" datastore:"app,noindex"` -// Fields []AuthenticationStore `json:"fields" datastore:"fields"` -// Usage []AuthenticationUsage `json:"usage" datastore:"usage"` -// WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"` -// NodeCount int64 `json:"node_count" datastore:"node_count"` -// OrgId string `json:"org_id" datastore:"org_id"` -// Created int64 `json:"created" datastore:"created"` -// Edited int64 `json:"edited" datastore:"edited"` -// Defined bool `json:"defined" datastore:"defined"` -//} -// -//type AuthenticationUsage struct { -// WorkflowId string `json:"workflow_id" datastore:"workflow_id"` -// Nodes []string `json:"nodes" datastore:"nodes"` -//} -// -//// An app inside Shuffle -//// Source string `json:"source" datastore:"soure" yaml:"source"` - downloadlocation -//type WorkflowApp struct { -// Name string `json:"name" yaml:"name" required:true datastore:"name"` -// IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"` -// ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"` -// Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"` -// AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"` -// SharingConfig string `json:"sharing_config" yaml:"sharing_config" datastore:"sharing_config"` -// Generated bool `json:"generated" yaml:"generated" required:false datastore:"generated"` -// Downloaded bool `json:"downloaded" yaml:"downloaded" required:false datastore:"downloaded"` -// Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` -// Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` -// Invalid bool `json:"invalid" yaml:"invalid" required:false datastore:"invalid"` -// Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` -// Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"` -// Owner string `json:"owner" datastore:"owner" yaml:"owner"` -// Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps -// PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"` -// Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"` -// Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"` -// SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` -// LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` -// ContactInfo struct { -// Name string `json:"name" datastore:"name" yaml:"name"` -// Url string `json:"url" datastore:"url" yaml:"url"` -// } `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false` -// ReferenceInfo struct { -// DocumentationUrl string `json:"documentation_url" datastore:"documentation_url"` -// GithubUrl string `json:"github_url" datastore:"github_url"` -// } -// FolderMount struct { -// FolderMount bool `json:"folder_mount" datastore:"folder_mount"` -// SourceFolder string `json:"source_folder" datastore:"source_folder"` -// DestinationFolder string `json:"destination_folder" datastore:"destination_folder"` -// } -// Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` -// Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"` -// Tags []string `json:"tags" yaml:"tags" required:false datastore:"activated"` -// Categories []string `json:"categories" yaml:"categories" required:false datastore:"categories"` -// Created int64 `json:"created" datastore:"created"` -// Edited int64 `json:"edited" datastore:"edited"` -// LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` -//} -// -//type WorkflowAppActionParameter struct { -// Description string `json:"description" datastore:"description,noindex" yaml:"description"` -// ID string `json:"id" datastore:"id" yaml:"id,omitempty"` -// Name string `json:"name" datastore:"name" yaml:"name"` -// Example string `json:"example" datastore:"example,noindex" yaml:"example"` -// Value string `json:"value" datastore:"value,noindex" yaml:"value,omitempty"` -// Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` -// Options []string `json:"options" datastore:"options" yaml:"options"` -// ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"` -// Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"` -// Required bool `json:"required" datastore:"required" yaml:"required"` -// Configuration bool `json:"configuration" datastore:"configuration" yaml:"configuration"` -// Tags []string `json:"tags" datastore:"tags" yaml:"tags"` -// Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` -// SkipMulticheck bool `json:"skip_multicheck" datastore:"skip_multicheck" yaml:"skip_multicheck"` -// ValueReplace []Valuereplace `json:"value_replace" datastore:"value_replace,noindex" yaml:"value_replace,omitempty"` -// UniqueToggled bool `json:"unique_toggled" datastore:"unique_toggled" yaml:"unique_toggled"` -//} -// -//type Valuereplace struct { -// Key string `json:"key" datastore:"key" yaml:"key"` -// Value string `json:"value" datastore:"value" yaml:"value"` -//} -// -//type SchemaDefinition struct { -// Type string `json:"type" datastore:"type"` -//} -// -//type WorkflowAppAction struct { -// Description string `json:"description" datastore:"description,noindex"` -// ID string `json:"id" datastore:"id" yaml:"id,omitempty"` -// Name string `json:"name" datastore:"name"` -// Label string `json:"label" datastore:"label"` -// NodeType string `json:"node_type" datastore:"node_type"` -// Environment string `json:"environment" datastore:"environment"` -// Sharing bool `json:"sharing" datastore:"sharing"` -// PrivateID string `json:"private_id" datastore:"private_id"` -// AppID string `json:"app_id" datastore:"app_id"` -// Tags []string `json:"tags" datastore:"tags" yaml:"tags"` -// Authentication []AuthenticationStore `json:"authentication" datastore:"authentication,noindex" yaml:"authentication,omitempty"` -// Tested bool `json:"tested" datastore:"tested" yaml:"tested"` -// Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"` -// ExecutionVariable struct { -// Description string `json:"description" datastore:"description,noindex"` -// ID string `json:"id" datastore:"id"` -// Name string `json:"name" datastore:"name"` -// Value string `json:"value" datastore:"value,noindex"` -// } `json:"execution_variable" datastore:"execution_variables"` -// Returns struct { -// Description string `json:"description" datastore:"returns" yaml:"description,omitempty"` -// Example string `json:"example" datastore:"example,noindex" yaml:"example"` -// ID string `json:"id" datastore:"id" yaml:"id,omitempty"` -// Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` -// } `json:"returns" datastore:"returns"` -// AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` -// Example string `json:"example,noindex" datastore:"example" yaml:"example"` -// AuthNotRequired bool `json:"auth_not_required" datastore:"auth_not_required" yaml:"auth_not_required"` -//} - -// FIXME: Generate a callback authentication ID? -// FIXME: Add org check .. -//type WorkflowExecution struct { -// Type string `json:"type" datastore:"type"` -// Status string `json:"status" datastore:"status"` -// Start string `json:"start" datastore:"start"` -// ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` -// ExecutionId string `json:"execution_id" datastore:"execution_id"` -// ExecutionSource string `json:"execution_source" datastore:"execution_source"` -// ExecutionParent string `json:"execution_parent" datastore:"execution_parent"` -// ExecutionOrg string `json:"execution_org" datastore:"execution_org"` -// WorkflowId string `json:"workflow_id" datastore:"workflow_id"` -// LastNode string `json:"last_node" datastore:"last_node"` -// Authorization string `json:"authorization" datastore:"authorization"` -// Result string `json:"result" datastore:"result,noindex"` -// StartedAt int64 `json:"started_at" datastore:"started_at"` -// CompletedAt int64 `json:"completed_at" datastore:"completed_at"` -// ProjectId string `json:"project_id" datastore:"project_id"` -// Locations []string `json:"locations" datastore:"locations"` -// Workflow Workflow `json:"workflow" datastore:"workflow,noindex"` -// Results []ActionResult `json:"results" datastore:"results,noindex"` -// ExecutionVariables []struct { -// Description string `json:"description" datastore:"description,noindex"` -// ID string `json:"id" datastore:"id"` -// Name string `json:"name" datastore:"name"` -// Value string `json:"value" datastore:"value,noindex"` -// } `json:"execution_variables,omitempty" datastore:"execution_variables,omitempty"` -// OrgId string `json:"org_id" datastore:"org_id"` -//} - -// This is for the nodes in a workflow, NOT the app action itself. -//type Action struct { -// AppName string `json:"app_name" datastore:"app_name"` -// AppVersion string `json:"app_version" datastore:"app_version"` -// AppID string `json:"app_id" datastore:"app_id"` -// Errors []string `json:"errors" datastore:"errors"` -// ID string `json:"id" datastore:"id"` -// IsValid bool `json:"is_valid" datastore:"is_valid"` -// IsStartNode bool `json:"isStartNode,omitempty" datastore:"isStartNode"` -// Sharing bool `json:"sharing,omitempty" datastore:"sharing"` -// PrivateID string `json:"private_id,omitempty" datastore:"private_id"` -// Label string `json:"label,omitempty" datastore:"label"` -// SmallImage string `json:"small_image,omitempty" datastore:"small_image,noindex" required:false yaml:"small_image"` -// LargeImage string `json:"large_image,omitempty" datastore:"large_image,noindex" yaml:"large_image" required:false` -// Environment string `json:"environment,omitempty" datastore:"environment"` -// Name string `json:"name" datastore:"name"` -// Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` -// ExecutionVariable struct { -// Description string `json:"description,omitempty" datastore:"description,noindex"` -// ID string `json:"id,omitempty" datastore:"id"` -// Name string `json:"name,omitempty" datastore:"name"` -// Value string `json:"value,omitempty" datastore:"value,noindex"` -// } `json:"execution_variable,omitempty" datastore:"execution_variable,omitempty"` -// Position struct { -// X float64 `json:"x,omitempty" datastore:"x"` -// Y float64 `json:"y,omitempty" datastore:"y"` -// } `json:"position,omitempty"` -// Priority int `json:"priority,omitempty" datastore:"priority"` -// AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` -// Example string `json:"example,omitempty" datastore:"example,noindex"` -// AuthNotRequired bool `json:"auth_not_required,omitempty" datastore:"auth_not_required" yaml:"auth_not_required"` -// Category string `json:"category" datastore:"category"` -//} -// -//// Added environment for location to execute -//type Trigger struct { -// AppName string `json:"app_name" datastore:"app_name"` -// Description string `json:"description" datastore:"description,noindex"` -// LongDescription string `json:"long_description" datastore:"long_description"` -// Status string `json:"status" datastore:"status"` -// AppVersion string `json:"app_version" datastore:"app_version"` -// Errors []string `json:"errors" datastore:"errors"` -// ID string `json:"id" datastore:"id"` -// IsValid bool `json:"is_valid" datastore:"is_valid"` -// IsStartNode bool `json:"isStartNode" datastore:"isStartNode"` -// Label string `json:"label" datastore:"label"` -// SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` -// LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` -// Environment string `json:"environment" datastore:"environment"` -// TriggerType string `json:"trigger_type" datastore:"trigger_type"` -// Name string `json:"name" datastore:"name"` -// Tags []string `json:"tags" datastore:"tags" yaml:"tags"` -// Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` -// Position struct { -// X float64 `json:"x" datastore:"x"` -// Y float64 `json:"y" datastore:"y"` -// } `json:"position"` -// Priority int `json:"priority" datastore:"priority"` -//} -// -//type Branch struct { -// DestinationID string `json:"destination_id" datastore:"destination_id"` -// ID string `json:"id" datastore:"id"` -// SourceID string `json:"source_id" datastore:"source_id"` -// Label string `json:"label" datastore:"label"` -// HasError bool `json:"has_errors" datastore: "has_errors"` -// Conditions []Condition `json:"conditions" datastore: "conditions,noindex"` -//} -// -//// Same format for a lot of stuff -//type Condition struct { -// Condition WorkflowAppActionParameter `json:"condition" datastore:"condition"` -// Source WorkflowAppActionParameter `json:"source" datastore:"source"` -// Destination WorkflowAppActionParameter `json:"destination" datastore:"destination"` -//} -// -//type Schedule struct { -// Name string `json:"name" datastore:"name"` -// Frequency string `json:"frequency" datastore:"frequency"` -// ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` -// Id string `json:"id" datastore:"id"` -// OrgId string `json:"org_id" datastore:"org_id"` -// Environment string `json:"environment" datastore:"environment"` -//} - -//type Workflow struct { -// Actions []Action `json:"actions" datastore:"actions,noindex"` -// Branches []Branch `json:"branches" datastore:"branches,noindex"` -// Triggers []Trigger `json:"triggers" datastore:"triggers,noindex"` -// Schedules []Schedule `json:"schedules" datastore:"schedules,noindex"` -// Configuration struct { -// ExitOnError bool `json:"exit_on_error" datastore:"exit_on_error"` -// StartFromTop bool `json:"start_from_top" datastore:"start_from_top"` -// } `json:"configuration,omitempty" datastore:"configuration"` -// Created int64 `json:"created" datastore:"created"` -// Edited int64 `json:"edited" datastore:"edited"` -// LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` -// Errors []string `json:"errors,omitempty" datastore:"errors"` -// Tags []string `json:"tags,omitempty" datastore:"tags"` -// ID string `json:"id" datastore:"id"` -// IsValid bool `json:"is_valid" datastore:"is_valid"` -// Name string `json:"name" datastore:"name"` -// Description string `json:"description" datastore:"description,noindex"` -// Start string `json:"start" datastore:"start"` -// Owner string `json:"owner" datastore:"owner"` -// Sharing string `json:"sharing" datastore:"sharing"` -// Org []Org `json:"org,omitempty" datastore:"org"` -// ExecutingOrg Org `json:"execution_org,omitempty" datastore:"execution_org"` -// OrgId string `json:"org_id,omitempty" datastore:"org_id"` -// WorkflowVariables []struct { -// Description string `json:"description" datastore:"description,noindex"` -// ID string `json:"id" datastore:"id"` -// Name string `json:"name" datastore:"name"` -// Value string `json:"value" datastore:"value,noindex"` -// } `json:"workflow_variables" datastore:"workflow_variables"` -// ExecutionVariables []struct { -// Description string `json:"description" datastore:"description,noindex"` -// ID string `json:"id" datastore:"id"` -// Name string `json:"name" datastore:"name"` -// Value string `json:"value" datastore:"value,noindex"` -// } `json:"execution_variables,omitempty" datastore:"execution_variables"` -// ExecutionEnvironment string `json:"execution_environment" datastore:"execution_environment"` -// PreviouslySaved bool `json:"previously_saved" datastore:"first_save"` -// Categories Categories `json:"categories" datastore:"categories"` -// ExampleArgument string `json:"example_argument" datastore:"example_argument,noindex"` -//} - -//type Category struct { -// Name string `json:"name" datastore:"name"` -// Description string `json:"description" datastore:"description"` -// Count int64 `json:"count" datastore:"count"` -//} -// -//type Categories struct { -// SIEM Category `json:"siem" datastore:"siem"` -// Communication Category `json:"communication" datastore:"communication"` -// Assets Category `json:"assets" datastore:"assets"` -// Cases Category `json:"cases" datastore:"cases"` -// Network Category `json:"network" datastore:"network"` -// Intel Category `json:"intel" datastore:"intel"` -// EDR Category `json:"edr" datastore:"edr"` -// Other Category `json:"other" datastore:"other"` -//} - -//type ActionResult struct { -// Action Action `json:"action" datastore:"action,noindex"` -// ExecutionId string `json:"execution_id" datastore:"execution_id"` -// Authorization string `json:"authorization" datastore:"authorization"` -// Result string `json:"result" datastore:"result,noindex"` -// StartedAt int64 `json:"started_at" datastore:"started_at"` -// CompletedAt int64 `json:"completed_at" datastore:"completed_at"` -// Status string `json:"status" datastore:"status"` -//} -// -//type Authentication struct { -// Required bool `json:"required" datastore:"required" yaml:"required" ` -// Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"` -//} -// -//type AuthenticationParams struct { -// Description string `json:"description" datastore:"description,noindex" yaml:"description"` -// ID string `json:"id" datastore:"id" yaml:"id"` -// Name string `json:"name" datastore:"name" yaml:"name"` -// Example string `json:"example" datastore:"example,noindex" yaml:"example"` -// Value string `json:"value,omitempty" datastore:"value,noindex" yaml:"value"` -// Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` -// Required bool `json:"required" datastore:"required" yaml:"required"` -// In string `json:"in" datastore:"in" yaml:"in"` -// Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` -// Scheme string `json:"scheme" datastore:"scheme" yaml:"scheme"` // Deprecated -//} -// -//type AuthenticationStore struct { -// Key string `json:"key" datastore:"key"` -// Value string `json:"value" datastore:"value,noindex"` -//} -// -//type ExecutionRequestWrapper struct { -// Data []ExecutionRequest `json:"data"` -//} -// -//type AppExecutionExample struct { -// AppName string `json:"app_name" datastore:"app_name"` -// AppVersion string `json:"app_version" datastore:"app_version"` -// AppAction string `json:"app_action" datastore:"app_action"` -// AppId string `json:"app_id" datastore:"app_id"` -// ExampleId string `json:"example_id" datastore:"example_id"` -// SuccessExamples []string `json:"success_examples" datastore:"success_examples,noindex"` -// FailureExamples []string `json:"failure_examples" datastore:"failure_examples,noindex"` -//} // Frequency = cronjob OR minutes between execution func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode, frequency, orgId string, body []byte) error { var err error @@ -515,13 +99,13 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode Body: ioutil.NopCloser(strings.NewReader(bodyWrapper)), } - _, _, err := handleExecution(workflowId, shuffle.Workflow{ExecutingOrg: shuffle.OrgMini{Id: orgId}}, request) + _, _, err := handleExecution(workflowId, shuffle.Workflow{ExecutingOrg: shuffle.OrgMini{Id: orgId}}, request, orgId) if err != nil { log.Printf("Failed to execute %s: %s", workflowId, err) } } - log.Printf("Starting frequency: %d", newfrequency) + log.Printf("[INFO] Starting frequency for execution: %d", newfrequency) jobret, err := newscheduler.Every(newfrequency).Seconds().NotImmediately().Run(job) if err != nil { log.Printf("Failed to schedule workflow: %s", err) @@ -561,15 +145,16 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode } func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } // FIXME: Add authentication? + // Cloud has auth. id := request.Header.Get("Org-Id") if len(id) == 0 { - log.Printf("No Org-Id header set - confirm") + log.Printf("[ERROR] No Org-Id header set - confirm") resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org-id header."}`))) return @@ -577,10 +162,10 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque //setWorkflowqueuetest(id) ctx := context.Background() - executionRequests, err := shuffle.GetWorkflowQueue(ctx, id, 10) + executionRequests, err := shuffle.GetWorkflowQueue(ctx, id, 100) if err != nil { - log.Printf("(1) Failed reading body for workflowqueue: %s", err) - resp.WriteHeader(401) + log.Printf("[WARNING] (1) Failed reading body for workflowqueue: %s", err) + resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Entity parsing error - confirm"}`))) return } @@ -594,8 +179,8 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque body, err := ioutil.ReadAll(request.Body) if err != nil { - log.Println("Failed reading body for stream result queue") - resp.WriteHeader(401) + log.Println("[WARNING] Failed reading body for stream result queue") + resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } @@ -605,16 +190,16 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque var removeExecutionRequests shuffle.ExecutionRequestWrapper err = json.Unmarshal(body, &removeExecutionRequests) if err != nil { - log.Printf("Failed executionrequest in queue unmarshaling: %s", err) - resp.WriteHeader(401) + log.Printf("[WARNING] Failed executionrequest in queue unmarshaling: %s", err) + resp.WriteHeader(400) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } if len(removeExecutionRequests.Data) == 0 { - log.Printf("No requests to fix remove from DB") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Some removal error"}`))) + log.Printf("[WARNING] No requests to fix remove from DB") + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Queue removal error"}`))) return } @@ -627,7 +212,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque err = shuffle.DeleteKeys(ctx, parsedId, ids) if err != nil { - log.Printf("[ERROR] Failed deleting %d execution keys for org %s", len(ids), id) + log.Printf("[ERROR] Failed deleting %d execution keys for org %s: %s", len(ids), id, err) } else { //log.Printf("[INFO] Deleted %d keys from org %s", len(ids), parsedId) } @@ -662,21 +247,208 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque // FIXME: Authenticate this one? Can org ID be auth enough? // (especially since we have a default: shuffle) func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } - id := request.Header.Get("Org-Id") - if len(id) == 0 { - log.Printf("[INFO] No org-id header set") + // This is really the environment's name - NOT org-id + orgId := request.Header.Get("Org-Id") + if len(orgId) == 0 { + log.Printf("[AUDIT] No org-id header set") resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org-id header."}`))) return } - ctx := context.Background() - executionRequests, err := shuffle.GetWorkflowQueue(ctx, id, 10) + environment := request.Header.Get("org") + if len(environment) == 0 { + //log.Printf("[AUDIT] No 'org' header set (get workflow queue). ") + /* + resp.WriteHeader(403) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org header. This can be done by setting the 'ORG' environment variable for Orborus to your Org ID in Shuffle"}`))) + return + */ + } + + orborusLabel := request.Header.Get("x-orborus-label") + + // This section is cloud custom for now + auth := request.Header.Get("Authorization") + if len(auth) == 0 { + //log.Printf("[AUDIT] No Authorization header set. Env: %s, org: %s", orgId, environment) + /* + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the auth header (only applicable for cloud for now)."}`))) + return + */ + } + + //log.Printf("[AUDIT] Get workflow queue for org %s, env %s, orborus label %s", orgId, environment, orborusLabel) + + ctx := shuffle.GetContext(request) + env, err := shuffle.GetEnvironment(ctx, orgId, "") + timeNow := time.Now().Unix() + if err == nil && len(env.Id) > 0 && len(env.Name) > 0 { + if time.Now().Unix() > env.Edited+60 { + env.RunningIp = request.RemoteAddr + env.Checkin = timeNow + err = shuffle.SetEnvironment(ctx, env) + if err != nil { + log.Printf("[WARNING] Failed updating environment: %s", err) + } + } + } + + //log.Printf("Found env: %#v", env) + if len(env.OrgId) > 0 { + environment = env.OrgId + } + + if request.Method == "POST" { + if rand.Intn(1) == 0 { + // Parse out body + body, err := ioutil.ReadAll(request.Body) + if err == nil { + + // Parse out CPU, memory and disk. + + var envData shuffle.OrborusStats + err = json.Unmarshal(body, &envData) + if err == nil && !envData.Swarm && !envData.Kubernetes && (envData.CPU > 0 || envData.Memory > 0 || envData.Disk > 0) { + + // Set the input in memory + envData.OrgId = orgId + envData.Environment = environment + envData.OrborusLabel = orborusLabel + envData.Timestamp = time.Now().Unix() + + if envData.CPU > 0 && envData.MaxCPU > 0 { + envData.CPUPercent = float64(envData.CPU) / float64(envData.MaxCPU) + } + + if envData.Memory > 0 && envData.MaxMemory > 0 { + envData.MemoryPercent = float64(envData.Memory) / float64(envData.MaxMemory) + } + + // Check if CPU percent constantly has stayed above X% for the last Y requests + percentageCheck := 90 + concurrentChecks := 2 + + //if int(envData.CPUPercent) > percentageCheck { + // Get cached data + percentages := []float64{} + cacheKey := fmt.Sprintf("%s_%s_percent", orgId, strings.ToLower(environment)) + + // Marshal float list into []byte + cacheData := []byte{} + cache, err := shuffle.GetCache(ctx, cacheKey) + if err == nil { + // Unmarshal into percentages + cacheData := []byte(cache.([]uint8)) + err = json.Unmarshal(cacheData, &percentages) + if err != nil { + log.Printf("[INFO] error in cache unmarshal for percentages: %s", err) + } + + if len(percentages) > concurrentChecks { + percentages = percentages[:concurrentChecks] + } + + percentages = append(percentages, envData.CPUPercent) + if len(percentages) > concurrentChecks { + //log.Printf("[INFO] Checking percentages: %v", percentages) + + // percentageCheck := 1 + sendAlert := true + for _, p := range percentages { + if int(p) < percentageCheck { + //log.Printf("[AUDIT] CPU percent is below %d: %d", percentageCheck, int(p)) + sendAlert = false + break + } + } + + if sendAlert { + log.Printf("[INFO] CPU percent has been above %d percent for the last 5 requests. Sending alert. Env: %s, org: %s", percentageCheck, environment, orgId) + + // Set notification + alert for organization + err = shuffle.CreateOrgNotification( + ctx, + fmt.Sprintf("CPU percent has been above %d percent", percentageCheck), + fmt.Sprintf("A environment %s has been using more than %d\\% CPU for the last 5 requests.", environment, percentageCheck), + fmt.Sprintf("/admin?tab=environments"), + environment, + true, + ) + + if err != nil { + log.Printf("[ERROR] error creating notification: %s", err) + } + + org, err := shuffle.GetOrg(ctx, environment) + if err == nil { + foundRecommendation := false + for _, recommendation := range org.Priorities { + if strings.Contains(recommendation.Name, "CPU") { + foundRecommendation = true + break + } + } + + if !foundRecommendation { + // Add to start of org.Priorities + org, _ = shuffle.AddPriority(*org, shuffle.Priority{ + Name: fmt.Sprintf("High CPU in environment %s", orgId), + Description: fmt.Sprintf("The environment %s has been using more than %d percent CPU. This indicates you may need to look at scaling.", orgId, percentageCheck), + Type: "scale", + Active: true, + URL: fmt.Sprintf("/admin?tab=environments"), + Severity: 1, + }, false) + + //Make last item the first item + org.Priorities = append([]shuffle.Priority{org.Priorities[len(org.Priorities)-1]}, org.Priorities[:len(org.Priorities)-1]...) + err = shuffle.SetOrg(ctx, *org, org.Id) + if err != nil { + log.Printf("[ERROR] Problem setting org: %s", err) + } + } + } + } + + if len(percentages) > 1 { + percentages = percentages[1:] + } + } + + // Marshal float list into []byte + } else { + //log.Printf("[ERROR] Failed getting cache: %s", err) + percentages = append(percentages, envData.CPUPercent) + } + + if len(percentages) > 0 { + //log.Printf("[DEBUG] Setting cache for %s: %#v", cacheKey, percentages) + cacheData, err = json.Marshal(percentages) + if err != nil { + log.Printf("[INFO] error in cache marshal: %s", err) + } + + // Add the new data + go shuffle.SetCache(ctx, cacheKey, cacheData, 5) + } + } + + //log.Printf("CPU percent: %f", envData.CPUPercent) + //log.Printf("Memory percent: %f", envData.MemoryPercent*100) + + go shuffle.SetenvStats(ctx, envData) + } + } + } + + executionRequests, err := shuffle.GetWorkflowQueue(ctx, orgId, 100) if err != nil { // Skipping as this comes up over and over //log.Printf("(2) Failed reading body for workflowqueue: %s", err) @@ -685,11 +457,48 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { return } + // Checking and updating the environment related to the first execution if len(executionRequests.Data) == 0 { executionRequests.Data = []shuffle.ExecutionRequest{} } else { - //log.Printf("[INFO] Executionrequests (%s): %d", id, len(executionRequests.Data)) - //log.Printf("IDS: %#v", executionRequests.Data[0].ExecutionId) + //log.Printf("In workflowqueue with %d", len(executionRequests.Data)) + + // Try again :) + if len(env.Id) == 0 && len(env.Name) == 0 { + foundId := "" + for _, requestData := range executionRequests.Data { + execution, err := shuffle.GetWorkflowExecution(ctx, requestData.ExecutionId) + if err == nil { + if len(execution.ExecutionOrg) > 0 { + foundId = execution.ExecutionOrg + break + } + } + } + + if len(orgId) > 0 { + env, err := shuffle.GetEnvironment(ctx, orgId, foundId) + if err != nil { + log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", orgId, err) + //resp.WriteHeader(401) + //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No env found matching %s"}`, id))) + //return + } else { + if timeNow > env.Edited+60 { + env.RunningIp = request.RemoteAddr + env.Checkin = timeNow + err = shuffle.SetEnvironment(ctx, env) + if err != nil { + log.Printf("[WARNING] Failed updating environment: %s", err) + } + } + } + } + } + + if len(executionRequests.Data) > 50 { + executionRequests.Data = executionRequests.Data[0:49] + } } newjson, err := json.Marshal(executionRequests) @@ -704,7 +513,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -727,15 +536,15 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { err = json.Unmarshal(body, &actionResult) if err != nil { log.Printf("[WARNING] Failed ActionResult unmarshaling (stream result): %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return + //resp.WriteHeader(401) + //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + //return } ctx := context.Background() workflowExecution, err := shuffle.GetWorkflowExecution(ctx, actionResult.ExecutionId) if err != nil { - log.Printf("Failed getting execution (streamresult) %s: %s", actionResult.ExecutionId, err) + log.Printf("[WARNING][%s] Failed getting execution (streamresult): %s", actionResult.ExecutionId, err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key or execution_id might not exist."}`))) return @@ -743,10 +552,56 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { // Authorization is done here if workflowExecution.Authorization != actionResult.Authorization { - log.Printf("[WARNING] Bad authorization key when getting stream results %s.", actionResult.ExecutionId) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key or execution_id might not exist."}`))) - return + user, err := shuffle.HandleApiAuthentication(resp, request) + if err != nil { + log.Printf("[WARNING] Api authentication failed in exec grabbing workflow: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key or execution_id might not exist."}`))) + return + } + + if len(workflowExecution.ExecutionOrg) > 0 && user.ActiveOrg.Id == workflowExecution.ExecutionOrg && user.Role == "admin" { + log.Printf("[DEBUG] User %s is in correct org. Allowing org continuation for execution!", user.Username) + } else { + log.Printf("[WARNING] Bad authorization key when getting stream results %s.", actionResult.ExecutionId) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key or execution_id might not exist."}`))) + return + } + } + + for _, action := range workflowExecution.Workflow.Actions { + found := false + for _, result := range workflowExecution.Results { + if result.Action.ID == action.ID { + found = true + break + } + } + + if found { + continue + } + + //log.Printf("[DEBUG] Maybe not handled yet: %s", action.ID) + cacheId := fmt.Sprintf("%s_%s_result", workflowExecution.ExecutionId, action.ID) + cache, err := shuffle.GetCache(ctx, cacheId) + if err != nil { + //log.Printf("[WARNING] Couldn't find in fix exec %s (2): %s", cacheId, err) + continue + } + + actionResult := shuffle.ActionResult{} + cacheData := []byte(cache.([]uint8)) + + // Just ensuring the data is good + err = json.Unmarshal(cacheData, &actionResult) + if err != nil { + continue + } else { + log.Printf("[DEBUG] APPENDING %s result to send to app or something\n\n\n\n", action.ID) + workflowExecution.Results = append(workflowExecution.Results, actionResult) + } } newjson, err := json.Marshal(workflowExecution) @@ -762,7 +617,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { } func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -789,16 +644,18 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "success"}`))) return } else { - //log.Printf("[WARNING] Handling other execution variant: %s", err) + log.Printf("[DEBUG] Handling other execution variant (subflow?): %s", err) } + log.Printf("[DEBUG] Got workflow result from %s of length %d.", request.RemoteAddr, len(body)) + var actionResult shuffle.ActionResult err = json.Unmarshal(body, &actionResult) if err != nil { log.Printf("[WARNING] Failed ActionResult unmarshaling (queue): %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return + //resp.WriteHeader(401) + //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + //return } //log.Printf("Received action: %#v", actionResult) @@ -843,63 +700,71 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { } } - if actionResult.Status == "WAITING" && actionResult.Action.AppName == "User Input" { - log.Printf("[INFO] SHOULD WAIT A BIT AND RUN CLOUD STUFF WITH USER INPUT! WAITING!") + /* + // Removed as UserInput is now handled as an app + if actionResult.Status == "WAITING" && actionResult.Action.AppName == "User Input" { + log.Printf("[INFO] SHOULD WAIT A BIT AND RUN USER INPUT! WAITING!") - var trigger shuffle.Trigger - err = json.Unmarshal([]byte(actionResult.Result), &trigger) - if err != nil { - log.Printf("Failed unmarshaling actionresult for user input: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } + var trigger shuffle.Trigger + err = json.Unmarshal([]byte(actionResult.Result), &trigger) + if err != nil { + log.Printf("[WARNING] Failed unmarshaling actionresult for user input: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } - orgId := workflowExecution.ExecutionOrg - if len(workflowExecution.OrgId) == 0 && len(workflowExecution.Workflow.OrgId) > 0 { - orgId = workflowExecution.Workflow.OrgId - } + orgId := workflowExecution.ExecutionOrg + if len(workflowExecution.OrgId) == 0 && len(workflowExecution.Workflow.OrgId) > 0 { + orgId = workflowExecution.Workflow.OrgId + } - err := handleUserInput(trigger, orgId, workflowExecution.Workflow.ID, workflowExecution.ExecutionId) - if err != nil { - log.Printf("Failed userinput handler: %s", err) - actionResult.Result = fmt.Sprintf("Cloud error: %s", err) - workflowExecution.Results = append(workflowExecution.Results, actionResult) - workflowExecution.Status = "ABORTED" - err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) - if err != nil { - log.Printf("Failed to set execution during wait") - } else { - log.Printf("Successfully set the execution to waiting.") + err := handleUserInput(trigger, orgId, workflowExecution.Workflow.ID, workflowExecution.ExecutionId) + if err != nil { + log.Printf("[WARNING] Failed userinput handler: %s", err) + + actionResult.Result = fmt.Sprintf(`{"success": false, "reason": "%s"}`, err) + + workflowExecution.Results = append(workflowExecution.Results, actionResult) + workflowExecution.Status = "ABORTED" + err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) + if err != nil { + log.Printf("[WARNING] Failed to set execution during wait: %s", err) + } else { + log.Printf("[INFO] Successfully set the execution %s to waiting.", workflowExecution.ExecutionId) + } + + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error: %s"}`, err))) + return + } else { + log.Printf("[INFO] Successful userinput handler") + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "CLOUD IS DONE"}`))) + + actionResult.Result = `{"success": True, "reason": "Waiting for user feedback based on configuration"}` + + workflowExecution.Results = append(workflowExecution.Results, actionResult) + workflowExecution.Status = actionResult.Status + err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) + if err != nil { + log.Printf("[WARNING] Failed setting userinput: %s", err) + } else { + log.Printf("[DEBUG] Successfully set the execution to waiting.") + } + } + + return } - - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error: %s"}`, err))) - } else { - log.Printf("Successful userinput handler") - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "CLOUD IS DONE"}`))) - - actionResult.Result = "Waiting for user feedback based on configuration" - - workflowExecution.Results = append(workflowExecution.Results, actionResult) - workflowExecution.Status = actionResult.Status - err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) - if err != nil { - log.Printf("Failed ") - } else { - log.Printf("Successfully set the execution to waiting.") - } - } - - return - } + */ runWorkflowExecutionTransaction(ctx, 0, workflowExecution.ExecutionId, actionResult, resp) } // Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult shuffle.ActionResult, resp http.ResponseWriter) { + log.Printf("[DEBUG] Running workflow execution transaction for %s", workflowExecutionId) + // Should start a tx for the execution here workflowExecution, err := shuffle.GetWorkflowExecution(ctx, workflowExecutionId) if err != nil { @@ -912,9 +777,15 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl //log.Printf("BASE LENGTH: %d", len(workflowExecution.Results)) workflowExecution, dbSave, err := shuffle.ParsedExecutionResult(ctx, *workflowExecution, actionResult, false, 0) if err != nil { - log.Printf("[ERROR] Failed running of parsedexecution: %s", err) + b, suberr := json.Marshal(actionResult) + if suberr != nil { + log.Printf("[ERROR] Failed running of parsedexecution: %s", err) + } else { + log.Printf("[ERROR] Failed running of parsedexecution: %s. Data: %s", err, string(b)) + } + resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution"}`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed updating execution"}`))) return } @@ -948,7 +819,7 @@ func JSONCheck(str string) bool { func handleExecutionStatistics(execution shuffle.WorkflowExecution) { // FIXME: CLEAN UP THE JSON THAT'S SAVED. - // https://github.com/frikky/Shuffle/issues/172 + // https://github.com/shuffle/Shuffle/issues/172 appResults := []shuffle.AppExecutionExample{} for _, result := range execution.Results { resultCheck := JSONCheck(result.Result) @@ -1026,7 +897,7 @@ func handleExecutionStatistics(execution shuffle.WorkflowExecution) { } func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1039,6 +910,13 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { return } + if user.Role == "org-reader" { + log.Printf("[WARNING] Org-reader doesn't have access to stop schedule: %s (%s)", user.Username, user.Id) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Read only user"}`)) + return + } + location := strings.Split(request.URL.String(), "/") var fileId string @@ -1083,7 +961,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { if item.TriggerType == "SCHEDULE" && item.Status != "uninitialized" { err = deleteSchedule(ctx, item.ID) if err != nil { - log.Printf("Failed to delete schedule: %s - is it started?", err) + log.Printf("[DEBUG] Failed to delete schedule: %s - is it started?", err) } } else if item.TriggerType == "WEBHOOK" { //err = removeWebhookFunction(ctx, item.ID) @@ -1093,7 +971,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { } else if item.TriggerType == "EMAIL" { err = shuffle.HandleOutlookSubRemoval(ctx, user, workflow.ID, item.ID) if err != nil { - log.Printf("Failed to delete OUTLOOK email sub (checking gmail after): %s", err) + log.Printf("[DEBUG] Failed to delete OUTLOOK email sub (checking gmail after): %s", err) } err = shuffle.HandleGmailSubRemoval(ctx, user, workflow.ID, item.ID) @@ -1101,14 +979,8 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { log.Printf("Failed to delete gmail email sub: %s", err) } } - - //err = increaseStatisticsField(ctx, "total_workflow_triggers", workflow.ID, -1, workflow.OrgId) - //if err != nil { - // log.Printf("Failed to increase total workflows: %s", err) - //} } - // FIXME - maybe delete workflow executions err = shuffle.DeleteKey(ctx, "workflow", fileId) if err != nil { log.Printf("[DEBUG]] Failed deleting key %s", fileId) @@ -1118,11 +990,6 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { } log.Printf("[INFO] Should have deleted workflow %s (%s)", workflow.Name, fileId) - //memcacheName := fmt.Sprintf("%s_%s", user.Username, fileId) - //memcache.Delete(ctx, memcacheName) - //memcacheName = fmt.Sprintf("%s_workflows", user.Username) - //memcache.Delete(ctx, memcacheName) - //cacheKey := fmt.Sprintf("%s_workflows", user.Id) cacheKey := fmt.Sprintf("%s_workflows", user.Id) shuffle.DeleteCache(ctx, cacheKey) log.Printf("[DEBUG] Cleared workflow cache for %s (%s)", user.Username, user.Id) @@ -1168,9 +1035,7 @@ func getWorkflowLocal(fileId string, request *http.Request) ([]byte, error) { return body, nil } -//// New execution with firestore - -func handleExecution(id string, workflow shuffle.Workflow, request *http.Request) (shuffle.WorkflowExecution, string, error) { +func handleExecution(id string, workflow shuffle.Workflow, request *http.Request, orgId string) (shuffle.WorkflowExecution, string, error) { //go func() { // log.Printf("\n\nPRE TIME: %s\n\n", time.Now().Format("2006-01-02 15:04:05")) // _ = <-time.After(time.Second * 60) @@ -1181,17 +1046,23 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request if workflow.ID == "" || workflow.ID != id { tmpworkflow, err := shuffle.GetWorkflow(ctx, id) if err != nil { - log.Printf("[WARNING] Failed getting the workflow locally (execution setup): %s", err) + //log.Printf("[WARNING] Failed getting the workflow locally (execution setup): %s", err) return shuffle.WorkflowExecution{}, "Failed getting workflow", err } workflow = *tmpworkflow } - if len(workflow.ExecutingOrg.Id) == 0 { - log.Printf("[INFO] Stopped execution because there is no executing org for workflow %s", workflow.ID) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow has no executing org defined"), errors.New("Workflow has no executing org defined") - } + /* + if len(workflow.ExecutingOrg.Id) == 0 { + if len(orgId) > 0 { + workflow.ExecutingOrg.Id = orgId + } else { + log.Printf("[INFO] Stopped execution because there is no executing org for workflow %s", workflow.ID) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow has no executing org defined"), errors.New("Workflow has no executing org defined") + } + } + */ if len(workflow.Actions) == 0 { workflow.Actions = []shuffle.Action{} @@ -1233,17 +1104,20 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request return shuffle.WorkflowExecution{}, fmt.Sprintf(`workflow %s is invalid`, workflow.ID), errors.New("Failed getting workflow") } - workflowBytes, err := json.Marshal(workflow) + workflowExecution, execInfo, _, err := shuffle.PrepareWorkflowExecution(ctx, workflow, request, 10) if err != nil { - log.Printf("Failed workflow unmarshal in execution: %s", err) - return shuffle.WorkflowExecution{}, "", err + if strings.Contains(fmt.Sprintf("%s", err), "User Input") { + // Special for user input callbacks + return workflowExecution, fmt.Sprintf("%s", err), nil + } else { + log.Printf("[WARNING] Failed in prepareExecution: %s", err) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed starting workflow: %s", err), err + } } - //log.Println(workflow) - var workflowExecution shuffle.WorkflowExecution - err = json.Unmarshal(workflowBytes, &workflowExecution.Workflow) + err = imageCheckBuilder(execInfo.ImageNames) if err != nil { - log.Printf("Failed execution unmarshaling: %s", err) + log.Printf("[ERROR] Failed building the required images from %#v: %s", execInfo.ImageNames, err) return shuffle.WorkflowExecution{}, "Failed unmarshal during execution", err } @@ -1919,39 +1793,26 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request return shuffle.WorkflowExecution{}, "Failed building missing Docker images", err } - //b, err := json.Marshal(workflowExecution) - //if err == nil { - // log.Printf("LEN: %d", len(string(b))) - // //workflowExecution.ExecutionOrg.SyncFeatures = Org{} - //} - - workflowExecution.Workflow.ExecutingOrg = shuffle.OrgMini{ - Id: workflowExecution.Workflow.ExecutingOrg.Id, - } - workflowExecution.Workflow.Org = []shuffle.OrgMini{ - workflowExecution.Workflow.ExecutingOrg, - } - //Org []Org `json:"org,omitempty" datastore:"org"` err = shuffle.SetWorkflowExecution(ctx, workflowExecution, true) if err != nil { - log.Printf("[WARNING] Error saving workflow execution for updates %s: %s", topic, err) + log.Printf("[WARNING] Error saving workflow execution for updates %s", err) return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed setting workflowexecution: %s", err), err } // Adds queue for onprem execution // FIXME - add specifics to executionRequest, e.g. specific environment (can run multi onprem) - if onpremExecution { + if execInfo.OnpremExecution { // FIXME - tmp name based on future companyname-companyId // This leads to issues with overlaps. Should set limits and such instead - for _, environment := range environments { + for _, environment := range execInfo.Environments { log.Printf("[INFO] Execution: %s should execute onprem with execution environment \"%s\". Workflow: %s", workflowExecution.ExecutionId, environment, workflowExecution.Workflow.ID) executionRequest := shuffle.ExecutionRequest{ ExecutionId: workflowExecution.ExecutionId, WorkflowId: workflowExecution.Workflow.ID, Authorization: workflowExecution.Authorization, - Environments: environments, + Environments: execInfo.Environments, } //executionRequestWrapper, err := getWorkflowQueue(ctx, environment) @@ -1964,6 +1825,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request //} //log.Printf("Execution request: %#v", executionRequest) + executionRequest.Priority = workflowExecution.Priority err = shuffle.SetWorkflowQueue(ctx, executionRequest, environment) if err != nil { log.Printf("[ERROR] Failed adding execution to db: %s", err) @@ -1972,7 +1834,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request } // Verifies and runs cloud executions - if cloudExec { + if execInfo.CloudExec { featuresList, err := handleVerifyCloudsync(workflowExecution.ExecutionOrg) if !featuresList.Workflows.Active || err != nil { log.Printf("Error: %s", err) @@ -2074,17 +1936,21 @@ func cloudExecuteAction(execution shuffle.WorkflowExecution) error { return nil } +// 1. Check CORS +// 2. Check authentication +// 3. Check authorization +// 4. Run the actual function func executeWorkflow(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } - user, err := shuffle.HandleApiAuthentication(resp, request) - if err != nil { - log.Printf("[INFO] Api authentication failed in execute workflow: %s", err) + user, userErr := shuffle.HandleApiAuthentication(resp, request) + if user.Role == "org-reader" { + log.Printf("[WARNING] Org-reader doesn't have access to run workflow: %s (%s)", user.Username, user.Id) resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) + resp.Write([]byte(`{"success": false, "reason": "Read only user"}`)) return } @@ -2099,6 +1965,9 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { } fileId = location[4] + if strings.Contains(fileId, "?") { + fileId = strings.Split(fileId, "?")[0] + } } if len(fileId) != 36 { @@ -2107,45 +1976,76 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { return } - //memcacheName := fmt.Sprintf("%s_%s", user.Username, fileId) + log.Printf("[INFO] Inside execute workflow for ID %s", fileId) + ctx := context.Background() workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil && workflow.ID == "" { log.Printf("[WARNING] Failed getting the workflow locally (execute workflow): %s", err) resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflow with ID %s doesn't exist."}`, fileId))) return } - if user.Id != workflow.Owner && user.Role != "scheduler" && user.Role != fmt.Sprintf("workflow_%s", fileId) { - if workflow.OrgId == user.ActiveOrg.Id && user.Role == "admin" { - log.Printf("[AUDIT] Letting user %s execute %s because they're admin of the same org", user.Username, workflow.ID) - } else { - log.Printf("[AUDIT] Wrong user (%s) for workflow %s (execute)", user.Username, workflow.ID) - resp.WriteHeader(401) + executionAuthValid := false + newOrgId := "" + if userErr != nil { + // Check if the execution data has correct info in it! Happens based on subflows. + // 1. Parent workflow contains this workflow ID in the source trigger? + // 2. Parent workflow's owner is same org? + // 3. Parent execution auth is correct + log.Printf("[INFO] Inside execute workflow access validation!") + + executionAuthValid, newOrgId = shuffle.RunExecuteAccessValidation(request, workflow) + if !executionAuthValid { + log.Printf("[INFO] Api authorization failed in execute workflow: %s", userErr) + resp.WriteHeader(403) resp.Write([]byte(`{"success": false}`)) return + } else { + log.Printf("[DEBUG] Execution of %s successfully validated and started based on subflow or user input execution", workflow.ID) + user.ActiveOrg = shuffle.OrgMini{ + Id: newOrgId, + } } } - log.Printf("[INFO] Starting execution of %s!", fileId) + if !executionAuthValid { + if user.Id != workflow.Owner && user.Role != "scheduler" && user.Role != fmt.Sprintf("workflow_%s", fileId) { + if workflow.OrgId == user.ActiveOrg.Id && user.Role == "admin" { + log.Printf("[AUDIT] Letting user %s execute %s because they're admin of the same org", user.Username, workflow.ID) + } else { + log.Printf("[AUDIT] Wrong user (%s) for workflow %s (execute)", user.Username, workflow.ID) + resp.WriteHeader(403) + resp.Write([]byte(`{"success": false}`)) + return + } + } + } + + log.Printf("[AUDIT] Starting execution of workflow '%s' by user %s (%s)!", fileId, user.Username, user.Id) user.ActiveOrg.Users = []shuffle.UserMini{} workflow.ExecutingOrg = user.ActiveOrg - workflowExecution, executionResp, err := handleExecution(fileId, *workflow, request) - + workflowExecution, executionResp, err := handleExecution(fileId, *workflow, request, user.ActiveOrg.Id) if err == nil { + if strings.Contains(executionResp, "User Input:") { + resp.WriteHeader(400) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, executionResp))) + return + } + resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization))) return } resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, executionResp))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "execution_id": "%s", "authorization": "%s", "reason": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization, executionResp))) } func stopSchedule(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -2158,6 +2058,13 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { return } + if user.Role == "org-reader" { + log.Printf("[WARNING] Org-reader doesn't have access to stop schedule: %s (%s)", user.Username, user.Id) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Read only user"}`)) + return + } + location := strings.Split(request.URL.String(), "/") var fileId string @@ -2288,7 +2195,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { } func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -2377,15 +2284,14 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { } func deleteSchedule(ctx context.Context, id string) error { - log.Printf("Should stop schedule %s!", id) + log.Printf("[DEBUG] Should stop schedule %s!", id) err := shuffle.DeleteKey(ctx, "schedules", id) if err != nil { - log.Printf("Failed to delete schedule: %s", err) + log.Printf("[ERROR] Failed to delete schedule: %s", err) return err } else { if value, exists := scheduledJobs[id]; exists { - log.Printf("STOPPING THIS SCHEDULE: %s", id) - // Looks like this does the trick? Hurr + // Stops the schedule properly value.Lock() } else { // FIXME - allow it to kind of stop anyway? @@ -2397,7 +2303,7 @@ func deleteSchedule(ctx context.Context, id string) error { } func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -2410,6 +2316,13 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { return } + if user.Role == "org-reader" { + log.Printf("[WARNING] Org-reader doesn't have access to schedule workflow: %s (%s)", user.Username, user.Id) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Read only user"}`)) + return + } + location := strings.Split(request.URL.String(), "/") var fileId string @@ -2481,14 +2394,19 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { // Finds the startnode for the specific schedule startNode := "" - for _, branch := range workflow.Branches { - if branch.SourceID == schedule.Id { - startNode = branch.DestinationID - } - } + if schedule.Start != "" { + startNode = schedule.Start + } else { - if startNode == "" { - startNode = workflow.Start + for _, branch := range workflow.Branches { + if branch.SourceID == schedule.Id { + startNode = branch.DestinationID + } + } + + if startNode == "" { + startNode = workflow.Start + } } //log.Printf("Startnode: %s", startNode) @@ -2577,15 +2495,15 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("Action: %#v", action) + //log.Printf("Starting Cloud schedule Action: %#v", action) err = executeCloudAction(action, org.SyncConfig.Apikey) if err != nil { - log.Printf("Failed cloud action START schedule: %s", err) + log.Printf("[WARNING] Failed cloud action START schedule: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } else { - log.Printf("Successfully set up cloud action schedule") + log.Printf("[INFO] Successfully set up cloud action schedule") resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Done"}`))) return @@ -2641,7 +2559,7 @@ func setExampleresult(ctx context.Context, result shuffle.AppExecutionExample) e } func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -2684,7 +2602,7 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { // Double unmarshal because of user apps newbody, err := json.Marshal(newapps) if err != nil { - log.Printf("Failed unmarshalling all newapps: %s", err) + log.Printf("[ERROR] Failed unmarshalling all newapps: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflow apps"}`))) return @@ -2714,7 +2632,7 @@ func handleGetfile(resp http.ResponseWriter, request *http.Request) ([]byte, err // Basically a search for apps that aren't activated yet func getSpecificApps(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -2789,14 +2707,14 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) { } func validateAppInput(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } // Just need to be logged in // FIXME - should have some permissions? - _, err := shuffle.HandleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new app: %s", err) resp.WriteHeader(401) @@ -2804,6 +2722,13 @@ func validateAppInput(resp http.ResponseWriter, request *http.Request) { return } + if user.Role == "org-reader" { + log.Printf("[WARNING] Org-reader doesn't have access to delete apps: %s (%s)", user.Username, user.Id) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Read only user"}`)) + return + } + filebytes, err := handleGetfile(resp, request) if err != nil { resp.WriteHeader(401) @@ -2866,7 +2791,7 @@ func loadGithubWorkflows(url, username, password, userId, branch, orgId string) storer := memory.NewStorage() r, err := git.Clone(storer, fs, cloneOptions) if err != nil { - log.Printf("Failed loading repo %s into memory (github workflows): %s", url, err) + log.Printf("[INFO] Failed loading repo %s into memory (github workflows): %s", url, err) return err } @@ -2903,7 +2828,7 @@ func loadGithubWorkflows(url, username, password, userId, branch, orgId string) } func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -2965,7 +2890,7 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { } func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -3026,8 +2951,9 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000, 0) appCounter := 0 if err != nil { - log.Printf("Failed to get existing generated apps") + log.Printf("[WARNING] Failed to get existing generated apps for OpenAPI verification: %s", err) } + for _, file := range dir { if len(onlyname) > 0 && file.Name() != onlyname { continue @@ -3136,7 +3062,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, if !found { err = shuffle.SetWorkflowAppDatastore(ctx, api, api.ID) if err != nil { - log.Printf("[WARNING] Failed setting workflowapp in loop: %s", err) + log.Printf("[WARNING] Failed setting workflowapp %s (%s) in loop: %s", api.Name, api.ID, err) continue } else { appCounter += 1 @@ -3166,7 +3092,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, } if appCounter > 0 { - log.Printf("Preloaded %d OpenApi apps in folder %s!", appCounter, extra) + //log.Printf("Preloaded %d OpenApi apps in folder %s!", appCounter, extra) } return nil @@ -3283,14 +3209,13 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra } func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } // Just need to be logged in - // FIXME - should have some permissions? - _, err := shuffle.HandleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new app: %s", err) resp.WriteHeader(401) @@ -3298,6 +3223,13 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { return } + if user.Role == "org-reader" { + log.Printf("[WARNING] Org-reader doesn't have access to set new workflowapp: %s (%s)", user.Username, user.Id) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Read only user"}`)) + return + } + body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("Error with body read: %s", err) @@ -3359,7 +3291,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { // Might require reflection into the python code to append the fields as well for index, action := range workflowapp.Actions { if action.AuthNotRequired { - log.Printf("Skipping auth setup: %s", action.Name) + log.Printf("[WARNING] Skipping auth setup for: %s", action.Name) continue } @@ -3407,7 +3339,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { err = shuffle.SetWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err != nil { - log.Printf("Failed setting workflowapp: %s", err) + log.Printf("[WARNING] Failed setting workflowapp: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -3430,6 +3362,7 @@ func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId // E.g. check email sms := "" email := "" + subflow := "" triggerType := "" triggerInformation := "" for _, item := range trigger.Parameters { @@ -3441,12 +3374,15 @@ func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId email = item.Value } else if item.Name == "sms" { sms = item.Value + } else if item.Name == "subflow" { + subflow = item.Value } } + _ = subflow if len(triggerType) == 0 { - log.Printf("No type specified for user input node") - return errors.New("No type specified for user input node") + log.Printf("[WARNING] No type specified for user input node") + //return errors.New("No type specified for user input node") } // FIXME: This is not the right time to send them, BUT it's well served for testing. Save -> send email / sms @@ -3478,6 +3414,7 @@ func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId log.Printf("[INFO] Should send email to %s during execution.", email) } + if strings.Contains(triggerType, "sms") { action := shuffle.CloudSyncJob{ Type: "user_input", @@ -3502,7 +3439,11 @@ func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId return err } - log.Printf("Should send SMS to %s during execution.", sms) + log.Printf("[DEBUG] Should send SMS to %s during execution.", sms) + } + + if strings.Contains(triggerType, "subflow") { + log.Printf("[DEBUG] Should run a subflow with the result for user input.") } return nil @@ -3522,6 +3463,13 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { return } + if user.Role == "org-reader" { + log.Printf("[WARNING] Org-reader doesn't have access to execute single action: %s (%s)", user.Username, user.Id) + resp.WriteHeader(403) + resp.Write([]byte(`{"success": false, "reason": "Read only user"}`)) + return + } + location := strings.Split(request.URL.String(), "/") var fileId string if location[1] == "api" { @@ -3551,6 +3499,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { return } + workflowExecution.Priority = 10 environments, err := shuffle.GetEnvironments(ctx, user.ActiveOrg.Id) environment := "Shuffle" if len(environments) >= 1 { @@ -3562,7 +3511,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("[INFO] Execution: %s should execute onprem with execution environment \"%s\". Workflow: %s", workflowExecution.ExecutionId, environment, workflowExecution.Workflow.ID) + log.Printf("[INFO] Execution (single action): %s should execute onprem with execution environment \"%s\". Workflow: %s", workflowExecution.ExecutionId, environment, workflowExecution.Workflow.ID) executionRequest := shuffle.ExecutionRequest{ ExecutionId: workflowExecution.ExecutionId, @@ -3571,6 +3520,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { Environments: []string{environment}, } + executionRequest.Priority = workflowExecution.Priority err = shuffle.SetWorkflowQueue(ctx, executionRequest, environment) if err != nil { log.Printf("[ERROR] Failed adding execution to db: %s", err) @@ -3618,6 +3568,11 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. // Folder? switch mode := file.Mode(); { case mode.IsDir(): + // Specific folder for skipping + if file.Name() == "unsupported" { + continue + } + tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name()) dir, err := fs.ReadDir(tmpExtra) if err != nil { @@ -3641,7 +3596,8 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. //buildFirst, buildLast, err := IterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate) if !forceUpdate { - return buildLaterFirst, buildLaterList, err + continue + //return buildLaterFirst, buildLaterList, err } } @@ -3659,7 +3615,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. fullPath = fmt.Sprintf("%s%s", extra, "api.yml") fileReader, err = fs.Open(fullPath) if err != nil { - log.Printf("Failed finding api.yaml/yml: %s", err) + log.Printf("[INFO] Failed finding api.yaml/yml for file %s: %s", filename, err) continue } } @@ -3715,7 +3671,8 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. err = gyaml.Unmarshal(appfileData, &workflowapp) if err != nil { log.Printf("[WARNING] Failed building workflowapp %s: %s", extra, err) - return buildLaterFirst, buildLaterList, errors.New(fmt.Sprintf("Failed building %s: %s", extra, err)) + continue + //return buildLaterFirst, buildLaterList, errors.New(fmt.Sprintf("Failed building %s: %s", extra, err)) //continue } @@ -3763,7 +3720,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. } } - workflowapp.ReferenceInfo.GithubUrl = fmt.Sprintf("https://github.com/frikky/shuffle-apps/tree/master/%s/%s", strings.ToLower(newName), workflowapp.AppVersion) + workflowapp.ReferenceInfo.GithubUrl = fmt.Sprintf("https://github.com/shuffle/shuffle-apps/tree/master/%s/%s", strings.ToLower(newName), workflowapp.AppVersion) tags := []string{ fmt.Sprintf("%s:%s_%s", baseDockerName, strings.ToLower(newName), workflowapp.AppVersion), @@ -3930,12 +3887,52 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. cacheKey = fmt.Sprintf("workflowapps-sorted-1000") shuffle.DeleteCache(ctx, cacheKey) + newSortedList := []shuffle.BuildLaterStruct{} + initApps := []string{ + "tools", + "http", + "email", + } + for _, buildLater := range buildLaterFirst { + found := false + for _, appname := range initApps { + for _, tag := range buildLater.Tags { + if strings.Contains(strings.ToLower(tag), appname) { + newSortedList = append(newSortedList, buildLater) + found = true + break + } + } + + if found { + break + } + } + } + + // Prepend newSortedList to buildLaterFirst + buildLaterFirst = append(newSortedList, buildLaterFirst...) + if len(extra) == 0 { log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst)) for _, item := range buildLaterFirst { err = buildImageMemory(fs, item.Tags, item.Extra, true) if err != nil { - log.Printf("Failed image build memory: %s", err) + orgId := "" + + log.Printf("[DEBUG] Failed image build memory. Creating notification with org %#v: %s", orgId, err) + + if len(item.Tags) > 0 { + err = shuffle.CreateOrgNotification( + ctx, + fmt.Sprintf("App failed to build"), + fmt.Sprintf("The app %s with image %s failed to build. Check backend logs with docker! docker logs shuffle-backend", item.Tags[0], item.Extra), + fmt.Sprintf("/apps"), + orgId, + false, + ) + } + } else { if len(item.Tags) > 0 { log.Printf("[INFO] Successfully built image %s", item.Tags[0]) @@ -3981,6 +3978,13 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { return } + if user.Role != "admin" { + log.Printf("[WARNING] Not admin during app loading: %s (%s).", user.Username, user.Id) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Not admin"}`)) + return + } + body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("Error with body read: %s", err) @@ -4004,8 +4008,8 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { var tmpBody tmpStruct err = json.Unmarshal(body, &tmpBody) if err != nil { - log.Printf("Error with unmarshal tmpBody: %s", err) - resp.WriteHeader(401) + log.Printf("[WARNING] Error with unmarshal app git clone: %s", err) + resp.WriteHeader(500) resp.Write([]byte(`{"success": false}`)) return } @@ -4032,20 +4036,20 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { storer := memory.NewStorage() r, err := git.Clone(storer, fs, cloneOptions) if err != nil { - log.Printf("Failed loading repo %s into memory (github workflows 2): %s", tmpBody.URL, err) - resp.WriteHeader(401) + log.Printf("[WARNING] Failed loading repo %s into memory (github apps 2): %s", tmpBody.URL, err) + resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } dir, err := fs.ReadDir("/") if err != nil { - log.Printf("FAiled reading folder: %s", err) + log.Printf("[WARNING] FAiled reading folder: %s", err) } _ = r if tmpBody.ForceUpdate { - log.Printf("[AUDIT] Running with force update from user %s (%s) for %s!", user.Username, user.Id, tmpBody.URL) + log.Printf("[AUDIT] Running app get with force update from user %s (%s) for %s!", user.Username, user.Id, tmpBody.URL) } else { log.Printf("[AUDIT] Updating apps with updates for user %s (%s) for %s (no force)", user.Username, user.Id, tmpBody.URL) } @@ -4054,9 +4058,19 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { if tmpBody.ForceUpdate { dockercli, err := dockerclient.NewEnvClient() if err == nil { - _, err := dockercli.ImagePull(ctx, "frikky/shuffle:app_sdk", types.ImagePullOptions{}) - if err != nil { - log.Printf("[WARNING] Failed to download apps with the new App SDK: %s", err) + + appSdk := os.Getenv("SHUFFLE_APP_SDK_VERSION") + if len(appSdk) == 0 { + _, err := dockercli.ImagePull(ctx, "frikky/shuffle:app_sdk", types.ImagePullOptions{}) + if err != nil { + log.Printf("[WARNING] Failed to download new App SDK: %s", err) + } + } else { + _, err := dockercli.ImagePull(ctx, fmt.Sprintf("%s/%s/shuffle-app_sdk:%s", "ghcr.io", "frikky", appSdk), types.ImagePullOptions{}) + if err != nil { + log.Printf("[WARNING] Failed to download new App SDK %s: %s", err) + } + } } else { log.Printf("[WARNING] Failed to download apps with the new App SDK because of docker cli: %s", err) diff --git a/backend/tests/files.sh b/backend/tests/files.sh index 64417941..5a699822 100755 --- a/backend/tests/files.sh +++ b/backend/tests/files.sh @@ -3,7 +3,7 @@ #curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -d '{"filename": "file.txt", "org_id": "b199646b-16d2-456d-9fd6-b9972e929466", "workflow_id": "global"}' # #echo -#curl http://localhost:5001/api/v1/files/e19cffe4-e2da-47e9-809e-904f5cb03687/upload -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -F 'shuffle_file=@files.sh' +#curl http://localhost:5001/api/v1/apps/upload -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -F 'shuffle_file=@files.sh' # #curl http://localhost:5001/api/v1/files/1915981b-b897-4db1-8a2e-44bc34cead3b/content -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" #curl http://localhost:5001/api/v1/files/e19cffe4-e2da-47e9-809e-904f5cb03687 -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" @@ -16,7 +16,7 @@ #r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS") -#curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" -d '{"filename": "rule2.yar", "org_id": "b4e88fe9-352b-47b4-b280-960181670acf", "workflow_id": "global", "namespace": "yara"}' -#curl http://localhost:5001/api/v1/files/5cb941ad-fa1c-4444-a685-92024b1fa31c/upload -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" -F 'shuffle_file=@upload.sh' +curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer 317f5066-395c-414d-aa3d-479cf27f47dd" -d '{"filename": "rule2.yar", "org_id": "292c7e25-40ad-4f05-904f-77d3c7b735e6", "workflow_id": "global", "namespace": "yara"}' +curl http://localhost:5001/api/v1/files/file_eb89e315-eb66-4d76-9df7-530fb003fc84/upload -H "Authorization: Bearer 317f5066-395c-414d-aa3d-479cf27f47dd" -F 'shuffle_file=@upload.sh' -curl http://localhost:5001/api/v1/files/namespaces/yara -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" --output rules.zip +#curl http://localhost:5001/api/v1/files/namespaces/yara -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" --output rules.zip diff --git a/backend/tests/hooks.sh b/backend/tests/hooks.sh index 738111af..42e0ede0 100755 --- a/backend/tests/hooks.sh +++ b/backend/tests/hooks.sh @@ -15,9 +15,9 @@ #curl -X POST "https://europe-west1-shuffle-241517.cloudfunctions.net/webhook_982995716e67c3a549092d3a3a7921cd" -H "Content-Type:application/json" -H "Authorization: Bearer 144308d0-6aab-4d4f-8bb2-75189281ee26" --data '{"name":"Keyboard Cat"}' -v ## GET HOOK -#curl http://localhost:5001/api/v1/hooks/b4ba07c9-45d4-41f2-b260-83c8e99eba0c -H "Authorization: Bearer 144308d0-6aab-4d4f-8bb2-75189281ee26" +#curl http://localhost:5001/api/v1/hooks/b4ba07c9-45d4-41f2-b260-83c8e99eba0c -H "Authorization: Bearer " -#curl https://shuffler.io/api/v1/hooks/b4ba07c9-45d4-41f2-b260-83c8e99eba0c -H "Authorization: Bearer 144308d0-6aab-4d4f-8bb2-75189281ee26" +#curl https://shuffler.io/api/v1/hooks/b4ba07c9-45d4-41f2-b260-83c8e99eba0c -H "Authorization: Bearer " #curl -X POST "https://europe-west1-shuffle-241517.cloudfunctions.net/webhook_3ceff795-ce9a-43a2-a2f5-d4401a6e772d" -H "Authorization: Bearer 144308d0-6aab-4d4f-8bb2-75189281ee26" --data 'wut' diff --git a/backend/tests/stop_execution.sh b/backend/tests/stop_execution.sh new file mode 100644 index 00000000..445f8eb4 --- /dev/null +++ b/backend/tests/stop_execution.sh @@ -0,0 +1 @@ +curl "http://localhost:5001/api/v1/environments/Shuffle/stop" -H "Authorization: Bearer e663cf93-7f10-4560-bef0-303f14aad982" diff --git a/backend/tests/upload.sh b/backend/tests/upload.sh new file mode 100644 index 00000000..c14c13ae --- /dev/null +++ b/backend/tests/upload.sh @@ -0,0 +1,4 @@ +# hello +this is line 2 +and 3 +Is it a python problem? diff --git a/docker-compose.yml b/docker-compose.yml index 3cb35784..04d07033 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,7 @@ version: '3' services: frontend: - #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:latest + image: ghcr.io/shuffle/shuffle-frontend:latest container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -16,8 +15,7 @@ services: depends_on: - backend backend: - #build: ./backend - image: ghcr.io/frikky/shuffle-backend:latest + image: ghcr.io/shuffle/shuffle-backend:latest container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -27,20 +25,16 @@ services: - shuffle volumes: - /var/run/docker.sock:/var/run/docker.sock - - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps - - ${SHUFFLE_FILE_LOCATION}:/shuffle-files - #- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate + - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z + - ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z env_file: .env environment: + #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - SHUFFLE_FILE_LOCATION=/shuffle-files restart: unless-stopped - #depends_on: - #- opensearch #Not necessary because dependancy is handled within the backend itself instead - #- database orborus: - #build: ./functions/onprem/orborus - image: ghcr.io/frikky/shuffle-orborus:latest + image: ghcr.io/shuffle/shuffle-orborus:latest container_name: shuffle-orborus hostname: shuffle-orborus networks: @@ -48,11 +42,9 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock environment: - - SHUFFLE_APP_SDK_VERSION=latest - - SHUFFLE_WORKER_VERSION=latest - - ORG_ID=${ORG_ID} + #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} + - BASE_URL=http://${OUTER_HOSTNAME}:5001 - DOCKER_API_VERSION=1.40 - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} @@ -61,10 +53,9 @@ services: - HTTPS_PROXY=${HTTPS_PROXY} - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY} - - SHUFFLE_ORBORUS_EXECUTION_TIMEOUT=600 - - SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=5 - - CLEANUP=${SHUFFLE_CONTAINER_AUTO_CLEANUP} restart: unless-stopped + security_opt: + - seccomp:unconfined opensearch: image: opensearchproject/opensearch:2.5.0 hostname: shuffle-opensearch @@ -72,13 +63,12 @@ services: environment: - bootstrap.memory_lock=true - "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - - plugins.security.disabled=true + - cluster.initial_master_nodes=shuffle-opensearch - cluster.routing.allocation.disk.threshold_enabled=false - cluster.name=shuffle-cluster - node.name=shuffle-opensearch - - discovery.seed_hosts=shuffle-opensearch - - cluster.initial_master_nodes=shuffle-opensearch - node.store.allow_mmap=false + - discovery.seed_hosts=shuffle-opensearch ulimits: memlock: soft: -1 @@ -87,12 +77,46 @@ services: soft: 65536 hard: 65536 volumes: - - ${DB_LOCATION}:/usr/share/opensearch/data:rw + - ${DB_LOCATION}:/usr/share/opensearch/data:z ports: - 9200:9200 networks: - shuffle restart: unless-stopped + #docker-socket-proxy: + # image: tecnativa/docker-socket-proxy + # container_name: shuffle-frontend + # hostname: docker-socket-proxy + # privileged: true + # environment: + # - SERVICES=1 + # - TASKS=1 + # - NETWORKS=1 + # - NODES=1 + # - BUILD=1 + # - IMAGES=1 + # - GRPC=1 + # - CONTAINERS=1 + # - PLUGINS=1 + # - SYSTEM=1 + # - VOLUMES=1 + # - INFO=1 + # - DISTRIBUTION=1 + # - POST=1 + # - AUTH=1 + # - SECRETS=1 + # - SWARM=1 + # volumes: + # - /var/run/docker.sock:/var/run/docker.sock + # networks: + # - shuffle networks: shuffle: driver: bridge + + # uncomment to set MTU for swarm mode. + # MTU should be whatever is your host's preferred MTU is. + # Refer to this doc to figure out what your host's MTU is: + # https://shuffler.io/docs/troubleshooting#TLS_timeout_error/Timeout_Errors/EOF_Errors + # driver_opts: + # com.docker.network.driver.mtu: 1460 diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 12ba0b27..b2d4f2ce 100755 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -8,7 +8,8 @@ ENV PATH /usr/src/app/node_modules/.bin:$PATH COPY package.json /usr/src/app/package.json -RUN yarn install +RUN yarn config set "strict-ssl" false -g +RUN yarn install --network-timeout 1000000 # copy only required files to not trigger rebuilding every time COPY ./certs /usr/src/app/certs/ @@ -17,10 +18,11 @@ COPY ./src /usr/src/app/src/ COPY ./*.sh /usr/src/app/ COPY ./*.json /usr/src/app/ +RUN rm -rf /usr/src/app/node_modules/webpack RUN yarn build # Production environment -FROM nginx:1.21.3 +FROM nginx:1.21.5 RUN mkdir -p /usr/share/nginx/html/build RUN mkdir -p /usr/share/nginx/html/css diff --git a/frontend/README.md b/frontend/README.md index 3bcf7832..fe883a45 100755 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,4 +1,5 @@ -# Certificate: +## Localhost Certificate info: + Creating a localhost certificate: ``` @@ -6,3 +7,14 @@ openssl genrsa -out privkey.pem 2048 openssl req -new -key privkey.pem -out certreq.csr openssl x509 -req -days 3650 -in certreq.csr -signkey privkey.pem -out fullchain.pem ``` + +## Using your own certificate +If you have your own .crt and .key file, you can do it like this: +``` +openssl x509 -in mycert.crt -out fullchain.cert.pem -outform PEM +``` + +The KEY file has to be named privkey.pem +``` +mv cert.key privkey.pem +``` diff --git a/frontend/package.json b/frontend/package.json index 14744ca8..7e25c17d 100755 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,21 +1,30 @@ { "name": "shuffler", "homepage": "https://shuffler.io", - "version": "0.9.24", + "version": "1.2.0", "private": true, "dependencies": { + "@codemirror/commands": "^6.2.2", + "@emotion/is-prop-valid": "^1.1.1", + "@emotion/react": "^11.7.0", + "@emotion/styled": "^11.6.0", + "@lezer/highlight": "^1.1.3", "@material-ui/core": "^4.5.2", - "@material-ui/data-grid": "^4.0.0-alpha.22", - "@material-ui/icons": "^4.11.2", + "@material-ui/icons": "^4.5.1", "@material-ui/lab": "^4.0.0-alpha.58", "@material-ui/styles": "^4.5.2", "@material-ui/utils": "^4.11.2", + "@metamask/detect-provider": "^1.2.0", + "@mui/icons-material": "^5.2.1", + "@mui/material": "^5.2.3", + "@mui/x-data-grid": "^5.17.11", + "@uiw/codemirror-themes": "^4.19.9", "@uiw/react-codemirror": "^3.2.1", "@use-it/interval": "^1.0.0", - "babel-eslint": "^10.1.0", - "class-transformer": "^0.3.1", - "create-react-app": "^2.0.3", - "cytoscape": "^3.11.0", + "algoliasearch": "^4.13.1", + "class-transformer": "^0.4.0", + "create-react-app": "^4.0.3", + "cytoscape": "^3.15.1", "cytoscape-clipboard": "^2.2.1", "cytoscape-cxtmenu": "^3.1.1", "cytoscape-edgehandles": "^3.6.0", @@ -39,6 +48,7 @@ "react": "^16.14.0", "react-alert": "^5.5.0", "react-alert-template-basic": "^1.0.0", + "react-alice-carousel": "^2.6.4", "react-avatar-editor": "^11.1.0", "react-beforeunload": "^2.2.1", "react-chartjs-2": "^2.11.1", @@ -47,35 +57,45 @@ "react-device-detect": "^1.9.10", "react-dom": "^16.14.0", "react-draggable": "^3.3.2", + "react-driftjs": "^1.2.2", "react-dropzone": "^10.1.10", - "react-ga": "^2.7.0", + "react-ga4": "^2.0.0", "react-iframe": "^1.8.0", + "react-instantsearch-dom": "^6.28.0", "react-json-pretty": "^2.2.0", "react-json-view": "^1.19.1", "react-markdown": "^4.2.2", "react-markdown-github": "^3.3.1", "react-powerhooks": "0.0.7", - "react-router": "^4.3.1", - "react-router-dom": "^4.3.1", + "react-router": "6.2.1", + "react-router-dom": "6.2.1", "react-scripts": "^4.0.1", + "react-shepherd": "^3.3.6", "reactstrap": "^7.1.0", + "reaviz": "^12.1.0", + "search-insights": "^2.2.1", "shellwords": "^0.1.1", "simplebar": "^4.2.3", "styled-components": "^4.4.0", - "webpack": "4.44.2", - "websocket": "^1.0.30", "yaml": "^1.7.2", "yamljs": "^0.3.0", "zone.js": "~0.11.4" }, "scripts": { - "start": "react-scripts start", + "start": "HTTPS=false&&PORT=3000 react-scripts --openssl-legacy-provider start", "build": "react-scripts build", "test": "react-scripts test", - "eject": "react-scripts eject" + "eject": "react-scripts eject", + "lint": "eslint 'src/**/*.{tsx,ts,js,jsx}'", + "lint_file": "eslint 'src/views/AngularWorkflow.jsx'" }, "eslintConfig": { - "extends": "react-app" + "extends": "react-app", + "rules": { + "jsx-a11y/img-redundant-alt": "off", + "no-redeclare": "off", + "no-loop-func": "off" + } }, "browserslist": [ ">0.2%", @@ -84,6 +104,11 @@ "not op_mini all" ], "devDependencies": { + "prettier": "2.4.1", "promise-window": "^1.2.1" + "@babel/core": "^7.15.8", + "babel-eslint": "^10.1.0", + "webpack": "^4.44.2", + "@babel/plugin-proposal-private-property-in-object": "^7.21.11" } } diff --git a/frontend/public/images/Arrow.png b/frontend/public/images/Arrow.png new file mode 100644 index 00000000..56c7d85c Binary files /dev/null and b/frontend/public/images/Arrow.png differ diff --git a/frontend/public/images/Shuffle_logo_new.png b/frontend/public/images/Shuffle_logo_new.png new file mode 100644 index 00000000..5b4a18be Binary files /dev/null and b/frontend/public/images/Shuffle_logo_new.png differ diff --git a/frontend/public/images/btn_google_light_focus_ios.svg b/frontend/public/images/btn_google_light_focus_ios.svg new file mode 100644 index 00000000..1f3ee4ff --- /dev/null +++ b/frontend/public/images/btn_google_light_focus_ios.svg @@ -0,0 +1,44 @@ + + + + btn_google_light_focus_ios + Created with Sketch. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/frontend/public/images/demo1.png b/frontend/public/images/demo1.png new file mode 100644 index 00000000..4cd8faf1 Binary files /dev/null and b/frontend/public/images/demo1.png differ diff --git a/frontend/public/images/demo2.png b/frontend/public/images/demo2.png new file mode 100644 index 00000000..2d98b193 Binary files /dev/null and b/frontend/public/images/demo2.png differ diff --git a/frontend/public/images/demo3.png b/frontend/public/images/demo3.png new file mode 100644 index 00000000..cbe2ed96 Binary files /dev/null and b/frontend/public/images/demo3.png differ diff --git a/frontend/public/images/detectionframework.png b/frontend/public/images/detectionframework.png new file mode 100644 index 00000000..30da65a5 Binary files /dev/null and b/frontend/public/images/detectionframework.png differ diff --git a/frontend/public/images/experienced.png b/frontend/public/images/experienced.png new file mode 100644 index 00000000..8e5f3e5a Binary files /dev/null and b/frontend/public/images/experienced.png differ diff --git a/frontend/public/images/finalize.gif b/frontend/public/images/finalize.gif new file mode 100644 index 00000000..f2656fa6 Binary files /dev/null and b/frontend/public/images/finalize.gif differ diff --git a/frontend/public/images/logo-algolia-nebula-blue-full.svg b/frontend/public/images/logo-algolia-nebula-blue-full.svg new file mode 100644 index 00000000..886c422e --- /dev/null +++ b/frontend/public/images/logo-algolia-nebula-blue-full.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/public/images/logos/orange_logo.svg b/frontend/public/images/logos/orange_logo.svg new file mode 100644 index 00000000..1024dd6a --- /dev/null +++ b/frontend/public/images/logos/orange_logo.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/frontend/public/images/social/discord.png b/frontend/public/images/social/discord.png new file mode 100644 index 00000000..c3a1b58d Binary files /dev/null and b/frontend/public/images/social/discord.png differ diff --git a/frontend/public/images/social/shuffle_logo_round.png b/frontend/public/images/social/shuffle_logo_round.png new file mode 100644 index 00000000..61c7f660 Binary files /dev/null and b/frontend/public/images/social/shuffle_logo_round.png differ diff --git a/frontend/public/images/testing.png b/frontend/public/images/testing.png new file mode 100644 index 00000000..032c94a8 Binary files /dev/null and b/frontend/public/images/testing.png differ diff --git a/frontend/public/images/welcome-to-shuffle.png b/frontend/public/images/welcome-to-shuffle.png new file mode 100644 index 00000000..cb976e31 Binary files /dev/null and b/frontend/public/images/welcome-to-shuffle.png differ diff --git a/frontend/public/images/welcome_cog.png b/frontend/public/images/welcome_cog.png new file mode 100644 index 00000000..c3eef260 Binary files /dev/null and b/frontend/public/images/welcome_cog.png differ diff --git a/frontend/public/index.html b/frontend/public/index.html index 2ece9bc4..748b7a1a 100755 --- a/frontend/public/index.html +++ b/frontend/public/index.html @@ -1,13 +1,15 @@ - - - - - + + + + + Shuffle -
diff --git a/frontend/run.sh b/frontend/run.sh index 819c0116..0ba2beea 100755 --- a/frontend/run.sh +++ b/frontend/run.sh @@ -1,11 +1,12 @@ #!/bin/sh docker stop shuffle-frontend docker rm shuffle-frontend -docker rmi frikky/shuffle:frontend +#docker rmi ghcr.io/frikky/shuffle-frontend:nightly echo "Running build for website" #sudo npm run build -docker build . -t frikky/shuffle:frontend +docker build . -t ghcr.io/frikky/shuffle-frontend:nightly +docker tag ghcr.io/frikky/shuffle-frontend:nightly ghcr.io/shuffle/shuffle-frontend:nightly echo "Starting server" # Rerun build locally for it to update :) diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index c4ef9a57..d4e116ba 100755 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -1,173 +1,735 @@ -import React, { useState, useEffect } from 'react'; +import React, { useState, useEffect } from "react"; -import { Route } from 'react-router'; -import { BrowserRouter } from 'react-router-dom'; -import { CookiesProvider } from 'react-cookie'; -import { removeCookies, useCookies } from 'react-cookie'; +//import { Route, Routes } from "react-router"; +import { Route, Routes, BrowserRouter } from "react-router-dom"; +import { CookiesProvider } from "react-cookie"; +import { removeCookies, useCookies } from "react-cookie"; -import EditSchedule from "./views/EditSchedule"; -import Schedules from "./views/Schedules"; -import Webhooks from "./views/Webhooks"; import Workflows from "./views/Workflows"; +import GettingStarted from "./views/GettingStarted"; import EditWebhook from "./views/EditWebhook"; import AngularWorkflow from "./views/AngularWorkflow"; -import Header from './components/Header'; -import theme from './theme' -import Apps from './views/Apps'; -import AppCreator from './views/AppCreator'; +import Header from "./components/Header.jsx"; +import theme from "./theme"; +import Apps from "./views/Apps"; +import AppCreator from "./views/AppCreator"; -import Dashboard from "./views/Dashboard"; +import Welcome from "./views/Welcome.jsx"; +import Dashboard from "./views/Dashboard.jsx"; +import DashboardView from "./views/DashboardViews.jsx"; import AdminSetup from "./views/AdminSetup"; import Admin from "./views/Admin"; import Docs from "./views/Docs"; import Introduction from "./views/Introduction"; import SetAuthentication from "./views/SetAuthentication"; import SetAuthenticationSSO from "./views/SetAuthenticationSSO"; +import Search from "./views/Search.jsx"; +import RunWorkflow from "./views/RunWorkflow.jsx"; import LandingPageNew from "./views/LandingpageNew"; import LoginPage from "./views/LoginPage"; import SettingsPage from "./views/SettingsPage"; +import KeepAlive from "./views/KeepAlive.jsx"; import MyView from "./views/MyView"; -import { createMuiTheme, MuiThemeProvider } from '@material-ui/core/styles'; +import { createMuiTheme, MuiThemeProvider } from "@material-ui/core/styles"; +import FrameworkWrapper from "./views/FrameworkWrapper.jsx"; import ScrollToTop from "./components/ScrollToTop"; import AlertTemplate from "./components/AlertTemplate"; -import { positions, Provider } from "react-alert"; -import {isMobile} from "react-device-detect"; +import { useAlert, positions, Provider } from "react-alert"; +import { isMobile } from "react-device-detect"; + +import detectEthereumProvider from "@metamask/detect-provider"; +import Drift from "react-driftjs"; +import DashboardPage from "./views/TempDashboard.jsx"; // Production - backend proxy forwarding in nginx -var globalUrl = window.location.origin +var globalUrl = window.location.origin; // CORS used for testing purposes. Should only happen with specific port and http -if ( window.location.port === "3000") { - globalUrl = "http://localhost:5001" - //globalUrl = "http://localhost:5002" +if (window.location.port === "3000") { + globalUrl = "http://localhost:5001"; + //globalUrl = "http://localhost:5002" } +// Development on Github Codespaces +if (globalUrl.includes("app.github.dev")) { + //globalUrl = globalUrl.replace("3000", "5001") + globalUrl = "https://frikky-shuffle-5gvr4xx62w64-5001.preview.app.github.dev" +} +//console.log("global: ", globalUrl) + const App = (message, props) => { - const [userdata, setUserData] = useState({}); - const [notifications, setNotifications] = useState([]) - const [cookies, setCookie, removeCookie] = useCookies([]) - const [isLoggedIn, setIsLoggedIn] = useState(false); - const [dataset, setDataset] = useState(false); - const [isLoaded, setIsLoaded] = useState(false); - const [curpath, setCurpath] = useState(typeof window === 'undefined' || window.location === undefined ? "" : window.location.pathname) - useEffect(() => { - if (dataset === false) { - getUserNotifications() - checkLogin() - setDataset(true) - } - }) + const [userdata, setUserData] = useState({}); + const [notifications, setNotifications] = useState([]) + const [cookies, setCookie, removeCookie] = useCookies([]) + const [isLoggedIn, setIsLoggedIn] = useState(false) + const [dataset, setDataset] = useState(false) + const [isLoaded, setIsLoaded] = useState(false) + const [curpath, setCurpath] = useState(typeof window === "undefined" || window.location === undefined ? "" : window.location.pathname) - if (isLoaded && !isLoggedIn && (!window.location.pathname.startsWith("/login") && (!window.location.pathname.startsWith("/docs") && (!window.location.pathname.startsWith("/adminsetup"))))) { - window.location = "/login" + + useEffect(() => { + if (dataset === false) { + getUserNotifications(); + checkLogin(); + setDataset(true); + } + }, []); + + if ( + isLoaded && + !isLoggedIn && + !window.location.pathname.startsWith("/login") && + !window.location.pathname.startsWith("/docs") && + !window.location.pathname.startsWith("/support") && + !window.location.pathname.startsWith("/detectionframework") && + !window.location.pathname.startsWith("/appframework") && + !window.location.pathname.startsWith("/adminsetup") && + !window.location.pathname.startsWith("/usecases") + ) { + window.location = "/login"; + } + + const getUserNotifications = () => { + fetch(`${globalUrl}/api/v1/users/notifications`, { + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + cors: "cors", + }) + .then((response) => response.json()) + .then((responseJson) => { + if ( + responseJson.success === true && + responseJson.notifications !== null && + responseJson.notifications !== undefined && + responseJson.notifications.length > 0 + ) { + //console.log("RESP: ", responseJson) + setNotifications(responseJson.notifications); + } + }) + .catch((error) => { + console.log("Failed getting notifications for user: ", error); + }); + }; + + const checkLogin = () => { + var baseurl = globalUrl; + fetch(`${globalUrl}/api/v1/getinfo`, { + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => response.json()) + .then((responseJson) => { + var userInfo = {}; + if (responseJson.success === true) { + //console.log("USER: ", responseJson); + + userInfo = responseJson; + setIsLoggedIn(true); + //console.log("Cookies: ", cookies) + // Updating cookie every request + for (var key in responseJson["cookies"]) { + setCookie( + responseJson["cookies"][key].key, + responseJson["cookies"][key].value, + { path: "/" } + ); + } + } + + // Handling Ethereum update + {/* + detectEthereumProvider().then((provider) => { + if ( + provider && + userInfo.eth_info !== undefined && + userInfo.eth_info !== null + ) { + if ( + userInfo.eth_info.account !== undefined && + userInfo.eth_info.account !== null && + userInfo.eth_info.account.length === 0 + ) { + userInfo.eth_info = {}; + var method = "eth_requestAccounts"; + var params = []; + provider + .request({ + method: method, + params, + }) + .then((result) => { + if ( + result !== undefined && + result !== null && + result.length > 0 + ) { + userInfo.eth_info.account = result[0]; + + // Getting and setting balance for the current user + method = "eth_getBalance"; + params = [userInfo.eth_info.account, "latest"]; + provider + .request({ + method: method, + params, + }) + .then((result) => { + if ( + result !== undefined && + result !== null && + result.length > 0 + ) { + userInfo.parsed_balance = + result / 1000000000000000000; + } else { + alert.error("Couldn't find balance: ", result); + } + // The result varies by RPC method. + // For example, this method will return a transaction hash hexadecimal string on success. + }) + .catch((error) => { + // If the request fails, the Promise will reject with an error. + alert.error( + "Failed getting info from ethereum API: " + error + ); + }); + } else { + alert.error("Couldn't find any user: ", result); + } + }) + .catch((error) => { + // If the request fails, the Promise will reject with an error. + alert.error( + "Failed getting info from ethereum API: " + error + ); + }); + } + + // Register hooks here + provider.on("message", (event) => { + alert.info("Message from MetaMask: ", event); + }); + + provider.on("chainChanged", (chainId) => { + console.log("Changed chain to: ", chainId); + + method = "eth_getBalance"; + params = [userInfo.eth_info.account, "latest"]; + provider + .request({ + method: method, + params, + }) + .then((result) => { + console.log("Got result: ", result); + if (result !== undefined && result !== null) { + userInfo.eth_info.balance = result; + userInfo.eth_info.parsed_balance = + result / 1000000000000000000; + console.log("INFO: ", userInfo); + setUserData(userInfo); + } else { + alert.error("Couldn't find balance: ", result); + } + }) + .catch((error) => { + // If the request fails, the Promise will reject with an error. + alert.error( + "Failed getting info from ethereum API: " + error + ); + }); + }); + } + }); + + if ( + userInfo.eth_info !== undefined && + userInfo.eth_info.balance !== undefined + ) { + //console.log(userInfo.eth_info.balance) + userInfo.eth_info.parsed_balance = + userInfo.eth_info.balance / 1000000000000000000; + } + */} + + //console.log("USER: ", userInfo) + setUserData(userInfo); + setIsLoaded(true); + }) + .catch((error) => { + setIsLoaded(true); + }); + }; + + // Dumb for content load (per now), but good for making the site not suddenly reload parts (ajax thingies) + + const options = { + timeout: 9000, + position: positions.BOTTOM_LEFT, + }; + + const handleFirstInteraction = (event) => { + console.log("First interaction: ", event) } - const getUserNotifications = () => { - fetch(`${globalUrl}/api/v1/notifications`, { - credentials: "include", - headers: { - 'Content-Type': 'application/json', - }, - }) - .then(response => response.json()) - .then(responseJson => { - if (responseJson.success === true && responseJson.notifications !== null && responseJson.notifications !== undefined && responseJson.notifications.length > 0) { - //console.log("RESP: ", responseJson) - setNotifications(responseJson.notifications) - } - }) - .catch(error => { - console.log("Failed getting notifications for user: ", error) - }); - } - - const checkLogin = () => { - var baseurl = globalUrl - fetch(baseurl + "/api/v1/users/getinfo", { - credentials: "include", - headers: { - 'Content-Type': 'application/json', - }, - }) - .then(response => response.json()) - .then(responseJson => { - if (responseJson.success === true) { - console.log(responseJson) - setUserData(responseJson) - setIsLoggedIn(true) - //console.log("Cookies: ", cookies) - - // Updating cookie every request - for (var key in responseJson["cookies"]) { - setCookie(responseJson["cookies"][key].key, responseJson["cookies"][key].value, { path: "/" }) + const includedData = + window.location.pathname === "/home" || + window.location.pathname === "/features" ? ( +
+ + } + /> + +
+ ) : ( +
+ + {!isLoaded ? null : + userdata.chat_disabled === true ? null : + } - } - setIsLoaded(true) - }) - .catch(error => { - setIsLoaded(true) - }); - } +
+ {/* +
+ */} + + + } + /> + + } + /> + } /> + + } + /> + {userdata.id !== undefined ? ( + + } + /> + ) : null} + + } + /> + + } + /> + + } + /> + + } + /> + + } + /> + + } + /> + + } + /> + + } + /> + + } + /> + + } + /> + } /> + } /> + + } + /> + + } + /> + + } + /> + + } + /> + + } + /> + + } + /> + + } + /> + + } + /> + + } + /> + + } + /> + + } + /> + + } + /> + +
+ ); - const includedData = window.location.pathname === "/home" || window.location.pathname === "/features" ? -
- } /> -
: -
- -
-
- } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - { window.location.pathname = "/docs/about" }} /> - } /> - } /> - } /> - } /> - } /> -
- - //
- // backgroundColor: "#213243", - // This is a mess hahahah - return ( - - - - - {includedData} - - - - - ); + //
+ // backgroundColor: "#213243", + // This is a mess hahahah + return ( + + + + + {includedData} + + + + + ); }; export default App; diff --git a/frontend/src/__test__/appdata.js b/frontend/src/__test__/appdata.js index 82772dbb..c454c44b 100755 --- a/frontend/src/__test__/appdata.js +++ b/frontend/src/__test__/appdata.js @@ -1,3 +1,1350 @@ -const Data = [{"name":"hive","is_valid":false,"id":"02828a90-658b-41c4-8726-e31da7e02fe9","id_":"02828a90-658b-41c4-8726-e31da7e02fe9","link":"","app_version":"1.0.0","description":"The Hive app allows for walkoff to generate or close cases in TheHive","environment":"cloud","contact_info":{"name":"FORGE Cyber","url":"https://github.com/"},"actions":[{"description":"creates a hive case","id_":"eb84008d-b0e0-41c6-a9b3-d7a51866bcd4","name":"create_case","node_type":"ACTION","environment":"cloud","parameters":[{"description":"log data to generate custom fields","id_":"b484aba8-0787-42e9-a97a-882a59e02f8f","name":"log_data","required":true,"schema":{"type":"string"}},{"description":"URL of TheHive","id_":"deb0a68d-479a-4472-b547-1eb82fb3888d","name":"url","required":true,"schema":{"type":"string"}},{"description":"API key to access TheHive","id_":"5116b359-c8da-41cf-80a4-4292a1a3c90b","name":"api_key","required":true,"schema":{"type":"string"}}],"returns":{"description":"","id_":"b040890c-5462-458b-8f54-054e6ce2a146","schema":{"type":"object"}}},{"description":"Updates a case data as well as severity","id_":"b7f64744-04e3-4166-990e-70106edc88cf","name":"update_case","node_type":"ACTION","environment":"cloud","parameters":[{"description":"json from trigger","id_":"ac37d451-ff45-4740-b5a7-de389afcb2b1","name":"input","required":true,"schema":{"type":"object"}},{"description":"id for case","id_":"69613390-78c9-4b5d-aa71-b6c4a4750f8f","name":"id","required":true,"schema":{"type":"string"}},{"description":"severity of change","id_":"b04523b0-e74e-44a3-9c51-055c1e53167a","name":"severity","required":true,"schema":{"type":"integer"}},{"description":"URL of TheHive","id_":"e415c1f9-d502-4a5d-9c8a-8cdefe0ebf88","name":"url","required":true,"schema":{"type":"string"}},{"description":"API key to access TheHive","id_":"2aa8e834-b889-467a-9172-373930b013cf","name":"api_key","required":true,"schema":{"type":"string"}}],"returns":{"description":"","id_":"","schema":{"type":""}}},{"description":"Closes a case in TheHive","id_":"240f26ef-5045-4711-94a5-af7a947ac5a4","name":"close_case","node_type":"ACTION","environment":"cloud","parameters":[{"description":"ID of case to close","id_":"71eb8b33-56dc-416f-8ae9-6bee56321f9b","name":"case_id","required":true,"schema":{"type":"string"}},{"description":"URL of TheHive","id_":"a3bbb859-d6fb-45ec-a08f-8895c8f8b0d5","name":"url","required":true,"schema":{"type":"string"}},{"description":"API key to access TheHive","id_":"660ba026-886f-413c-809a-8a48961d9228","name":"api_key","required":true,"schema":{"type":"string"}},{"description":"Resolution status of the case to close.","id_":"6b9604d8-7358-4362-b0f4-0d570a366724","name":"resolution_status","required":true,"schema":{"type":"string"}},{"description":"Impact status of the case to close. The impact status is only captured when resolution status is TruePositive","id_":"11651f4f-4521-4f1b-a2c7-66078f9be886","name":"impact_status","required":true,"schema":{"type":"string"}},{"description":"Tags to add to the case once closed. Comma separated string.","id_":"01a0951f-92f8-485a-b8a2-0664a7e03862","name":"tags","required":true,"schema":{"type":"string"}},{"description":"Explanation of why the case was closed.","id_":"fa67a17b-b6e0-42e6-8988-03a16484c4b0","name":"summary","required":true,"schema":{"type":"string"}}],"returns":{"description":"","id_":"ebba5506-32c8-4bd2-a004-f32c8ec9cdf6","schema":{"type":"object"}}}]},{"name":"walk_off","is_valid":false,"id":"1f52e17e-c11b-4cf4-abdb-a216211a0d27","id_":"1f52e17e-c11b-4cf4-abdb-a216211a0d27","link":"","app_version":"1.0.0","description":"An example of a Walkoff App specification","environment":"cloud","contact_info":{"name":"Walkoff Team","url":"https://github.com/nsacyber/walkoff"},"actions":[{"description":"Connect to Walkoff","id_":"a9926e7f-18f9-4b93-b805-27cdb5a232ff","name":"connect","node_type":"ACTION","environment":"cloud","parameters":[{"description":"Timeout on the request (in seconds)","id_":"300136af-3c30-4e2b-babe-272f845942a0","name":"timeout","required":true,"schema":{"type":"number"}},{"description":"username","id_":"896de900-9ca1-4108-a034-016e9c6ffe54","name":"username","required":true,"schema":{"type":"string"}},{"description":"password","id_":"e75e75d4-8a6b-4c24-8ffd-61b435683edb","name":"password","required":true,"schema":{"type":"string"}}],"returns":{"description":"","id_":"de121927-243f-4f09-9331-c48ca25e85d1","schema":{"type":"string"}}},{"description":"Disconnect from Walkoff","id_":"acd81f9b-353c-4749-a33a-2091da2d307c","name":"disconnect","node_type":"ACTION","environment":"cloud","parameters":[{"description":"Timeout on the request (in seconds)","id_":"76819a03-5272-4421-99cd-1e53f95963f0","name":"timeout","required":true,"schema":{"type":"number"}},{"description":"refresh token","id_":"b583836e-eff4-4e7c-8430-286bfa81e43f","name":"refresh_token","required":true,"schema":{"type":"string"}}],"returns":{"description":"","id_":"d6cc2470-2dc9-43f9-a2f9-fb123c590426","schema":{"type":"string"}}},{"description":"Gets a list of all the users loaded on the system","id_":"6610a111-687e-45f5-a60d-26048594cdb1","name":"get_all_users","node_type":"ACTION","environment":"cloud","parameters":[{"description":"Timeout on the request (in seconds)","id_":"89b9a459-f756-40d8-b2c0-1a7c57987fee","name":"timeout","required":true,"schema":{"type":"number"}},{"description":"Access Token","id_":"496c7956-964b-4977-88ed-27db81a94cfe","name":"access_token","required":true,"schema":{"type":"string"}}],"returns":{"description":"","id_":"642ecea4-e747-4dfe-9ca7-4f290847c0ed","schema":{"type":"string"}}},{"description":"Gets a list of all the workflows loaded on the system","id_":"7cb34d71-2619-40b1-a565-17274fea924e","name":"get_all_workflows","node_type":"ACTION","environment":"cloud","parameters":[{"description":"Timeout on the request (in seconds)","id_":"e4553287-018d-465a-8599-a605fdb5fb71","name":"timeout","required":true,"schema":{"type":"number"}},{"description":"Access Token","id_":"f9193b36-cc5b-4a5f-babf-7df321837f57","name":"access_token","required":true,"schema":{"type":"string"}}],"returns":{"description":"","id_":"684de06e-b048-4332-88c5-9555ca41e2a1","schema":{"type":"string"}}},{"description":"Executes a workflow","id_":"9aaf7af4-fa4b-45d1-ac99-5722f0487aef","name":"execute_workflow","node_type":"ACTION","environment":"cloud","parameters":[{"description":"ID of the workflow","id_":"2f9c0aaf-82ea-4fab-9c26-99268cc9735e","name":"workflow_id","required":true,"schema":{"type":"string"}},{"description":"Timeout on the request (in seconds)","id_":"35fc2b33-0e03-4b48-9e1c-73aa8a0b4ad7","name":"timeout","required":true,"schema":{"type":"number"}},{"description":"Access Token","id_":"463425b6-ead9-4783-ad40-e04e6d30bc6e","name":"access_token","required":true,"schema":{"type":"string"}}],"returns":{"description":"","id_":"fda59148-a2b1-4180-a34d-8beef5d68929","schema":{"type":"string"}}},{"description":"Log out of Walkoff","id_":"160693dc-38c4-4a7c-a36b-bd529e69118b","name":"shutdown","node_type":"ACTION","environment":"cloud","parameters":[{"description":"refresh Token","id_":"a24a2395-7696-48cc-a1b8-b71c881c1180","name":"refresh_token","required":true,"schema":{"type":"string"}},{"description":"Timeout on the request (in seconds)","id_":"4b0b1d88-203b-4ba3-956f-822def41d0cb","name":"timeout","required":true,"schema":{"type":"number"}}],"returns":{"description":"","id_":"7836185e-786b-4d13-8c9b-b29d89d61155","schema":{"type":"string"}}}]},{"name":"ip_addr_utils","is_valid":false,"id":"4f00d85c-1fc8-4db2-a1d2-fab1eff5d02e","id_":"4f00d85c-1fc8-4db2-a1d2-fab1eff5d02e","link":"","app_version":"1.0.0","description":"An IP address app that will allow users to specify ip addresses and will format them correctly","environment":"cloud","contact_info":{"name":"Walkoff Team","url":"https://github.com/nsacyber/walkoff"},"actions":[{"description":"Sets the timestamp fro which scipt outputs will be filed under","id_":"5404d09b-08d8-469b-9087-e6b22892188b","name":"set_timestamp","node_type":"ACTION","environment":"cloud","parameters":[],"returns":{"description":"","id_":"8945dbff-0920-4b04-8682-e9086bcae5a4","schema":{"type":"string"}}},{"description":"Converts ip address from CIDR notation to individual IP's for easier integration with other apps.","id_":"d6c39b8b-63b7-4198-82d2-b8a4f3ed95f2","name":"cidr_to_array","node_type":"ACTION","environment":"cloud","parameters":[{"description":"list of hosts to execute on","id_":"14a43bde-2a88-4cf7-987c-e7bb62c202c0","name":"ip_array","required":true,"schema":{"type":"array"}}],"returns":{"description":"","id_":"22c7aac2-321e-451a-a9f4-61b9c6656cb7","schema":{"type":"array"}}}]},{"name":"power_shell","is_valid":false,"id":"5b570c0b-4f31-4f2e-93ac-da39b78502bb","id_":"5b570c0b-4f31-4f2e-93ac-da39b78502bb","link":"","app_version":"1.0.0","description":"A power shell app that can run commands on a remote host.","environment":"cloud","contact_info":{"name":"Walkoff Team","url":"https://github.com/nsacyber/walkoff"},"actions":[{"description":"Sets the timestamp fro which scipt outputs will be filed under","id_":"8f0cb2b7-1cd1-4284-8790-f8234ec9a1f9","name":"set_timestamp","node_type":"ACTION","environment":"cloud","parameters":[],"returns":{"description":"","id_":"b58197d0-42fc-40bc-a40f-b1bea1c44b01","schema":{"type":"string"}}},{"description":"Executes powershell scripts on remote devices (Scripts located in \"scripts\" directory within app).","id_":"cb1fb39a-9cd9-485d-8c2a-e4fe138035e8","name":"exec_command_prompt_from_file","node_type":"ACTION","environment":"cloud","parameters":[{"description":"list of hosts to execute on","id_":"13fd6552-5b2b-4081-89f9-8e9dad5bdea2","name":"hosts","required":true,"schema":{"type":"array"}},{"description":"filename in which scripts will be located","id_":"7a51a4e2-2c7d-4c7f-9aca-dc2d1316f2db","name":"local_file_name","required":true,"schema":{"type":"string"}},{"description":"Username for remote host","id_":"6ea5d033-308b-453e-84ff-416011b459d7","name":"username","required":true,"schema":{"type":"string"}},{"description":"Password for remote host user","id_":"b71eb882-4b22-456e-980d-84cbba55216b","name":"password","required":true,"schema":{"type":"string"}},{"description":"transport type","id_":"78c824c8-97f9-46ea-ba2a-ada98127555b","name":"transport","required":true,"schema":{"type":"string"}},{"description":"whether server certificate should be validated","id_":"ca847f0c-26de-4289-a13b-0aca5ac55d1c","name":"server_cert_validation","required":true,"schema":{"type":"boolean"}},{"description":"Will encrypt the WinRM messages if set to True and \"transport auth\" supports message encryption","id_":"2747d198-2fd2-47a0-8854-4a3dfc5baa77","name":"message_encryption","required":true,"schema":{"type":"string"}}],"returns":{"description":"","id_":"","schema":{"type":""}}},{"description":"Executes the powershell command on remote devices.","id_":"70717155-5377-40d1-989b-3db7b90c5975","name":"exec_command_prompt","node_type":"ACTION","environment":"cloud","parameters":[{"description":"list of hosts to execute on","id_":"b151934c-dea4-4ba9-a993-1e527093b6e4","name":"hosts","required":true,"schema":{"type":"array"}},{"description":"list of commands to execute","id_":"fa6386e4-0e45-453e-8510-6d4affdb7e31","name":"commands","required":true,"schema":{"type":"array"}},{"description":"Username for remote host","id_":"a2c56d5d-48eb-455f-8e4b-155855ac4fa4","name":"username","required":true,"schema":{"type":"string"}},{"description":"Password for remote host user","id_":"66b4b42e-bd81-4180-8ff8-84fae5e590f2","name":"password","required":true,"schema":{"type":"string"}},{"description":"transport type","id_":"d0a7b275-8a0f-4421-9b75-7eb3dbc6ff38","name":"transport","required":true,"schema":{"type":"string"}},{"description":"whether server certificate should be validated","id_":"5df9e652-9e3c-437d-b81f-92e8e0702312","name":"server_cert_validation","required":true,"schema":{"type":"boolean"}},{"description":"Will encrypt the WinRM messages if set to True and the transport auth supports message encryption","id_":"64f64521-5ded-4f0b-8590-f9c6395cd98f","name":"message_encryption","required":true,"schema":{"type":"string"}}],"returns":{"description":"","id_":"","schema":{"type":""}}},{"description":"Executes the powershell script on remote devices based on script file passed in.","id_":"d0fabf12-9db6-4a0e-8785-5248b1035051","name":"exec_powershell_script_from_file","node_type":"ACTION","environment":"cloud","parameters":[{"description":"list of hosts to execute on","id_":"09299ae2-38a0-4fb3-b064-fd3597424d67","name":"hosts","required":true,"schema":{"type":"array"}},{"description":"type of shell you want to execute","id_":"3d076cba-93d2-4f58-abfd-1475b354444d","name":"shell_type","required":true,"schema":{"type":"string"}},{"description":"filename in which scripts will be located","id_":"b82484c2-861c-4193-925a-e23bd882938c","name":"local_file_name","required":true,"schema":{"type":"string"}},{"description":"Username for remote host","id_":"79f42b59-2218-4d41-b13e-154b2ed89105","name":"username","required":true,"schema":{"type":"string"}},{"description":"Password for remote host user","id_":"35ac706c-d375-45ab-add5-40d9b846100c","name":"password","required":true,"schema":{"type":"string"}},{"description":"transport type","id_":"aa9a2b80-7836-4810-82fa-3c270d9e18f6","name":"transport","required":true,"schema":{"type":"string"}},{"description":"whether server certificate should be validated","id_":"39cbd4d7-a0cf-4b9c-99bc-5b7c46cfc8df","name":"server_cert_validation","required":true,"schema":{"type":"boolean"}},{"description":"Will encrypt the WinRM messages if set to True and the transport auth supports message encryption","id_":"c7c161e7-c5db-49b8-a7de-b10a203c551b","name":"message_encryption","required":true,"schema":{"type":"string"}}],"returns":{"description":"","id_":"","schema":{"type":""}}},{"description":"Executes the powershell command/script on remote devices.","id_":"98c0cd0f-a05a-4b26-809c-5dbf3ecb791f","name":"exec_powershell_script","node_type":"ACTION","environment":"cloud","parameters":[{"description":"list of hosts to execute on","id_":"9e69e9b0-4639-43dd-9b4c-e4cb710ce63c","name":"hosts","required":true,"schema":{"type":"array"}},{"description":"type of shell you want to execute","id_":"6dce7ac6-b0ac-4d48-a79a-e9db795d751b","name":"shell_type","required":true,"schema":{"type":"string"}},{"description":"script in the form of array commands","id_":"700bb930-582e-4cbe-85e9-f5656461ecd5","name":"arguments","required":true,"schema":{"type":"array"}},{"description":"Username for remote host","id_":"22f0f58f-7e3c-43f6-afca-8e6800a6052e","name":"username","required":true,"schema":{"type":"string"}},{"description":"Password for remote host user","id_":"5b53cb8d-eaa4-4d31-8142-31151406100c","name":"password","required":true,"schema":{"type":"string"}},{"description":"transport type","id_":"635e350e-7a4d-4b94-aa0e-3e0a57ba5577","name":"transport","required":true,"schema":{"type":"string"}},{"description":"whether server certificate should be validated","id_":"07ec9b94-f626-42e7-a5a8-6e9b68b7bf7d","name":"server_cert_validation","required":true,"schema":{"type":"boolean"}},{"description":"Will encrypt the WinRM messages if set to True and the transport auth supports message encryption","id_":"1ec509eb-0444-4f08-971e-1da23983d588","name":"message_encryption","required":true,"schema":{"type":"string"}}],"returns":{"description":"","id_":"762934c3-8de3-4261-850f-38b12effc90a","schema":{"type":"string"}}}]},{"name":"ssh","is_valid":false,"id":"68bfeb11-4e8f-4d46-9cf5-5f681be05858","id_":"68bfeb11-4e8f-4d46-9cf5-5f681be05858","link":"","app_version":"1.0.0","description":"Executes ssh shell commands via SSH","environment":"cloud","contact_info":{"name":"Walkoff Team","url":"https://github.com/nsacyber/walkoff"},"actions":[{"description":"Execute command on remote server with SSH client","id_":"2ec132a9-1c32-42eb-8260-bcfac47901fb","name":"exec_command","node_type":"ACTION","environment":"cloud","parameters":[{"description":"hosts or hostnames of the remote server","id_":"d94d2878-9208-445a-a9b9-852d05ef5b0c","name":"hosts","required":true,"schema":{"type":"array"}},{"description":"port number","id_":"022cd027-b0f2-49ee-a512-da8a07ed2bfc","name":"port","required":true,"schema":{"type":"integer"}},{"description":"json array of arguments","id_":"2d8dff25-dbc9-4656-9de3-73280015eaaf","name":"args","required":true,"schema":{"type":"array"}},{"description":"username to login with","id_":"89145276-3837-4e48-b9ab-0aa76500ba72","name":"username","required":true,"schema":{"type":"string"}},{"description":"password to login with","id_":"564fb3d7-89ff-41df-b3ed-6ba23d7b3f3e","name":"password","required":true,"schema":{"type":"string"}}],"returns":{"description":"","id_":"1395434e-52c2-4428-a1ac-966a315af6e0","schema":{"type":"string"}}},{"description":"Run a local bash command","id_":"baa63db8-c8d6-4dea-b774-81fe4b8dddfb","name":"exec_local_command","node_type":"ACTION","environment":"cloud","parameters":[{"description":"source path of the file to copy","id_":"fe99ea27-1176-4ca5-8de7-a5754f429784","name":"command","required":true,"schema":{"type":"string"}}],"returns":{"description":"","id_":"ac58f084-4234-4380-b438-a2c9239344f2","schema":{"type":"string"}}},{"description":"Copy remote file to remote host using sftp","id_":"87f34206-3b72-4d01-b64a-9bacdf822526","name":"sftp_copy","node_type":"ACTION","environment":"cloud","parameters":[{"description":"source path of the file to copy","id_":"43f07eb3-7e1b-4ea3-8684-862111011394","name":"src_path","required":true,"schema":{"type":"string"}},{"description":"remote path of the file destination","id_":"1a53170b-15b7-4cf8-9dff-9309d56bdacf","name":"dest_path","required":true,"schema":{"type":"string"}},{"description":"host or hostname of the remote server","id_":"4d78b34e-09ae-48c3-9eeb-f5042903bc04","name":"src_host","required":true,"schema":{"type":"string"}},{"description":"port number","id_":"29ddd615-05d3-4531-9869-23ca7b366133","name":"src_port","required":true,"schema":{"type":"integer"}},{"description":"username to login with","id_":"f9db153e-0c9f-4808-9814-9277f56b2f47","name":"src_username","required":true,"schema":{"type":"string"}},{"description":"password to login with","id_":"a6337fa3-0704-49ca-8920-a49229bed77e","name":"src_password","required":true,"schema":{"type":"string"}},{"description":"host or hostname of the remote server","id_":"04399d82-0ef6-4bdc-b563-344df67219a2","name":"dest_host","required":true,"schema":{"type":"string"}},{"description":"port number","id_":"262df217-34fa-4165-a753-e71488099729","name":"dest_port","required":true,"schema":{"type":"integer"}},{"description":"username to login with","id_":"4fde41af-112b-4b33-88dc-ce3219b444f3","name":"dest_username","required":true,"schema":{"type":"string"}},{"description":"password to login with","id_":"d9382c2a-b64c-4ad8-b7ca-488aa8e440fb","name":"dest_password","required":true,"schema":{"type":"string"}}],"returns":{"description":"","id_":"9b484424-f7d7-42ef-b553-f40ec1805806","schema":{"type":"string"}}},{"description":"runs the specified shell script on the remote server(s)","id_":"bb32494e-b808-4d9b-8f9a-f366a9bfd21e","name":"run_shell_script_file","node_type":"ACTION","environment":"cloud","parameters":[{"description":"local path of the shell script to run","id_":"41df393d-3435-4e0c-9c7e-dd6531fdc0a6","name":"local_file_name","required":true,"schema":{"type":"string"}},{"description":"hosts of the remote server","id_":"f2bd124e-1dbb-4b87-a12f-ef7f25d6eb2d","name":"hosts","required":true,"schema":{"type":"array"}},{"description":"port number","id_":"eb2a8834-ea05-4d54-8900-e723e2056132","name":"port","required":true,"schema":{"type":"integer"}},{"description":"username to login with","id_":"ba961ec7-2200-420e-9142-7b237c046809","name":"username","required":true,"schema":{"type":"string"}},{"description":"password to login with","id_":"65b6d98a-5b23-4d5d-ae57-022057038bae","name":"password","required":true,"schema":{"type":"string"}}],"returns":{"description":"","id_":"94e55d3b-ef12-4114-8c60-07e3e6417df8","schema":{"type":"string"}}}]},{"name":"Builtin","is_valid":false,"id":"e34f0c67-83e9-443a-b9e7-7b152b4b16f6","id_":"e34f0c67-83e9-443a-b9e7-7b152b4b16f6","link":"","app_version":"1.0.0","description":"Walkoff built-in functions useful in workflow development.","environment":"cloud","contact_info":{"name":"Walkoff Team","url":"https://github.com/nsacyber/walkoff"},"actions":[{"description":"Takes input from an API Call and triggers the rest of the workflow to beign executing again.","id_":"c3e959c6-2c97-48bb-bf00-4082bf812a5d","name":"Trigger","node_type":"TRIGGER","environment":"cloud","parameters":[],"returns":{"description":"","id_":"6265f75b-e9a7-4d0d-b0e8-7c4da5ac5e65","schema":{"type":"string"}}},{"description":"Takes input from a previous action and chooses which branch to take according to your logic.","id_":"8b260c29-dd20-405d-b1a6-29d2e67d43e5","name":"Condition","node_type":"CONDITION","environment":"cloud","parameters":[],"returns":{"description":"","id_":"a4c44cdd-fd3b-46ef-b151-51c1f686fa40","schema":{"type":"string"}}}]},{"name":"hello_world","is_valid":false,"id":"e66d38eb-19b4-4801-abf2-38248b3b2786","id_":"e66d38eb-19b4-4801-abf2-38248b3b2786","link":"","app_version":"1.0.0","description":"An example of a Walkoff App specification","environment":"cloud","contact_info":{"name":"Walkoff Team","url":"https://github.com/nsacyber/walkoff"},"actions":[{"description":"Returns Hello World from the hostname the action is run on","id_":"ac0e2250-20b1-46a3-93ec-1718f9973cc4","name":"hello_world","node_type":"ACTION","environment":"cloud","parameters":[],"returns":{"description":"","id_":"66f4cd0e-7a97-4185-a167-3956dcf3f627","schema":{"type":"string"}}},{"description":"Returns a random float between 0.0 and 1.0","id_":"3817fd8b-1370-4ce8-b874-a526e3c204de","name":"random_number","node_type":"ACTION","environment":"cloud","parameters":[],"returns":{"description":"","id_":"c788404e-c637-4704-bf9e-1b861060e97e","schema":{"type":"number"}}},{"description":"returns the outputs from the trigger data if it's in Json format.","id_":"7fae3591-ab32-402d-8dc4-26e8b802c661","name":"repeat_trigger_as_json","node_type":"ACTION","environment":"cloud","parameters":[{"description":"message to hold output from","id_":"0f0a76ba-b590-4150-ae03-02d0e797f7e4","name":"call","required":true,"schema":{"type":"string"}}],"returns":{"description":"","id_":"1ab56d07-0ce1-4fe5-be01-d70337d9d589","schema":{"type":"object"}}},{"description":"Repeats the call parameter","id_":"23984f43-7593-4c7b-81b6-77852e498add","name":"repeat_back_to_me","node_type":"ACTION","environment":"cloud","parameters":[{"description":"message to repeat","id_":"bedf6d97-958c-4acd-b8a9-a72e19c5d54b","name":"call","required":true,"schema":{"type":"string"}}],"returns":{"description":"","id_":"78708871-00b5-49e6-a4b4-d5e265d89f36","schema":{"type":"string"}}},{"description":"Increments the number parameter by 1","id_":"86624c25-bb66-4fc3-b66f-dc8d394f09bd","name":"return_plus_one","node_type":"ACTION","environment":"cloud","parameters":[{"description":"number to increment","id_":"45af1005-e7f0-46c3-ac09-28748f022a17","name":"number","required":true,"schema":{"type":"number"}}],"returns":{"description":"","id_":"c84fc303-4d27-4091-9500-cb8cd83d6f05","schema":{"type":"number"}}},{"description":"Pause execution by the seconds parameter","id_":"bf787802-6039-4010-bc98-2d6d1dbdce21","name":"pause","node_type":"ACTION","environment":"cloud","parameters":[{"description":"seconds to pause for","id_":"9adcc6f4-0559-47c2-9496-02c8fb3fd58a","name":"seconds","required":true,"schema":{"type":"number"}}],"returns":{"description":"","id_":"","schema":{"type":""}}},{"description":"Echo the data parameter","id_":"0a9fec6a-5bf5-4266-a264-dae7fe52c0d5","name":"echo_array","node_type":"ACTION","environment":"cloud","parameters":[{"description":"array to echo","id_":"8ee260ee-f181-40b3-9d10-f1821f03228c","name":"data","required":true,"schema":{"type":"array"}}],"returns":{"description":"","id_":"bde96402-3d66-43d8-a418-8c0859cb4d01","schema":{"type":"array"}}},{"description":"echos the given JSON object","id_":"e0927a9c-0e6c-429a-965b-40d0804c13f3","name":"echo_json","node_type":"ACTION","environment":"cloud","parameters":[{"description":"The data to echo","id_":"ba719062-90a0-42dd-8ef3-eaa304a57667","name":"data","required":true,"schema":{"type":"object"}}],"returns":{"description":"","id_":"fd4f9c20-e4a0-43df-9a14-7bf0046f391f","schema":{"type":"object"}}}]},{"name":"nmap","is_valid":false,"id":"fc3d231c-9437-4ba9-8fe8-8ba199626197","id_":"fc3d231c-9437-4ba9-8fe8-8ba199626197","link":"","app_version":"1.0.0","description":"A simple app to interact with map","environment":"cloud","contact_info":{"name":"Walkoff Team","url":"https://github.com/nsacyber/walkoff"},"actions":[{"description":"looks into xml nmap for osfamily","id_":"09840ebb-f72a-43e4-b49e-ab5a10d56d96","name":"parse_xml_for_windows_from_file","node_type":"ACTION","environment":"cloud","parameters":[{"description":"nmap output as xml filename","id_":"2bfccd6f-bd75-4ad0-b8df-5ff74aa64761","name":"nmap_file","required":true,"schema":{"type":"string"}}],"returns":{"description":"os","id_":"9c6900a5-7e20-4033-9f3e-61ce8b95ab5f","schema":{"type":"array"}}},{"description":"transforms xml nmap results into json","id_":"150e28cb-3333-49ad-859c-b61e2b758ff7","name":"xml_to_json","node_type":"ACTION","environment":"cloud","parameters":[{"description":"nmap output either as xml filename or string","id_":"5fb18897-fe0e-473f-a83d-3c11f3d8b2a1","name":"nmap_out","required":true,"schema":{"type":"string"}},{"description":"whether the previous parameter is a filename or string","id_":"f87d9bd6-e85a-4f6d-8300-902e3ba29347","name":"is_file","required":true,"schema":{"type":"boolean"}}],"returns":{"description":"xml string on nmap output","id_":"1cd901cc-2101-4df1-9759-bf74fcdb7b9b","schema":{"type":"string"}}},{"description":"retrieves the hosts and ports from an nmap scan for use with OpenVAS","id_":"f743dfa4-2b06-4d1b-bbc7-55d34b3ce499","name":"ports_and_hosts_from_json","node_type":"ACTION","environment":"cloud","parameters":[{"description":"json string or filename","id_":"ca686f90-c947-4473-9d39-abc8bc4895fd","name":"nmap_json","required":true,"schema":{"type":"string"}},{"description":"whether or not first input is a filename or not","id_":"e605313e-f9c5-4335-8ce7-d46abd422e68","name":"is_file","required":true,"schema":{"type":"boolean"}}],"returns":{"description":"","id_":"040f32fc-d020-469f-9f4c-47928b076688","schema":{"type":"string"}}},{"description":"Runs an nmap scan, returns results as string or filename","id_":"ada798da-fff9-4ccf-85f2-24e2edb7722a","name":"run_scan","node_type":"ACTION","environment":"cloud","parameters":[{"description":"The target(s) to scan, comma separated values, CIDR supported","id_":"72704c22-a9c9-457f-ae33-e7c17e758e7d","name":"targets","required":true,"schema":{"type":"array"}},{"description":"see nmap manpage -- some options require root","id_":"51ea8e70-adae-47a1-9241-87674b4712c1","name":"options","required":true,"schema":{"type":"string"}}],"returns":{"description":"xml string on nmap output","id_":"9c18b2a2-c023-4102-bd46-ddeb31a5430c","schema":{"type":"array"}}},{"description":"Gets the list of active hosts on a network from an nmap scan","id_":"48b4814f-47ea-4c66-a5db-cacc9cd305b6","name":"get_hosts_from_scan","node_type":"ACTION","environment":"cloud","parameters":[{"description":"The target (or targets in CIDR notation) to scan","id_":"e48d60f7-4590-4dd2-98ce-cd112ab9df2f","name":"targets","required":true,"schema":{"type":"array"}},{"description":"","id_":"2eb29670-701e-4622-99f2-80e4b51cb06e","name":"options","required":true,"schema":{"type":"string"}}],"returns":{"description":"xml string on nmap output","id_":"1f710e7d-ef6d-4a28-8ceb-8cbc49f6e197","schema":{"type":"string"}}},{"description":"looks into xml nmap for osfamily to match Linux","id_":"f5425571-2de1-4a84-9221-3829d118617a","name":"parse_xml_for_linux","node_type":"ACTION","environment":"cloud","parameters":[{"description":"nmap output as xml array","id_":"e2e7d0c7-13e8-49f9-85f2-da8bfc73308e","name":"nmap_arr","required":true,"schema":{"type":"array"}}],"returns":{"description":"os","id_":"785f146f-095d-4008-a068-0c3146fdf4f0","schema":{"type":"array"}}},{"description":"looks into xml nmap for osfamily to match Windows","id_":"cf35f64c-43b4-4f23-ae75-70d912f4c1d5","name":"parse_xml_for_windows","node_type":"ACTION","environment":"cloud","parameters":[{"description":"nmap output as xml array","id_":"ef1ce558-2d2a-4d30-8e96-c4ad4c18fae9","name":"nmap_arr","required":true,"schema":{"type":"array"}}],"returns":{"description":"os","id_":"6d21bd12-c076-459e-970d-cd2f39545efb","schema":{"type":"array"}}},{"description":"looks into xml nmap for osfamily","id_":"2baa37be-59cb-4e8a-a67b-1abae654ce4b","name":"parse_xml_for_linux_from_file","node_type":"ACTION","environment":"cloud","parameters":[{"description":"nmap output as xml filename","id_":"aa6324a9-efd2-49c7-a181-90f67b39deb9","name":"nmap_file","required":true,"schema":{"type":"string"}}],"returns":{"description":"os","id_":"96b98b54-2272-4065-b427-2eadf140cb12","schema":{"type":"array"}}}]}] +const Data = [ + { + name: "hive", + is_valid: false, + id: "02828a90-658b-41c4-8726-e31da7e02fe9", + id_: "02828a90-658b-41c4-8726-e31da7e02fe9", + link: "", + app_version: "1.0.0", + description: + "The Hive app allows for walkoff to generate or close cases in TheHive", + environment: "cloud", + contact_info: { name: "FORGE Cyber", url: "https://github.com/" }, + actions: [ + { + description: "creates a hive case", + id_: "eb84008d-b0e0-41c6-a9b3-d7a51866bcd4", + name: "create_case", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "log data to generate custom fields", + id_: "b484aba8-0787-42e9-a97a-882a59e02f8f", + name: "log_data", + required: true, + schema: { type: "string" }, + }, + { + description: "URL of TheHive", + id_: "deb0a68d-479a-4472-b547-1eb82fb3888d", + name: "url", + required: true, + schema: { type: "string" }, + }, + { + description: "API key to access TheHive", + id_: "5116b359-c8da-41cf-80a4-4292a1a3c90b", + name: "api_key", + required: true, + schema: { type: "string" }, + }, + ], + returns: { + description: "", + id_: "b040890c-5462-458b-8f54-054e6ce2a146", + schema: { type: "object" }, + }, + }, + { + description: "Updates a case data as well as severity", + id_: "b7f64744-04e3-4166-990e-70106edc88cf", + name: "update_case", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "json from trigger", + id_: "ac37d451-ff45-4740-b5a7-de389afcb2b1", + name: "input", + required: true, + schema: { type: "object" }, + }, + { + description: "id for case", + id_: "69613390-78c9-4b5d-aa71-b6c4a4750f8f", + name: "id", + required: true, + schema: { type: "string" }, + }, + { + description: "severity of change", + id_: "b04523b0-e74e-44a3-9c51-055c1e53167a", + name: "severity", + required: true, + schema: { type: "integer" }, + }, + { + description: "URL of TheHive", + id_: "e415c1f9-d502-4a5d-9c8a-8cdefe0ebf88", + name: "url", + required: true, + schema: { type: "string" }, + }, + { + description: "API key to access TheHive", + id_: "2aa8e834-b889-467a-9172-373930b013cf", + name: "api_key", + required: true, + schema: { type: "string" }, + }, + ], + returns: { description: "", id_: "", schema: { type: "" } }, + }, + { + description: "Closes a case in TheHive", + id_: "240f26ef-5045-4711-94a5-af7a947ac5a4", + name: "close_case", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "ID of case to close", + id_: "71eb8b33-56dc-416f-8ae9-6bee56321f9b", + name: "case_id", + required: true, + schema: { type: "string" }, + }, + { + description: "URL of TheHive", + id_: "a3bbb859-d6fb-45ec-a08f-8895c8f8b0d5", + name: "url", + required: true, + schema: { type: "string" }, + }, + { + description: "API key to access TheHive", + id_: "660ba026-886f-413c-809a-8a48961d9228", + name: "api_key", + required: true, + schema: { type: "string" }, + }, + { + description: "Resolution status of the case to close.", + id_: "6b9604d8-7358-4362-b0f4-0d570a366724", + name: "resolution_status", + required: true, + schema: { type: "string" }, + }, + { + description: + "Impact status of the case to close. The impact status is only captured when resolution status is TruePositive", + id_: "11651f4f-4521-4f1b-a2c7-66078f9be886", + name: "impact_status", + required: true, + schema: { type: "string" }, + }, + { + description: + "Tags to add to the case once closed. Comma separated string.", + id_: "01a0951f-92f8-485a-b8a2-0664a7e03862", + name: "tags", + required: true, + schema: { type: "string" }, + }, + { + description: "Explanation of why the case was closed.", + id_: "fa67a17b-b6e0-42e6-8988-03a16484c4b0", + name: "summary", + required: true, + schema: { type: "string" }, + }, + ], + returns: { + description: "", + id_: "ebba5506-32c8-4bd2-a004-f32c8ec9cdf6", + schema: { type: "object" }, + }, + }, + ], + }, + { + name: "walk_off", + is_valid: false, + id: "1f52e17e-c11b-4cf4-abdb-a216211a0d27", + id_: "1f52e17e-c11b-4cf4-abdb-a216211a0d27", + link: "", + app_version: "1.0.0", + description: "An example of a Walkoff App specification", + environment: "cloud", + contact_info: { + name: "Walkoff Team", + url: "https://github.com/nsacyber/walkoff", + }, + actions: [ + { + description: "Connect to Walkoff", + id_: "a9926e7f-18f9-4b93-b805-27cdb5a232ff", + name: "connect", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "Timeout on the request (in seconds)", + id_: "300136af-3c30-4e2b-babe-272f845942a0", + name: "timeout", + required: true, + schema: { type: "number" }, + }, + { + description: "username", + id_: "896de900-9ca1-4108-a034-016e9c6ffe54", + name: "username", + required: true, + schema: { type: "string" }, + }, + { + description: "password", + id_: "e75e75d4-8a6b-4c24-8ffd-61b435683edb", + name: "password", + required: true, + schema: { type: "string" }, + }, + ], + returns: { + description: "", + id_: "de121927-243f-4f09-9331-c48ca25e85d1", + schema: { type: "string" }, + }, + }, + { + description: "Disconnect from Walkoff", + id_: "acd81f9b-353c-4749-a33a-2091da2d307c", + name: "disconnect", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "Timeout on the request (in seconds)", + id_: "76819a03-5272-4421-99cd-1e53f95963f0", + name: "timeout", + required: true, + schema: { type: "number" }, + }, + { + description: "refresh token", + id_: "b583836e-eff4-4e7c-8430-286bfa81e43f", + name: "refresh_token", + required: true, + schema: { type: "string" }, + }, + ], + returns: { + description: "", + id_: "d6cc2470-2dc9-43f9-a2f9-fb123c590426", + schema: { type: "string" }, + }, + }, + { + description: "Gets a list of all the users loaded on the system", + id_: "6610a111-687e-45f5-a60d-26048594cdb1", + name: "get_all_users", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "Timeout on the request (in seconds)", + id_: "89b9a459-f756-40d8-b2c0-1a7c57987fee", + name: "timeout", + required: true, + schema: { type: "number" }, + }, + { + description: "Access Token", + id_: "496c7956-964b-4977-88ed-27db81a94cfe", + name: "access_token", + required: true, + schema: { type: "string" }, + }, + ], + returns: { + description: "", + id_: "642ecea4-e747-4dfe-9ca7-4f290847c0ed", + schema: { type: "string" }, + }, + }, + { + description: "Gets a list of all the workflows loaded on the system", + id_: "7cb34d71-2619-40b1-a565-17274fea924e", + name: "get_all_workflows", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "Timeout on the request (in seconds)", + id_: "e4553287-018d-465a-8599-a605fdb5fb71", + name: "timeout", + required: true, + schema: { type: "number" }, + }, + { + description: "Access Token", + id_: "f9193b36-cc5b-4a5f-babf-7df321837f57", + name: "access_token", + required: true, + schema: { type: "string" }, + }, + ], + returns: { + description: "", + id_: "684de06e-b048-4332-88c5-9555ca41e2a1", + schema: { type: "string" }, + }, + }, + { + description: "Executes a workflow", + id_: "9aaf7af4-fa4b-45d1-ac99-5722f0487aef", + name: "execute_workflow", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "ID of the workflow", + id_: "2f9c0aaf-82ea-4fab-9c26-99268cc9735e", + name: "workflow_id", + required: true, + schema: { type: "string" }, + }, + { + description: "Timeout on the request (in seconds)", + id_: "35fc2b33-0e03-4b48-9e1c-73aa8a0b4ad7", + name: "timeout", + required: true, + schema: { type: "number" }, + }, + { + description: "Access Token", + id_: "463425b6-ead9-4783-ad40-e04e6d30bc6e", + name: "access_token", + required: true, + schema: { type: "string" }, + }, + ], + returns: { + description: "", + id_: "fda59148-a2b1-4180-a34d-8beef5d68929", + schema: { type: "string" }, + }, + }, + { + description: "Log out of Walkoff", + id_: "160693dc-38c4-4a7c-a36b-bd529e69118b", + name: "shutdown", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "refresh Token", + id_: "a24a2395-7696-48cc-a1b8-b71c881c1180", + name: "refresh_token", + required: true, + schema: { type: "string" }, + }, + { + description: "Timeout on the request (in seconds)", + id_: "4b0b1d88-203b-4ba3-956f-822def41d0cb", + name: "timeout", + required: true, + schema: { type: "number" }, + }, + ], + returns: { + description: "", + id_: "7836185e-786b-4d13-8c9b-b29d89d61155", + schema: { type: "string" }, + }, + }, + ], + }, + { + name: "ip_addr_utils", + is_valid: false, + id: "4f00d85c-1fc8-4db2-a1d2-fab1eff5d02e", + id_: "4f00d85c-1fc8-4db2-a1d2-fab1eff5d02e", + link: "", + app_version: "1.0.0", + description: + "An IP address app that will allow users to specify ip addresses and will format them correctly", + environment: "cloud", + contact_info: { + name: "Walkoff Team", + url: "https://github.com/nsacyber/walkoff", + }, + actions: [ + { + description: + "Sets the timestamp fro which scipt outputs will be filed under", + id_: "5404d09b-08d8-469b-9087-e6b22892188b", + name: "set_timestamp", + node_type: "ACTION", + environment: "cloud", + parameters: [], + returns: { + description: "", + id_: "8945dbff-0920-4b04-8682-e9086bcae5a4", + schema: { type: "string" }, + }, + }, + { + description: + "Converts ip address from CIDR notation to individual IP's for easier integration with other apps.", + id_: "d6c39b8b-63b7-4198-82d2-b8a4f3ed95f2", + name: "cidr_to_array", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "list of hosts to execute on", + id_: "14a43bde-2a88-4cf7-987c-e7bb62c202c0", + name: "ip_array", + required: true, + schema: { type: "array" }, + }, + ], + returns: { + description: "", + id_: "22c7aac2-321e-451a-a9f4-61b9c6656cb7", + schema: { type: "array" }, + }, + }, + ], + }, + { + name: "power_shell", + is_valid: false, + id: "5b570c0b-4f31-4f2e-93ac-da39b78502bb", + id_: "5b570c0b-4f31-4f2e-93ac-da39b78502bb", + link: "", + app_version: "1.0.0", + description: "A power shell app that can run commands on a remote host.", + environment: "cloud", + contact_info: { + name: "Walkoff Team", + url: "https://github.com/nsacyber/walkoff", + }, + actions: [ + { + description: + "Sets the timestamp fro which scipt outputs will be filed under", + id_: "8f0cb2b7-1cd1-4284-8790-f8234ec9a1f9", + name: "set_timestamp", + node_type: "ACTION", + environment: "cloud", + parameters: [], + returns: { + description: "", + id_: "b58197d0-42fc-40bc-a40f-b1bea1c44b01", + schema: { type: "string" }, + }, + }, + { + description: + 'Executes powershell scripts on remote devices (Scripts located in "scripts" directory within app).', + id_: "cb1fb39a-9cd9-485d-8c2a-e4fe138035e8", + name: "exec_command_prompt_from_file", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "list of hosts to execute on", + id_: "13fd6552-5b2b-4081-89f9-8e9dad5bdea2", + name: "hosts", + required: true, + schema: { type: "array" }, + }, + { + description: "filename in which scripts will be located", + id_: "7a51a4e2-2c7d-4c7f-9aca-dc2d1316f2db", + name: "local_file_name", + required: true, + schema: { type: "string" }, + }, + { + description: "Username for remote host", + id_: "6ea5d033-308b-453e-84ff-416011b459d7", + name: "username", + required: true, + schema: { type: "string" }, + }, + { + description: "Password for remote host user", + id_: "b71eb882-4b22-456e-980d-84cbba55216b", + name: "password", + required: true, + schema: { type: "string" }, + }, + { + description: "transport type", + id_: "78c824c8-97f9-46ea-ba2a-ada98127555b", + name: "transport", + required: true, + schema: { type: "string" }, + }, + { + description: "whether server certificate should be validated", + id_: "ca847f0c-26de-4289-a13b-0aca5ac55d1c", + name: "server_cert_validation", + required: true, + schema: { type: "boolean" }, + }, + { + description: + 'Will encrypt the WinRM messages if set to True and "transport auth" supports message encryption', + id_: "2747d198-2fd2-47a0-8854-4a3dfc5baa77", + name: "message_encryption", + required: true, + schema: { type: "string" }, + }, + ], + returns: { description: "", id_: "", schema: { type: "" } }, + }, + { + description: "Executes the powershell command on remote devices.", + id_: "70717155-5377-40d1-989b-3db7b90c5975", + name: "exec_command_prompt", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "list of hosts to execute on", + id_: "b151934c-dea4-4ba9-a993-1e527093b6e4", + name: "hosts", + required: true, + schema: { type: "array" }, + }, + { + description: "list of commands to execute", + id_: "fa6386e4-0e45-453e-8510-6d4affdb7e31", + name: "commands", + required: true, + schema: { type: "array" }, + }, + { + description: "Username for remote host", + id_: "a2c56d5d-48eb-455f-8e4b-155855ac4fa4", + name: "username", + required: true, + schema: { type: "string" }, + }, + { + description: "Password for remote host user", + id_: "66b4b42e-bd81-4180-8ff8-84fae5e590f2", + name: "password", + required: true, + schema: { type: "string" }, + }, + { + description: "transport type", + id_: "d0a7b275-8a0f-4421-9b75-7eb3dbc6ff38", + name: "transport", + required: true, + schema: { type: "string" }, + }, + { + description: "whether server certificate should be validated", + id_: "5df9e652-9e3c-437d-b81f-92e8e0702312", + name: "server_cert_validation", + required: true, + schema: { type: "boolean" }, + }, + { + description: + "Will encrypt the WinRM messages if set to True and the transport auth supports message encryption", + id_: "64f64521-5ded-4f0b-8590-f9c6395cd98f", + name: "message_encryption", + required: true, + schema: { type: "string" }, + }, + ], + returns: { description: "", id_: "", schema: { type: "" } }, + }, + { + description: + "Executes the powershell script on remote devices based on script file passed in.", + id_: "d0fabf12-9db6-4a0e-8785-5248b1035051", + name: "exec_powershell_script_from_file", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "list of hosts to execute on", + id_: "09299ae2-38a0-4fb3-b064-fd3597424d67", + name: "hosts", + required: true, + schema: { type: "array" }, + }, + { + description: "type of shell you want to execute", + id_: "3d076cba-93d2-4f58-abfd-1475b354444d", + name: "shell_type", + required: true, + schema: { type: "string" }, + }, + { + description: "filename in which scripts will be located", + id_: "b82484c2-861c-4193-925a-e23bd882938c", + name: "local_file_name", + required: true, + schema: { type: "string" }, + }, + { + description: "Username for remote host", + id_: "79f42b59-2218-4d41-b13e-154b2ed89105", + name: "username", + required: true, + schema: { type: "string" }, + }, + { + description: "Password for remote host user", + id_: "35ac706c-d375-45ab-add5-40d9b846100c", + name: "password", + required: true, + schema: { type: "string" }, + }, + { + description: "transport type", + id_: "aa9a2b80-7836-4810-82fa-3c270d9e18f6", + name: "transport", + required: true, + schema: { type: "string" }, + }, + { + description: "whether server certificate should be validated", + id_: "39cbd4d7-a0cf-4b9c-99bc-5b7c46cfc8df", + name: "server_cert_validation", + required: true, + schema: { type: "boolean" }, + }, + { + description: + "Will encrypt the WinRM messages if set to True and the transport auth supports message encryption", + id_: "c7c161e7-c5db-49b8-a7de-b10a203c551b", + name: "message_encryption", + required: true, + schema: { type: "string" }, + }, + ], + returns: { description: "", id_: "", schema: { type: "" } }, + }, + { + description: + "Executes the powershell command/script on remote devices.", + id_: "98c0cd0f-a05a-4b26-809c-5dbf3ecb791f", + name: "exec_powershell_script", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "list of hosts to execute on", + id_: "9e69e9b0-4639-43dd-9b4c-e4cb710ce63c", + name: "hosts", + required: true, + schema: { type: "array" }, + }, + { + description: "type of shell you want to execute", + id_: "6dce7ac6-b0ac-4d48-a79a-e9db795d751b", + name: "shell_type", + required: true, + schema: { type: "string" }, + }, + { + description: "script in the form of array commands", + id_: "700bb930-582e-4cbe-85e9-f5656461ecd5", + name: "arguments", + required: true, + schema: { type: "array" }, + }, + { + description: "Username for remote host", + id_: "22f0f58f-7e3c-43f6-afca-8e6800a6052e", + name: "username", + required: true, + schema: { type: "string" }, + }, + { + description: "Password for remote host user", + id_: "5b53cb8d-eaa4-4d31-8142-31151406100c", + name: "password", + required: true, + schema: { type: "string" }, + }, + { + description: "transport type", + id_: "635e350e-7a4d-4b94-aa0e-3e0a57ba5577", + name: "transport", + required: true, + schema: { type: "string" }, + }, + { + description: "whether server certificate should be validated", + id_: "07ec9b94-f626-42e7-a5a8-6e9b68b7bf7d", + name: "server_cert_validation", + required: true, + schema: { type: "boolean" }, + }, + { + description: + "Will encrypt the WinRM messages if set to True and the transport auth supports message encryption", + id_: "1ec509eb-0444-4f08-971e-1da23983d588", + name: "message_encryption", + required: true, + schema: { type: "string" }, + }, + ], + returns: { + description: "", + id_: "762934c3-8de3-4261-850f-38b12effc90a", + schema: { type: "string" }, + }, + }, + ], + }, + { + name: "ssh", + is_valid: false, + id: "68bfeb11-4e8f-4d46-9cf5-5f681be05858", + id_: "68bfeb11-4e8f-4d46-9cf5-5f681be05858", + link: "", + app_version: "1.0.0", + description: "Executes ssh shell commands via SSH", + environment: "cloud", + contact_info: { + name: "Walkoff Team", + url: "https://github.com/nsacyber/walkoff", + }, + actions: [ + { + description: "Execute command on remote server with SSH client", + id_: "2ec132a9-1c32-42eb-8260-bcfac47901fb", + name: "exec_command", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "hosts or hostnames of the remote server", + id_: "d94d2878-9208-445a-a9b9-852d05ef5b0c", + name: "hosts", + required: true, + schema: { type: "array" }, + }, + { + description: "port number", + id_: "022cd027-b0f2-49ee-a512-da8a07ed2bfc", + name: "port", + required: true, + schema: { type: "integer" }, + }, + { + description: "json array of arguments", + id_: "2d8dff25-dbc9-4656-9de3-73280015eaaf", + name: "args", + required: true, + schema: { type: "array" }, + }, + { + description: "username to login with", + id_: "89145276-3837-4e48-b9ab-0aa76500ba72", + name: "username", + required: true, + schema: { type: "string" }, + }, + { + description: "password to login with", + id_: "564fb3d7-89ff-41df-b3ed-6ba23d7b3f3e", + name: "password", + required: true, + schema: { type: "string" }, + }, + ], + returns: { + description: "", + id_: "1395434e-52c2-4428-a1ac-966a315af6e0", + schema: { type: "string" }, + }, + }, + { + description: "Run a local bash command", + id_: "baa63db8-c8d6-4dea-b774-81fe4b8dddfb", + name: "exec_local_command", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "source path of the file to copy", + id_: "fe99ea27-1176-4ca5-8de7-a5754f429784", + name: "command", + required: true, + schema: { type: "string" }, + }, + ], + returns: { + description: "", + id_: "ac58f084-4234-4380-b438-a2c9239344f2", + schema: { type: "string" }, + }, + }, + { + description: "Copy remote file to remote host using sftp", + id_: "87f34206-3b72-4d01-b64a-9bacdf822526", + name: "sftp_copy", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "source path of the file to copy", + id_: "43f07eb3-7e1b-4ea3-8684-862111011394", + name: "src_path", + required: true, + schema: { type: "string" }, + }, + { + description: "remote path of the file destination", + id_: "1a53170b-15b7-4cf8-9dff-9309d56bdacf", + name: "dest_path", + required: true, + schema: { type: "string" }, + }, + { + description: "host or hostname of the remote server", + id_: "4d78b34e-09ae-48c3-9eeb-f5042903bc04", + name: "src_host", + required: true, + schema: { type: "string" }, + }, + { + description: "port number", + id_: "29ddd615-05d3-4531-9869-23ca7b366133", + name: "src_port", + required: true, + schema: { type: "integer" }, + }, + { + description: "username to login with", + id_: "f9db153e-0c9f-4808-9814-9277f56b2f47", + name: "src_username", + required: true, + schema: { type: "string" }, + }, + { + description: "password to login with", + id_: "a6337fa3-0704-49ca-8920-a49229bed77e", + name: "src_password", + required: true, + schema: { type: "string" }, + }, + { + description: "host or hostname of the remote server", + id_: "04399d82-0ef6-4bdc-b563-344df67219a2", + name: "dest_host", + required: true, + schema: { type: "string" }, + }, + { + description: "port number", + id_: "262df217-34fa-4165-a753-e71488099729", + name: "dest_port", + required: true, + schema: { type: "integer" }, + }, + { + description: "username to login with", + id_: "4fde41af-112b-4b33-88dc-ce3219b444f3", + name: "dest_username", + required: true, + schema: { type: "string" }, + }, + { + description: "password to login with", + id_: "d9382c2a-b64c-4ad8-b7ca-488aa8e440fb", + name: "dest_password", + required: true, + schema: { type: "string" }, + }, + ], + returns: { + description: "", + id_: "9b484424-f7d7-42ef-b553-f40ec1805806", + schema: { type: "string" }, + }, + }, + { + description: "runs the specified shell script on the remote server(s)", + id_: "bb32494e-b808-4d9b-8f9a-f366a9bfd21e", + name: "run_shell_script_file", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "local path of the shell script to run", + id_: "41df393d-3435-4e0c-9c7e-dd6531fdc0a6", + name: "local_file_name", + required: true, + schema: { type: "string" }, + }, + { + description: "hosts of the remote server", + id_: "f2bd124e-1dbb-4b87-a12f-ef7f25d6eb2d", + name: "hosts", + required: true, + schema: { type: "array" }, + }, + { + description: "port number", + id_: "eb2a8834-ea05-4d54-8900-e723e2056132", + name: "port", + required: true, + schema: { type: "integer" }, + }, + { + description: "username to login with", + id_: "ba961ec7-2200-420e-9142-7b237c046809", + name: "username", + required: true, + schema: { type: "string" }, + }, + { + description: "password to login with", + id_: "65b6d98a-5b23-4d5d-ae57-022057038bae", + name: "password", + required: true, + schema: { type: "string" }, + }, + ], + returns: { + description: "", + id_: "94e55d3b-ef12-4114-8c60-07e3e6417df8", + schema: { type: "string" }, + }, + }, + ], + }, + { + name: "Builtin", + is_valid: false, + id: "e34f0c67-83e9-443a-b9e7-7b152b4b16f6", + id_: "e34f0c67-83e9-443a-b9e7-7b152b4b16f6", + link: "", + app_version: "1.0.0", + description: "Walkoff built-in functions useful in workflow development.", + environment: "cloud", + contact_info: { + name: "Walkoff Team", + url: "https://github.com/nsacyber/walkoff", + }, + actions: [ + { + description: + "Takes input from an API Call and triggers the rest of the workflow to beign executing again.", + id_: "c3e959c6-2c97-48bb-bf00-4082bf812a5d", + name: "Trigger", + node_type: "TRIGGER", + environment: "cloud", + parameters: [], + returns: { + description: "", + id_: "6265f75b-e9a7-4d0d-b0e8-7c4da5ac5e65", + schema: { type: "string" }, + }, + }, + { + description: + "Takes input from a previous action and chooses which branch to take according to your logic.", + id_: "8b260c29-dd20-405d-b1a6-29d2e67d43e5", + name: "Condition", + node_type: "CONDITION", + environment: "cloud", + parameters: [], + returns: { + description: "", + id_: "a4c44cdd-fd3b-46ef-b151-51c1f686fa40", + schema: { type: "string" }, + }, + }, + ], + }, + { + name: "hello_world", + is_valid: false, + id: "e66d38eb-19b4-4801-abf2-38248b3b2786", + id_: "e66d38eb-19b4-4801-abf2-38248b3b2786", + link: "", + app_version: "1.0.0", + description: "An example of a Walkoff App specification", + environment: "cloud", + contact_info: { + name: "Walkoff Team", + url: "https://github.com/nsacyber/walkoff", + }, + actions: [ + { + description: + "Returns Hello World from the hostname the action is run on", + id_: "ac0e2250-20b1-46a3-93ec-1718f9973cc4", + name: "hello_world", + node_type: "ACTION", + environment: "cloud", + parameters: [], + returns: { + description: "", + id_: "66f4cd0e-7a97-4185-a167-3956dcf3f627", + schema: { type: "string" }, + }, + }, + { + description: "Returns a random float between 0.0 and 1.0", + id_: "3817fd8b-1370-4ce8-b874-a526e3c204de", + name: "random_number", + node_type: "ACTION", + environment: "cloud", + parameters: [], + returns: { + description: "", + id_: "c788404e-c637-4704-bf9e-1b861060e97e", + schema: { type: "number" }, + }, + }, + { + description: + "returns the outputs from the trigger data if it's in Json format.", + id_: "7fae3591-ab32-402d-8dc4-26e8b802c661", + name: "repeat_trigger_as_json", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "message to hold output from", + id_: "0f0a76ba-b590-4150-ae03-02d0e797f7e4", + name: "call", + required: true, + schema: { type: "string" }, + }, + ], + returns: { + description: "", + id_: "1ab56d07-0ce1-4fe5-be01-d70337d9d589", + schema: { type: "object" }, + }, + }, + { + description: "Repeats the call parameter", + id_: "23984f43-7593-4c7b-81b6-77852e498add", + name: "repeat_back_to_me", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "message to repeat", + id_: "bedf6d97-958c-4acd-b8a9-a72e19c5d54b", + name: "call", + required: true, + schema: { type: "string" }, + }, + ], + returns: { + description: "", + id_: "78708871-00b5-49e6-a4b4-d5e265d89f36", + schema: { type: "string" }, + }, + }, + { + description: "Increments the number parameter by 1", + id_: "86624c25-bb66-4fc3-b66f-dc8d394f09bd", + name: "return_plus_one", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "number to increment", + id_: "45af1005-e7f0-46c3-ac09-28748f022a17", + name: "number", + required: true, + schema: { type: "number" }, + }, + ], + returns: { + description: "", + id_: "c84fc303-4d27-4091-9500-cb8cd83d6f05", + schema: { type: "number" }, + }, + }, + { + description: "Pause execution by the seconds parameter", + id_: "bf787802-6039-4010-bc98-2d6d1dbdce21", + name: "pause", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "seconds to pause for", + id_: "9adcc6f4-0559-47c2-9496-02c8fb3fd58a", + name: "seconds", + required: true, + schema: { type: "number" }, + }, + ], + returns: { description: "", id_: "", schema: { type: "" } }, + }, + { + description: "Echo the data parameter", + id_: "0a9fec6a-5bf5-4266-a264-dae7fe52c0d5", + name: "echo_array", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "array to echo", + id_: "8ee260ee-f181-40b3-9d10-f1821f03228c", + name: "data", + required: true, + schema: { type: "array" }, + }, + ], + returns: { + description: "", + id_: "bde96402-3d66-43d8-a418-8c0859cb4d01", + schema: { type: "array" }, + }, + }, + { + description: "echos the given JSON object", + id_: "e0927a9c-0e6c-429a-965b-40d0804c13f3", + name: "echo_json", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "The data to echo", + id_: "ba719062-90a0-42dd-8ef3-eaa304a57667", + name: "data", + required: true, + schema: { type: "object" }, + }, + ], + returns: { + description: "", + id_: "fd4f9c20-e4a0-43df-9a14-7bf0046f391f", + schema: { type: "object" }, + }, + }, + ], + }, + { + name: "nmap", + is_valid: false, + id: "fc3d231c-9437-4ba9-8fe8-8ba199626197", + id_: "fc3d231c-9437-4ba9-8fe8-8ba199626197", + link: "", + app_version: "1.0.0", + description: "A simple app to interact with map", + environment: "cloud", + contact_info: { + name: "Walkoff Team", + url: "https://github.com/nsacyber/walkoff", + }, + actions: [ + { + description: "looks into xml nmap for osfamily", + id_: "09840ebb-f72a-43e4-b49e-ab5a10d56d96", + name: "parse_xml_for_windows_from_file", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "nmap output as xml filename", + id_: "2bfccd6f-bd75-4ad0-b8df-5ff74aa64761", + name: "nmap_file", + required: true, + schema: { type: "string" }, + }, + ], + returns: { + description: "os", + id_: "9c6900a5-7e20-4033-9f3e-61ce8b95ab5f", + schema: { type: "array" }, + }, + }, + { + description: "transforms xml nmap results into json", + id_: "150e28cb-3333-49ad-859c-b61e2b758ff7", + name: "xml_to_json", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "nmap output either as xml filename or string", + id_: "5fb18897-fe0e-473f-a83d-3c11f3d8b2a1", + name: "nmap_out", + required: true, + schema: { type: "string" }, + }, + { + description: + "whether the previous parameter is a filename or string", + id_: "f87d9bd6-e85a-4f6d-8300-902e3ba29347", + name: "is_file", + required: true, + schema: { type: "boolean" }, + }, + ], + returns: { + description: "xml string on nmap output", + id_: "1cd901cc-2101-4df1-9759-bf74fcdb7b9b", + schema: { type: "string" }, + }, + }, + { + description: + "retrieves the hosts and ports from an nmap scan for use with OpenVAS", + id_: "f743dfa4-2b06-4d1b-bbc7-55d34b3ce499", + name: "ports_and_hosts_from_json", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "json string or filename", + id_: "ca686f90-c947-4473-9d39-abc8bc4895fd", + name: "nmap_json", + required: true, + schema: { type: "string" }, + }, + { + description: "whether or not first input is a filename or not", + id_: "e605313e-f9c5-4335-8ce7-d46abd422e68", + name: "is_file", + required: true, + schema: { type: "boolean" }, + }, + ], + returns: { + description: "", + id_: "040f32fc-d020-469f-9f4c-47928b076688", + schema: { type: "string" }, + }, + }, + { + description: "Runs an nmap scan, returns results as string or filename", + id_: "ada798da-fff9-4ccf-85f2-24e2edb7722a", + name: "run_scan", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: + "The target(s) to scan, comma separated values, CIDR supported", + id_: "72704c22-a9c9-457f-ae33-e7c17e758e7d", + name: "targets", + required: true, + schema: { type: "array" }, + }, + { + description: "see nmap manpage -- some options require root", + id_: "51ea8e70-adae-47a1-9241-87674b4712c1", + name: "options", + required: true, + schema: { type: "string" }, + }, + ], + returns: { + description: "xml string on nmap output", + id_: "9c18b2a2-c023-4102-bd46-ddeb31a5430c", + schema: { type: "array" }, + }, + }, + { + description: + "Gets the list of active hosts on a network from an nmap scan", + id_: "48b4814f-47ea-4c66-a5db-cacc9cd305b6", + name: "get_hosts_from_scan", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "The target (or targets in CIDR notation) to scan", + id_: "e48d60f7-4590-4dd2-98ce-cd112ab9df2f", + name: "targets", + required: true, + schema: { type: "array" }, + }, + { + description: "", + id_: "2eb29670-701e-4622-99f2-80e4b51cb06e", + name: "options", + required: true, + schema: { type: "string" }, + }, + ], + returns: { + description: "xml string on nmap output", + id_: "1f710e7d-ef6d-4a28-8ceb-8cbc49f6e197", + schema: { type: "string" }, + }, + }, + { + description: "looks into xml nmap for osfamily to match Linux", + id_: "f5425571-2de1-4a84-9221-3829d118617a", + name: "parse_xml_for_linux", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "nmap output as xml array", + id_: "e2e7d0c7-13e8-49f9-85f2-da8bfc73308e", + name: "nmap_arr", + required: true, + schema: { type: "array" }, + }, + ], + returns: { + description: "os", + id_: "785f146f-095d-4008-a068-0c3146fdf4f0", + schema: { type: "array" }, + }, + }, + { + description: "looks into xml nmap for osfamily to match Windows", + id_: "cf35f64c-43b4-4f23-ae75-70d912f4c1d5", + name: "parse_xml_for_windows", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "nmap output as xml array", + id_: "ef1ce558-2d2a-4d30-8e96-c4ad4c18fae9", + name: "nmap_arr", + required: true, + schema: { type: "array" }, + }, + ], + returns: { + description: "os", + id_: "6d21bd12-c076-459e-970d-cd2f39545efb", + schema: { type: "array" }, + }, + }, + { + description: "looks into xml nmap for osfamily", + id_: "2baa37be-59cb-4e8a-a67b-1abae654ce4b", + name: "parse_xml_for_linux_from_file", + node_type: "ACTION", + environment: "cloud", + parameters: [ + { + description: "nmap output as xml filename", + id_: "aa6324a9-efd2-49c7-a181-90f67b39deb9", + name: "nmap_file", + required: true, + schema: { type: "string" }, + }, + ], + returns: { + description: "os", + id_: "96b98b54-2272-4065-b427-2eadf140cb12", + schema: { type: "array" }, + }, + }, + ], + }, +]; export default Data; diff --git a/frontend/src/__test__/environmentdata.js b/frontend/src/__test__/environmentdata.js index 7551141b..706d41ce 100755 --- a/frontend/src/__test__/environmentdata.js +++ b/frontend/src/__test__/environmentdata.js @@ -1,3 +1,6 @@ -const data = [{"name": "cloud", "type": "cloud"}, {"name": "onprem", "type": "onprem"}] +const data = [ + { name: "cloud", type: "cloud" }, + { name: "onprem", type: "onprem" }, +]; -export default data; +export default data; diff --git a/frontend/src/__test__/scheduledata.js b/frontend/src/__test__/scheduledata.js index 49e05e90..73f5d5ae 100755 --- a/frontend/src/__test__/scheduledata.js +++ b/frontend/src/__test__/scheduledata.js @@ -1,30 +1,29 @@ const Data = { - "src": { - "name": "Get Tickets", - "description": "Get tickets", - "outputparameters": [{ - "name": "SymptomDescription", - "schema": {"type": "string"}}, - {"name": "DetailedDescription", - "schema": {"type": "string"}}, - {"name": "EventSource", - "schema": {"type": "string"} - }] - }, - "dst": { - "name": "Create alert", - "description": "Create alert in TheHive", - "inputparameters": [{ - "name": "title", - "required": true, - "schema": {"type": "string"}}, - {"name": "description", - "required": true, - "schema": {"type": "string"}}, - {"name": "source", - "required": true, - "schema": {"type": "string"} - }]} + src: { + name: "Get Tickets", + description: "Get tickets", + outputparameters: [ + { + name: "SymptomDescription", + schema: { type: "string" }, + }, + { name: "DetailedDescription", schema: { type: "string" } }, + { name: "EventSource", schema: { type: "string" } }, + ], + }, + dst: { + name: "Create alert", + description: "Create alert in TheHive", + inputparameters: [ + { + name: "title", + required: true, + schema: { type: "string" }, + }, + { name: "description", required: true, schema: { type: "string" } }, + { name: "source", required: true, schema: { type: "string" } }, + ], + }, }; export default Data; diff --git a/frontend/src/__test__/webhookdata.js b/frontend/src/__test__/webhookdata.js index 1e3a8c55..2142a3f5 100755 --- a/frontend/src/__test__/webhookdata.js +++ b/frontend/src/__test__/webhookdata.js @@ -1,15 +1,15 @@ const data = { - "id":"8ccf0bec1fde018771ab685d2a40bd52", - "info":{ - "url":"", - "name":"testing", - "description":"wut" - }, - "transforms":{}, - "actions": {}, - "type":"webhook", - "status":"uninitialized", - "running":false -} + id: "8ccf0bec1fde018771ab685d2a40bd52", + info: { + url: "", + name: "testing", + description: "wut", + }, + transforms: {}, + actions: {}, + type: "webhook", + status: "uninitialized", + running: false, +}; export default data; diff --git a/frontend/src/__test__/workflowdata.js b/frontend/src/__test__/workflowdata.js index 2e2d0254..d1df8ea8 100755 --- a/frontend/src/__test__/workflowdata.js +++ b/frontend/src/__test__/workflowdata.js @@ -1,3 +1,169 @@ -const data = {"actions":[{"app_name":"hello_world","app_version":"1.0.0","errors":null,"id":"70574332-da82-cf17-c723-75fa7b8493c2","is_valid":true,"label":"hello_world","environment":"onprem","name":"hello_world","parameters":null,"position":{"x":353.7438792397648,"y":260.6717930890377},"priority":0},{"app_name":"hello_world","app_version":"1.0.0","errors":null,"id":"30522433-56ed-53c3-575d-766e282e1d3e","is_valid":true,"label":"random_number","environment":"cloud","name":"random_number","parameters":null,"position":{"x":458.30040774503794,"y":104.27580103487651},"priority":0},{"app_name":"hello_world","app_version":"1.0.0","errors":null,"id":"5b7ac5b5-9514-02b9-ebe0-998c0843b104","is_valid":false,"label":"hello_world_2","environment":"onprem","name":"hello_world","parameters":null,"position":{"x":414.7256019053981,"y":-140.46450482659628},"priority":0},{"app_name":"hello_world","app_version":"1.0.0","errors":null,"id":"7e6e7a19-4636-cebc-91c4-052a3769a18b","is_valid":true,"label":"hello_world_3","environment":"cloud","name":"hello_world","parameters":null,"position":{"x":83.59752786243806,"y":50.232317715020734},"priority":0},{"app_name":"hello_world","app_version":"1.0.0","errors":null,"id":"edbf927d-5a00-2405-28ed-47982cdf5110","is_valid":true,"label":"hello_world_4","environment":"cloud","name":"hello_world","parameters":null,"position":{"x":-147.30681300186404,"y":89.16690830150289},"priority":0},{"app_name":"hello_world","app_version":"1.0.0","errors":null,"id":"4844a855-1e2b-669d-fc72-5f398321ac5d","is_valid":false,"label":"hello_world_5","environment":"onprem","name":"hello_world","parameters":null,"position":{"x":130.24982593523967,"y":233.8325632286361},"priority":0},{"app_name":"hello_world","app_version":"1.0.0","errors":null,"id":"6d1d3f8a-1ac9-3db4-0e0f-2fe32e9d3c09","is_valid":true,"label":"hello_world_6","environment":"cloud","name":"hello_world","parameters":null,"position":{"x":83.551088005629,"y":-105.15867327274223},"priority":0},{"app_name":"hello_world","app_version":"1.0.0","errors":null,"id":"469d8c2b-52ac-e397-9a29-becccd04aed8","is_valid":true,"label":"hello_world_7","environment":"cloud","name":"hello_world","parameters":null,"position":{"x":314.4987657226086,"y":10.167183586257954},"priority":0}],"branches":[{"destination_id":"30522433-56ed-53c3-575d-766e282e1d3e","id":"4bcb9795-94e6-7d5f-2074-0d5b27784e0b","source_id":"70574332-da82-cf17-c723-75fa7b8493c2"},{"destination_id":"5b7ac5b5-9514-02b9-ebe0-998c0843b104","id":"fe0ab8e4-a535-61cd-3c09-8fd3d8e40769","source_id":"30522433-56ed-53c3-575d-766e282e1d3e"},{"destination_id":"469d8c2b-52ac-e397-9a29-becccd04aed8","id":"8b9ee9bc-b0ab-0bb6-af61-46d4594b2663","source_id":"30522433-56ed-53c3-575d-766e282e1d3e"},{"destination_id":"6d1d3f8a-1ac9-3db4-0e0f-2fe32e9d3c09","id":"c204d5ef-9cc1-d906-9988-86a624c57783","source_id":"469d8c2b-52ac-e397-9a29-becccd04aed8"},{"destination_id":"6d1d3f8a-1ac9-3db4-0e0f-2fe32e9d3c09","id":"1ffb3934-60ec-8f80-5cee-3ddc0a37fdb6","source_id":"5b7ac5b5-9514-02b9-ebe0-998c0843b104"},{"destination_id":"edbf927d-5a00-2405-28ed-47982cdf5110","id":"9c7fb048-9d0d-cb84-9ba0-be729af9b4d1","source_id":"6d1d3f8a-1ac9-3db4-0e0f-2fe32e9d3c09"},{"destination_id":"edbf927d-5a00-2405-28ed-47982cdf5110","id":"e3ab104e-fc8b-3af5-8daa-bfa57bcf9690","source_id":"7e6e7a19-4636-cebc-91c4-052a3769a18b"},{"destination_id":"7e6e7a19-4636-cebc-91c4-052a3769a18b","id":"b6626081-22dd-3af3-b899-480f60d886ca","source_id":"30522433-56ed-53c3-575d-766e282e1d3e"},{"destination_id":"4844a855-1e2b-669d-fc72-5f398321ac5d","id":"4275cf97-0447-bbda-0c80-ab20d389de1a","source_id":"edbf927d-5a00-2405-28ed-47982cdf5110"}],"conditions":[],"triggers":[],"transforms":[],"description":"asd","id":"2f299808-0f1b-4ae0-97fc-ac17483dfcf7","id":"2f299808-0f1b-4ae0-97fc-ac17483dfcf7","is_valid":true,"name":"test2","start":"70574332-da82-cf17-c723-75fa7b8493c2","owner":{"username":"","id":"","orgs":""},"execution_org":{"name":"","org":"","users":null,"id":""},"workflow_variables":null} +const data = { + actions: [ + { + app_name: "hello_world", + app_version: "1.0.0", + errors: null, + id: "70574332-da82-cf17-c723-75fa7b8493c2", + is_valid: true, + label: "hello_world", + environment: "onprem", + name: "hello_world", + parameters: null, + position: { x: 353.7438792397648, y: 260.6717930890377 }, + priority: 0, + }, + { + app_name: "hello_world", + app_version: "1.0.0", + errors: null, + id: "30522433-56ed-53c3-575d-766e282e1d3e", + is_valid: true, + label: "random_number", + environment: "cloud", + name: "random_number", + parameters: null, + position: { x: 458.30040774503794, y: 104.27580103487651 }, + priority: 0, + }, + { + app_name: "hello_world", + app_version: "1.0.0", + errors: null, + id: "5b7ac5b5-9514-02b9-ebe0-998c0843b104", + is_valid: false, + label: "hello_world_2", + environment: "onprem", + name: "hello_world", + parameters: null, + position: { x: 414.7256019053981, y: -140.46450482659628 }, + priority: 0, + }, + { + app_name: "hello_world", + app_version: "1.0.0", + errors: null, + id: "7e6e7a19-4636-cebc-91c4-052a3769a18b", + is_valid: true, + label: "hello_world_3", + environment: "cloud", + name: "hello_world", + parameters: null, + position: { x: 83.59752786243806, y: 50.232317715020734 }, + priority: 0, + }, + { + app_name: "hello_world", + app_version: "1.0.0", + errors: null, + id: "edbf927d-5a00-2405-28ed-47982cdf5110", + is_valid: true, + label: "hello_world_4", + environment: "cloud", + name: "hello_world", + parameters: null, + position: { x: -147.30681300186404, y: 89.16690830150289 }, + priority: 0, + }, + { + app_name: "hello_world", + app_version: "1.0.0", + errors: null, + id: "4844a855-1e2b-669d-fc72-5f398321ac5d", + is_valid: false, + label: "hello_world_5", + environment: "onprem", + name: "hello_world", + parameters: null, + position: { x: 130.24982593523967, y: 233.8325632286361 }, + priority: 0, + }, + { + app_name: "hello_world", + app_version: "1.0.0", + errors: null, + id: "6d1d3f8a-1ac9-3db4-0e0f-2fe32e9d3c09", + is_valid: true, + label: "hello_world_6", + environment: "cloud", + name: "hello_world", + parameters: null, + position: { x: 83.551088005629, y: -105.15867327274223 }, + priority: 0, + }, + { + app_name: "hello_world", + app_version: "1.0.0", + errors: null, + id: "469d8c2b-52ac-e397-9a29-becccd04aed8", + is_valid: true, + label: "hello_world_7", + environment: "cloud", + name: "hello_world", + parameters: null, + position: { x: 314.4987657226086, y: 10.167183586257954 }, + priority: 0, + }, + ], + branches: [ + { + destination_id: "30522433-56ed-53c3-575d-766e282e1d3e", + id: "4bcb9795-94e6-7d5f-2074-0d5b27784e0b", + source_id: "70574332-da82-cf17-c723-75fa7b8493c2", + }, + { + destination_id: "5b7ac5b5-9514-02b9-ebe0-998c0843b104", + id: "fe0ab8e4-a535-61cd-3c09-8fd3d8e40769", + source_id: "30522433-56ed-53c3-575d-766e282e1d3e", + }, + { + destination_id: "469d8c2b-52ac-e397-9a29-becccd04aed8", + id: "8b9ee9bc-b0ab-0bb6-af61-46d4594b2663", + source_id: "30522433-56ed-53c3-575d-766e282e1d3e", + }, + { + destination_id: "6d1d3f8a-1ac9-3db4-0e0f-2fe32e9d3c09", + id: "c204d5ef-9cc1-d906-9988-86a624c57783", + source_id: "469d8c2b-52ac-e397-9a29-becccd04aed8", + }, + { + destination_id: "6d1d3f8a-1ac9-3db4-0e0f-2fe32e9d3c09", + id: "1ffb3934-60ec-8f80-5cee-3ddc0a37fdb6", + source_id: "5b7ac5b5-9514-02b9-ebe0-998c0843b104", + }, + { + destination_id: "edbf927d-5a00-2405-28ed-47982cdf5110", + id: "9c7fb048-9d0d-cb84-9ba0-be729af9b4d1", + source_id: "6d1d3f8a-1ac9-3db4-0e0f-2fe32e9d3c09", + }, + { + destination_id: "edbf927d-5a00-2405-28ed-47982cdf5110", + id: "e3ab104e-fc8b-3af5-8daa-bfa57bcf9690", + source_id: "7e6e7a19-4636-cebc-91c4-052a3769a18b", + }, + { + destination_id: "7e6e7a19-4636-cebc-91c4-052a3769a18b", + id: "b6626081-22dd-3af3-b899-480f60d886ca", + source_id: "30522433-56ed-53c3-575d-766e282e1d3e", + }, + { + destination_id: "4844a855-1e2b-669d-fc72-5f398321ac5d", + id: "4275cf97-0447-bbda-0c80-ab20d389de1a", + source_id: "edbf927d-5a00-2405-28ed-47982cdf5110", + }, + ], + conditions: [], + triggers: [], + transforms: [], + description: "asd", + id: "2f299808-0f1b-4ae0-97fc-ac17483dfcf7", + id: "2f299808-0f1b-4ae0-97fc-ac17483dfcf7", + is_valid: true, + name: "test2", + start: "70574332-da82-cf17-c723-75fa7b8493c2", + owner: { username: "", id: "", orgs: "" }, + execution_org: { name: "", org: "", users: null, id: "" }, + workflow_variables: null, +}; -export default data; +export default data; diff --git a/frontend/src/charts.js b/frontend/src/charts.js index 4751047f..56690e80 100755 --- a/frontend/src/charts.js +++ b/frontend/src/charts.js @@ -23,7 +23,7 @@ let chart1_2_options = { maintainAspectRatio: false, legend: { - display: false + display: false, }, tooltips: { backgroundColor: "#f5f5f5", @@ -33,7 +33,7 @@ let chart1_2_options = { xPadding: 12, mode: "nearest", intersect: 0, - position: "nearest" + position: "nearest", }, responsive: true, scales: { @@ -43,15 +43,15 @@ let chart1_2_options = { gridLines: { drawBorder: false, color: "rgba(29,140,248,0.0)", - zeroLineColor: "transparent" + zeroLineColor: "transparent", }, ticks: { suggestedMin: 60, suggestedMax: 125, padding: 20, - fontColor: "#9a9a9a" - } - } + fontColor: "#9a9a9a", + }, + }, ], xAxes: [ { @@ -59,22 +59,22 @@ let chart1_2_options = { gridLines: { drawBorder: false, color: "rgba(29,140,248,0.1)", - zeroLineColor: "transparent" + zeroLineColor: "transparent", }, ticks: { padding: 20, - fontColor: "#9a9a9a" - } - } - ] - } + fontColor: "#9a9a9a", + }, + }, + ], + }, }; // ######################################### // // // used inside src/views/Dashboard.js // ######################################### let chartExample1 = { - data1: canvas => { + data1: (canvas) => { let ctx = canvas.getContext("2d"); let gradientStroke = ctx.createLinearGradient(0, 230, 0, 50); @@ -96,7 +96,7 @@ let chartExample1 = { "SEP", "OCT", "NOV", - "DEC" + "DEC", ], datasets: [ { @@ -114,12 +114,12 @@ let chartExample1 = { pointHoverRadius: 4, pointHoverBorderWidth: 15, pointRadius: 4, - data: [100, 70, 90, 70, 85, 60, 75, 60, 90, 80, 110, 100] - } - ] + data: [100, 70, 90, 70, 85, 60, 75, 60, 90, 80, 110, 100], + }, + ], }; }, - data2: canvas => { + data2: (canvas) => { let ctx = canvas.getContext("2d"); let gradientStroke = ctx.createLinearGradient(0, 230, 0, 50); @@ -141,7 +141,7 @@ let chartExample1 = { "SEP", "OCT", "NOV", - "DEC" + "DEC", ], datasets: [ { @@ -159,12 +159,12 @@ let chartExample1 = { pointHoverRadius: 4, pointHoverBorderWidth: 15, pointRadius: 4, - data: [80, 120, 105, 110, 95, 105, 90, 100, 80, 95, 70, 120] - } - ] + data: [80, 120, 105, 110, 95, 105, 90, 100, 80, 95, 70, 120], + }, + ], }; }, - data3: canvas => { + data3: (canvas) => { let ctx = canvas.getContext("2d"); let gradientStroke = ctx.createLinearGradient(0, 230, 0, 50); @@ -186,7 +186,7 @@ let chartExample1 = { "SEP", "OCT", "NOV", - "DEC" + "DEC", ], datasets: [ { @@ -204,19 +204,19 @@ let chartExample1 = { pointHoverRadius: 4, pointHoverBorderWidth: 15, pointRadius: 4, - data: [60, 80, 65, 130, 80, 105, 90, 130, 70, 115, 60, 130] - } - ] + data: [60, 80, 65, 130, 80, 105, 90, 130, 70, 115, 60, 130], + }, + ], }; }, - options: chart1_2_options + options: chart1_2_options, }; // ######################################### // // // used inside src/views/Dashboard.js // ######################################### let chartExample2 = { - data: canvas => { + data: (canvas) => { let ctx = canvas.getContext("2d"); let gradientStroke = ctx.createLinearGradient(0, 230, 0, 50); @@ -243,19 +243,19 @@ let chartExample2 = { pointHoverRadius: 4, pointHoverBorderWidth: 15, pointRadius: 4, - data: [80, 100, 70, 80, 120, 80] - } - ] + data: [80, 100, 70, 80, 120, 80], + }, + ], }; }, - options: chart1_2_options + options: chart1_2_options, }; // ######################################### // // // used inside src/views/Dashboard.js // ######################################### let chartExample3 = { - data: canvas => { + data: (canvas) => { let ctx = canvas.getContext("2d"); let gradientStroke = ctx.createLinearGradient(0, 230, 0, 50); @@ -276,15 +276,15 @@ let chartExample3 = { borderWidth: 2, borderDash: [], borderDashOffset: 0.0, - data: [53, 20, 10, 80, 100, 45] - } - ] + data: [53, 20, 10, 80, 100, 45], + }, + ], }; }, options: { maintainAspectRatio: false, legend: { - display: false + display: false, }, tooltips: { backgroundColor: "#f5f5f5", @@ -294,7 +294,7 @@ let chartExample3 = { xPadding: 12, mode: "nearest", intersect: 0, - position: "nearest" + position: "nearest", }, responsive: true, scales: { @@ -303,38 +303,38 @@ let chartExample3 = { gridLines: { drawBorder: false, color: "rgba(225,78,202,0.1)", - zeroLineColor: "transparent" + zeroLineColor: "transparent", }, ticks: { suggestedMin: 60, suggestedMax: 120, padding: 20, - fontColor: "#9e9e9e" - } - } + fontColor: "#9e9e9e", + }, + }, ], xAxes: [ { gridLines: { drawBorder: false, color: "rgba(225,78,202,0.1)", - zeroLineColor: "transparent" + zeroLineColor: "transparent", }, ticks: { padding: 20, - fontColor: "#9e9e9e" - } - } - ] - } - } + fontColor: "#9e9e9e", + }, + }, + ], + }, + }, }; // ######################################### // // // used inside src/views/Dashboard.js // ######################################### const chartExample4 = { - data: canvas => { + data: (canvas) => { let ctx = canvas.getContext("2d"); let gradientStroke = ctx.createLinearGradient(0, 230, 0, 50); @@ -361,15 +361,15 @@ const chartExample4 = { pointHoverRadius: 4, pointHoverBorderWidth: 15, pointRadius: 4, - data: [90, 27, 60, 12, 80] - } - ] + data: [90, 27, 60, 12, 80], + }, + ], }; }, options: { maintainAspectRatio: false, legend: { - display: false + display: false, }, tooltips: { @@ -380,7 +380,7 @@ const chartExample4 = { xPadding: 12, mode: "nearest", intersect: 0, - position: "nearest" + position: "nearest", }, responsive: true, scales: { @@ -390,15 +390,15 @@ const chartExample4 = { gridLines: { drawBorder: false, color: "rgba(29,140,248,0.0)", - zeroLineColor: "transparent" + zeroLineColor: "transparent", }, ticks: { suggestedMin: 50, suggestedMax: 125, padding: 20, - fontColor: "#9e9e9e" - } - } + fontColor: "#9e9e9e", + }, + }, ], xAxes: [ @@ -407,21 +407,21 @@ const chartExample4 = { gridLines: { drawBorder: false, color: "rgba(0,242,195,0.1)", - zeroLineColor: "transparent" + zeroLineColor: "transparent", }, ticks: { padding: 20, - fontColor: "#9e9e9e" - } - } - ] - } - } + fontColor: "#9e9e9e", + }, + }, + ], + }, + }, }; module.exports = { chartExample1, // in src/views/Dashboard.js chartExample2, // in src/views/Dashboard.js chartExample3, // in src/views/Dashboard.js - chartExample4 // in src/views/Dashboard.js + chartExample4, // in src/views/Dashboard.js }; diff --git a/frontend/src/components/AlertPopup.js b/frontend/src/components/AlertPopup.js index 67b54596..340d03e2 100755 --- a/frontend/src/components/AlertPopup.js +++ b/frontend/src/components/AlertPopup.js @@ -1,26 +1,19 @@ -import React, { useEffect} from 'react'; +import React, { useEffect } from "react"; const Popup = (props) => { - const { data } = props; + const { data } = props; - const popupStyle = { - position: "fixed", - width: "300px", - height: "50px", - backgroundColor: "black", - color: "white", - } + const popupStyle = { + position: "fixed", + width: "300px", + height: "50px", + backgroundColor: "black", + color: "white", + }; - const popupData = -
- HEY -
+ const popupData =
HEY
; - return ( -
- {popupData} -
- ) -} + return
{popupData}
; +}; -export default Popup +export default Popup; diff --git a/frontend/src/components/AlertTemplate.js b/frontend/src/components/AlertTemplate.js index 66f4b1c7..4dfa06d0 100755 --- a/frontend/src/components/AlertTemplate.js +++ b/frontend/src/components/AlertTemplate.js @@ -1,46 +1,48 @@ -import React from 'react' -import InfoIcon from '@material-ui/icons/Info'; -import CheckIcon from '@material-ui/icons/Check'; -import ErrorOutlineIcon from '@material-ui/icons/ErrorOutline'; -import CloseIcon from '@material-ui/icons/Close'; -import Typography from '@material-ui/core/Typography'; +import React from "react"; +import InfoIcon from "@material-ui/icons/Info"; +import CheckIcon from "@material-ui/icons/Check"; +import ErrorOutlineIcon from "@material-ui/icons/ErrorOutline"; +import CloseIcon from "@material-ui/icons/Close"; +import Typography from "@material-ui/core/Typography"; const alertStyle = { - backgroundColor: 'rgba(0,0,0,0.9)', - color: 'white', + backgroundColor: "rgba(0,0,0,0.9)", + color: "white", padding: 15, - textTransform: 'uppercase', - borderRadius: '3px', - display: 'flex', - justifyContent: 'space-between', - alignItems: 'center', - boxShadow: '0px 2px 2px 2px rgba(0, 0, 0, 0.03)', + textTransform: "uppercase", + borderRadius: "3px", + display: "flex", + justifyContent: "space-between", + alignItems: "center", + boxShadow: "0px 2px 2px 2px rgba(0, 0, 0, 0.03)", width: 300, - boxSizing: 'border-box', - zIndex: 100001, - overflow: "hidden", -} + boxSizing: "border-box", + zIndex: 100001, + overflow: "hidden", +}; const buttonStyle = { - marginLeft: '20px', - border: 'none', - backgroundColor: 'transparent', - cursor: 'pointer', - color: '#FFFFFF' -} + marginLeft: "20px", + border: "none", + backgroundColor: "transparent", + cursor: "pointer", + color: "#FFFFFF", +}; const AlertTemplate = ({ message, options, style, close }) => { return (
- {options.type === 'info' && } - {options.type === 'success' && } - {options.type === 'error' && } - {message} + {options.type === "info" && } + {options.type === "success" && } + {options.type === "error" && ( + + )} + {message}
- ) -} + ); +}; -export default AlertTemplate +export default AlertTemplate; diff --git a/frontend/src/components/AppGrid1.jsx b/frontend/src/components/AppGrid1.jsx new file mode 100644 index 00000000..b64610c5 --- /dev/null +++ b/frontend/src/components/AppGrid1.jsx @@ -0,0 +1,377 @@ +import React, {useEffect, useState} from 'react'; + +import ReactGA from 'react-ga4'; +import { useTheme } from '@material-ui/core/styles'; +import {Link} from 'react-router-dom'; + +import { Search as SearchIcon, CloudQueue as CloudQueueIcon, Code as CodeIcon } from '@material-ui/icons'; + +import algoliasearch from 'algoliasearch/lite'; +import { InstantSearch, Configure, connectSearchBox, connectHits, connectHitInsights } from 'react-instantsearch-dom'; + +import aa from 'search-insights' + +import { + Zoom, + Grid, + Paper, + TextField, + ButtonBase, + InputAdornment, + Typography, + Button, + Tooltip +} from '@material-ui/core'; + +const searchClient = algoliasearch("JNSS5CFDZZ", "db08e40265e2941b9a7d8f644b6e5240") +//const searchClient = algoliasearch("L55H18ZINA", "a19be455e7e75ee8f20a93d26b9fc6d6") +const AppGrid1 = props => { + const { maxRows, showName, showSuggestion, isMobile, globalUrl, parsedXs, userdata, searchValue } = props + + const isCloud = + window.location.host === "localhost:3000" || + window.location.host === "shuffler.io"; + + const rowHandler = maxRows === undefined || maxRows === null ? 50 : maxRows + const xs = parsedXs === undefined || parsedXs === null ? isMobile ? 6 : 2 : parsedXs + const theme = useTheme(); + //const [apps, setApps] = React.useState([]); + //const [filteredApps, setFilteredApps] = React.useState([]); + const [formMail, setFormMail] = React.useState(""); + const [message, setMessage] = React.useState(""); + const [formMessage, setFormMessage] = React.useState(""); + + const buttonStyle = {borderRadius: 30, height: 50, width: 220, margin: isMobile ? "15px auto 15px auto" : 20, fontSize: 18,} + + const innerColor = "rgba(255,255,255,0.65)" + const borderRadius = 3 + window.title = "Shuffle | Apps | Find and integrate any app" + + const submitContact = (email, message) => { + const data = { + "firstname": "", + "lastname": "", + "title": "", + "companyname": "", + "email": email, + "phone": "", + "message": message, + } + + const errorMessage = "Something went wrong. Please contact frikky@shuffler.io directly." + + fetch(globalUrl+"/api/v1/contact", { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(data), + }) + .then(response => response.json()) + .then(response => { + if (response.success === true) { + setFormMessage(response.reason) + //alert.info("Thanks for submitting!") + } else { + setFormMessage(errorMessage) + } + + setFormMail("") + setMessage("") + }) + .catch(error => { + setFormMessage(errorMessage) + console.log(error) + }); + } + + const SearchBox = ({currentRefinement, refine, isSearchStalled} ) => { + + useEffect(() => { + if (window !== undefined && window.location !== undefined && window.location.search !== undefined && window.location.search !== null) { + const urlSearchParams = new URLSearchParams(window.location.search) + const params = Object.fromEntries(urlSearchParams.entries()) + const foundQuery = {searchValue} + if (foundQuery !== null && foundQuery !== undefined) { + console.log("Got query: ", foundQuery) + refine(foundQuery) + } + } + }, []) + + return ( +
+ + + + ), + }} + autoComplete='off' + type="hidden" + color="primary" + defaultValue={currentRefinement} + placeholder="Find Apps..." + id="shuffle_search_field" + onChange={(event) => { + refine(event.currentTarget.value) + }} + limit={5} + /> + {/*isSearchStalled ? 'My search is stalled' : ''*/} + + ) + } + + var workflowDelay = -50 + const Hits = ({ hits, insights }) => { + const [mouseHoverIndex, setMouseHoverIndex] = useState(-1) + var counted = 0 + + //console.log(hits) + //var curhits = hits + //if (hits.length > 0 && defaultApps.length === 0) { + // setDefaultApps(hits) + //} + + //const [defaultApps, setDefaultApps] = React.useState([]) + //console.log(hits) + //if (hits.length > 0 && hits.length !== innerHits.length) { + // setInnerHits(hits) + //} + + return ( + + {hits.map((data, index) => { + + workflowDelay += 50 + + const paperStyle = { + backgroundColor: index === mouseHoverIndex ? "rgba(255,255,255,0.8)" : theme.palette.inputColor, + color: index === mouseHoverIndex ? theme.palette.inputColor : "rgba(255,255,255,0.8)", + border: `1px solid ${innerColor}`, + padding: 15, + cursor: "pointer", + position: "relative", + minHeight: 116, + } + + if (counted === 12/xs*rowHandler) { + return null + } + + counted += 1 + var parsedname = "" + for (var key = 0; key < data.name.length; key++) { + var character = data.name.charAt(key) + if (character === character.toUpperCase()) { + //console.log(data.name[key], data.name[key+1]) + if (data.name.charAt(key+1) !== undefined && data.name.charAt(key+1) === data.name.charAt(key+1).toUpperCase()) { + } else { + parsedname += " " + } + } + + parsedname += character + } + + parsedname = (parsedname.charAt(0).toUpperCase()+parsedname.substring(1)).replaceAll("_", " ") + + return ( + + + + { + setMouseHoverIndex(index) + /* + ReactGA.event({ + category: "app_grid_view", + action: `search_bar_click`, + label: "", + }) + */ + }} onMouseOut={() => { + setMouseHoverIndex(-1) + }} onClick={() => { + if (isCloud) { + ReactGA.event({ + category: "app_grid_view", + action: `app_${parsedname}_${data.id}_click`, + label: "", + }) + } + + //const searchClient = algoliasearch("L55H18ZINA", "a19be455e7e75ee8f20a93d26b9fc6d6") + console.log(searchClient) + aa('init', { + appId: searchClient.appId, + apiKey: searchClient.transporter.queryParameters["x-algolia-api-key"] + }) + + const timestamp = new Date().getTime() + aa('sendEvents', [ + { + eventType: 'click', + eventName: 'Product Clicked', + index: 'appsearch', + objectIDs: [data.objectID], + timestamp: timestamp, + queryID: data.__queryID, + positions: [data.__position], + userToken: userdata === undefined || userdata === null || userdata.id === undefined ? "unauthenticated" : userdata.id, + } + ]) + + }}> + + {data.name} + +
+ {index === mouseHoverIndex || showName === true ? + parsedname + : + null + } + {data.generated ? + + {data.invalid ? + + : + + } + + : + + + + } + + + + + ) + })} + + ) + } + + const CustomSearchBox = connectSearchBox(SearchBox) + const CustomHits = connectHits(Hits) + //const CustomHits = connectHitInsights(aa)(Hits) + const selectButtonStyle = { + minWidth: 150, + maxWidth: 150, + minHeight: 50, + } + + return ( +
+ {/* +
+ +
+ */} +
+ +
+ +
+ + + + {showSuggestion === true ? +
+ + Can't find what you're looking for? + +
+ setFormMail(e.target.value)} + /> + setMessage(e.target.value)} + /> +
+ + {formMessage} +
+ : null + } + + + + Search by + + + Algolia logo + + +
+
+ ) +} + +export default AppGrid1; diff --git a/frontend/src/components/Billing.jsx b/frontend/src/components/Billing.jsx new file mode 100644 index 00000000..b59b5e4a --- /dev/null +++ b/frontend/src/components/Billing.jsx @@ -0,0 +1,317 @@ +import React, { useState, useEffect } from "react"; +import ReactGA from 'react-ga4'; + +import { useTheme } from "@material-ui/core/styles"; +import { + Paper, + Typography, + Divider, + Button, + Grid, + Card, +} from "@material-ui/core"; + +import { useAlert } from "react-alert"; +import { typecost, typecost_single, } from "../views/HandlePaymentNew.jsx"; + +const Billing = (props) => { + const { globalUrl, userdata, serverside, billingInfo, stripeKey, selectedOrganization, handleGetOrg, } = props; + console.log("Billing: ", billingInfo); + const theme = useTheme(); + const alert = useAlert(); + + const stripe = typeof window === 'undefined' || window.location === undefined ? "" : props.stripeKey === undefined ? "" : window.Stripe ? window.Stripe(props.stripeKey) : "" + console.log("Stripe: ", stripe) + + const paperStyle = { + padding: 20, + height: "100%", + width: "100%", + backgroundColor: theme.palette.surfaceColor, + border: "1px solid rgba(255,255,255,0.3)", + marginRight: 10, + } + + const isCloud = + window.location.host === "localhost:3002" || + window.location.host === "shuffler.io"; + + billingInfo.subscription = { + "active": true, + "name": "Pay as you go", + "price": typecost_single, + "currency": "USD", + "currency_text": "$", + "interval": "app run / month", + "description": "Pay as you go", + "features": [ + "Includes 10.000 app run/month for free. ", + "Pay for what you use with no minimum commitment and cancel anytime.", + ], + "limit": 10000, + } + + + const handleStripeRedirect = () => { + //var priceItem = "price_1MRNF1DzMUgUjxHSfFTUb2Xh" + if (stripe == "") { + console.log("Stripe not loaded") + return + } + + var priceItem = "price_1MROFrDzMUgUjxHShcSxgHO1" + + const successUrl = `${window.location.origin}/admin?admin_tab=billing&payment=success` + const failUrl = `${window.location.origin}/admin?admin_tab=billing&payment=failure` + var checkoutObject = { + lineItems: [ + { + price: priceItem, + quantity: 1 + }, + ], + mode: "subscription", + billingAddressCollection: "auto", + successUrl: successUrl, + cancelUrl: failUrl, + clientReferenceId: props.userdata.active_org.id, + } + //submitType: "donate", + + stripe.redirectToCheckout(checkoutObject) + .then(function (result) { + console.log("SUCCESS STRIPE?: ", result) + + ReactGA.event({ + category: "pricing", + action: "add_card_success", + label: "", + }) + }) + .catch(function(error) { + console.error("STRIPE ERROR: ", error) + + ReactGA.event({ + category: "pricing", + action: "add_card_error", + label: "", + }) + }); + } + + const cancelSubscriptions = (subscription_id) => { + const orgId = selectedOrganization.id; + const data = { + subscription_id: subscription_id, + action: "cancel", + org_id: selectedOrganization.id, + }; + + const url = globalUrl + `/api/v1/orgs/${orgId}/cancel`; + fetch(url, { + mode: "cors", + method: "POST", + body: JSON.stringify(data), + credentials: "include", + crossDomain: true, + withCredentials: true, + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + }) + .then(function (response) { + if (response.status !== 200) { + console.log("Error in response"); + } + + if (handleGetOrg != undefined) { + handleGetOrg(selectedOrganization.id); + } + + return response.json(); + }) + .then(function (responseJson) { + if (responseJson.success !== undefined && responseJson.success) { + alert.success("Successfully stopped subscription!"); + } else { + alert.error("Failed stopping subscription. Please contact us."); + } + }) + .catch(function (error) { + console.log("Error: ", error); + alert.error("Failed stopping subscription. Please contact us."); + }); + }; + + const SubscriptionObject = (props) => { + const { globalUrl, userdata, serverside, billingInfo, stripeKey, selectedOrganization, handleGetOrg, subscription, } = props; + + console.log("Sub: ", subscription) + var top_text = "Base Access" + if (subscription.limit === undefined && subscription.level !== undefined) { + + subscription.name = "Enterprise" + subscription.currency_text = "$" + subscription.price = subscription.level*180 + subscription.limit = subscription.level*100000 + subscription.interval = subscription.recurrence + subscription.features = [ + "Includes " + subscription.limit + " app runs/month. ", + "Multi-Tenancy and Region-Selection", + "And all other features from /pricing", + ] + } + + if (subscription.name === "Enterprise" && subscription.active === true) { + top_text = "Current Plan" + } + + return ( + +
+ + {top_text} + +
+ +
+ + {subscription.name} + +
+ + {subscription.currency_text}{subscription.price} + + + / {subscription.interval} + +
+ + Features + +
    + {subscription.features !== undefined && subscription.features !== null ? + subscription.features.map((feature, index) => { + return ( +
  • + + {feature} + +
    • + ) + }) + : null} +
+
+ {/*subscription.name === "Pay as you go" && subscription.limit <= 10000 ? + + + You are not subscribed to any plan and are using the free plan with max 10,000 apps per month. Activate billing to de-activate this limit. + + + + : null*/} + + ) + } + + + return ( +
+ + Billing + + + We use Stripe to manage subscriptions and do not store any of your billing information. You can manage your subscription and billing information below. + +
+ {billingInfo.subscription !== undefined && billingInfo.subscription !== null ? + + : null} + {isCloud && + selectedOrganization.subscriptions !== undefined && + selectedOrganization.subscriptions !== null && + selectedOrganization.subscriptions.length > 0 ? + + selectedOrganization.subscriptions + .reverse() + .map((sub, index) => { + return ( + + ) + }) + : null} + {/* + + + Quantity: {sub.level} +
+ Recurrence: {sub.recurrence} +
+ {sub.active ? ( +
+ Started:{" "} + {new Date(sub.startdate * 1000).toISOString()} +
+ +
+ ) : ( +
+ Cancelled:{" "} + {new Date( + sub.cancellationdate * 1000 + ).toISOString()} +
+ + Status: Deactivated + +
+ )} + + + */} +
+
+ ) +} + +export default Billing; diff --git a/frontend/src/components/Dropzone.js b/frontend/src/components/Dropzone.jsx similarity index 57% rename from frontend/src/components/Dropzone.js rename to frontend/src/components/Dropzone.jsx index 621e74b5..6ea9f0de 100755 --- a/frontend/src/components/Dropzone.js +++ b/frontend/src/components/Dropzone.jsx @@ -1,19 +1,19 @@ -import React, { useRef, useState } from 'react'; -import { useEffect } from 'react'; -import BackupIcon from '@material-ui/icons/Backup'; +import React, { useRef, useState } from "react"; +import { useEffect } from "react"; +import BackupIcon from "@material-ui/icons/Backup"; const dragOverStyle = { - backgroundColor: 'rgba(0,0,0,0.8)', - border: '5px dashed white', - borderRadius: '8px', - width: '100%', - height: '100%', - position: 'absolute', - overflow: 'hidden', + backgroundColor: "rgba(0,0,0,0.8)", + border: "5px dashed white", + borderRadius: "8px", + width: "100%", + height: "100%", + position: "absolute", + overflow: "hidden", zIndex: 100, - display: 'flex', - alignItems: 'center', - justifyContent: 'center' + display: "flex", + alignItems: "center", + justifyContent: "center", }; const Dropzone = ({ children, style, onDrop }) => { @@ -56,21 +56,21 @@ const Dropzone = ({ children, style, onDrop }) => { useEffect(() => { if (!dropzoneRef.current) return; - dropzoneRef.current.addEventListener('dragover', handleDragOver); - dropzoneRef.current.addEventListener('dragenter', handleDragEnter); - dropzoneRef.current.addEventListener('dragleave', handleDragLeave); - dropzoneRef.current.addEventListener('drop', handleDrop); + dropzoneRef.current.addEventListener("dragover", handleDragOver); + dropzoneRef.current.addEventListener("dragenter", handleDragEnter); + dropzoneRef.current.addEventListener("dragleave", handleDragLeave); + dropzoneRef.current.addEventListener("drop", handleDrop); return () => { - dropzoneRef.current.removeEventListener('dragover', handleDragOver); - dropzoneRef.current.removeEventListener('dragenter', handleDragEnter); - dropzoneRef.current.removeEventListener('dragleave', handleDragLeave); - dropzoneRef.current.removeEventListener('drop', handleDrop); + dropzoneRef.current.removeEventListener("dragover", handleDragOver); + dropzoneRef.current.removeEventListener("dragenter", handleDragEnter); + dropzoneRef.current.removeEventListener("dragleave", handleDragLeave); + dropzoneRef.current.removeEventListener("drop", handleDrop); }; }, [dropzoneRef]); return ( -
+
{dragging && (
diff --git a/frontend/src/components/FAQ.jsx b/frontend/src/components/FAQ.jsx new file mode 100644 index 00000000..ad3aab2c --- /dev/null +++ b/frontend/src/components/FAQ.jsx @@ -0,0 +1,23 @@ +import React, {useState} from 'react'; + + +const FAQItem = (props) => { + const { question, answer } = props + + const [isExpanded, setIsExpanded] = useState(false) + + return ( + { + setIsExpanded(!isExpanded) + }}> + + {question} + + + {answer} + + + ) +} + +export default FAQItem; diff --git a/frontend/src/components/FooterNew.js b/frontend/src/components/FooterNew.js index aa68d486..482c45b7 100755 --- a/frontend/src/components/FooterNew.js +++ b/frontend/src/components/FooterNew.js @@ -1,54 +1,54 @@ -import React from 'react'; +import React from "react"; //import List from '@material-ui/core/List'; //import ListItem from '@material-ui/core/ListItem'; //borderTop: "1px solid #385F71" const FooterStyle = { - right: "0", - left: "0", - bottom: "0", - height: "130px", - backgroundColor: 'rgba(15, 14, 31, 1)', + right: "0", + left: "0", + bottom: "0", + height: "130px", + backgroundColor: "rgba(15, 14, 31, 1)", }; const FooterInfo = { - maxWidth: '1150px', - minWidth: '768px', - textAlign: 'center', - margin: 'auto', + maxWidth: "1150px", + minWidth: "768px", + textAlign: "center", + margin: "auto", }; const hrefStyle = { - color: "#bdbdbd", - textDecoration: "none" -} - -const Footer = props => { - return ( -
-
- -
-
- ); + color: "#bdbdbd", + textDecoration: "none", }; -const Box = props => { - return( - - ); +const Footer = (props) => { + return ( +
+
+ +
+
+ ); +}; + +const Box = (props) => { + return ( + + ); }; export default Footer; diff --git a/frontend/src/components/Header.js b/frontend/src/components/Header.js index 449c057b..0934f2e6 100755 --- a/frontend/src/components/Header.js +++ b/frontend/src/components/Header.js @@ -1,250 +1,320 @@ -import React, {useState} from 'react'; -import {BrowserView, MobileView} from "react-device-detect"; +import React, { useState } from "react"; +import { BrowserView, MobileView } from "react-device-detect"; -import {Link} from 'react-router-dom'; +import { Link } from "react-router-dom"; -import { useTheme } from '@material-ui/core/styles'; +import { useTheme } from "@material-ui/core/styles"; -import { Chip, Badge, Typography, Paper, Tooltip, List, Avatar, Menu, ListItem, MenuItem, Select, Button, IconButton, Grid } from '@material-ui/core'; -import { MeetingRoom as MeetingRoomIcon, Settings as SettingsIcon, Notifications as NotificationsIcon, Home as HomeIcon, Polymer as PolymerIcon, Apps as AppsIcon, Description as DescriptionIcon} from '@material-ui/icons'; +import { + Chip, + Badge, + Typography, + Paper, + Tooltip, + List, + Avatar, + Menu, + ListItem, + MenuItem, + Select, + Button, + IconButton, + Grid, +} from "@material-ui/core"; + +import { + MeetingRoom as MeetingRoomIcon, + Settings as SettingsIcon, + Notifications as NotificationsIcon, + Home as HomeIcon, + Polymer as PolymerIcon, + Apps as AppsIcon, + Description as DescriptionIcon, + HelpOutline as HelpOutlineIcon, +} from "@material-ui/icons"; + +import { + Analytics as AnalyticsIcon, + Lightbulb as LightbulbIcon, +} from "@mui/icons-material"; //import LogoutIcon from '@mui/icons-material/Logout'; import { useAlert } from "react-alert"; +import SearchField from '../components/Searchfield' -const hoverColor = "#f85a3e" -const hoverOutColor = "#e8eaf6" +const hoverColor = "#f85a3e"; +const hoverOutColor = "#e8eaf6"; -const Header = props => { - const { globalUrl, setNotifications, notifications, isLoggedIn, removeCookie, homePage, isLoaded, userdata, cookies } = props; - const theme = useTheme(); +const Header = (props) => { + const { + globalUrl, + setNotifications, + notifications, + isLoggedIn, + removeCookie, + homePage, + isLoaded, + userdata, + cookies, + } = props; + const theme = useTheme(); - const [HomeHoverColor, setHomeHoverColor] = useState(hoverOutColor); - const [SoarHoverColor, setSoarHoverColor] = useState(hoverOutColor); - const [LoginHoverColor, setLoginHoverColor] = useState(hoverOutColor); - const [DocsHoverColor, setDocsHoverColor] = useState(hoverOutColor); + const [HomeHoverColor, setHomeHoverColor] = useState(hoverOutColor); + const [SoarHoverColor, setSoarHoverColor] = useState(hoverOutColor); + const [LoginHoverColor, setLoginHoverColor] = useState(hoverOutColor); + const [DocsHoverColor, setDocsHoverColor] = useState(hoverOutColor); const [HelpHoverColor, setHelpHoverColor] = useState(hoverOutColor); const [anchorEl, setAnchorEl] = React.useState(null); const [anchorElAvatar, setAnchorElAvatar] = React.useState(null); - const alert = useAlert() + const alert = useAlert(); - const hrefStyle = { - color: hoverOutColor, - textDecoration: "none", - } + const hrefStyle = { + color: hoverOutColor, + textDecoration: "none", + }; const handleClose = () => { setAnchorEl(null); setAnchorElAvatar(null); }; - const clearNotifications = () => { - // Don't really care about the logout + const clearNotifications = () => { + // Don't really care about the logout fetch(`${globalUrl}/api/v1/notifications/clear`, { - credentials: "include", - method: 'GET', - headers: { - 'Content-Type': 'application/json', - }, - }) - .then(function(response) { - if (response.status !== 200) { - console.log("Error in response") - } + credentials: "include", + method: "GET", + headers: { + "Content-Type": "application/json", + }, + }) + .then(function (response) { + if (response.status !== 200) { + console.log("Error in response"); + } - return response.json(); - }).then(function(responseJson) { - if (responseJson.success === true) { - setNotifications([]) - handleClose() - } else { - alert.error("Failed dismissing notifications. Please try again later.") - } - }) - .catch(error => { - console.log("error in notification dismissal: ", error) - //removeCookie("session_token", {path: "/"}) - }) - } + return response.json(); + }) + .then(function (responseJson) { + if (responseJson.success === true) { + setNotifications([]); + handleClose(); + } else { + alert.error( + "Failed dismissing notifications. Please try again later." + ); + } + }) + .catch((error) => { + console.log("error in notification dismissal: ", error); + //removeCookie("session_token", {path: "/"}) + }); + }; - const dismissNotification = (alert_id) => { - // Don't really care about the logout + const dismissNotification = (alert_id) => { + // Don't really care about the logout fetch(`${globalUrl}/api/v1/notifications/${alert_id}/markasread`, { - credentials: "include", - method: 'GET', - headers: { - 'Content-Type': 'application/json', - }, - }) - .then(function(response) { - if (response.status !== 200) { - console.log("Error in response") - } + credentials: "include", + method: "GET", + headers: { + "Content-Type": "application/json", + }, + }) + .then(function (response) { + if (response.status !== 200) { + console.log("Error in response"); + } - return response.json(); - }).then(function(responseJson) { - if (responseJson.success === true) { - const newNotifications = notifications.filter(data => data.id !== alert_id) - console.log("NEW NOTIFICATIONS: ", newNotifications) - setNotifications(newNotifications) - } else { - alert.error("Failed dismissing notification. Please try again later.") - } - }) - .catch(error => { - console.log("error in notification dismissal: ", error) - //removeCookie("session_token", {path: "/"}) - }) - } + return response.json(); + }) + .then(function (responseJson) { + if (responseJson.success === true) { + const newNotifications = notifications.filter( + (data) => data.id !== alert_id + ); + console.log("NEW NOTIFICATIONS: ", newNotifications); + setNotifications(newNotifications); + } else { + alert.error( + "Failed dismissing notification. Please try again later." + ); + } + }) + .catch((error) => { + console.log("error in notification dismissal: ", error); + //removeCookie("session_token", {path: "/"}) + }); + }; - // DEBUG HERE - const handleClickLogout = () => { - console.log("COOKIES: ", cookies, "Remover: ", removeCookie) - // Don't really care about the logout - fetch(globalUrl+"/api/v1/logout", { - credentials: "include", - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - }) - .then(() => { - // Log out anyway - //cookies.remove("session_token") - //window.location.pathname = "/" - console.log("Should've logged out") - removeCookie("session_token", {path: "/"}) - removeCookie("session_token", {path: "/workflows"}) - window.location.reload() - }) - .catch(error => { - console.log("Error in logout: ", error) - removeCookie("session_token", {path: "/"}) - window.location.reload() - //removeCookie("session_token", {path: "/"}) - }) - } + // DEBUG HERE + const handleClickLogout = () => { + console.log("COOKIES: ", cookies, "Remover: ", removeCookie); + // Don't really care about the logout + fetch(globalUrl + "/api/v1/logout", { + credentials: "include", + method: "POST", + headers: { + "Content-Type": "application/json", + }, + }) + .then(() => { + // Log out anyway + //cookies.remove("session_token") + //window.location.pathname = "/" + console.log("Should've logged out"); + removeCookie("session_token", { path: "/" }); + removeCookie("session_token", { path: "/workflows" }); + window.location.reload(); + }) + .catch((error) => { + console.log("Error in logout: ", error); + removeCookie("session_token", { path: "/" }); + window.location.reload(); + //removeCookie("session_token", {path: "/"}) + }); + }; - const handleClickChangeOrg = (orgId) => { - // Don't really care about the logout - //name: org.name, - //orgId = "asd" - const data = { - org_id: orgId, - } + const handleClickChangeOrg = (orgId) => { + // Don't really care about the logout + //name: org.name, + //orgId = "asd" + const data = { + org_id: orgId, + }; + + localStorage.setItem("getting_started_sidebar", "open"); fetch(`${globalUrl}/api/v1/orgs/${orgId}/change`, { - mode: 'cors', - method: 'POST', - body: JSON.stringify(data), - credentials: 'include', - crossDomain: true, - withCredentials: true, - headers: { - 'Content-Type': 'application/json; charset=utf-8', - }, - }) - .then(function(response) { - if (response.status !== 200) { - console.log("Error in response") - } + mode: "cors", + method: "POST", + body: JSON.stringify(data), + credentials: "include", + crossDomain: true, + withCredentials: true, + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + }) + .then(function (response) { + if (response.status !== 200) { + console.log("Error in response"); + } - return response.json(); - }).then(function(responseJson) { - if (responseJson.success !== undefined && responseJson.success) { - setTimeout(() => { - window.location.reload() - }, 2000) - alert.success("Successfully changed active organization - refreshing!") - } else { - alert.error("Failed changing org: ", responseJson.reason) - } - }) - .catch(error => { - console.log("error changing: ", error) - //removeCookie("session_token", {path: "/"}) - }) - } + return response.json(); + }) + .then(function (responseJson) { + if (responseJson.success !== undefined && responseJson.success) { + setTimeout(() => { + window.location.reload(); + }, 2000); + alert.success( + "Successfully changed active organization - refreshing!" + ); + } else { + alert.error("Failed changing org: ", responseJson.reason); + } + }) + .catch((error) => { + console.log("error changing: ", error); + //removeCookie("session_token", {path: "/"}) + }); + }; - // Rofl this is weird - const handleDocsHover = () => { - setDocsHoverColor(hoverColor) - } + // Rofl this is weird + const handleDocsHover = () => { + setDocsHoverColor(hoverColor); + }; - const handleDocsHoverOut = () => { - setDocsHoverColor(hoverOutColor) - } + const handleDocsHoverOut = () => { + setDocsHoverColor(hoverOutColor); + }; - const handleHomeHover = () => { - setHomeHoverColor(hoverColor) - } + const handleHomeHover = () => { + setHomeHoverColor(hoverColor); + }; - const handleHelpHover = () => { - setHelpHoverColor(hoverColor) - } + const handleHelpHover = () => { + setHelpHoverColor(hoverColor); + }; - const handleHelpHoverOut = () => { - setHelpHoverColor(hoverOutColor) - } - - const handleSoarHover = () => { - setSoarHoverColor(hoverColor) - } + const handleHelpHoverOut = () => { + setHelpHoverColor(hoverOutColor); + }; - const handleSoarHoverOut = () => { - setSoarHoverColor(hoverOutColor) - } + const handleSoarHover = () => { + setSoarHoverColor(hoverColor); + }; - const handleHomeHoverOut = () => { - setHomeHoverColor(hoverOutColor) - } + const handleSoarHoverOut = () => { + setSoarHoverColor(hoverOutColor); + }; - const handleLoginHover = () => { - setLoginHoverColor(hoverColor) - } + const handleHomeHoverOut = () => { + setHomeHoverColor(hoverOutColor); + }; - const handleLoginHoverOut = () => { - setLoginHoverColor(hoverOutColor) - } + const handleLoginHover = () => { + setLoginHoverColor(hoverColor); + }; + const handleLoginHoverOut = () => { + setLoginHoverColor(hoverOutColor); + }; const handleClick = (event) => { setAnchorEl(event.currentTarget); }; + const chipStyle = { + backgroundColor: "#3d3f43", + height: 30, + marginRight: 5, + paddingLeft: 5, + paddingRight: 5, + height: 28, + cursor: "pointer", + borderColor: "#3d3f43", + color: "white", + }; - const chipStyle = { - backgroundColor: "#3d3f43", height: 30, marginRight: 5, paddingLeft: 5, paddingRight: 5, height: 28, cursor: "pointer", borderColor: "#3d3f43", color: "white", - } + const notificationWidth = 350; + const NotificationItem = (props) => { + const { data } = props; - const notificationWidth = 350 - const NotificationItem = (props) => { - const {data} = props - - return ( - - {/* + return ( + + {/* {new Date(data.updated_at).toISOString()} */} - {data.reference_url !== undefined && data.reference_url !== null && data.reference_url.length > 0 ? - - - {data.title} - - - : - - {data.title} - - } + {data.reference_url !== undefined && + data.reference_url !== null && + data.reference_url.length > 0 ? ( + + {data.title} + + ) : ( + {data.title} + )} - {data.image !== undefined && data.image !== null && data.image.length > 0 ? - {data.title} - : - null - } - - {data.description} - - {/*data.tags !== undefined && data.tags !== null && data.tags.length > 0 ? + {data.image !== undefined && + data.image !== null && + data.image.length > 0 ? ( + {data.title} + ) : null} + {data.description} + {/*data.tags !== undefined && data.tags !== null && data.tags.length > 0 ? data.tags.map((tag, index) => { return ( { ) }) : null */} - {data.read === false ? - - : null} - - ) - } + {data.read === false ? ( + + ) : null} + + ); + }; - const notificationMenu = - - { - setAnchorEl(event.currentTarget); - }}> - - - - - { - handleClose() - }} - > - -
- - Your Notifications ({notifications.length}) - - {notifications.length > 1 ? - - : null} -
- - Notifications are made by Shuffle to help you discover issues or improvements. - - - {notifications.map((data, index) => { - return ( - - ) - })} - - + const notificationMenu = ( + + { + setAnchorEl(event.currentTarget); + }} + > + + + + + { + handleClose(); + }} + > + +
+ + Your Notifications ({notifications.length}) + + {notifications.length > 1 ? ( + + ) : null} +
+ + Notifications are made by Shuffle to help you discover issues or + improvements. + + + {notifications.map((data, index) => { + return ; + })} + + + ); - // Should be based on some path - const avatarMenu = - - { - setAnchorElAvatar(event.currentTarget); - }}> - - - { - handleClose() - }} - > - { - event.preventDefault() - handleClose() - }}> - - Settings - - - { - event.preventDefault() - handleClose() - handleClickLogout() - }}> -  Logout - - - + // Should be based on some path + const avatarMenu = ( + + { + setAnchorElAvatar(event.currentTarget); + }} + > + + + { + handleClose(); + }} + > + { + event.preventDefault(); + handleClose(); + }} + > + + About + + + { + event.preventDefault(); + handleClose(); + }} + > + + Get Started + + + { + event.preventDefault(); + handleClose(); + }} + > + + Use Cases + + + { + event.preventDefault(); + handleClose(); + }} + > + + Settings + + + { + event.preventDefault(); + handleClose(); + handleClickLogout(); + }} + > +  Logout + + + + ); - - // Handle top bar or something - const logoCheck = !homePage ? null : null - //
- const loginTextBrowser = !isLoggedIn ? -
- - - -
- About -
- - - -
- - - -
Login
- - - -
-
- : -
-
- - - -
- - Workflows -
- - - - -
- - Apps -
- - - {/* + // Handle top bar or something + const logoCheck = !homePage ? null : null; + //
+ const loginTextBrowser = !isLoggedIn ? ( +
+ + + +
+ About +
+ + + + {!isLoaded ? null : + userdata.chat_disabled === true ? null : +
+ +
+ } +
+ + + +
+ Login +
+ + + +
+
+ ) : ( +
+
+ + + +
+ + Workflows +
+ + + + +
+ + Apps +
+ + + {/*
Dashboard
 */} - - -
- - Docs -
- - - {/* + + +
+ + Docs +
+ + + {/*
@@ -425,146 +619,265 @@ const Header = props => { */} - {/* + {/*
Configure
 */} - -
-
- {avatarMenu} - {notificationMenu} - {userdata === undefined || userdata.admin === undefined || userdata.admin === null || !userdata.admin ? null : - - - - } - {userdata === undefined || userdata.orgs === undefined || userdata.orgs === null || userdata.orgs.length <= 1 ? null : - { + handleClickChangeOrg(e.target.value); + }} + > + {userdata.orgs.map((data, index) => { + if ( + data.name === undefined || + data.name === null || + data.name.length === 0 + ) { + return null; + } - const imagesize = 22 - const imageStyle = {width: imagesize, height: imagesize, pointerEvents: "none", marginRight: 10, marginLeft: data.creator_org !== undefined && data.creator_org.length > 0 ? 20 : 0} - const image = data.image === "" ? - {data.name} - : - {data.name} + const imagesize = 22 - return ( - + //if (data.creator_org !== undefined && data.creator_org !== null && data.creator_org.length > 0 && data.fixed !== true) { + var skipOrg = false + if (data.creator_org !== undefined && data.creator_org !== null && data.creator_org.length > 0) { + // Finds the parent org + for (var key in userdata.child_orgs) { + if (data.child_orgs[key].id === data.creator_org) { + skipOrg = true + break + } + } - -
- {image} {data.name} -
- - - ) - })} - - } -
-
+ if (skipOrg) { + return null + } + } - //console.log("USR: ", userdata.orgs) + // Reordering to have suborgs with access under original org + if (data.child_orgs !== undefined && data.child_orgs !== null) { + var cnt = 0 + for (var key in data.child_orgs) { + const childorg = data.child_orgs[key] + const foundIndex = userdata.orgs.findIndex(item => item.id === childorg.id) + if (foundIndex !== -1) { + const newindex = parseInt(index)+parseInt(cnt) - const loginTextMobile = !isLoggedIn ? -
- - - -
- - - - - -
- - - - -
- About -
- - - -
- : -
-
- - - -
Shuffle
- - - - -
Workflows
- - - - -
Apps
- - - {/* + var newitem = userdata.orgs[foundIndex] + newitem.fixed = true + userdata.orgs.splice(newindex+1, 0, newitem) + userdata.orgs.splice(foundIndex+1, 1) + } else { + console.log("ORG NOT FOUND IN LIST: ", childorg) + } + + // This is stupid :) + cnt += 1 + } + } + + //console.log("ORG: ", data) + const imageStyle = { + width: imagesize, + height: imagesize, + pointerEvents: "none", + marginRight: 10, + marginLeft: data.creator_org !== undefined && data.creator_org !== null && data.creator_org.length > 0 ? 20 : 0, + } + + const parsedTitle = data.creator_org !== undefined && data.creator_org !== null && data.creator_org.length > 0 ? `Suborg of ${data.creator_org}` : "" + + const image = data.image === "" ? + {data.name} + : + {data.name} + + return ( + + + +
+ {image} {data.name} +
+ + + ) + })} + + )} +
+
+ ); + + //console.log("USR: ", userdata.orgs) + + const loginTextMobile = !isLoggedIn ? ( +
+ + + +
+ + + + + +
+ + + + +
+ About +
+ + + +
+ ) : ( +
+
+ + + +
+ Shuffle +
+ + + + +
+ Workflows +
+ + + + +
+ Apps +
+ + + {/*
Configure
 */} - -
-
- {avatarMenu} -
-
+ +
+
+ {avatarMenu} +
+
+ ); - // - const loadedCheck = -
- - {loginTextBrowser} - - - {loginTextMobile} - -
- //
- return ( -
- {loadedCheck} -
- ) -} + // + const loadedCheck = ( +
+ {loginTextBrowser} + {loginTextMobile} +
+ ); + //
+ return ( +
+ {loadedCheck} +
+ ); +}; export default Header; diff --git a/frontend/src/components/LandingpageUsecases.jsx b/frontend/src/components/LandingpageUsecases.jsx new file mode 100644 index 00000000..9fd5d016 --- /dev/null +++ b/frontend/src/components/LandingpageUsecases.jsx @@ -0,0 +1,245 @@ +import React, { useState, useEffect } from 'react'; +import {isMobile} from "react-device-detect"; +import AppFramework, { usecases } from "../components/AppFramework.jsx"; +import {Link} from 'react-router-dom'; +import ReactGA from 'react-ga4'; + +import { Button, LinearProgress, Typography } from '@material-ui/core'; + +export const securityFramework = [ + { + image: , + text: "Cases", + description: "Case management" + }, + { + image: + , + text: "SIEM", + description: "Case management" + }, + { + image: + , + text: "Assets", + description: "Case management" + }, + { + image: + , + text: "IAM", + description: "Case management" + }, + { + image: , + text: "Intel", + description: "Case management" + }, + { + image: + , + text: "Comms", + description: "Case management" + }, + { + image: + , + text: "Network", + description: "Case management" + }, + { + image: + , + text: "EDR & AV", + description: "Case management" + }, +] + +const LandingpageUsecases = (props) => { + const [selectedUsecase, setSelectedUsecase] = useState("Phishing") + const usecasekeys = usecases === undefined || usecases === null ? [] : Object.keys(usecases) + const buttonBackground = "linear-gradient(to right, #f86a3e, #f34079)" + const buttonStyle = {borderRadius: 25, height: 50, width: 260, margin: isMobile ? "15px auto 15px auto" : 20, fontSize: 18, backgroundImage: buttonBackground} + + const HandleTitle = (props) => { + const { usecases, selectedUsecase, setSelecedUsecase } = props + const [progress, setProgress] = useState(0) + + useEffect(() => { + const timer = setInterval(() => { + setProgress((oldProgress) => { + if (oldProgress >= 105) { + const foundIndex = usecasekeys.findIndex(key => key === selectedUsecase) + var newitem = usecasekeys[foundIndex+1] + if (newitem === undefined || newitem === 0) { + newitem = usecasekeys[1] + } + + setSelectedUsecase(newitem) + return -18 + } + + if (oldProgress >= 65) { + return oldProgress + 3 + } + + if (oldProgress >= 80) { + return oldProgress + 1 + } + + return oldProgress + 6 + }) + }, 165) + + return () => { + clearInterval(timer) + } + }, []) + + if (usecases === null || usecases === undefined || usecases.length === 0) { + return null + } + + const modifier = isMobile ? 17 : 22 + return ( + + Handle
+ + {selectedUsecase} + + + with confidence + + ) + } + + const parsedWidth = isMobile ? "100%" : 1100 + return ( +
+
+
+ + + + + {/*Security Automation is Hard*/} + + + Connecting your everchanging environment is hard. We get it! That's why we built Shuffle, where you can use and share your security workflows to everyones benefit. + {/*Shuffle is an automation platform where you don't need to be an expert to automate. Get access to our large pool of security playbooks, apps and people.*/} + +
+ {isMobile ? null : + + + + } + {isMobile ? null : + + + + } +
+
+ {isMobile ? null : +
+ +
+ } + {isMobile ? null : +
+ + + + + +
+ } +
+
+ {isMobile ? + + + + : null + } + {/*isMobile ? + + + + : null*/} +
+ {isMobile ? null : +
+ {securityFramework.map((data, index) => { + return ( +
+ + + {data.image} + + + + {data.text} + +
+ ) + })} +
+ } +
+ ) +} + +export default LandingpageUsecases; diff --git a/frontend/src/components/LoginPopup.js b/frontend/src/components/LoginPopup.js index 232b4976..1807ae9a 100755 --- a/frontend/src/components/LoginPopup.js +++ b/frontend/src/components/LoginPopup.js @@ -1,139 +1,176 @@ /* eslint-disable react/no-multi-comp */ -import React, {useState} from 'react'; +import React, { useState } from "react"; -import DialogTitle from '@material-ui/core/DialogTitle'; -import Dialog from '@material-ui/core/Dialog'; -import TextField from '@material-ui/core/TextField'; -import Button from '@material-ui/core/Button'; +import DialogTitle from "@material-ui/core/DialogTitle"; +import Dialog from "@material-ui/core/Dialog"; +import TextField from "@material-ui/core/TextField"; +import Button from "@material-ui/core/Button"; -const LoginDialog = props => { - const { classes, onClose, open, globalUrl, isLoggedIn, setIsLoggedIn, ...other } = props; +const LoginDialog = (props) => { + const { + classes, + onClose, + open, + globalUrl, + isLoggedIn, + setIsLoggedIn, + ...other + } = props; - const [username, setUsername] = useState(""); - const [password, setPassword] = useState(""); - //const [selectedValue, setSelectedValue] = useState(false); + const [username, setUsername] = useState(""); + const [password, setPassword] = useState(""); + //const [selectedValue, setSelectedValue] = useState(false); - // Used to swap from login to register. True = login, false = register - const [loginCheck, setLoginCheck] = useState(true); + // Used to swap from login to register. True = login, false = register + const [loginCheck, setLoginCheck] = useState(true); - // Error messages etc - const [loginInfo, setLoginInfo] = useState(""); + // Error messages etc + const [loginInfo, setLoginInfo] = useState(""); - const handleValidateForm = () => { - return (username.length > 1 && password.length > 8); - } + const handleValidateForm = () => { + return username.length > 1 && password.length > 8; + }; - const onSubmit = (e) => { - e.preventDefault() + const onSubmit = (e) => { + e.preventDefault(); - // Just use this one? - var data = '{"username": "' + username + '", "password": "' + password + '"}'; - var baseurl = globalUrl - if (loginCheck) { - var url = baseurl+'/login'; - fetch(url, { - method: 'POST', - body: data, - headers: { - 'Content-Type': 'application/json', - }, - }) - .then(response => - response.json().then(responseJson => { - console.log(responseJson) - //console.log(e) - if (responseJson["success"] === false) { - setLoginInfo(responseJson["reason"]) - } else { - setLoginInfo("Successful login :)") - onClose() - setIsLoggedIn(true) - } - }), - ) - .catch(error => { - setLoginInfo("Error in userdata") - }); - } else { - url = baseurl+'/register'; - fetch(url, { - method: 'POST', - body: data, - headers: { - 'Content-Type': 'application/json', - }, - }) - .then(response => - response.json().then(responseJson => { - if (responseJson["success"] === false) { - setLoginInfo(responseJson["reason"]) - } else { - setLoginInfo("Successful register :)") - onClose() - setIsLoggedIn(true) - } - }), - ) - .catch(error => { - setLoginInfo("Error in userdata") - }); - } - } + // Just use this one? + var data = + '{"username": "' + username + '", "password": "' + password + '"}'; + var baseurl = globalUrl; + if (loginCheck) { + var url = baseurl + "/login"; + fetch(url, { + method: "POST", + body: data, + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => + response.json().then((responseJson) => { + console.log(responseJson); + //console.log(e) + if (responseJson["success"] === false) { + setLoginInfo(responseJson["reason"]); + } else { + setLoginInfo("Successful login :)"); + onClose(); + setIsLoggedIn(true); + } + }) + ) + .catch((error) => { + setLoginInfo("Error in userdata"); + }); + } else { + url = baseurl + "/register"; + fetch(url, { + method: "POST", + body: data, + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => + response.json().then((responseJson) => { + if (responseJson["success"] === false) { + setLoginInfo(responseJson["reason"]); + } else { + setLoginInfo("Successful register :)"); + onClose(); + setIsLoggedIn(true); + } + }) + ) + .catch((error) => { + setLoginInfo("Error in userdata"); + }); + } + }; - const onChangeUser = (e) => { - setUsername(e.target.value) - } + const onChangeUser = (e) => { + setUsername(e.target.value); + }; - const onChangePass = (e) => { - setPassword(e.target.value) - } + const onChangePass = (e) => { + setPassword(e.target.value); + }; - const onClickRegister = () => { - setLoginCheck(!loginCheck) - } + const onClickRegister = () => { + setLoginCheck(!loginCheck); + }; - //var loginChange = loginCheck ? (

Want to register? Click here.

) : (

Go back to login? Click here.

); - var formtitle = loginCheck ?
Login
:
Register
- var formButton = loginCheck ?
Click to Register
:
Click to Login
+ //var loginChange = loginCheck ? (

Want to register? Click here.

) : (

Go back to login? Click here.

); + var formtitle = loginCheck ?
Login
:
Register
; + var formButton = loginCheck ? ( +
Click to Register
+ ) : ( +
Click to Login
+ ); - return ( - - {formtitle} -
- Username -
- -
- Password -
- -
-
- - - -
- {loginInfo} -
-
- -
- - ); -} + return ( + + {formtitle} +
+ Username +
+ +
+ Password +
+ +
+
+ + + +
+ {loginInfo} +
+
+ +
+ + ); +}; export default LoginDialog; diff --git a/frontend/src/components/NestedMenu.jsx b/frontend/src/components/NestedMenu.jsx index cabf1b24..35d770a5 100755 --- a/frontend/src/components/NestedMenu.jsx +++ b/frontend/src/components/NestedMenu.jsx @@ -1,9 +1,9 @@ -import React, {useState, useRef, useImperativeHandle} from 'react' -import {makeStyles} from '@material-ui/core/styles' -import Menu, {MenuProps} from '@material-ui/core/Menu' -import MenuItem, {MenuItemProps} from '@material-ui/core/MenuItem' -import ArrowRight from '@material-ui/icons/ArrowRight' -import clsx from 'clsx' +import React, { useState, useRef, useImperativeHandle } from "react"; +import { makeStyles } from "@material-ui/core/styles"; +import Menu, { MenuProps } from "@material-ui/core/Menu"; +import MenuItem, { MenuItemProps } from "@material-ui/core/MenuItem"; +import ArrowRight from "@material-ui/icons/ArrowRight"; +import clsx from "clsx"; // @@ -39,15 +39,15 @@ import clsx from 'clsx' // /** // * @see https://material-ui.com/api/list-item/ // */ -// button: true; +// button: true; //} -const TRANSPARENT = 'rgba(0,0,0,0)' +const TRANSPARENT = "rgba(0,0,0,0)"; const useMenuItemStyles = makeStyles((theme) => ({ root: (props: any) => ({ - backgroundColor: props.open ? theme.palette.action.hover : TRANSPARENT - }) -})) + backgroundColor: props.open ? theme.palette.action.hover : TRANSPARENT, + }), +})); /** * Use as a drop-in replacement for `` when you need to add cascading @@ -55,11 +55,11 @@ const useMenuItemStyles = makeStyles((theme) => ({ */ //const NestedMenuItem = React.forwardRef( const NestedMenuItem = (props, ref) => { - console.log(props, ref) - //function NestedMenuItem(props, ref) { + console.log(props, ref); + //function NestedMenuItem(props, ref) { const { parentMenuOpen, - component = 'div', + component = "div", label, rightIcon = , children, @@ -68,94 +68,100 @@ const NestedMenuItem = (props, ref) => { MenuProps = {}, ContainerProps: ContainerPropsProp = {}, ...MenuItemProps - } = props + } = props; - const [isSubMenuOpen, setIsSubMenuOpen] = useState(false) + const [isSubMenuOpen, setIsSubMenuOpen] = useState(false); - const {ref: containerRefProp, ...ContainerProps} = ContainerPropsProp + const { ref: containerRefProp, ...ContainerProps } = ContainerPropsProp; + const menuItemRef = useRef < HTMLLIElement > null; + useImperativeHandle(ref, () => menuItemRef.current); + const containerRef = useRef < HTMLDivElement > null; + useImperativeHandle(containerRefProp, () => containerRef.current); + const menuContainerRef = useRef < HTMLDivElement > null; - const menuItemRef = useRef(null) - useImperativeHandle(ref, () => menuItemRef.current) - const containerRef = useRef(null) - useImperativeHandle(containerRefProp, () => containerRef.current) - const menuContainerRef = useRef(null) - - console.log("PAST THIS: ", containerRefProp, menuItemRef, containerRef, menuContainerRef, ContainerProps) + console.log( + "PAST THIS: ", + containerRefProp, + menuItemRef, + containerRef, + menuContainerRef, + ContainerProps + ); const handleMouseEnter = (event: React.MouseEvent) => { - setIsSubMenuOpen(true) + setIsSubMenuOpen(true); if (ContainerProps?.onMouseEnter) { - ContainerProps.onMouseEnter(event) + ContainerProps.onMouseEnter(event); } - } + }; const handleMouseLeave = (event: React.MouseEvent) => { - setIsSubMenuOpen(false) + setIsSubMenuOpen(false); if (ContainerProps?.onMouseLeave) { - ContainerProps.onMouseLeave(event) + ContainerProps.onMouseLeave(event); } - } + }; // Check if any immediate children are active const isSubmenuFocused = () => { - const active = containerRef.current?.ownerDocument?.activeElement + const active = containerRef.current?.ownerDocument?.activeElement; for (const child of menuContainerRef.current?.children ?? []) { if (child === active) { - return true + return true; } } - return false - } + return false; + }; const handleFocus = (event: React.FocusEvent) => { if (event.target === containerRef.current) { - setIsSubMenuOpen(true) + setIsSubMenuOpen(true); } if (ContainerProps?.onFocus) { - ContainerProps.onFocus(event) + ContainerProps.onFocus(event); } - } + }; const handleKeyDown = (event: React.KeyboardEvent) => { - if (event.key === 'Escape') { - return + if (event.key === "Escape") { + return; } if (isSubmenuFocused()) { - event.stopPropagation() + event.stopPropagation(); } - const active = containerRef.current?.ownerDocument?.activeElement + const active = containerRef.current?.ownerDocument?.activeElement; - if (event.key === 'ArrowLeft' && isSubmenuFocused()) { - containerRef.current?.focus() + if (event.key === "ArrowLeft" && isSubmenuFocused()) { + containerRef.current?.focus(); } if ( - event.key === 'ArrowRight' && + event.key === "ArrowRight" && event.target === containerRef.current && event.target === active ) { - console.log("MENU: ", menuContainerRef) - const firstChild = menuContainerRef.current.children[0] - console.log("FIRST: ", firstChild) - firstChild.focus() + console.log("MENU: ", menuContainerRef); + const firstChild = menuContainerRef.current.children[0]; + console.log("FIRST: ", firstChild); + firstChild.focus(); } - } + }; - const open = isSubMenuOpen && parentMenuOpen - const menuItemClasses = useMenuItemStyles({open}) + const open = isSubMenuOpen && parentMenuOpen; + const menuItemClasses = useMenuItemStyles({ open }); // Root element must have a `tabIndex` attribute for keyboard navigation - let tabIndex + let tabIndex; if (!props.disabled) { - tabIndex = tabIndexProp !== undefined ? tabIndexProp : -1 + tabIndex = tabIndexProp !== undefined ? tabIndexProp : -1; } - console.log("PAST 2! ", tabIndex) + console.log("PAST 2! ", tabIndex); return (
{ { - setIsSubMenuOpen(false) + setIsSubMenuOpen(false); }} > -
+
{children}
- ) -} + ); +}; -export default NestedMenuItem +export default NestedMenuItem; diff --git a/frontend/src/components/NestedMenuItem.jsx b/frontend/src/components/NestedMenuItem.jsx new file mode 100644 index 00000000..e48c2988 --- /dev/null +++ b/frontend/src/components/NestedMenuItem.jsx @@ -0,0 +1,202 @@ +import React, {useState, useRef, useImperativeHandle} from 'react' +import {makeStyles} from '@material-ui/core/styles' +import Menu, {MenuProps} from '@material-ui/core/Menu' +import MenuItem, {MenuItemProps} from '@material-ui/core/MenuItem' +import ArrowRight from '@material-ui/icons/ArrowRight' +import clsx from 'clsx' + +export interface NestedMenuItemProps extends Omit { + /** + * Open state of parent ``, used to close decendent menus when the + * root menu is closed. + */ + parentMenuOpen: boolean + /** + * Component for the container element. + * @default 'div' + */ + component?: React.ElementType + /** + * Effectively becomes the `children` prop passed to the `` + * element. + */ + label?: React.ReactNode + /** + * @default + */ + rightIcon?: React.ReactNode + /** + * Props passed to container element. + */ + ContainerProps?: React.HTMLAttributes & + React.RefAttributes + /** + * Props passed to sub `` element + */ + MenuProps?: Omit + /** + * @see https://material-ui.com/api/list-item/ + */ + button?: true | undefined +} + +const TRANSPARENT = 'rgba(0,0,0,0)' +const useMenuItemStyles = makeStyles((theme) => ({ + root: (props: any) => ({ + backgroundColor: props.open ? theme.palette.action.hover : TRANSPARENT + }) +})) + +/** + * Use as a drop-in replacement for `` when you need to add cascading + * menu elements as children to this component. + */ +const NestedMenuItem = React.forwardRef< + HTMLLIElement | null, + NestedMenuItemProps +>(function NestedMenuItem(props, ref) { + const { + parentMenuOpen, + component = 'div', + label, + rightIcon = , + children, + className, + tabIndex: tabIndexProp, + MenuProps = {}, + ContainerProps: ContainerPropsProp = {}, + ...MenuItemProps + } = props + + const {ref: containerRefProp, ...ContainerProps} = ContainerPropsProp + + const menuItemRef = useRef(null) + useImperativeHandle(ref, () => menuItemRef.current) + + const containerRef = useRef(null) + useImperativeHandle(containerRefProp, () => containerRef.current) + + const menuContainerRef = useRef(null) + + const [isSubMenuOpen, setIsSubMenuOpen] = useState(false) + + const handleMouseEnter = (event: React.MouseEvent) => { + setIsSubMenuOpen(true) + + if (ContainerProps?.onMouseEnter) { + ContainerProps.onMouseEnter(event) + } + } + const handleMouseLeave = (event: React.MouseEvent) => { + setIsSubMenuOpen(false) + + if (ContainerProps?.onMouseLeave) { + ContainerProps.onMouseLeave(event) + } + } + + // Check if any immediate children are active + const isSubmenuFocused = () => { + const active = containerRef.current?.ownerDocument?.activeElement + for (const child of menuContainerRef.current?.children ?? []) { + if (child === active) { + return true + } + } + return false + } + + const handleFocus = (event: React.FocusEvent) => { + if (event.target === containerRef.current) { + setIsSubMenuOpen(true) + } + + if (ContainerProps?.onFocus) { + ContainerProps.onFocus(event) + } + } + + const handleKeyDown = (event: React.KeyboardEvent) => { + if (event.key === 'Escape') { + return + } + + if (isSubmenuFocused()) { + event.stopPropagation() + } + + const active = containerRef.current?.ownerDocument?.activeElement + + if (event.key === 'ArrowLeft' && isSubmenuFocused()) { + containerRef.current?.focus() + } + + if ( + event.key === 'ArrowRight' && + event.target === containerRef.current && + event.target === active + ) { + const firstChild = menuContainerRef.current?.children[0] as + | HTMLElement + | undefined + firstChild?.focus() + } + } + + const open = isSubMenuOpen && parentMenuOpen + const menuItemClasses = useMenuItemStyles({open}) + + // Root element must have a `tabIndex` attribute for keyboard navigation + let tabIndex + if (!props.disabled) { + tabIndex = tabIndexProp !== undefined ? tabIndexProp : -1 + } + + return ( +
+ + {label} + {rightIcon} + + { + setIsSubMenuOpen(false) + }} + > +
+ {children} +
+ +
+ ) +}) + +export default NestedMenuItem diff --git a/frontend/src/components/Newsletter.jsx b/frontend/src/components/Newsletter.jsx new file mode 100644 index 00000000..ccadf77b --- /dev/null +++ b/frontend/src/components/Newsletter.jsx @@ -0,0 +1,102 @@ +import React, {useState} from 'react'; +import { useTheme } from '@material-ui/core/styles'; +import {isMobile} from "react-device-detect"; +import ReactGA from 'react-ga4'; + +import {TextField, Typography, Button} from '@material-ui/core'; + +const Newsletter = (props) => { + const { globalUrl, } = props; + + const theme = useTheme(); + const [email, setEmail] = useState(""); + const [msg, setMsg] = useState(""); + const [buttonActive, setButtonActive] = useState(true); + const buttonStyle = {minWidth: 300, borderRadius: 30, height: 60, width: 140, margin: isMobile ? "15px auto 15px auto" : "20px 20px 20px 10px", fontSize: 18,} + + const newsletterSignup = (inemail) => { + if (inemail.length < 4) { + setMsg("Invalid email") + setButtonActive(true) + return + } + + setButtonActive(false) + const data = {"email": inemail} + const url = globalUrl+'/api/v1/functions/newsletter_signup' + fetch(url, { + method: 'POST', + body: JSON.stringify(data), + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + }) + .then(response => + response.json().then(responseJson => { + setButtonActive(true) + setMsg(responseJson["reason"]) + if (responseJson["success"] === false) { + } else { + setEmail("") + } + }), + ) + .catch(error => { + setMsg("Something went wrong: ", error.toString()) + setButtonActive(true) + }); + } + + return ( +
+ + Security Automation Newsletter + + + Defensive security is 99% noise. Join us to sift through it. + +
+ { + setEmail(e.target.value) + }} + placeholder="Your email" + id="standard-required" + margin="normal" + variant="outlined" + /> +
+ +
+ {msg} +
+ ) +} + + +export default Newsletter; diff --git a/frontend/src/components/Oauth2Auth.jsx b/frontend/src/components/Oauth2Auth.jsx index 9832b990..e33a8f1f 100755 --- a/frontend/src/components/Oauth2Auth.jsx +++ b/frontend/src/components/Oauth2Auth.jsx @@ -325,7 +325,61 @@ const AuthenticationOauth2 = (props) => { console.log("Adding authorization from user side") state += `%26authorization%3d${userAuth}`; } + // write:request:jira-service-management + } + + const handleOauth2Request = (client_id, client_secret, oauth_url, scopes, admin_consent, prompt) => { + setButtonClicked(true); + //console.log("SCOPES: ", scopes); + + client_id = client_id.trim() + client_secret = client_secret.trim() + oauth_url = oauth_url.trim() + + var resources = ""; + if (scopes !== undefined && (scopes !== null) & (scopes.length > 0)) { + console.log("IN scope 1") + if (offlineAccess === true && !scopes.includes("offline_access")) { + + console.log("IN scope 2") + if (!authenticationType.redirect_uri.includes("google")) { + console.log("Appending offline access") + scopes.push("offline_access") + } + } + + resources = scopes.join(" "); + //resources = scopes.join(","); + } + + const authentication_url = authenticationType.token_uri; + //console.log("AUTH: ", authenticationType) + //console.log("SCOPES2: ", resources) + const redirectUri = `${window.location.protocol}//${window.location.host}/set_authentication`; + const workflowId = workflow !== undefined ? workflow.id : ""; + var state = `workflow_id%3D${workflowId}%26reference_action_id%3d${selectedAction.app_id}%26app_name%3d${selectedAction.app_name}%26app_id%3d${selectedAction.app_id}%26app_version%3d${selectedAction.app_version}%26authentication_url%3d${authentication_url}%26scope%3d${resources}%26client_id%3d${client_id}%26client_secret%3d${client_secret}`; + if (oauth_url !== undefined && oauth_url !== null && oauth_url.length > 0) { + state += `%26oauth_url%3d${oauth_url}`; + console.log("ADDING OAUTH2 URL: ", state); + } + + if ( + authenticationType.refresh_uri !== undefined && + authenticationType.refresh_uri !== null && + authenticationType.refresh_uri.length > 0 + ) { + state += `%26refresh_uri%3d${authenticationType.refresh_uri}`; + } else { + state += `%26refresh_uri%3d${authentication_url}`; + } + + // No prompt forcing + //var url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=login&scope=${resources}&state=${state}&access_type=offline`; + var defaultPrompt = "login" + if (prompt !== undefined && prompt !== null && prompt.length > 0) { + defaultPrompt = prompt + } // Check for org_id const orgId = urlParams.get("org_id"); if (orgId !== undefined && orgId !== null && orgId.length > 0) { @@ -470,7 +524,6 @@ const AuthenticationOauth2 = (props) => { } else { alert.info( "Field " + selectedApp.authentication.parameters[key].name.replace("_basic", "", -1).replace("_", " ", -1) + " can't be empty" - ); return; } diff --git a/frontend/src/components/OrgHeaderexpanded.jsx b/frontend/src/components/OrgHeaderexpanded.jsx new file mode 100644 index 00000000..f2691e54 --- /dev/null +++ b/frontend/src/components/OrgHeaderexpanded.jsx @@ -0,0 +1,702 @@ +import React, { useEffect } from "react"; + +import { makeStyles } from "@material-ui/styles"; +import { useTheme } from "@material-ui/core/styles"; +import { useAlert } from "react-alert"; + +import { + FormControl, + InputLabel, + Paper, + OutlinedInput, + Checkbox, + Card, + Tooltip, + FormControlLabel, + Typography, + Switch, + Select, + MenuItem, + Divider, + TextField, + Button, + Tabs, + Tab, + Grid, +} from "@material-ui/core"; + +import IconButton from "@material-ui/core/IconButton"; +import ExpandLessIcon from "@material-ui/icons/ExpandLess"; +import ExpandMoreIcon from "@material-ui/icons/ExpandMore"; +import SaveIcon from "@material-ui/icons/Save"; + +const useStyles = makeStyles({ + notchedOutline: { + borderColor: "#f85a3e !important", + }, +}); + +const OrgHeaderexpanded = (props) => { + const { + userdata, + selectedOrganization, + setSelectedOrganization, + globalUrl, + isCloud, + adminTab, + } = props; + + const theme = useTheme(); + const alert = useAlert(); + const classes = useStyles(); + const defaultBranch = "master"; + + const [orgName, setOrgName] = React.useState(selectedOrganization.name); + const [orgDescription, setOrgDescription] = React.useState( + selectedOrganization.description + ); + + const [appDownloadUrl, setAppDownloadUrl] = React.useState( + selectedOrganization.defaults === undefined + ? "https://github.com/frikky/shuffle-apps" + : selectedOrganization.defaults.app_download_repo === undefined || + selectedOrganization.defaults.app_download_repo.length === 0 + ? "https://github.com/frikky/shuffle-apps" + : selectedOrganization.defaults.app_download_repo + ); + const [appDownloadBranch, setAppDownloadBranch] = React.useState( + selectedOrganization.defaults === undefined + ? defaultBranch + : selectedOrganization.defaults.app_download_branch === undefined || + selectedOrganization.defaults.app_download_branch.length === 0 + ? defaultBranch + : selectedOrganization.defaults.app_download_branch + ); + const [workflowDownloadUrl, setWorkflowDownloadUrl] = React.useState( + selectedOrganization.defaults === undefined + ? "https://github.com/frikky/shuffle-apps" + : selectedOrganization.defaults.workflow_download_repo === undefined || + selectedOrganization.defaults.workflow_download_repo.length === 0 + ? "https://github.com/frikky/shuffle-workflows" + : selectedOrganization.defaults.workflow_download_repo + ); + const [workflowDownloadBranch, setWorkflowDownloadBranch] = React.useState( + selectedOrganization.defaults === undefined + ? defaultBranch + : selectedOrganization.defaults.workflow_download_branch === undefined || + selectedOrganization.defaults.workflow_download_branch.length === 0 + ? defaultBranch + : selectedOrganization.defaults.workflow_download_branch + ); + const [ssoEntrypoint, setSsoEntrypoint] = React.useState( + selectedOrganization.sso_config === undefined + ? "" + : selectedOrganization.sso_config.sso_entrypoint === undefined || + selectedOrganization.sso_config.sso_entrypoint.length === 0 + ? "" + : selectedOrganization.sso_config.sso_entrypoint + ); + const [ssoCertificate, setSsoCertificate] = React.useState( + selectedOrganization.sso_config === undefined + ? "" + : selectedOrganization.sso_config.sso_certificate === undefined || + selectedOrganization.sso_config.sso_certificate.length === 0 + ? "" + : selectedOrganization.sso_config.sso_certificate + ); + const [notificationWorkflow, setNotificationWorkflow] = React.useState( + selectedOrganization.defaults === undefined + ? "" + : selectedOrganization.defaults.notification_workflow === undefined || + selectedOrganization.defaults.notification_workflow.length === 0 + ? "" + : selectedOrganization.defaults.notification_workflow + ); + + const [documentationReference, setDocumentationReference] = React.useState( + selectedOrganization.defaults === undefined + ? "" + : selectedOrganization.defaults.documentation_reference === undefined || + selectedOrganization.defaults.documentation_reference.length === 0 + ? "" + : selectedOrganization.defaults.documentation_reference + ); + const [openidClientId, setOpenidClientId] = React.useState( + selectedOrganization.sso_config === undefined + ? "" + : selectedOrganization.sso_config.client_id === undefined || + selectedOrganization.sso_config.client_id.length === 0 + ? "" + : selectedOrganization.sso_config.client_id + ); + const [openidClientSecret, setOpenidClientSecret] = React.useState( + selectedOrganization.sso_config === undefined + ? "" + : selectedOrganization.sso_config.client_secret === undefined || + selectedOrganization.sso_config.client_secret.length === 0 + ? "" + : selectedOrganization.sso_config.client_secret + ); + const [openidAuthorization, setOpenidAuthorization] = React.useState( + selectedOrganization.sso_config === undefined + ? "" + : selectedOrganization.sso_config.openid_authorization === undefined || + selectedOrganization.sso_config.openid_authorization.length === 0 + ? "" + : selectedOrganization.sso_config.openid_authorization + ); + const [openidToken, setOpenidToken] = React.useState( + selectedOrganization.sso_config === undefined + ? "" + : selectedOrganization.sso_config.openid_token === undefined || + selectedOrganization.sso_config.openid_token.length === 0 + ? "" + : selectedOrganization.sso_config.openid_token + ) + + const handleEditOrg = ( + name, + description, + orgId, + image, + defaults, + sso_config + ) => { + + const data = { + name: name, + description: description, + org_id: orgId, + image: image, + defaults: defaults, + sso_config: sso_config, + }; + + const url = globalUrl + `/api/v1/orgs/${selectedOrganization.id}`; + fetch(url, { + mode: "cors", + method: "POST", + body: JSON.stringify(data), + credentials: "include", + crossDomain: true, + withCredentials: true, + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + }) + .then((response) => + response.json().then((responseJson) => { + if (responseJson["success"] === false) { + alert.error("Failed updating org: ", responseJson.reason); + } else { + alert.success("Successfully edited org!"); + } + }) + ) + .catch((error) => { + alert.error("Err: " + error.toString()); + }); + }; + + const orgSaveButton = ( + + + + ); + + return ( +
+ + + + Notification Workflow ID + { + setNotificationWorkflow(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + Org Documentation reference + { + setDocumentationReference(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + {isCloud ? null : + + OpenID connect + + + + Client ID + 0 + } + id="outlined-with-placeholder" + margin="normal" + variant="outlined" + placeholder="The OpenID client ID from the identity provider" + value={openidClientId} + onChange={(e) => { + setOpenidClientId(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + Client Secret (optional) + 0 + } + id="outlined-with-placeholder" + margin="normal" + variant="outlined" + placeholder="The OpenID client secret - DONT use this if dealing with implicit auth / PKCE" + value={openidClientSecret} + onChange={(e) => { + setOpenidClientSecret(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + + + Authorization URL + { + setOpenidAuthorization(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + Token URL + { + setOpenidToken(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + } + {/*isCloud ? null : */} + + SAML SSO (v1.1) + + + + SSO Entrypoint (IdP) + 0 + } + id="outlined-with-placeholder" + margin="normal" + variant="outlined" + placeholder="The entrypoint URL from your provider" + value={ssoEntrypoint} + onChange={(e) => { + setSsoEntrypoint(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + SSO Certificate (X509) + { + setSsoCertificate(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + {isCloud ? + + IdP URL for Shuffle: https://shuffler.io/api/v1/login_sso + + : null} + + {isCloud ? null : ( + + + App Download URL + { + setAppDownloadUrl(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + )} + {isCloud ? null : ( + + + App Download Branch + { + setAppDownloadBranch(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + )} + {isCloud ? null : ( + + + Workflow Download URL + { + setWorkflowDownloadUrl(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + )} + {isCloud ? null : ( + + + Workflow Download Branch + { + setWorkflowDownloadBranch(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + )} + +
+ {orgSaveButton} +
+ {/* + + {expanded ? + + : + + } + + */} + +
+ ) +} + +export default OrgHeaderexpanded; \ No newline at end of file diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 1871dee7..1d628133 100755 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -1595,7 +1595,7 @@ const ParsedAction = (props) => { maxWidth: "95%", fontSize: "1em", }, - disableUnderline: true, + disableUnderline: true, endAdornment: hideExtraTypes ? null : ( diff --git a/frontend/src/components/RenderCytoscape.js b/frontend/src/components/RenderCytoscape.js index 3a88dcf8..18c9815c 100755 --- a/frontend/src/components/RenderCytoscape.js +++ b/frontend/src/components/RenderCytoscape.js @@ -1,79 +1,82 @@ -import React, {useState, useEffect, useLayoutEffect} from 'react'; -import * as cytoscape from 'cytoscape'; -import CytoscapeComponent from 'react-cytoscapejs'; -import cystyle from '../defaultCytoscapeStyle'; +import React, { useState, useEffect, useLayoutEffect } from "react"; +import * as cytoscape from "cytoscape"; +import CytoscapeComponent from "react-cytoscapejs"; +import cystyle from "../defaultCytoscapeStyle"; -const surfaceColor = "#27292D" +const surfaceColor = "#27292D"; const CytoscapeWrapper = (props) => { const { globalUrl, inworkflow } = props; - const [elements, setElements] = useState([]) - const [workflow, setWorkflow] = useState(inworkflow) - const [cy, setCy] = React.useState() - const bodyWidth = 200 - const bodyHeight = 150 + const [elements, setElements] = useState([]); + const [workflow, setWorkflow] = useState(inworkflow); + const [cy, setCy] = React.useState(); + const bodyWidth = 200; + const bodyHeight = 150; - const setupGraph = () => { - const actions = workflow.actions.map(action => { - const node = {} - node.position = action.position - node.data = action + const setupGraph = () => { + const actions = workflow.actions.map((action) => { + const node = {}; + node.position = action.position; + node.data = action; - node.data._id = action["id"] - node.data.type = "ACTION" - node.isStartNode = action["id"] === workflow.start + node.data._id = action["id"]; + node.data.type = "ACTION"; + node.isStartNode = action["id"] === workflow.start; + var example = ""; + if ( + action.example !== undefined && + action.example !== null && + action.example.length > 0 + ) { + example = action.example; + } - var example = "" - if (action.example !== undefined && action.example !== null && action.example.length > 0) { - example = action.example - } + node.data.example = example; + return node; + }); - node.data.example = example - return node; - }) + const triggers = workflow.triggers.map((trigger) => { + const node = {}; + node.position = trigger.position; + node.data = trigger; - const triggers = workflow.triggers.map(trigger => { - const node = {} - node.position = trigger.position - node.data = trigger + node.data._id = trigger["id"]; + node.data.type = "TRIGGER"; - node.data._id = trigger["id"] - node.data.type = "TRIGGER" + return node; + }); - return node; - }) + // FIXME - tmp branch update + var insertedNodes = [].concat(actions, triggers); + const edges = workflow.branches.map((branch, index) => { + //workflow.branches[index].conditions = [{ - // FIXME - tmp branch update - var insertedNodes = [].concat(actions, triggers) - const edges = workflow.branches.map((branch, index) => { - //workflow.branches[index].conditions = [{ + const edge = {}; + var conditions = workflow.branches[index].conditions; + if (conditions === undefined || conditions === null) { + conditions = []; + } - const edge = { }; - var conditions = workflow.branches[index].conditions - if (conditions === undefined || conditions === null) { - conditions = [] - } + var label = ""; + if (conditions.length === 1) { + label = conditions.length + " condition"; + } else if (conditions.length > 1) { + label = conditions.length + " conditions"; + } - var label = "" - if (conditions.length === 1) { - label = conditions.length+" condition" - } else if (conditions.length > 1) { - label = conditions.length+" conditions" - } + edge.data = { + id: branch.id, + _id: branch.id, + source: branch.source_id, + target: branch.destination_id, + label: label, + conditions: conditions, + hasErrors: branch.has_errors, + }; - edge.data = { - id: branch.id, - _id: branch.id, - source: branch.source_id, - target: branch.destination_id, - label: label, - conditions: conditions, - hasErrors: branch.has_errors - }; - - // This is an attempt at prettier edges. The numbers are weird to work with. - /* + // This is an attempt at prettier edges. The numbers are weird to work with. + /* //http://manual.graphspace.org/projects/graphspace-python/en/latest/demos/edge-types.html const sourcenode = actions.find(node => node.data._id === branch.source_id) const destinationnode = actions.find(node => node.data._id === branch.destination_id) @@ -96,51 +99,59 @@ const CytoscapeWrapper = (props) => { } */ - return edge; - }) + return edge; + }); - setWorkflow(workflow) + setWorkflow(workflow); - // Verifies if a branch is valid and skips others - var newedges = [] - for (var key in edges) { - var item = edges[key] + // Verifies if a branch is valid and skips others + var newedges = []; + for (var key in edges) { + var item = edges[key]; - const sourcecheck = insertedNodes.find(data => data.data.id === item.data.source) - const destcheck = insertedNodes.find(data => data.data.id === item.data.target) - if (sourcecheck === undefined || destcheck === undefined) { - continue - } + const sourcecheck = insertedNodes.find( + (data) => data.data.id === item.data.source + ); + const destcheck = insertedNodes.find( + (data) => data.data.id === item.data.target + ); + if (sourcecheck === undefined || destcheck === undefined) { + continue; + } - newedges.push(item) - } + newedges.push(item); + } - insertedNodes = insertedNodes.concat(newedges) - setElements(insertedNodes) - } + insertedNodes = insertedNodes.concat(newedges); + setElements(insertedNodes); + }; - if (elements.length === 0) { - setupGraph() - } + if (elements.length === 0) { + setupGraph(); + } - return ( - { - // FIXME: There's something specific loading when - // you do the first hover of a node. Why is this different? - //console.log("CY: ", incy) - setCy(incy) - }} - /> - ) -} + return ( + { + // FIXME: There's something specific loading when + // you do the first hover of a node. Why is this different? + //console.log("CY: ", incy) + setCy(incy); + }} + /> + ); +}; -export default CytoscapeWrapper +export default CytoscapeWrapper; diff --git a/frontend/src/components/ScrollToTop.jsx b/frontend/src/components/ScrollToTop.jsx index 88d2d158..2e19a282 100755 --- a/frontend/src/components/ScrollToTop.jsx +++ b/frontend/src/components/ScrollToTop.jsx @@ -1,25 +1,33 @@ -import { useEffect } from 'react'; -import { withRouter } from 'react-router-dom'; +import { useEffect } from "react"; +//import { withRouter } from "react-router-dom"; +import { useLocation } from "react-router-dom"; + +// ensures scrolling happens in the right way on different pages and when changing +function ScrollToTop({ getUserNotifications, curpath, setCurpath, history }) { + let location = useLocation(); -function ScrollToTop({getUserNotifications, setCurpath, history }) { useEffect(() => { - const unlisten = history.listen(() => { - window.scroll({ - top: 0, + // Custom handler for certain scroll mechanics + // + console.log("OLD: ", curpath, "NeW: ", window.location.pathname) + if (curpath === window.location.pathname && curpath === "/usecases") { + } else { + + window.scroll({ + top: 0, left: 0, behavior: "smooth", }); - setCurpath(window.location.pathname) - getUserNotifications() - }); - return () => { - unlisten(); - } - }, []); + setCurpath(window.location.pathname); + getUserNotifications(); + } + }, [location]); - return (null); + return null; } // https://stackoverflow.com/questions/36904185/react-router-scroll-to-top-on-every-transition -export default withRouter(ScrollToTop); +//export default withRouter(ScrollToTop); +// https://v5.reactrouter.com/web/api/Hooks/uselocation +export default ScrollToTop; diff --git a/frontend/src/components/Searchfield.js b/frontend/src/components/Searchfield.js new file mode 100644 index 00000000..43fa7934 --- /dev/null +++ b/frontend/src/components/Searchfield.js @@ -0,0 +1,648 @@ +import React, {useState, useEffect, useRef} from 'react'; + +import { useNavigate, Link, useParams } from "react-router-dom"; +import { useTheme } from '@material-ui/core/styles'; +import SearchIcon from '@material-ui/icons/Search'; + +import { + Chip, + IconButton, + TextField, + InputAdornment, + List, + Card, + ListItem, + ListItemAvatar, + ListItemText, + Avatar, + Typography, + Tooltip, +} from '@material-ui/core'; + +import { + AvatarGroup, +} from "@mui/material" + +import {Close as CloseIcon, Folder as FolderIcon, Polymer as PolymerIcon, LibraryBooks as LibraryBooksIcon} from '@material-ui/icons' + +import algoliasearch from 'algoliasearch/lite'; +import aa from 'search-insights' +import { InstantSearch, Configure, connectSearchBox, connectHits, Index } from 'react-instantsearch-dom'; +//import { InstantSearch, SearchBox, Hits, connectSearchBox, connectHits, Index } from 'react-instantsearch-dom'; + +// https://www.algolia.com/doc/api-reference/widgets/search-box/react/ +const chipStyle = { + backgroundColor: "#3d3f43", height: 30, marginRight: 5, paddingLeft: 5, paddingRight: 5, height: 28, cursor: "pointer", borderColor: "#3d3f43", color: "white", +} + +const searchClient = algoliasearch("JNSS5CFDZZ", "db08e40265e2941b9a7d8f644b6e5240") +const SearchField = props => { + const { serverside, userdata } = props + + const theme = useTheme(); + let navigate = useNavigate(); + const borderRadius = 3 + const node = useRef() + const [searchOpen, setSearchOpen] = useState(false) + const [oldPath, setOldPath] = useState("") + + if (serverside === true) { + return null + } + + if (window !== undefined && window.location !== undefined && window.location.pathname === "/search") { + return null + } + + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + + if (window.location.pathname !== oldPath) { + setSearchOpen(false) + setOldPath(window.location.pathname) + } + + //useEffect(() => { + // if (searchOpen) { + // var tarfield = document.getElementById("shuffle_search_field") + // tarfield.focus() + // } + //}, searchOpen) + + const SearchBox = ({currentRefinement, refine, isSearchStalled, } ) => { + + /* + endAdornment: ( + { + event.preventDefault() + }}> + { + setSearchOpen(false) + }} /> + + ), + */ + + return ( +