marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #538 from frikky/launch

Browse Source 
0.9.25 :D
This commit is contained in:
Frikky
2021-10-15 01:23:57 +02:00
committed by GitHub
parent 3205e8066a 6c03beeca9
commit 3eac4eacb9
85 changed files with 9387 additions and 3795 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+8
View File
@@ -25,11 +25,16 @@ SHUFFLE_APP_HOTLOAD_FOLDER=./shuffle-apps
SHUFFLE_APP_HOTLOAD_LOCATION=./shuffle-apps
SHUFFLE_FILE_LOCATION=./shuffle-files
# Encryption modifier. This HAS to be set to encrypt any authentication being used in Shuffle. This is put together with other relevant values to ensure multiple parts are needed to decrypt.
# If this key is lost or changed, you will have to reauthenticate all apps.
SHUFFLE_ENCRYPTION_MODIFIER=
# Other configs
BACKEND_HOSTNAME=shuffle-backend
BACKEND_PORT=5001
FRONTEND_PORT=3001
FRONTEND_PORT_HTTPS=3443
# CHANGE THIS IF YOU WANT GOOD LOCAL EXECUTIONS:
OUTER_HOSTNAME=shuffle-backend
DB_LOCATION=./shuffle-database
@@ -41,6 +46,8 @@ HTTP_PROXY=
HTTPS_PROXY=
SHUFFLE_PASS_WORKER_PROXY=TRUE
SHUFFLE_PASS_APP_PROXY=FALSE
TZ=Europe/Amsterdam # Timezone-handler in Orborus, Worker and Apps
ORBORUS_CONTAINER_NAME= # Used to FIND the containername. cgroup v2: issue 501
SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io
SHUFFLE_BASE_IMAGE_NAME=frikky
@@ -60,4 +67,5 @@ SHUFFLE_OPENSEARCH_CERTIFICATE_FILE=
SHUFFLE_OPENSEARCH_APIKEY=
SHUFFLE_OPENSEARCH_CLOUDID=
SHUFFLE_OPENSEARCH_PROXY=
SHUFFLE_OPENSEARCH_INDEX_PREFIX=
SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY=true
.github/CONTRIBUTING.md
+10 -1
View File
@@ -16,7 +16,7 @@ As with everything else, app creation for Shuffle is made as accessibl as possib
Workflows are where the magic of Shuffle automation happens. Our current ones [are outlined here](https://github.com/frikky/security-openapis), and will be automatically imported into Shuffle instances in the future. They are split into Prepare and Response, but don't necessarily have to be. If you'd like to talk about workflow creation or use-cases in general, either Open a [new issue](https://github.com/frikky/shuffle-workflows/issues/new) or send us an email at [frikky@shuffler.io](mailto:frikky@shuffler.io)
#### Documentation (Markdown)
Documentation is essential to any product, and Shuffle is no exception. Documentation in Shuffle uses markdown and is located in the [shuffle-docs](https://github.com/frikky/shuffle-docs/tree/master/docs) repository. These are then loaded into Shuffle when someone visits [https://shuffler/docs/about](https://shuffler/docs/about), then cached for later use. If you make an edit, expect it on our website in about an hour.
Documentation is essential to any product, and Shuffle is no exception. Documentation in Shuffle uses markdown and is located in the [shuffle-docs](https://github.com/frikky/shuffle-docs/tree/master/docs) repository. These are then loaded into Shuffle when someone visits [https://shuffler/docs/about](https://shuffler/docs/about), then cached for later use. If you make an edit, expect it on our website in about an hour.
#### Frontend (ReactJS)
The frontend of Shuffle is what everyone sees when they log in. Our goal here is to make it easy to get started and keep going with Shuffle - removing any blockers from the point of accessibility. If you'd like to get started, find [an issue](https://github.com/frikky/Shuffle/issues) and check the [installation guide](https://github.com/frikky/Shuffle/blob/master/install-guide.md#local-development-installation) for setting it up locally without Docker.
@@ -24,6 +24,15 @@ The frontend of Shuffle is what everyone sees when they log in. Our goal here is
#### Backend (Golang)
The backend of Shuffle is our REST API Server that runs in the background, handling all the API-calls in general, whether from users or apps. If you'd like to get started, find [an issue](https://github.com/frikky/Shuffle/issues) and check the [installation guide](https://github.com/frikky/Shuffle/blob/master/install-guide.md#local-development-installation) for setting it up locally without Docker.
#### Scaling (Golang & Python)
Shuffle runs using Docker, and is built to scale. There are many areas that may revolve around scaling, but the main issues come down to how we use Docker in our [architecture](https://shuffler.io/docs/architecture). If you want to help by submitting Helm charts (K8s), Docker swarm configurations, blogposts, or talk about code changes that would help scaling - please reach out (or just start building!), and we can discuss the possibilities. Make sure to read about the architecture first :)
#### Testing
Whether it's security testing, code testing or CI/CD, we could always need another hand. E.g. an example of CI/CD used for apps can be found [here](https://github.com/Shuffle/Shuffle-apps/blob/master/.github/workflows/ci.yaml), but we don't at all limit the scope to Github actions. If you find a security issue, whether open source or not, please contact [security@shuffler.io](mailto:security@shuffler.io) or [contact us on our website](https://shuffler.io/contact).
#### Community
What is a product without a community? Want to help out? Whether it be through blogposts, videos or community management, don't hesitate to [reach out](https://shuffler.io/contact) if you would like to help, and get a more keen understanding of how we work. (PS: We're hiring)
## Working on an issue
**Shuffle** uses the [GitHub flow](https://guides.github.com/introduction/flow/index.html). All project changes are made through pull requests.
.github/install-guide.md
+21 -24
View File
@@ -18,13 +18,6 @@ cd Shuffle
3. Fix prerequisites for the Opensearch database (Elasticsearch):
```
sudo chown 1000:1000 -R shuffle-database # Required for Opensearch
sudo sysctl -w vm.max_map_count=262144 # https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html
# To make the changes permanent, do:
# 1. Open the file /etc/sysctl.conf
# 2. Go to the bottom of the file
# 3. Add this line:
vm.max_map_count=262144
```
4. Run docker-compose.
@@ -38,26 +31,20 @@ When you're done, skip to the "After installation" step below.
This step is for setting up with Docker on windows from scratch.
1. Make sure you have [Docker](https://docs.docker.com/docker-for-windows/install/) and [docker-compose](https://docs.docker.com/compose/install/) installed. WSL2 may be required.
2. Go to https://github.com/frikky/shuffle/releases and download the latest .zip release (or install git)
3. Unzip the folder and enter it
4. Open the .env file and change the line with "OUTER_HOSTNAME" to contain your IP:
```
OUTER_HOSTNAME=YOUR.IP.HERE
```
5. Configure max memory (WSL) by opening a new CMD/Powershell window. Required for Elasticsearch
```
wsl -d docker-desktop
sysctl -w vm.max_map_count=262144
echo "vm.max_map_count = 262144" > /etc/sysctl.d/99-docker-desktop.conf
echo -e "\nvm.max_map_count = 262144\n" >> /etc/sysctl.d/00-alpine.conf
# https://stackoverflow.com/questions/42111566/elasticsearch-in-windows-docker-image-vm-max-map-count
```
6. Run docker-compose
```
docker-compose up -d
docker compose up -d
```
### Configurations (proxies, default users etc.)
@@ -79,9 +66,11 @@ https://shuffler.io/docs/configuration
* Default database location is in the same folder: ./shuffle-database
# Local development installation
Local development is pretty straight forward with **ReactJS** and **Golang**. This part is intended to help you run the code for development purposes.
Local development is pretty straight forward with **ReactJS** and **Golang**. This part is intended to help you run the code for development purposes. We recommend having Shuffle running with the Docker-compose, then manually running the portion that you want to test and/or edit.
**PS: You have to stop the Backend Docker container to get this one working**
**PPS: Use the "Launch" branch when developing to get it set up easier**
## Frontend - ReactJS /w cytoscape
@@ -96,17 +85,25 @@ npm start
http://localhost:5001 - REST API - requires [>=go1.13](https://golang.org/dl/)
```bash
export SHUFFLE_OPENSEARCH_URL="http://localhost:9200"
export SHUFFLE_ELASTIC=true
cd backend/go-app
go run *.go
```
Large portions of the backend is written in another repository - [shuffle-shared](https://github.com/frikky/shuffle-shared). If you want to update any of this code and test in realtime, we recommend following these steps:
1. Clone shuffle-shared to a local repository
2. Open the Shuffle backend's go.mod file (./shuffle/backend/go.mod) (**NOT** in shuffle-shared)
3. Change the following line to point to your directory AFTER the =>
```
//replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared
```
4. Make the changes you want, then restart the backend server!
5. With your changes made, make a pull request :fire:
**WINDOWS USERS:** You'll have to to add the "export" part as an environment variable.
## Database - Datastore
Based on Google datastore
```
docker run -p 8000:8000 google/cloud-sdk gcloud beta emulators datastore start --project=shuffle --host-port 0.0.0.0:8000 --no-store-on-disk
```
## Database - Opensearch
Make sure this is running through the docker-compose, and that the backend points to it with SHUFFLE_OPENSEARCH_URL defined
## Orborus
Execution of Workflows:
.github/workflows/ci.yaml
+66
View File
@@ -0,0 +1,66 @@
name: ci
on:
push:
branches: master
jobs:
main:
runs-on: ubuntu-latest
continue-on-error: ${{ matrix.experimental }}
strategy:
fail-fast: false
matrix:
include:
- app: frontend
path: frontend
version: 0.8.3
experimental: true
- app: backend
path: backend
version: 0.8.3
experimental: false
- app: orborus
path: functions/onprem/orborus
version: 0.8.0
experimental: false
- app: database
path: backend/database
version: 0.8.0
experimental: false
steps:
-
name: Checkout
uses: actions/checkout@v2
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
# Use below configuration for ghcr.io
# with:
# registry: ghcr.io
# username: ${{ github.repository_owner }}
# password: ${{ secrets.CR_PAT }}
-
name: Build and push
id: docker_build
uses: docker/build-push-action@v2
env:
BUILDX_NO_DEFAULT_LOAD: true
with:
context: ${{ matrix.path }}/
file: ${{ matrix.path }}/Dockerfile
platforms: linux/amd64,linux/arm64
#,linux/386 - no node image I guess?
push: true
tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.app }}:${{ matrix.version }}
-
name: Image digest
run: echo ${{ steps.docker_build.outputs.digest }}
backend/Dockerfile
+2 -9
View File
@@ -1,4 +1,4 @@
FROM golang:1.16.0-buster as builder
FROM golang:1.17.2-buster as builder
# Add files
RUN mkdir /app
@@ -7,16 +7,11 @@ WORKDIR /app
ADD ./go-app/main.go /app
ADD ./go-app/walkoff.go /app
ADD ./go-app/docker.go /app
ADD ./go-app/oauth2.go /app
ADD ./go-app/go.mod /app
# Required files for code generation
ADD ./app_sdk/app_base.py /app_sdk
ADD ./app_sdk_kali/app_base.py /app_sdk_kali
ADD ./app_sdk_kali/static_baseline.py /app_sdk_kali
ADD ./app_sdk_blackarch/app_base.py /app_sdk_blackarch
ADD ./app_sdk_blackarch/static_baseline.py /app_sdk_blackarch
ADD ./app_gen /app_gen
RUN go get -v
@@ -27,12 +22,10 @@ RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o webapp .
FROM alpine:latest as certs
RUN apk --update add ca-certificates
FROM alpine:3.12
FROM alpine:3.14.2
COPY --from=builder /app/ /app
COPY --from=builder /app_sdk/ /app_sdk
COPY --from=builder /app_sdk_kali/ /app_sdk_kali
COPY --from=builder /app_sdk_blackarch/ /app_sdk_blackarch
COPY --from=builder /app_gen/ /app_gen
COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
backend/app_gen/openapi/baseline/Dockerfile
+14 -11
View File
@@ -2,24 +2,27 @@
FROM frikky/shuffle:app_sdk as base
# We're going to stage away all of the bloat from the build tools so lets create a builder stage
FROM base as builder
#FROM base as builder
# Install all alpine build tools needed for our pip installs
RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev
#RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev
# Install all of our pip packages in a single directory that we can copy to our base image later
RUN mkdir /install
WORKDIR /install
COPY requirements.txt /requirements.txt
RUN pip install --prefix="/install" -r /requirements.txt
#RUN mkdir /install
#WORKDIR /install
#COPY requirements.txt /requirements.txt
#
## Switch back to our base image and copy in all of our built packages and source code
#FROM base
#COPY --from=builder /install /usr/local
# Switch back to our base image and copy in all of our built packages and source code
FROM base
COPY --from=builder /install /usr/local
COPY src /app
# Install any binary dependencies needed in our final image - this can be a lot of different stuff
RUN apk --no-cache add --update libmagic
#RUN apk --no-cache add --update libmagic
WORKDIR /
COPY requirements.txt /requirements.txt
RUN pip install --prefix="/usr/local" -r /requirements.txt
COPY src /app
# Finally, lets run our app!
WORKDIR /app
backend/app_sdk/Dockerfile
+9 -5
View File
@@ -1,16 +1,20 @@
FROM python:3.9.4-alpine as base
#FROM python:3.9.1-alpine as base
FROM python:3.10.0-alpine as base
FROM base as builder
RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev
RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev tzdata coreutils
RUN mkdir /install
WORKDIR /install
FROM base
#--no-cache
RUN apk update && apk add --update tzdata libmagic alpine-sdk libffi libffi-dev musl-dev openssl-dev coreutils
COPY --from=builder /install /usr/local
COPY requirements.txt /requirements.txt
RUN pip3 install -r /requirements.txt
FROM base
COPY --from=builder /install /usr/local
COPY __init__.py /app/walkoff_app_sdk/__init__.py
COPY app_base.py /app/walkoff_app_sdk/app_base.py
backend/app_sdk/Dockerfile_blackarch
+19
View File
@@ -0,0 +1,19 @@
FROM peterclemenko/blackarch as base
FROM base as builder
RUN /bin/pacman -Syu --noconfirm
RUN /bin/pacman -Sy --noconfirm base-devel libffi musl openssl python python-pip -y
RUN mkdir /install
WORKDIR /install
COPY requirements.txt /requirements.txt
RUN pip install --prefix="/install" -r /requirements.txt
FROM base
COPY --from=builder /install /usr/local
COPY __init__.py /app/walkoff_app_sdk/__init__.py
COPY app_base.py /app/walkoff_app_sdk/app_base.py
backend/app_sdk/Dockerfile_kali
+19
View File
@@ -0,0 +1,19 @@
FROM kalilinux/kali-rolling as base
FROM base as builder
RUN apt-get update
RUN apt-get dist-upgrade -y
RUN apt install build-essential libffi-dev musl-dev openssl python3 python3-pip -y
RUN mkdir /install
WORKDIR /install
COPY requirements.txt /requirements.txt
RUN pip install --prefix="/install" -r /requirements.txt
FROM base
COPY --from=builder /install /usr/local
COPY __init__.py /app/walkoff_app_sdk/__init__.py
COPY app_base.py /app/walkoff_app_sdk/app_base.py
backend/app_sdk/app_base.py
+710 -489
View File
File diff suppressed because it is too large Load Diff
backend/app_sdk/build.sh
+2 -2
View File
@@ -3,10 +3,10 @@
### DEFAULT
NAME=shuffle-app_sdk
VERSION=0.8.104
VERSION=0.9.23
docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly
#docker push frikky/$NAME:$VERSION
#docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION
backend/app_sdk/requirements.txt
+3 -1
View File
@@ -1,2 +1,4 @@
urllib3==1.25.9
urllib3==1.26.5
requests==2.25.1
MarkupSafe==2.0.1
liquidpy==0.7.1
backend/database/Dockerfile
-5
View File
@@ -1,5 +0,0 @@
# 2. docker run -p 8000:8000
FROM google/cloud-sdk
EXPOSE 8000
CMD ["gcloud", "beta", "emulators", "datastore", "start", "--project=shuffle", "--host-port", "0.0.0.0:8000", "--data-dir=/etc/shuffle"]
backend/database/opensearch/docker-compose.yml
+34
View File
@@ -0,0 +1,34 @@
version: '3'
services:
opensearch-node1:
image: opensearchproject/opensearch:latest
hostname: shuffle-database
container_name: shuffle-opensearch
environment:
- cluster.name=shuffle-cluster
- node.name=shuffle-opensearch
- discovery.seed_hosts=shuffle-opensearch
- cluster.initial_master_nodes=shuffle-opensearch
- bootstrap.memory_lock=true # along with the memlock settings below, disables swapping
- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
- cluster.routing.allocation.disk.threshold_enabled=false
- opendistro_security.disabled=true
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
hard: 65536
volumes:
- ~/git/shuffle/shuffle-database:/usr/share/opensearch/data
ports:
- 9200:9200
networks:
- opensearch-net
volumes:
opensearch-data1:
networks:
opensearch-net:
backend/go-app/docker.go
+51 -50
View File
@@ -2,7 +2,7 @@ package main
// Docker
import (
"github.com/frikky/shuffle-shared"
"github.com/shuffle/shuffle-shared"
"archive/tar"
//"bufio"
@@ -220,19 +220,18 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
// docker build --build-arg http_proxy=http://my.proxy.url
// Attempt at setting name according to #359: https://github.com/frikky/Shuffle/issues/359
labels := map[string]string{}
target := ""
if len(tags) > 0 {
if strings.Contains(tags[0], ":") {
version := strings.Split(tags[0], ":")
if len(version) == 2 {
target = fmt.Sprintf("shuffle-build-%s", version[1])
tags = append(tags, target)
labels["name"] = target
}
}
}
//target := ""
//if len(tags) > 0 {
// if strings.Contains(tags[0], ":") {
// version := strings.Split(tags[0], ":")
// if len(version) == 2 {
// target = fmt.Sprintf("shuffle-build-%s", version[1])
// tags = append(tags, target)
// labels["name"] = target
// }
// }
//}
_ = labels
buildOptions := types.ImageBuildOptions{
Remove: true,
Tags: tags,
@@ -261,47 +260,49 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
//log.Printf("RESPONSE: %#v", imageBuildResponse)
//log.Printf("Response: %#v", imageBuildResponse.Body)
//log.Printf("IMAGERESPONSE: %#v", imageBuildResponse.Body)
log.Printf("[DEBUG] IMAGERESPONSE: %#v", imageBuildResponse.Body)
defer imageBuildResponse.Body.Close()
buildBuf := new(strings.Builder)
_, newerr := io.Copy(buildBuf, imageBuildResponse.Body)
if newerr != nil {
log.Printf("Failed reading Docker build STDOUT: %s", newerr)
} else {
log.Printf("STRING: %s", buildBuf.String())
if strings.Contains(buildBuf.String(), "errorDetail") {
log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), strings.Join(tags, "\n"))
if imageBuildResponse.Body != nil {
defer imageBuildResponse.Body.Close()
buildBuf := new(strings.Builder)
_, newerr := io.Copy(buildBuf, imageBuildResponse.Body)
if newerr != nil {
log.Printf("[WARNING] Failed reading Docker build STDOUT: %s", newerr)
} else {
log.Printf("[INFO] STRING: %s", buildBuf.String())
if strings.Contains(buildBuf.String(), "errorDetail") {
log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), strings.Join(tags, "\n"))
// Handles pulling of the same image if applicable
// This fixes some issues with older versions of Docker which can't build
// on their own ( <17.05 )
pullOptions := types.ImagePullOptions{}
downloaded := false
for _, image := range tags {
// Is this ok? Not sure. Tags shouldn't be controlled here prolly.
image = strings.ToLower(image)
// Handles pulling of the same image if applicable
// This fixes some issues with older versions of Docker which can't build
// on their own ( <17.05 )
pullOptions := types.ImagePullOptions{}
downloaded := false
for _, image := range tags {
// Is this ok? Not sure. Tags shouldn't be controlled here prolly.
image = strings.ToLower(image)
newImage := fmt.Sprintf("%s/%s", registryName, image)
log.Printf("[INFO] Pulling image %s", newImage)
reader, err := client.ImagePull(ctx, newImage, pullOptions)
if err != nil {
log.Printf("[ERROR] Failed getting image %s: %s", newImage, err)
continue
newImage := fmt.Sprintf("%s/%s", registryName, image)
log.Printf("[INFO] Pulling image %s", newImage)
reader, err := client.ImagePull(ctx, newImage, pullOptions)
if err != nil {
log.Printf("[ERROR] Failed getting image %s: %s", newImage, err)
continue
}
// Attempt to retag the image to not contain registry...
//newBuf := buildBuf
downloaded = true
io.Copy(os.Stdout, reader)
log.Printf("[INFO] Successfully downloaded and built %s", newImage)
}
// Attempt to retag the image to not contain registry...
//newBuf := buildBuf
downloaded = true
io.Copy(os.Stdout, reader)
log.Printf("[INFO] Successfully downloaded and built %s", newImage)
if !downloaded {
return errors.New(fmt.Sprintf("Failed to build / download images %s", strings.Join(tags, ",")))
}
//baseDockerName
}
if !downloaded {
return errors.New(fmt.Sprintf("Failed to build / download images %s", strings.Join(tags, ",")))
}
//baseDockerName
}
}
@@ -777,7 +778,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
newClient, err := newdockerclient.NewClientFromEnv()
if err != nil {
log.Printf("[WARNING] Failed setting up docker env: %s", newClient)
log.Printf("[ERROR] Failed setting up docker env: %#v", newClient)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't make docker client"}`)))
return
@@ -791,7 +792,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
}
if err := newClient.ExportImage(opts); err != nil {
log.Printf("[WARNING] FAILED to save image to file: %s", err)
log.Printf("[ERROR] FAILED to save image to file: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't export image"}`)))
return
backend/go-app/go.mod
+6 -2
View File
@@ -2,7 +2,7 @@ module shuffle
go 1.13
replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared
replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared
//replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi
@@ -12,6 +12,7 @@ require (
cloud.google.com/go/pubsub v1.3.1
cloud.google.com/go/storage v1.12.0
github.com/Masterminds/semver v1.5.0 // indirect
github.com/RobotsAndPencils/go-saml v0.0.0-20170520135329-fb13cb52a46b // indirect
github.com/algolia/algoliasearch-client-go/v3 v3.18.1 // indirect
github.com/basgys/goxml2json v1.1.0
github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect
@@ -22,7 +23,7 @@ require (
github.com/docker/go-units v0.4.0 // indirect
github.com/elastic/go-elasticsearch/v7 v7.13.1 // indirect
github.com/frikky/kin-openapi v0.39.0
github.com/frikky/shuffle-shared v0.0.69
github.com/frikky/shuffle-shared v0.1.15
github.com/fsouza/go-dockerclient v1.7.2
github.com/ghodss/yaml v1.0.0
github.com/go-git/go-billy/v5 v5.0.0
@@ -31,8 +32,11 @@ require (
github.com/gorilla/handlers v1.4.2 // indirect
github.com/gorilla/mux v1.8.0
github.com/h2non/filetype v1.0.12
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.1.15
go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423
backend/go-app/main.go
+175 -203
View File
@@ -1,12 +1,13 @@
package main
import (
"github.com/frikky/shuffle-shared"
"github.com/shuffle/shuffle-shared"
"bufio"
"bytes"
"context"
"crypto/md5"
//"crypto/tls"
//"crypto/x509"
"encoding/hex"
"encoding/json"
@@ -48,6 +49,7 @@ import (
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/storage/memory"
//githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
// Random
xj "github.com/basgys/goxml2json"
@@ -57,7 +59,7 @@ import (
"gopkg.in/yaml.v3"
// PROXY overrides
// "gopkg.in/src-d/go-git.v4/plumbing/transport/client"
//"gopkg.in/src-d/go-git.v4/plumbing/transport/client"
// githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
// Web
@@ -578,23 +580,6 @@ func redirect(w http.ResponseWriter, req *http.Request) {
http.StatusTemporaryRedirect)
}
func parseLoginParameters(resp http.ResponseWriter, request *http.Request) (loginStruct, error) {
body, err := ioutil.ReadAll(request.Body)
if err != nil {
return loginStruct{}, err
}
var t loginStruct
err = json.Unmarshal(body, &t)
if err != nil {
return loginStruct{}, err
}
return t, nil
}
// No more emails :)
func checkUsername(Username string) error {
// Stupid first check of email loool
@@ -729,6 +714,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) {
// Only admin can CREATE users, but if there are no users, anyone can make (first)
ctx := context.Background()
users, countErr := shuffle.GetAllUsers(ctx)
count := len(users)
user, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
@@ -739,16 +725,19 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) {
}
}
apikey := ""
if count != 0 {
if user.Role != "admin" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Can't register without being admin (2)"}`))
return
}
} else {
apikey = uuid.NewV4().String()
}
// Gets a struct of Username, password
data, err := parseLoginParameters(resp, request)
data, err := shuffle.ParseLoginParameters(resp, request)
if err != nil {
log.Printf("Invalid params: %s", err)
resp.WriteHeader(401)
@@ -808,7 +797,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) {
err = shuffle.SetEnvironment(ctx, &item)
if err != nil {
log.Printf("[WARNING] Failed setting up new environment for new org: %s")
log.Printf("[WARNING] Failed setting up new environment for new org: %s", err)
}
currentOrg = shuffle.OrgMini{
@@ -819,7 +808,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) {
}
}
err = createNewUser(data.Username, data.Password, role, "", currentOrg)
err = createNewUser(data.Username, data.Password, role, apikey, currentOrg)
if err != nil {
log.Printf("[WARNING] Failed registering user: %s", err)
resp.WriteHeader(401)
@@ -828,7 +817,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) {
}
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
resp.Write([]byte(fmt.Sprintf(`{"success": true, "apikey": "%s"}`, apikey)))
log.Printf("[INFO] %s Successfully registered.", data.Username)
}
@@ -854,6 +843,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) {
userInfo, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in handleInfo: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
@@ -976,38 +966,66 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) {
}
err = shuffle.SetUser(ctx, &userInfo, true)
if err != nil {
log.Printf("Error patching User for activeOrg: %s", err)
log.Printf("[INFO] Error patching User for activeOrg: %s", err)
}
}
}
// FIXME: Remove this dependency by updating users' orgs when org itself is updated
org, err := shuffle.GetOrg(ctx, userInfo.ActiveOrg.Id)
if err == nil {
userInfo.ActiveOrg = shuffle.OrgMini{
Id: org.Id,
Name: org.Name,
Id: org.Id,
Name: org.Name,
CreatorOrg: org.CreatorOrg,
Role: userInfo.ActiveOrg.Role,
Image: org.Image,
}
userInfo.ActiveOrg.Users = []shuffle.UserMini{}
}
userInfo.ActiveOrg.Users = []shuffle.UserMini{}
currentOrg, err := json.Marshal(userInfo.ActiveOrg)
if err != nil {
currentOrg = []byte("{}")
userOrgs := []shuffle.OrgMini{}
for _, item := range userInfo.Orgs {
if item == userInfo.ActiveOrg.Id {
userOrgs = append(userOrgs, userInfo.ActiveOrg)
continue
}
org, err := shuffle.GetOrg(ctx, item)
if err == nil {
userOrgs = append(userOrgs, shuffle.OrgMini{
Id: org.Id,
Name: org.Name,
CreatorOrg: org.CreatorOrg,
Image: org.Image,
})
} else {
log.Printf("[WARNING] Failed to get org %s for user %s", item, userInfo.Username)
}
}
returnData := fmt.Sprintf(`{
"success": true,
"username": "%s",
"admin": %s,
"tutorials": [],
"id": "%s",
"orgs": [%s],
"active_org": %s,
"cookies": [{"key": "session_token", "value": "%s", "expiration": %d}]
}`, userInfo.Username, parsedAdmin, userInfo.Id, currentOrg, currentOrg, userInfo.Session, expiration.Unix())
returnValue := shuffle.HandleInfo{
Success: true,
Username: userInfo.Username,
Admin: parsedAdmin,
Id: userInfo.Id,
Orgs: userOrgs,
ActiveOrg: userInfo.ActiveOrg,
Cookies: []shuffle.SessionCookie{
shuffle.SessionCookie{
Key: "session_token",
Value: userInfo.Session,
Expiration: expiration.Unix(),
},
},
}
returnData, err := json.Marshal(returnValue)
if err != nil {
log.Printf("[WARNING] Failed marshalling info in handleinfo: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write([]byte(returnData))
@@ -1147,8 +1165,10 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) {
return
}
//ssoUrl = org.SSOConfig.SOSOEntrypoint
redirectUri := shuffle.SSOUrl
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "redirect"}`)))
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "redirect", "sso_url": "%s"}`, redirectUri)))
}
func handleLogin(resp http.ResponseWriter, request *http.Request) {
@@ -1158,7 +1178,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) {
}
// Gets a struct of Username, password
data, err := parseLoginParameters(resp, request)
data, err := shuffle.ParseLoginParameters(resp, request)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
@@ -1935,6 +1955,10 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
// 1. Get callback data
// 2. Load the configuration
// 3. Execute the workflow
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
path := strings.Split(request.URL.String(), "/")
if len(path) < 4 {
@@ -1949,8 +1973,10 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
location := strings.Split(request.URL.String(), "/")
var hookId string
var queries string
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("[INFO] Couldn't handle location. Too short in webhook: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
@@ -1959,10 +1985,20 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
hookId = location[4]
}
if strings.Contains(hookId, "?") {
splitter := strings.Split(hookId, "?")
hookId = splitter[0]
if len(splitter) > 1 {
queries = splitter[1]
}
}
// ID: webhook_<UID>
if len(hookId) != 44 {
log.Printf("[INFO] Couldn't handle hookId. Too short in webhook: %d", len(hookId))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "message": "ID not valid"}`))
resp.Write([]byte(`{"success": false, "reason": "Hook ID not valid"}`))
return
}
@@ -1986,19 +2022,19 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
if hook.Status == "stopped" {
log.Printf("[WARNING] Not running %s because hook status is stopped", hook.Id)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "The webhook isn't running. Click start to start it"}`)))
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "The webhook isn't running. Is it running?"}`)))
return
}
if len(hook.Workflows) == 0 {
log.Printf("Not running because hook isn't connected to any workflows")
log.Printf("[DEBUG] Not running because hook isn't connected to any workflows")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No workflows are defined"}`)))
return
}
if hook.Environment == "cloud" {
log.Printf("This should trigger in the cloud. Duplicate action allowed onprem.")
log.Printf("[DEBUG] This should trigger in the cloud. Duplicate action allowed onprem.")
}
// Check auth
@@ -2014,12 +2050,16 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Body data error: %s", err)
log.Printf("[DEBUG] Body data error: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if len(queries) > 0 && len(body) == 0 {
body = []byte(queries)
}
//log.Printf("BODY: %s", parsedBody)
// This is a specific fix for MSteams and may fix other things as well
@@ -2567,7 +2607,6 @@ func findValidScheduleAppFolders(rootAppFolder string) ([]string, error) {
appFiles, err := ioutil.ReadDir(appFolderLocation)
if err != nil {
// Invalid app folder (deleted within a few MS lol)
log.Printf("%s", err)
invalidRootFolders = append(invalidRootFolders, rootfile.Name())
continue
}
@@ -2691,121 +2730,6 @@ func validateAppYaml(fileLocation string) error {
return nil
}
func handleSendalert(resp http.ResponseWriter, request *http.Request) {
user, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in sendalert: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "mail" && user.Role != "admin" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "You don't have access to send mail"}`))
return
}
// ReferenceExecution and below are for execution continuations when user inputs arrive
type mailcheck struct {
Targets []string `json:"targets"`
Body string `json:"body"`
Subject string `json:"subject"`
Type string `json:"type"`
SenderCompany string `json:"sender_company"`
ReferenceExecution string `json:"reference_execution"`
WorkflowId string `json:"workflow_id"`
ExecutionType string `json:"execution_type"`
Start string `json:"start"`
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Body data error on mail: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
var mailbody mailcheck
err = json.Unmarshal(body, &mailbody)
if err != nil {
log.Printf("Unmarshal error on mail: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
ctx := context.Background()
confirmMessage := `
You have a new alert from shuffler.io!
%s
Please contact us at shuffler.io or frikky@shuffler.io if there is an issue with this message.`
parsedBody := fmt.Sprintf(confirmMessage, mailbody.Body)
// FIXME - Make a continuation email here - might need more info from worker
// making the request, e.g. what the next start-node is and execution_id for
// how to make the links
if mailbody.Type == "User input" {
authkey := uuid.NewV4().String()
log.Printf("Should handle differentiator for user input in email!")
log.Printf("%#v", mailbody)
url := "https://shuffler.io"
//url := "http://localhost:5001"
continueUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute?authorization=%s&start=%s&reference_execution=%s&answer=true", url, mailbody.WorkflowId, authkey, mailbody.Start, mailbody.ReferenceExecution)
stopUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute?authorization=%s&start=%s&reference_execution=%s&answer=false", url, mailbody.WorkflowId, authkey, mailbody.Start, mailbody.ReferenceExecution)
//item := &memcache.Item{
// Key: authkey,
// Value: []byte(fmt.Sprintf(`{"role": "workflow_%s"}`, mailbody.WorkflowId)),
// Expiration: time.Minute * 1200,
//}
//if err := memcache.Add(ctx, item); err == memcache.ErrNotStored {
// if err := memcache.Set(ctx, item); err != nil {
// log.Printf("Error setting new user item: %v", err)
// }
//} else if err != nil {
// log.Printf("error adding item: %v", err)
//} else {
// log.Printf("Set cache for %s", item.Key)
//}
parsedBody = fmt.Sprintf(`
Action required!
%s
If this is TRUE click this: %s
IF THIS IS FALSE, click this: %s
Please contact us at shuffler.io or frikky@shuffler.io if there is an issue with this message.
`, mailbody.Body, continueUrl, stopUrl)
}
msg := &mail.Message{
Sender: "Shuffle <frikky@shuffler.io>",
To: mailbody.Targets,
Subject: fmt.Sprintf("Shuffle - %s - %s", mailbody.Type, mailbody.Subject),
Body: parsedBody,
}
log.Println(msg.Body)
if err := mail.Send(ctx, msg); err != nil {
log.Printf("Couldn't send email: %v", err)
}
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
func setBadMemcache(ctx context.Context, path string) {
// Add to cache if it doesn't exist
//item := &memcache.Item{
@@ -3133,7 +3057,6 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) {
swagger, err := swaggerLoader.LoadSwaggerFromData(body)
if err != nil {
log.Printf("[ERROR] Swagger validation error: %s", err)
//log.Printf("%s", string(body))
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed verifying openapi"}`))
return
@@ -3161,7 +3084,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) {
//log.Printf("Should generate yaml")
swagger, api, pythonfunctions, err := shuffle.GenerateYaml(swagger, newmd5)
if err != nil {
log.Printf("Failed building and generating yaml: %s", err)
log.Printf("[WARNING] Failed building and generating yaml (buildapp): %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed building and parsing yaml"}`))
return
@@ -3331,9 +3254,12 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) {
log.Printf("[ERROR] Failed saving app %s to database: %s", newmd5, err)
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%"}`, err)))
return
}
shuffle.SetOpenApiDatastore(ctx, api.ID, parsed)
} else {
//log.Printf("
}
// Backup every single one
@@ -3358,6 +3284,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) {
shuffle.DeleteCache(ctx, cacheKey)
shuffle.DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
log.Printf("[DEBUG] Successfully built app %s (%s)", api.Name, api.ID)
if len(user.Id) > 0 {
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, api.ID)))
@@ -3468,7 +3395,7 @@ func handleAppHotload(ctx context.Context, location string, forceUpdate bool) er
}
//log.Printf("Reading app folder: %#v", dir)
_, _, err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate)
_, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate)
if err != nil {
log.Printf("[WARNING] Githubfolders error: %s", err)
return err
@@ -3552,7 +3479,7 @@ func handleCloudJob(job shuffle.CloudSyncJob) error {
if job.Action == "execute" {
// FIXME: Get the email
ctx := context.Background()
maildata := shuffle.MailData{}
maildata := shuffle.MailDataOutlook{}
err := json.Unmarshal([]byte(job.ThirdItem), &maildata)
if err != nil {
log.Printf("Maildata unmarshal error: %s", err)
@@ -3568,13 +3495,13 @@ func handleCloudJob(job shuffle.CloudSyncJob) error {
redirectDomain := "localhost:5001"
redirectUrl := fmt.Sprintf("http://%s/api/v1/triggers/outlook/register", redirectDomain)
outlookClient, _, err := getOutlookClient(ctx, "", hook.OauthToken, redirectUrl)
outlookClient, _, err := shuffle.GetOutlookClient(ctx, "", hook.OauthToken, redirectUrl)
if err != nil {
log.Printf("Oauth client failure - triggerauth: %s", err)
return err
}
emails, err := getOutlookEmail(outlookClient, maildata)
emails, err := shuffle.GetOutlookEmail(outlookClient, maildata)
//log.Printf("EMAILS: %d", len(emails))
//log.Printf("INSIDE GET OUTLOOK EMAIL!: %#v, %s", emails, err)
@@ -3841,6 +3768,15 @@ func runInitCloudSetup() {
func runInitEs(ctx context.Context) {
log.Printf("[DEBUG] Starting INIT setup (ES)")
httpProxy := os.Getenv("HTTP_PROXY")
if len(httpProxy) > 0 {
log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
}
httpsProxy := os.Getenv("HTTPS_PROXY")
if len(httpsProxy) > 0 {
log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
}
defaultEnv := os.Getenv("ORG_ID")
if len(defaultEnv) == 0 {
defaultEnv = "Shuffle"
@@ -3941,17 +3877,11 @@ func runInitEs(ctx context.Context) {
log.Printf("[WARNING] Failed getting schedules during service init: %s", err)
} else {
log.Printf("[INFO] Setting up %d schedule(s)", len(schedules))
url := &url.URL{}
for _, schedule := range schedules {
if schedule.Environment == "cloud" {
log.Printf("Skipping cloud schedule")
continue
}
//log.Printf("Schedule: %#v", schedule)
job := func() {
//log.Printf("[INFO] Running schedule %s with interval %d.", schedule.Id, schedule.Seconds)
//log.Printf("ARG: %s", schedule.WrappedArgument)
url := &url.URL{}
job := func(schedule shuffle.ScheduleOld) func() {
return func() {
log.Printf("[INFO] Running schedule %s with interval %d.", schedule.Id, schedule.Seconds)
request := &http.Request{
URL: url,
@@ -3964,9 +3894,17 @@ func runInitEs(ctx context.Context) {
log.Printf("[WARNING] Failed to execute %s: %s", schedule.WorkflowId, err)
}
}
}
for _, schedule := range schedules {
if schedule.Environment == "cloud" {
log.Printf("Skipping cloud schedule")
continue
}
//log.Printf("Schedule: %#v", schedule)
//log.Printf("Schedule time: every %d seconds", schedule.Seconds)
jobret, err := newscheduler.Every(schedule.Seconds).Seconds().NotImmediately().Run(job)
jobret, err := newscheduler.Every(schedule.Seconds).Seconds().NotImmediately().Run(job(schedule))
if err != nil {
log.Printf("Failed to schedule workflow: %s", err)
}
@@ -4044,7 +3982,7 @@ func runInitEs(ctx context.Context) {
for _, org := range activeOrgs {
if !org.CloudSync {
log.Printf("[WARNING] Skipping org %s because sync isn't set (1).", org.Id)
log.Printf("[WARNING] Skipping org syncCheck for %s because sync isn't set (1).", org.Id)
continue
}
@@ -4083,6 +4021,17 @@ func runInitEs(ctx context.Context) {
// FIXME: Isn't this a little backwards?
workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000)
log.Printf("[INFO] Getting and validating workflowapps. Got %d with err %#v", len(workflowapps), err)
// accept any certificate (might be useful for testing)
//customGitClient := &http.Client{
// Transport: &http.Transport{
// TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
// },
// Timeout: 15 * time.Second,
//}
//client.InstallProtocol("http", githttp.NewClient(customGitClient))
//client.InstallProtocol("https", githttp.NewClient(customGitClient))
if err != nil && len(workflowapps) == 0 {
log.Printf("[WARNING] Failed getting apps (runInit): %s", err)
} else if err == nil {
@@ -4108,6 +4057,7 @@ func runInitEs(ctx context.Context) {
Password: password,
}
}
branch := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_BRANCH")
if len(branch) > 0 && branch != "master" && branch != "main" {
cloneOptions.ReferenceName = plumbing.ReferenceName(branch)
@@ -4118,18 +4068,18 @@ func runInitEs(ctx context.Context) {
r, err := git.Clone(storer, fs, cloneOptions)
if err != nil {
log.Printf("Failed loading repo into memory (init): %s", err)
log.Printf("[WARNING] Failed loading repo into memory (init): %s", err)
}
dir, err := fs.ReadDir("")
if err != nil {
log.Printf("Failed reading folder: %s", err)
log.Printf("[WARNING] Failed reading folder (init): %s", err)
}
_ = r
//iterateAppGithubFolders(fs, dir, "", "testing")
// FIXME: Get all the apps?
_, _, err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate)
_, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate)
if err != nil {
log.Printf("[WARNING] Error from app load in init: %s", err)
}
@@ -4154,7 +4104,7 @@ func runInitEs(ctx context.Context) {
}
_, err = git.Clone(storer, fs, cloneOptions)
if err != nil {
log.Printf("Failed loading repo %s into memory: %s", apis, err)
log.Printf("[WARNING] Failed loading repo %s into memory: %s", apis, err)
} else {
log.Printf("[INFO] Finished git clone. Looking for updates to the repo.")
dir, err := fs.ReadDir("")
@@ -4784,7 +4734,7 @@ func runInit(ctx context.Context) {
//iterateAppGithubFolders(fs, dir, "", "testing")
// FIXME: Get all the apps?
_, _, err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate)
_, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate)
if err != nil {
log.Printf("[WARNING] Error from app load in init: %s", err)
}
@@ -4933,6 +4883,7 @@ func handleStopCloudSync(syncUrl string, org shuffle.Org) (*shuffle.Org, error)
return &org, err
}
// FIXME: If it says bad API-key, stop cloud sync for the Org
if newresp.StatusCode != 200 {
return &org, errors.New(fmt.Sprintf("Got status code %d when disabling org remotely. Expected 200. Contact support.", newresp.StatusCode))
}
@@ -5210,7 +5161,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) {
// 2. If cloud env found, enable it (un-archive)
// 3. If it doesn't create it
environments, err := shuffle.GetEnvironments(ctx, org.Id)
log.Printf("GETTING ENVS: %#s", environments)
log.Printf("GETTING ENVS: %#v", environments)
if err == nil {
// Don't disable, this will be deleted entirely
@@ -5304,8 +5255,8 @@ func migrateDatabase(resp http.ResponseWriter, request *http.Request) {
}
ctx := context.Background()
es := shuffle.GetEsConfig()
_, err := shuffle.RunInit(*dbclient, *es, storage.Client{}, gceProject, "onprem", false, "")
//es := shuffle.GetEsConfig()
_, err := shuffle.RunInit(*dbclient, storage.Client{}, gceProject, "onprem", false, "")
if err != nil {
log.Printf("[WARNING] Failed to start migration because of init issues: %s", err)
resp.WriteHeader(401)
@@ -5387,7 +5338,7 @@ func migrateDatabase(resp http.ResponseWriter, request *http.Request) {
envSuccess := 0
hookSuccess := 0
scheduleSuccess := 0
_, err = shuffle.RunInit(*dbclient, *es, storage.Client{}, gceProject, "onprem", false, "elasticsearch")
_, err = shuffle.RunInit(*dbclient, storage.Client{}, gceProject, "onprem", false, "elasticsearch")
for _, item := range orgs {
err = shuffle.SetOrg(ctx, item, item.Id)
@@ -5569,7 +5520,7 @@ func makeWorkflowPublic(resp http.ResponseWriter, request *http.Request) {
if workflow.OrgId == user.ActiveOrg.Id && user.Role == "admin" {
log.Printf("[AUDIT] User %s is accessing workflow %s as admin (public)", user.Username, workflow.ID)
} else {
log.Printf("[WARNING] Wrong user (%s) for workflow %s (public)", user.Username, workflow.ID)
log.Printf("[AUDIT] Wrong user (%s) for workflow %s (public)", user.Username, workflow.ID)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
@@ -5668,14 +5619,14 @@ func initHandlers() {
log.Fatalf("[DEBUG] Database client error during init: %s", err)
}
es := shuffle.GetEsConfig()
//es := shuffle.GetEsConfig()
elasticConfig := "elasticsearch"
if strings.ToLower(os.Getenv("SHUFFLE_ELASTIC")) == "false" {
elasticConfig = ""
}
for {
_, err = shuffle.RunInit(*dbclient, *es, storage.Client{}, gceProject, "onprem", true, elasticConfig)
_, err = shuffle.RunInit(*dbclient, storage.Client{}, gceProject, "onprem", true, elasticConfig)
if err != nil {
log.Printf("[ERROR] Error in initial database connection. Retrying in 5 seconds. %s", err)
time.Sleep(5 * time.Second)
@@ -5746,8 +5697,8 @@ func initHandlers() {
r.HandleFunc("/api/v1/apps/{appId}", shuffle.DeleteWorkflowApp).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appId}/config", shuffle.GetWorkflowAppConfig).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/get_existing", loadSpecificApps).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/download_remote", loadSpecificApps).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/get_existing", LoadSpecificApps).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/download_remote", LoadSpecificApps).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/validate", validateAppInput).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps", getWorkflowApps).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps", setNewWorkflowApp).Methods("PUT", "OPTIONS")
@@ -5775,8 +5726,6 @@ func initHandlers() {
r.HandleFunc("/api/v1/workflows/download_remote", loadSpecificWorkflows).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/execute", executeWorkflow).Methods("GET", "POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/schedule/{schedule}", stopSchedule).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/outlook", createOutlookSub).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/outlook/{triggerId}", handleDeleteOutlookSub).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}", deleteWorkflow).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}", shuffle.SaveWorkflow).Methods("PUT", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}", shuffle.GetSpecificWorkflow).Methods("GET", "OPTIONS")
@@ -5794,20 +5743,35 @@ func initHandlers() {
r.HandleFunc("/api/v1/get_openapi/{key}", getOpenapi).Methods("GET", "OPTIONS")
// Specific triggers
r.HandleFunc("/api/v1/triggers/outlook/register", handleNewOutlookRegister).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/outlook", shuffle.HandleCreateOutlookSub).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/outlook/{triggerId}", shuffle.HandleDeleteOutlookSub).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/triggers/outlook/register", shuffle.HandleNewOutlookRegister).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/triggers/outlook/getFolders", shuffle.HandleGetOutlookFolders).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/triggers/outlook/{key}", handleGetSpecificTrigger).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/triggers/outlook/{key}", shuffle.HandleGetSpecificTrigger).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/triggers/gmail/register", shuffle.HandleNewGmailRegister).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/triggers/gmail/getFolders", shuffle.HandleGetGmailFolders).Methods("GET", "OPTIONS")
//r.HandleFunc("/api/v1/triggers/gmail/routing", handleGmailRouting).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/triggers/gmail/{key}", shuffle.HandleGetSpecificTrigger).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/gmail", shuffle.HandleCreateGmailSub).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/gmail/{triggerId}", shuffle.HandleDeleteGmailSub).Methods("DELETE", "OPTIONS")
//r.HandleFunc("/api/v1/triggers/gmail/{key}", handleGetSpecificGmailTrigger).Methods("GET", "OPTIONS")
//r.HandleFunc("/api/v1/triggers/outlook/getFolders", shuffle.HandleGetOutlookFolders).Methods("GET", "OPTIONS")
//r.HandleFunc("/api/v1/triggers/outlook/{key}", handleGetSpecificTrigger).Methods("GET", "OPTIONS")
//r.HandleFunc("/api/v1/triggers/outlook/{key}/callback", handleOutlookCallback).Methods("POST", "OPTIONS")
//r.HandleFunc("/api/v1/stats/{key}", handleGetSpecificStats).Methods("GET", "OPTIONS")
// EVERYTHING below here is NEW for 0.8.0 (written 25.05.2021)
r.HandleFunc("/api/v1/workflows/{key}/publish", makeWorkflowPublic).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/cloud/setup", handleCloudSetup).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/orgs", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS")
//r.HandleFunc("/api/v1/orgs", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS")
//r.HandleFunc("/api/v1/orgs/", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleGetOrg).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleEditOrg).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/create_sub_org", shuffle.HandleCreateSubOrg).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/change", shuffle.HandleChangeUserOrg).Methods("POST", "OPTIONS") // Swaps to the org
// This is a new API that validates if a key has been seen before.
// Not sure what the best course of action is for it.
@@ -5819,11 +5783,13 @@ func initHandlers() {
// Docker orborus specific - downloads an image
r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/migrate_database", migrateDatabase).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/login_sso", shuffle.HandleSSO).Methods("POST", "OPTIONS")
// Important for email, IDS etc. Create this by:
// PS: For cloud, this has to use cloud storage.
// https://developer.box.com/reference/get-files-id-content/
// 1. Creating the "get file" option. Make it possible to run this in the frontend.
r.HandleFunc("/api/v1/files/namespaces/{namespace}", shuffle.HandleGetFileNamespace).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}/content", shuffle.HandleGetFileContent).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/files/create", shuffle.HandleCreateFile).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}/upload", shuffle.HandleUploadFile).Methods("POST", "OPTIONS")
@@ -5831,6 +5797,12 @@ func initHandlers() {
r.HandleFunc("/api/v1/files/{fileId}", shuffle.HandleDeleteFile).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/files", shuffle.HandleGetFiles).Methods("GET", "OPTIONS")
// Introduced in 0.9.21 to handle notifications for e.g. failed Workflow
r.HandleFunc("/api/v1/notifications", shuffle.HandleGetNotifications).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/notifications/clear", shuffle.HandleClearNotifications).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS")
//r.HandleFunc("/api/v1/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS")
http.Handle("/", r)
}
backend/go-app/main_test.go
+340
View File
@@ -0,0 +1,340 @@
package main
import (
"github.com/shuffle/shuffle-shared"
"bytes"
"context"
"log"
"net/http"
"net/http/httptest"
"reflect"
"runtime"
"testing"
"time"
"cloud.google.com/go/datastore"
"cloud.google.com/go/storage"
"google.golang.org/api/option"
"google.golang.org/grpc"
)
type endpoint struct {
handler http.HandlerFunc
path string
method string
}
func init() {
ctx := context.Background()
dbclient, err := datastore.NewClient(ctx, gceProject, option.WithGRPCDialOption(grpc.WithNoProxy()))
if err != nil {
log.Fatalf("[DEBUG] Database client error during init: %s", err)
}
_, err = shuffle.RunInit(*dbclient, storage.Client{}, gceProject, "onprem", true, "elasticsearch")
log.Printf("INIT")
}
// TestTestAuthenticationRequired tests that the handlers in the `handlers`
// variable returns 401 Unauthorized when called without credentials.
func TestAuthenticationRequired(t *testing.T) {
handlers := []endpoint{
{handler: shuffle.HandleNewOutlookRegister, path: "/functions/outlook/register", method: "GET"},
{handler: shuffle.HandleGetOutlookFolders, path: "/functions/outlook/getFolders", method: "GET"},
{handler: shuffle.HandleApiGeneration, path: "/api/v1/users/generateapikey", method: "GET"},
{handler: handleLogin, path: "/api/v1/users/login", method: "POST"}, // prob not this one
// handleRegister generates nil pointer exception. Not necessary for this anyway.
//{handler: handleRegister, path: "/api/v1/users/register", method: "POST"},
{handler: shuffle.HandleGetUsers, path: "/api/v1/users/getusers", method: "GET"},
{handler: handleInfo, path: "/api/v1/users/getinfo", method: "GET"},
{handler: shuffle.HandleSettings, path: "/api/v1/users/getsettings", method: "GET"},
{handler: shuffle.HandleUpdateUser, path: "/api/v1/users/updateuser", method: "PUT"},
{handler: shuffle.DeleteUser, path: "/api/v1/users/123", method: "DELETE"},
{handler: shuffle.HandlePasswordChange, path: "/api/v1/users/passwordchange", method: "POST"},
{handler: shuffle.HandleGetUsers, path: "/api/v1/users", method: "GET"},
{handler: shuffle.HandleGetEnvironments, path: "/api/v1/getenvironments", method: "GET"},
{handler: shuffle.HandleSetEnvironments, path: "/api/v1/setenvironments", method: "PUT"},
// handleWorkflowQueue generates nil pointer exception
//{handler: handleWorkflowQueue, path: "/api/v1/streams", method: "POST"},
// handleGetStreamResults generates nil pointer exception
//{handler: handleGetStreamResults, path: "/api/v1/streams/results", method: "POST"},
{handler: handleAppHotloadRequest, path: "/api/v1/apps/run_hotload", method: "GET"},
{handler: LoadSpecificApps, path: "/api/v1/apps/get_existing", method: "POST"},
{handler: shuffle.UpdateWorkflowAppConfig, path: "/api/v1/apps/123", method: "PATCH"},
{handler: validateAppInput, path: "/api/v1/apps/validate", method: "POST"},
{handler: shuffle.DeleteWorkflowApp, path: "/api/v1/apps/123", method: "DELETE"},
{handler: shuffle.GetWorkflowAppConfig, path: "/api/v1/apps/123/config", method: "GET"},
{handler: getWorkflowApps, path: "/api/v1/apps", method: "GET"},
{handler: setNewWorkflowApp, path: "/api/v1/apps", method: "PUT"},
//{handler: shuffle.GetSpecificApps, path: "/api/v1/apps/search", method: "POST"},
{handler: shuffle.GetAppAuthentication, path: "/api/v1/apps/authentication", method: "GET"},
{handler: shuffle.AddAppAuthentication, path: "/api/v1/apps/authentication", method: "PUT"},
{handler: shuffle.DeleteAppAuthentication, path: "/api/v1/apps/authentication/123", method: "DELETE"},
{handler: validateAppInput, path: "/api/v1/workflows/apps/validate", method: "POST"},
{handler: getWorkflowApps, path: "/api/v1/workflows/apps", method: "GET"},
{handler: setNewWorkflowApp, path: "/api/v1/workflows/apps", method: "PUT"},
{handler: shuffle.GetWorkflows, path: "/api/v1/workflows", method: "GET"},
{handler: shuffle.SetNewWorkflow, path: "/api/v1/workflows", method: "POST"},
{handler: handleGetWorkflowqueue, path: "/api/v1/workflows/queue", method: "GET"},
{handler: handleGetWorkflowqueueConfirm, path: "/api/v1/workflows/queue/confirm", method: "POST"},
{handler: shuffle.HandleGetSchedules, path: "/api/v1/workflows/schedules", method: "GET"},
{handler: loadSpecificWorkflows, path: "/api/v1/workflows/download_remote", method: "POST"},
{handler: executeWorkflow, path: "/api/v1/workflows/123/execute", method: "GET"},
{handler: scheduleWorkflow, path: "/api/v1/workflows/123/schedule", method: "POST"},
{handler: stopSchedule, path: "/api/v1/workflows/123/schedule/abc", method: "DELETE"},
// createOutlookSub generates nil pointer exception
{handler: shuffle.HandleCreateOutlookSub, path: "/api/v1/workflows/123/outlook", method: "POST"},
// handleDeleteOutlookSub generates nil pointer exception
{handler: shuffle.HandleDeleteOutlookSub, path: "/api/v1/workflows/123/outlook/abc", method: "DELETE"},
{handler: shuffle.GetWorkflowExecutions, path: "/api/v1/workflows/123/executions", method: "GET"},
{handler: shuffle.AbortExecution, path: "/api/v1/workflows/123/executions/abc/abort", method: "GET"},
{handler: shuffle.GetSpecificWorkflow, path: "/api/v1/workflows/123", method: "GET"},
{handler: shuffle.SaveWorkflow, path: "/api/v1/workflows/123", method: "PUT"},
{handler: deleteWorkflow, path: "/api/v1/workflows/123", method: "DELETE"},
{handler: shuffle.HandleNewHook, path: "/api/v1/hooks/new", method: "POST"},
{handler: handleWebhookCallback, path: "/api/v1/hooks/123", method: "POST"},
{handler: shuffle.HandleDeleteHook, path: "/api/v1/hooks/123/delete", method: "DELETE"},
{handler: shuffle.HandleGetSpecificTrigger, path: "/api/v1/triggers/123", method: "GET"},
//{handler: shuffle.HandleGetSpecificStats, path: "/api/v1/stats/123", method: "GET"},
{handler: verifySwagger, path: "/api/v1/verify_swagger", method: "POST"},
{handler: verifySwagger, path: "/api/v1/verify_openapi", method: "POST"},
{handler: echoOpenapiData, path: "/api/v1/get_openapi_uri", method: "POST"},
{handler: echoOpenapiData, path: "/api/v1/validate_openapi", method: "POST"},
{handler: shuffle.ValidateSwagger, path: "/api/v1/validate_openapi", method: "POST"},
{handler: getOpenapi, path: "/api/v1/get_openapi", method: "GET"},
//{handler: shuffle.CleanupExecutions, path: "/api/v1/execution_cleanup", method: "GET"},
{handler: handleCloudSetup, path: "/api/v1/cloud/setup", method: "POST"},
{handler: shuffle.HandleGetOrgs, path: "/api/v1/orgs", method: "POST"},
{handler: shuffle.HandleGetFileContent, path: "/api/v1/files/{fileId}/content", method: "POST"},
}
var err error
ctx := context.Background()
// Most handlers requires database access in order to not crash or cause
// nil pointer issues.
// To start a local database instance, run:
// docker-compose up database
// To let the tests know about the database, run:
// DATASTORE_EMULATOR_HOST=0.0.0.0:8000 go test
dbclient, err = datastore.NewClient(ctx, gceProject, option.WithGRPCDialOption(grpc.WithNoProxy()))
if err != nil {
t.Fatal(err)
}
dummyBody := bytes.NewBufferString("dummy")
for _, e := range handlers {
log.Printf("Endpoint: %#v", e.path)
req, err := http.NewRequest(e.method, e.path, dummyBody)
if err != nil {
t.Fatal(err)
}
rr := httptest.NewRecorder()
handler := http.HandlerFunc(e.handler)
timeoutHandler := http.TimeoutHandler(handler, 2*time.Second, `Request Timeout.`)
timeoutHandler.ServeHTTP(rr, req)
funcName := getFunctionNameFromFunction(e.handler)
if status := rr.Code; status != http.StatusUnauthorized {
t.Errorf("%s handler returned wrong status code: got %v want %v",
funcName, status, http.StatusUnauthorized)
}
}
}
func TestAuthenticationNotRequired(t *testing.T) {
// All of these return 200 OK when user not logged in
handlers := []endpoint{
{handler: checkAdminLogin, path: "/api/v1/users/checkusers", method: "GET"},
{handler: shuffle.HandleLogout, path: "/api/v1/users/logout", method: "POST"},
{handler: shuffle.GetDocList, path: "/api/v1/docs", method: "GET"},
{handler: shuffle.GetDocs, path: "/api/v1/docs/123", method: "GET"},
{handler: healthCheckHandler, path: "/api/v1/_ah/health"},
}
for _, e := range handlers {
log.Printf("Endpoint: %#v", e.path)
req, err := http.NewRequest(e.method, e.path, nil)
if err != nil {
t.Fatal(err)
}
rr := httptest.NewRecorder()
handler := http.HandlerFunc(e.handler)
timeoutHandler := http.TimeoutHandler(handler, 2*time.Second, `Request Timeout.`)
timeoutHandler.ServeHTTP(rr, req)
funcName := getFunctionNameFromFunction(e.handler)
if status := rr.Code; status != http.StatusOK {
t.Errorf("%s handler returned wrong status code: got %v want %v",
funcName, status, http.StatusOK)
}
}
}
// TestCors tests that all endpoints returns the same CORS headers when hit
// with an OPTIONS type request.
// It feels very fragile to test headers like this, especially for the
// "Access-Control-Allow-Origin", but this test should be helpful while
// refactoring the CORS logic into a middleware, and that's the reason this
// test exists right now. It might change after the refactor because our
// requirements might change after the refactor.
func TestCors(t *testing.T) {
handlers := []endpoint{
{handler: shuffle.HandleNewOutlookRegister, path: "/functions/outlook/register", method: "GET"},
{handler: shuffle.HandleGetOutlookFolders, path: "/functions/outlook/getFolders", method: "GET"},
{handler: shuffle.HandleApiGeneration, path: "/api/v1/users/generateapikey", method: "GET"},
{handler: handleLogin, path: "/api/v1/users/login", method: "POST"}, // prob not this one
// handleRegister generates nil pointer exception
{handler: handleRegister, path: "/api/v1/users/register", method: "POST"},
{handler: shuffle.HandleGetUsers, path: "/api/v1/users/getusers", method: "GET"},
{handler: handleInfo, path: "/api/v1/users/getinfo", method: "GET"},
{handler: shuffle.HandleSettings, path: "/api/v1/users/getsettings", method: "GET"},
{handler: shuffle.HandleUpdateUser, path: "/api/v1/users/updateuser", method: "PUT"},
{handler: shuffle.DeleteUser, path: "/api/v1/users/123", method: "DELETE"},
// handlePasswordChange generates nil pointer exception
{handler: shuffle.HandlePasswordChange, path: "/api/v1/users/passwordchange", method: "POST"},
{handler: shuffle.HandleGetUsers, path: "/api/v1/users", method: "GET"},
{handler: shuffle.HandleGetEnvironments, path: "/api/v1/getenvironments", method: "GET"},
{handler: shuffle.HandleSetEnvironments, path: "/api/v1/setenvironments", method: "PUT"},
// handleWorkflowQueue generates nil pointer exception
{handler: handleWorkflowQueue, path: "/api/v1/streams", method: "POST"},
// handleGetStreamResults generates nil pointer exception
{handler: handleGetStreamResults, path: "/api/v1/streams/results", method: "POST"},
{handler: handleAppHotloadRequest, path: "/api/v1/apps/run_hotload", method: "GET"},
{handler: LoadSpecificApps, path: "/api/v1/apps/get_existing", method: "POST"},
{handler: shuffle.UpdateWorkflowAppConfig, path: "/api/v1/apps/123", method: "PATCH"},
{handler: validateAppInput, path: "/api/v1/apps/validate", method: "POST"},
{handler: shuffle.DeleteWorkflowApp, path: "/api/v1/apps/123", method: "DELETE"},
{handler: shuffle.GetWorkflowAppConfig, path: "/api/v1/apps/123/config", method: "GET"},
{handler: getWorkflowApps, path: "/api/v1/apps", method: "GET"},
{handler: setNewWorkflowApp, path: "/api/v1/apps", method: "PUT"},
//{handler: shuffle.GetSpecificApps, path: "/api/v1/apps/search", method: "POST"},
{handler: shuffle.GetAppAuthentication, path: "/api/v1/apps/authentication", method: "GET"},
{handler: shuffle.AddAppAuthentication, path: "/api/v1/apps/authentication", method: "PUT"},
{handler: shuffle.DeleteAppAuthentication, path: "/api/v1/apps/authentication/123", method: "DELETE"},
{handler: validateAppInput, path: "/api/v1/workflows/apps/validate", method: "POST"},
{handler: getWorkflowApps, path: "/api/v1/workflows/apps", method: "GET"},
{handler: setNewWorkflowApp, path: "/api/v1/workflows/apps", method: "PUT"},
{handler: shuffle.GetWorkflows, path: "/api/v1/workflows", method: "GET"},
{handler: shuffle.SetNewWorkflow, path: "/api/v1/workflows", method: "POST"},
{handler: handleGetWorkflowqueue, path: "/api/v1/workflows/queue", method: "GET"},
{handler: handleGetWorkflowqueueConfirm, path: "/api/v1/workflows/queue/confirm", method: "POST"},
{handler: shuffle.HandleGetSchedules, path: "/api/v1/workflows/schedules", method: "GET"},
{handler: loadSpecificWorkflows, path: "/api/v1/workflows/download_remote", method: "POST"},
{handler: executeWorkflow, path: "/api/v1/workflows/123/execute", method: "GET"},
{handler: scheduleWorkflow, path: "/api/v1/workflows/123/schedule", method: "POST"},
{handler: stopSchedule, path: "/api/v1/workflows/123/schedule/abc", method: "DELETE"},
// createOutlookSub generates nil pointer exception
{handler: shuffle.HandleCreateOutlookSub, path: "/api/v1/workflows/123/outlook", method: "POST"},
// handleDeleteOutlookSub generates nil pointer exception
{handler: shuffle.HandleDeleteOutlookSub, path: "/api/v1/workflows/123/outlook/abc", method: "DELETE"},
{handler: shuffle.GetWorkflowExecutions, path: "/api/v1/workflows/123/executions", method: "GET"},
{handler: shuffle.AbortExecution, path: "/api/v1/workflows/123/executions/abc/abort", method: "GET"},
{handler: shuffle.GetSpecificWorkflow, path: "/api/v1/workflows/123", method: "GET"},
{handler: shuffle.SaveWorkflow, path: "/api/v1/workflows/123", method: "PUT"},
{handler: deleteWorkflow, path: "/api/v1/workflows/123", method: "DELETE"},
{handler: shuffle.HandleNewHook, path: "/api/v1/hooks/new", method: "POST"},
{handler: handleWebhookCallback, path: "/api/v1/hooks/123", method: "POST"},
{handler: shuffle.HandleDeleteHook, path: "/api/v1/hooks/123/delete", method: "DELETE"},
{handler: shuffle.HandleGetSpecificTrigger, path: "/api/v1/triggers/123", method: "GET"},
//{handler: shuffle.HandleGetSpecificStats, path: "/api/v1/stats/123", method: "GET"},
{handler: verifySwagger, path: "/api/v1/verify_swagger", method: "POST"},
{handler: verifySwagger, path: "/api/v1/verify_openapi", method: "POST"},
{handler: echoOpenapiData, path: "/api/v1/get_openapi_uri", method: "POST"},
{handler: echoOpenapiData, path: "/api/v1/validate_openapi", method: "POST"},
{handler: shuffle.ValidateSwagger, path: "/api/v1/validate_openapi", method: "POST"},
{handler: getOpenapi, path: "/api/v1/get_openapi", method: "GET"},
//{handler: shuffle.CleanupExecutions, path: "/api/v1/execution_cleanup", method: "GET"},
{handler: handleCloudSetup, path: "/api/v1/cloud/setup", method: "POST"},
{handler: shuffle.HandleGetOrgs, path: "/api/v1/orgs", method: "POST"},
{handler: shuffle.HandleGetFileContent, path: "/api/v1/files/{fileId}/content", method: "POST"},
}
//r := initHandlers(context.TODO())
initHandlers()
outerLoop:
for _, e := range handlers {
log.Printf("Endpoint: %#v", e.path)
req, err := http.NewRequest("OPTIONS", e.path, nil)
req.Header.Add("Origin", "http://localhost:3000")
req.Header.Add("Access-Control-Request-Method", "POST")
req.Header.Add("Access-Control-Request-Headers", "Content-Type, Accept, X-Requested-With, remember-me")
// OPTIONS /resource/foo
// Access-Control-Request-Method: DELETE
// Access-Control-Request-Headers: origin, x-requested-with
// Origin: https://foo.bar.org
if err != nil {
t.Errorf("Failure in OPTIONS setup: %s", err)
continue
}
rr := httptest.NewRecorder()
//timeoutHandler := http.TimeoutHandler(r, 2*time.Second, `Request Timeout`)
//timeoutHandler.ServeHTTP(rr, req)
funcName := getFunctionNameFromFunction(e.handler)
if status := rr.Code; status != http.StatusOK {
t.Errorf("%s handler returned wrong status code: got %v want %v",
funcName, status, http.StatusOK)
continue
}
want := map[string]string{
"Vary": "Origin",
"Access-Control-Allow-Headers": "Content-Type, Accept, X-Requested-With, Remember-Me",
"Access-Control-Allow-Methods": "POST",
"Access-Control-Allow-Credentials": "true",
"Access-Control-Allow-Origin": "http://localhost:3000",
}
// Remember to use canonical header name if accessing the headers array
// directly:
// v := r.Header[textproto.CanonicalMIMEHeaderKey("foo")]
// When using Header().Get(h), h will automatically be converted to canonical format.
for key, value := range want {
got := rr.Header().Get(key)
if got != value {
t.Errorf("%s handler returned wrong value for '%s' header: got '%v' want '%v'",
funcName, key, got, value)
continue outerLoop
}
}
}
}
func getFunctionNameFromFunction(f interface{}) string {
return runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
}
backend/go-app/oauth2.go
-1033
View File
File diff suppressed because it is too large Load Diff
backend/go-app/run.sh
Executable
+1
View File
@@ -0,0 +1 @@
go run main.go walkoff.go docker.go
backend/go-app/walkoff.go
+689 -666
View File
File diff suppressed because it is too large Load Diff
backend/tests/files.sh
+16 -10
View File
@@ -1,16 +1,22 @@
curl http://192.168.3.6:5001/api/v1/files/create -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -d '{"filename": "file.txt", "org_id": "b199646b-16d2-456d-9fd6-b9972e929466", "workflow_id": "global"}'
curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -d '{"filename": "file.txt", "org_id": "b199646b-16d2-456d-9fd6-b9972e929466", "workflow_id": "global"}'
echo
curl http://localhost:5001/api/v1/files/e19cffe4-e2da-47e9-809e-904f5cb03687/upload -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -F 'shuffle_file=@files.sh'
curl http://localhost:5001/api/v1/files/1915981b-b897-4db1-8a2e-44bc34cead3b/content -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl http://localhost:5001/api/v1/files/e19cffe4-e2da-47e9-809e-904f5cb03687 -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XDELETE http://localhost:5001/api/v1/files/e19cffe4-e2da-47e9-809e-904f5cb03687 -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
#curl http://192.168.3.6:5001/api/v1/files/create -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -d '{"filename": "file.txt", "org_id": "b199646b-16d2-456d-9fd6-b9972e929466", "workflow_id": "global"}'
#
#curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -d '{"filename": "file.txt", "org_id": "b199646b-16d2-456d-9fd6-b9972e929466", "workflow_id": "global"}'
#
#echo
#curl http://localhost:5001/api/v1/files/e19cffe4-e2da-47e9-809e-904f5cb03687/upload -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -F 'shuffle_file=@files.sh'
#
#curl http://localhost:5001/api/v1/files/1915981b-b897-4db1-8a2e-44bc34cead3b/content -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
#curl http://localhost:5001/api/v1/files/e19cffe4-e2da-47e9-809e-904f5cb03687 -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
#curl -XDELETE http://localhost:5001/api/v1/files/e19cffe4-e2da-47e9-809e-904f5cb03687 -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
#r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS")
#r.HandleFunc("/api/v1/files/create", handleCreateFile).Methods("POST", "OPTIONS")
#r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS")
#r.HandleFunc("/api/v1/files/{fileId}", handleGetFileMeta).Methods("GET", "OPTIONS")
#r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS")
#curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" -d '{"filename": "rule2.yar", "org_id": "b4e88fe9-352b-47b4-b280-960181670acf", "workflow_id": "global", "namespace": "yara"}'
#curl http://localhost:5001/api/v1/files/5cb941ad-fa1c-4444-a685-92024b1fa31c/upload -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" -F 'shuffle_file=@upload.sh'
curl http://localhost:5001/api/v1/files/namespaces/yara -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" --output rules.zip
docker-compose.yml
+8 -23
View File
@@ -25,18 +25,18 @@ services:
- "${BACKEND_PORT}:5001"
networks:
- shuffle
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps
- ${SHUFFLE_FILE_LOCATION}:/shuffle-files
#- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate
env_file: .env
env_file: .env
environment:
- SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps
- SHUFFLE_FILE_LOCATION=/shuffle-files
restart: unless-stopped
#depends_on:
#- opensearch
#- opensearch #Not necessary because dependancy is handled within the backend itself instead
#- database
orborus:
#build: ./functions/onprem/orborus
@@ -62,11 +62,11 @@ services:
- SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY}
- SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY}
- SHUFFLE_ORBORUS_EXECUTION_TIMEOUT=600
- SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=50
- SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=5
- CLEANUP=${SHUFFLE_CONTAINER_AUTO_CLEANUP}
restart: unless-stopped
opensearch:
image: opensearchproject/opensearch:1.0.0
image: opensearchproject/opensearch:1.1.0
hostname: shuffle-opensearch
container_name: shuffle-opensearch
environment:
@@ -84,7 +84,7 @@ services:
soft: -1
hard: -1
nofile:
soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
soft: 65536
hard: 65536
volumes:
- ${DB_LOCATION}:/usr/share/opensearch/data:rw
@@ -93,21 +93,6 @@ services:
networks:
- shuffle
restart: unless-stopped
#database:
# #build: ./backend/database
# image: frikky/shuffle:database
# container_name: shuffle-database
# hostname: shuffle-database
# ports:
# - "8000:8000"
# networks:
# - shuffle
# environment:
# - _JAVA_OPTIONS="-Xmx2g"
# restart: unless-stopped
# volumes:
# - ${DB_LOCATION}:/etc/shuffle
networks:
shuffle:
driver: bridge
frontend/Dockerfile
+1 -2
View File
@@ -8,7 +8,6 @@ ENV PATH /usr/src/app/node_modules/.bin:$PATH
COPY package.json /usr/src/app/package.json
#RUN npm install --verbose
RUN yarn install
# copy only required files to not trigger rebuilding every time
@@ -21,7 +20,7 @@ COPY ./*.json /usr/src/app/
RUN yarn build
# Production environment
FROM nginx:1.21
FROM nginx:1.21.3
RUN mkdir -p /usr/share/nginx/html/build
RUN mkdir -p /usr/share/nginx/html/css
frontend/package.json
+10 -8
View File
@@ -1,14 +1,16 @@
{
"name": "shuffler",
"homepage": "https://shuffler.io",
"version": "0.8.92",
"version": "0.9.24",
"private": true,
"dependencies": {
"@material-ui/core": "^4.5.2",
"@material-ui/data-grid": "^4.0.0-alpha.22",
"@material-ui/icons": "^4.5.1",
"@material-ui/icons": "^4.11.2",
"@material-ui/lab": "^4.0.0-alpha.58",
"@material-ui/styles": "^4.5.2",
"@material-ui/utils": "^4.11.2",
"@uiw/react-codemirror": "^3.2.1",
"@use-it/interval": "^1.0.0",
"babel-eslint": "^10.1.0",
"class-transformer": "^0.3.1",
@@ -17,23 +19,23 @@
"cytoscape-clipboard": "^2.2.1",
"cytoscape-cxtmenu": "^3.1.1",
"cytoscape-edgehandles": "^3.6.0",
"cytoscape-grid-guide": "~2.1.2",
"cytoscape-grid-guide": "~2.3.3",
"cytoscape-node-html-label": "^1.1.5",
"cytoscape-panzoom": "^2.5.2",
"cytoscape-undo-redo": "^1.3.2",
"d3": "~4.10.0",
"d3": "^7.1.1",
"dotenv": "^6.1.0",
"downshift": "^3.3.5",
"github-markdown-css": "^3.0.1",
"import": "0.0.6",
"interweave": "^11.2.0",
"material-icons": "^0.3.1",
"material-icons": "^0.7.7",
"material-icons-react": "^1.0.4",
"material-ui-chip-input": "^2.0.0-beta.2",
"material-ui-nested-menu-item": "^1.0.2",
"md5-file": "^4.0.0",
"mdbreact": "^4.21.1",
"moment": "~2.20.1",
"moment": "^2.29.1",
"react": "^16.14.0",
"react-alert": "^5.5.0",
"react-alert-template-basic": "^1.0.0",
@@ -64,10 +66,10 @@
"websocket": "^1.0.30",
"yaml": "^1.7.2",
"yamljs": "^0.3.0",
"zone.js": "~0.8.26"
"zone.js": "~0.11.4"
},
"scripts": {
"start": "set HTTPS=true&&react-scripts start",
"start": "react-scripts start",
"build": "react-scripts build",
"test": "react-scripts test",
"eject": "react-scripts eject"
frontend/public/images/Shuffle_logo.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 217 KiB

frontend/src/App.jsx
+32 -6
View File
@@ -22,6 +22,8 @@ import AdminSetup from "./views/AdminSetup";
import Admin from "./views/Admin";
import Docs from "./views/Docs";
import Introduction from "./views/Introduction";
import SetAuthentication from "./views/SetAuthentication";
import SetAuthenticationSSO from "./views/SetAuthenticationSSO";
import LandingPageNew from "./views/LandingpageNew";
import LoginPage from "./views/LoginPage";
@@ -40,14 +42,15 @@ import {isMobile} from "react-device-detect";
var globalUrl = window.location.origin
// CORS used for testing purposes. Should only happen with specific port and http
if (window.location.protocol == "http:" && window.location.port === "3000") {
if ( window.location.port === "3000") {
globalUrl = "http://localhost:5001"
//globalUrl = "http://localhost:5002"
}
const App = (message, props) => {
const [userdata, setUserData] = useState({});
const [cookies, setCookie, removeCookie] = useCookies([]);
const [notifications, setNotifications] = useState([])
const [cookies, setCookie, removeCookie] = useCookies([])
const [isLoggedIn, setIsLoggedIn] = useState(false);
const [dataset, setDataset] = useState(false);
const [isLoaded, setIsLoaded] = useState(false);
@@ -55,6 +58,7 @@ const App = (message, props) => {
useEffect(() => {
if (dataset === false) {
getUserNotifications()
checkLogin()
setDataset(true)
}
@@ -64,6 +68,25 @@ const App = (message, props) => {
window.location = "/login"
}
const getUserNotifications = () => {
fetch(`${globalUrl}/api/v1/notifications`, {
credentials: "include",
headers: {
'Content-Type': 'application/json',
},
})
.then(response => response.json())
.then(responseJson => {
if (responseJson.success === true && responseJson.notifications !== null && responseJson.notifications !== undefined && responseJson.notifications.length > 0) {
//console.log("RESP: ", responseJson)
setNotifications(responseJson.notifications)
}
})
.catch(error => {
console.log("Failed getting notifications for user: ", error)
});
}
const checkLogin = () => {
var baseurl = globalUrl
fetch(baseurl + "/api/v1/users/getinfo", {
@@ -75,7 +98,7 @@ const App = (message, props) => {
.then(response => response.json())
.then(responseJson => {
if (responseJson.success === true) {
//console.log(responseJson.success)
console.log(responseJson)
setUserData(responseJson)
setIsLoggedIn(true)
//console.log("Cookies: ", cookies)
@@ -104,9 +127,10 @@ const App = (message, props) => {
<Route exact path="/home" render={props => <LandingPageNew isLoaded={isLoaded} {...props} />} />
</div> :
<div style={{ backgroundColor: "#1F2023", color: "rgba(255, 255, 255, 0.65)", minHeight: "100vh" }}>
<ScrollToTop setCurpath={setCurpath} />
<Header cookies={cookies} removeCookie={removeCookie} isLoaded={isLoaded} globalUrl={globalUrl} setIsLoggedIn={setIsLoggedIn} isLoggedIn={isLoggedIn} userdata={userdata} {...props} />
<Route exact path="/login" render={props => <LoginPage isLoggedIn={isLoggedIn} setIsLoggedIn={setIsLoggedIn} register={true} isLoaded={isLoaded} globalUrl={globalUrl} setCookie={setCookie} cookies={cookies} {...props} />} />
<ScrollToTop getUserNotifications={getUserNotifications} setCurpath={setCurpath} />
<Header notifications={notifications} setNotifications={setNotifications} checkLogin={checkLogin} cookies={cookies} removeCookie={removeCookie} isLoaded={isLoaded} globalUrl={globalUrl} setIsLoggedIn={setIsLoggedIn} isLoggedIn={isLoggedIn} userdata={userdata} {...props} />
<div style={{height: 60}}/>
<Route exact path="/login" render={props => <LoginPage isLoggedIn={isLoggedIn} setIsLoggedIn={setIsLoggedIn} register={true} isLoaded={isLoaded} globalUrl={globalUrl} setCookie={setCookie} cookies={cookies} checkLogin={checkLogin} {...props} />} />
<Route exact path="/admin" render={props => <Admin userdata={userdata} isLoggedIn={isLoggedIn} setIsLoggedIn={setIsLoggedIn} register={true} isLoaded={isLoaded} globalUrl={globalUrl} setCookie={setCookie} cookies={cookies} {...props} />} />
<Route exact path="/admin/:key" render={props => <Admin isLoggedIn={isLoggedIn} setIsLoggedIn={setIsLoggedIn} register={true} isLoaded={isLoaded} globalUrl={globalUrl} setCookie={setCookie} cookies={cookies} {...props} />} />
<Route exact path="/settings" render={props => <SettingsPage isLoaded={isLoaded} userdata={userdata} globalUrl={globalUrl} {...props} />} />
@@ -125,6 +149,8 @@ const App = (message, props) => {
<Route exact path="/docs" render={props => { window.location.pathname = "/docs/about" }} />
<Route exact path="/introduction" render={props => <Introduction isLoaded={isLoaded} globalUrl={globalUrl} {...props} />} />
<Route exact path="/introduction/:key" render={props => <Introduction isLoaded={isLoaded} globalUrl={globalUrl} {...props} />} />
<Route exact path="/set_authentication" render={props => <SetAuthentication userdata={userdata} isLoggedIn={isLoggedIn} setIsLoggedIn={setIsLoggedIn} register={true} isLoaded={isLoaded} globalUrl={globalUrl} setCookie={setCookie} cookies={cookies} {...props} />} />
<Route exact path="/login_sso" render={props => <SetAuthenticationSSO userdata={userdata} isLoggedIn={isLoggedIn} setIsLoggedIn={setIsLoggedIn} register={true} isLoaded={isLoaded} globalUrl={globalUrl} setCookie={setCookie} cookies={cookies} {...props} />} />
<Route exact path="/" render={props => <LoginPage isLoggedIn={isLoggedIn} setIsLoggedIn={setIsLoggedIn} register={true} isLoaded={isLoaded} globalUrl={globalUrl} setCookie={setCookie} cookies={cookies} {...props} />} />
</div>
frontend/src/components/AlertTemplate.js
+1 -1
View File
@@ -15,7 +15,7 @@ const alertStyle = {
justifyContent: 'space-between',
alignItems: 'center',
boxShadow: '0px 2px 2px 2px rgba(0, 0, 0, 0.03)',
width: 400,
width: 300,
boxSizing: 'border-box',
zIndex: 100001,
overflow: "hidden",
frontend/src/components/ConfigureWorkflow.jsx
+46 -6
View File
@@ -11,8 +11,8 @@ import { FixName } from "../views/Apps.jsx";
// Triggers
//
// Specifically used for UNSAVED workflows only?
const Workflow = (props) => {
const { globalUrl, theme, workflow, appAuthentication, setSelectedAction, setAuthenticationModalOpen, setSelectedApp, apps, selectedAction,setConfigureWorkflowModalOpen, saveWorkflow, newWebhook, submitSchedule, referenceUrl, isCloud, } = props
const ConfigureWorkflow = (props) => {
const { globalUrl, theme, workflow, appAuthentication, setSelectedAction, setAuthenticationModalOpen, setSelectedApp, apps, selectedAction,setConfigureWorkflowModalOpen, saveWorkflow, newWebhook, submitSchedule, referenceUrl, isCloud, setAuthenticationType, alert, } = props
const [requiredActions, setRequiredActions] = React.useState([])
const [requiredVariables, setRequiredVariables] = React.useState([])
const [requiredTriggers, setRequiredTriggers] = React.useState([])
@@ -90,7 +90,7 @@ const Workflow = (props) => {
const app = apps.find(app => app.name === action.app_name && (app.app_version === action.app_version || (app.loop_versions !== null && app.loop_versions.includes(action.app_version))))
if (app === undefined || app === null) {
console.log("App not found!")
console.log("App not found: ", action.app_name)
newaction.must_activate = true
} else {
@@ -374,6 +374,17 @@ const Workflow = (props) => {
<CircularProgress />
:
<Button color="primary" variant="contained" onClick={() => {
setAuthenticationType(action.app.authentication.type === "oauth2" && action.app.authentication.redirect_uri !== undefined && action.app.authentication.redirect_uri !== null ?
{
"type": "oauth2",
"redirect_uri": action.app.authentication.redirect_uri,
"token_uri": action.app.authentication.token_uri,
"scope": action.app.authentication.scope,
} : {
"type": ""
}
)
setItemChanged(true)
setSelectedAction(action.action)
setSelectedApp(action.app)
@@ -384,8 +395,8 @@ const Workflow = (props) => {
:
null}
{action.must_activate ?
<Button disabled={true} color="primary" variant="contained" onClick={() => {
console.log("SHOULD ACTIVATE: ", action)
<Button color="primary" variant="contained" onClick={() => {
activateApp(action.app_id, action.app_name, action.app_version)
setItemChanged(true)
}}>
Activate
@@ -395,6 +406,35 @@ const Workflow = (props) => {
)
}
const activateApp = (app_id, app_name, app_version) => {
fetch(`${globalUrl}/api/v1/apps/app_id/activate?app_name=${app_name}&app_version=${app_version}`, {
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
},
credentials: "include",
})
.then((response) => {
if (response.status !== 200) {
//window.location.pathname = "/search"
//alert.error("Failed to find this app. Is it public?")
}
return response.json()
})
.then((responseJson) => {
if (responseJson.success === false) {
alert.error("Failed to activate the app")
} else {
alert.success("App activated for your organization!")
}
})
.catch(error => {
alert.error(error.toString())
});
}
return (
<div>
<Typography variant="h6">{workflow.name}</Typography>
@@ -467,4 +507,4 @@ const Workflow = (props) => {
)
}
export default Workflow
export default ConfigureWorkflow
frontend/src/components/Header.js
+280 -29
View File
@@ -3,26 +3,18 @@ import {BrowserView, MobileView} from "react-device-detect";
import {Link} from 'react-router-dom';
import List from '@material-ui/core/List';
import Avatar from '@material-ui/core/Avatar';
import Menu from '@material-ui/core/Menu';
import ListItem from '@material-ui/core/ListItem';
import MenuItem from '@material-ui/core/MenuItem';
import Select from '@material-ui/core/Select';
import Button from '@material-ui/core/Button';
import IconButton from '@material-ui/core/IconButton';
import HomeIcon from '@material-ui/icons/Home';
import PolymerIcon from '@material-ui/icons/Polymer';
import AppsIcon from '@material-ui/icons/Apps';
import DescriptionIcon from '@material-ui/icons/Description';
import Grid from '@material-ui/core/Grid';
import { useTheme } from '@material-ui/core/styles';
import { Chip, Badge, Typography, Paper, Tooltip, List, Avatar, Menu, ListItem, MenuItem, Select, Button, IconButton, Grid } from '@material-ui/core';
import { MeetingRoom as MeetingRoomIcon, Settings as SettingsIcon, Notifications as NotificationsIcon, Home as HomeIcon, Polymer as PolymerIcon, Apps as AppsIcon, Description as DescriptionIcon} from '@material-ui/icons';
//import LogoutIcon from '@mui/icons-material/Logout';
import { useAlert } from "react-alert";
const hoverColor = "#f85a3e"
const hoverOutColor = "#e8eaf6"
const Header = props => {
const { globalUrl, isLoggedIn, removeCookie, homePage, isLoaded, userdata, cookies } = props;
const { globalUrl, setNotifications, notifications, isLoggedIn, removeCookie, homePage, isLoaded, userdata, cookies } = props;
const theme = useTheme();
const [HomeHoverColor, setHomeHoverColor] = useState(hoverOutColor);
@@ -31,12 +23,78 @@ const Header = props => {
const [DocsHoverColor, setDocsHoverColor] = useState(hoverOutColor);
const [HelpHoverColor, setHelpHoverColor] = useState(hoverOutColor);
const [anchorEl, setAnchorEl] = React.useState(null);
const [anchorElAvatar, setAnchorElAvatar] = React.useState(null);
const alert = useAlert()
const hrefStyle = {
color: hoverOutColor,
textDecoration: "none",
}
const handleClose = () => {
setAnchorEl(null);
setAnchorElAvatar(null);
};
const clearNotifications = () => {
// Don't really care about the logout
fetch(`${globalUrl}/api/v1/notifications/clear`, {
credentials: "include",
method: 'GET',
headers: {
'Content-Type': 'application/json',
},
})
.then(function(response) {
if (response.status !== 200) {
console.log("Error in response")
}
return response.json();
}).then(function(responseJson) {
if (responseJson.success === true) {
setNotifications([])
handleClose()
} else {
alert.error("Failed dismissing notifications. Please try again later.")
}
})
.catch(error => {
console.log("error in notification dismissal: ", error)
//removeCookie("session_token", {path: "/"})
})
}
const dismissNotification = (alert_id) => {
// Don't really care about the logout
fetch(`${globalUrl}/api/v1/notifications/${alert_id}/markasread`, {
credentials: "include",
method: 'GET',
headers: {
'Content-Type': 'application/json',
},
})
.then(function(response) {
if (response.status !== 200) {
console.log("Error in response")
}
return response.json();
}).then(function(responseJson) {
if (responseJson.success === true) {
const newNotifications = notifications.filter(data => data.id !== alert_id)
console.log("NEW NOTIFICATIONS: ", newNotifications)
setNotifications(newNotifications)
} else {
alert.error("Failed dismissing notification. Please try again later.")
}
})
.catch(error => {
console.log("error in notification dismissal: ", error)
//removeCookie("session_token", {path: "/"})
})
}
// DEBUG HERE
const handleClickLogout = () => {
console.log("COOKIES: ", cookies, "Remover: ", removeCookie)
@@ -65,6 +123,47 @@ const Header = props => {
})
}
const handleClickChangeOrg = (orgId) => {
// Don't really care about the logout
//name: org.name,
//orgId = "asd"
const data = {
org_id: orgId,
}
fetch(`${globalUrl}/api/v1/orgs/${orgId}/change`, {
mode: 'cors',
method: 'POST',
body: JSON.stringify(data),
credentials: 'include',
crossDomain: true,
withCredentials: true,
headers: {
'Content-Type': 'application/json; charset=utf-8',
},
})
.then(function(response) {
if (response.status !== 200) {
console.log("Error in response")
}
return response.json();
}).then(function(responseJson) {
if (responseJson.success !== undefined && responseJson.success) {
setTimeout(() => {
window.location.reload()
}, 2000)
alert.success("Successfully changed active organization - refreshing!")
} else {
alert.error("Failed changing org: ", responseJson.reason)
}
})
.catch(error => {
console.log("error changing: ", error)
//removeCookie("session_token", {path: "/"})
})
}
// Rofl this is weird
const handleDocsHover = () => {
setDocsHoverColor(hoverColor)
@@ -111,25 +210,129 @@ const Header = props => {
setAnchorEl(event.currentTarget);
};
const handleClose = () => {
setAnchorEl(null);
};
const chipStyle = {
backgroundColor: "#3d3f43", height: 30, marginRight: 5, paddingLeft: 5, paddingRight: 5, height: 28, cursor: "pointer", borderColor: "#3d3f43", color: "white",
}
const notificationWidth = 350
const NotificationItem = (props) => {
const {data} = props
// Should be based on some path
const avatarMenu =
<span>
<IconButton color="primary" style={{marginRight: 15, }} aria-controls="simple-menu" aria-haspopup="true" onClick={(event) => {
return (
<Paper style={{backgroundColor: theme.palette.surfaceColor, width: notificationWidth, padding: 25, borderBottom: "1px solid rgba(255,255,255,0.4)"}}>
{/*<Typography variant="h6">
{new Date(data.updated_at).toISOString()}
</Typography >*/}
{data.reference_url !== undefined && data.reference_url !== null && data.reference_url.length > 0 ?
<Link to={data.reference_url} style={{color: "#f86a3e", textDecoration: "none",}}>
<Typography variant="h6">
{data.title}
</Typography >
</Link>
:
<Typography variant="h6">
{data.title}
</Typography >
}
{data.image !== undefined && data.image !== null && data.image.length > 0 ?
<img alt={data.title} src={data.image} style={{height: 100, width: 100, }} />
:
null
}
<Typography variant="body1">
{data.description}
</Typography >
{/*data.tags !== undefined && data.tags !== null && data.tags.length > 0 ?
data.tags.map((tag, index) => {
return (
<Chip
key={index}
style={chipStyle}
label={tag}
onClick={() => {
}}
variant="outlined"
color="primary"
/>
)
})
: null */}
{data.read === false ?
<Button color="primary" variant="contained" style={{marginTop: 15}} onClick={() => {
dismissNotification(data.id)
}}>
Dismiss
</Button>
: null}
</Paper>
)
}
const notificationMenu =
<span style={{zIndex: 10001}}>
<IconButton color="primary" style={{zIndex: 10001, marginRight: 15, }} aria-controls="simple-menu" aria-haspopup="true" onClick={(event) => {
setAnchorEl(event.currentTarget);
}}>
<Avatar style={{height: 35, width: 35,}} alt="Your username here" src="" />
<Badge badgeContent={notifications.length} color="primary">
<NotificationsIcon color="secondary" style={{height: 35, width: 35,}} alt="Your username here" src="" />
</Badge>
</IconButton>
<Menu
id="simple-menu"
anchorEl={anchorEl}
keepMounted
open={Boolean(anchorEl)}
style={{zIndex: 10002, maxHeight: "90vh", overflowX: "hidden", overflowY: "auto",}}
PaperProps={{
style: {
backgroundColor: theme.palette.surfaceColor,
}
}}
onClose={() => {
handleClose()
}}
>
<Paper style={{backgroundColor: theme.palette.surfaceColor, width: notificationWidth, padding: 25, borderBottom: "3px solid rgba(255,255,255,0.4)"}}>
<div style={{display: "flex", marginBottom: 5, }}>
<Typography variant="h6">
Your Notifications ({notifications.length})
</Typography>
{notifications.length > 1 ?
<Button color="primary" variant="contained" style={{marginLeft: 30, }} onClick={() => {
clearNotifications()
}}>
Flush
</Button>
: null}
</div>
<Typography variant="body2">
Notifications are made by Shuffle to help you discover issues or improvements.
</Typography >
</Paper>
{notifications.map((data, index) => {
return (
<NotificationItem data={data} key={index} />
)
})}
</Menu>
</span>
// Should be based on some path
const avatarMenu =
<span style={{zIndex: 10001}}>
<IconButton color="primary" style={{zIndex: 10001, marginRight: 15, }} aria-controls="simple-menu" aria-haspopup="true" onClick={(event) => {
setAnchorElAvatar(event.currentTarget);
}}>
<Avatar style={{height: 35, width: 35,}} alt="Your username here" src="" />
</IconButton>
<Menu
id="simple-menu"
anchorEl={anchorElAvatar}
keepMounted
open={Boolean(anchorElAvatar)}
style={{zIndex: 10012}}
onClose={() => {
handleClose()
}}
@@ -139,7 +342,7 @@ const Header = props => {
handleClose()
}}>
<Link to="/settings" style={hrefStyle}>
Settings
<SettingsIcon /> Settings
</Link>
</MenuItem>
<MenuItem style={{color: "white"}} onClick={(event) => {
@@ -147,7 +350,7 @@ const Header = props => {
handleClose()
handleClickLogout()
}}>
Logout
<MeetingRoomIcon /> &nbsp;Logout
</MenuItem>
</Menu>
</span>
@@ -155,8 +358,9 @@ const Header = props => {
// Handle top bar or something
const logoCheck = !homePage ? null : null
//<div style={{position: "fixed", top: 0, left: 0, display: "flex"}}>
const loginTextBrowser = !isLoggedIn ?
<div style={{display: "flex"}}>
<div style={{display: "flex"}}>
<List style={{display: "flex", flexDirect: "row"}} component="nav">
<ListItem style={{textAlign: "center", marginLeft: "0px"}}>
<Link to ="/docs/about" style={hrefStyle}>
@@ -232,6 +436,7 @@ const Header = props => {
</div>
<div style={{flex: "10", display: "flex", flexDirection: "row-reverse"}}>
{avatarMenu}
{notificationMenu}
{userdata === undefined || userdata.admin === undefined || userdata.admin === null || !userdata.admin ? null :
<Link to="/admin" style={hrefStyle}>
<Button color="primary" variant="contained" style={{marginRight: 15, marginTop: 12}}>
@@ -239,9 +444,55 @@ const Header = props => {
</Button>
</Link>
}
{userdata === undefined || userdata.orgs === undefined || userdata.orgs === null || userdata.orgs.length <= 1 ? null :
<Select
SelectDisplayProps={{
style: {
marginLeft: 10,
maxWidth: 200,
overflow: "hidden",
}
}}
value={userdata.active_org.id}
fullWidth
style={{zIndex: 10012, marginTop: 5, backgroundColor: theme.palette.surfaceColor, marginRight: 15, color: "white", height: 50, width: 200}}
MenuProps={{
style: {zIndex: 10012}
}}
onChange={(e) => {
handleClickChangeOrg(e.target.value)
}}
>
{userdata.orgs.map((data, index) => {
if (data.name === undefined || data.name === null || data.name.length === 0) {
return null
}
const imagesize = 22
const imageStyle = {width: imagesize, height: imagesize, pointerEvents: "none", marginRight: 10, marginLeft: data.creator_org !== undefined && data.creator_org.length > 0 ? 20 : 0}
const image = data.image === "" ?
<img alt={data.name} src={theme.palette.defaultImage} style={imageStyle} />
:
<img alt={data.name} src={data.image} style={imageStyle} />
return (
<MenuItem key={index} disabled={data.id === userdata.active_org.id} style={{backgroundColor: theme.palette.inputColor, color: "white", zIndex: 10013,}} value={data.id}>
<Tooltip color="primary" title={`Suborg of ${data.creator_org}`} placement="left">
<div style={{display: "flex"}}>
{image} {data.name}
</div>
</Tooltip>
</MenuItem>
)
})}
</Select>
}
</div>
</div>
//console.log("USR: ", userdata.orgs)
const loginTextMobile = !isLoggedIn ?
<div style={{display: "flex"}}>
<List style={{display: "flex", flexDirection: "row"}} component="nav">
@@ -300,7 +551,7 @@ const Header = props => {
// <Divider style={{height: "1px", width: "100%", backgroundColor: "rgb(91, 96, 100)"}}/>
const loadedCheck =
<div style={{minHeight: 60}}>
<div style={{minHeight: 60, }}>
<BrowserView>
{loginTextBrowser}
</BrowserView>
@@ -310,10 +561,10 @@ const Header = props => {
</div>
// <div style={{backgroundImage: "linear-gradient(-90deg,#342f78 0,#29255e 50%,#1b1947 100%"}}>
return (
<div>
{loadedCheck}
<div style={{width: "100%", position: "fixed", minHeight: 60, top: 0, zIndex: 10000, backgroundColor: "inherit",}}>
{loadedCheck}
</div>
)
)
}
export default Header;
frontend/src/components/Oauth2Auth.jsx
+417
View File
@@ -0,0 +1,417 @@
import React, {useRef, useState, useEffect, useLayoutEffect} from 'react';
import { useTheme } from '@material-ui/core/styles';
import { v4 as uuidv4 } from 'uuid';
import { ListItemText, TextField, Drawer, Button, Paper, Grid, Tabs, InputAdornment, Tab, ButtonBase, Tooltip, Select, MenuItem, Divider, Dialog, Modal, DialogActions, DialogTitle, InputLabel, DialogContent, FormControl, IconButton, Menu, Input, FormGroup, FormControlLabel, Typography, Checkbox, Breadcrumbs, CircularProgress, Switch, Fade } from '@material-ui/core';
import { LockOpen as LockOpenIcon } from '@material-ui/icons';
const ITEM_HEIGHT = 55
const ITEM_PADDING_TOP = 8
const MenuProps = {
PaperProps: {
style: {
maxHeight: ITEM_HEIGHT * 4.5 + ITEM_PADDING_TOP,
minWidth: 500,
maxWidth: 500,
scrollX: "auto",
},
},
}
const AuthenticationOauth2 = (props) => {
const { saveWorkflow, selectedApp, workflow, selectedAction, authenticationType, getAppAuthentication, appAuthentication, setSelectedAction, setNewAppAuth, setAuthenticationModalOpen} = props;
const theme = useTheme();
//const [update, setUpdate] = React.useState("|")
const [defaultConfigSet, setDefaultConfigSet] = React.useState(authenticationType.client_id !== undefined && authenticationType.client_id !== null && authenticationType.client_id.length > 0 && authenticationType.client_secret !== undefined && authenticationType.client_secret !== null && authenticationType.client_secret.length > 0)
const [clientId, setClientId] = React.useState(defaultConfigSet ? authenticationType.client_id : "")
const [clientSecret, setClientSecret] = React.useState(defaultConfigSet ? authenticationType.client_secret : "")
const [oauthUrl, setOauthUrl] = React.useState("")
const [buttonClicked, setButtonClicked] = React.useState(false)
const [selectedScopes, setSelectedScopes] = React.useState([])
const allscopes = authenticationType.scope !== undefined ? authenticationType.scope: []
const [manuallyConfigure, setManuallyConfigure] = React.useState(defaultConfigSet ? false : true)
const [authenticationOption, setAuthenticationOptions] = React.useState({
app: JSON.parse(JSON.stringify(selectedApp)),
fields: {},
label: "",
usage: [{
workflow_id: workflow.id,
}],
id: uuidv4(),
active: true,
})
if (selectedApp.authentication === undefined) {
return null
}
const handleOauth2Request = (client_id, client_secret, oauth_url, scopes) => {
setButtonClicked(true)
console.log("SCOPES: ", scopes)
var resources = ""
if (scopes !== undefined && scopes !== null & scopes.length > 0) {
resources = scopes.join(",")
}
const authentication_url = authenticationType.token_uri
console.log("SCOPES2: ", resources)
const redirectUri = `${window.location.protocol}//${window.location.host}/set_authentication`
var state = `workflow_id%3D${workflow.id}%26reference_action_id%3d${selectedAction.app_id}%26app_name%3d${selectedAction.app_name}%26app_id%3d${selectedAction.app_id}%26app_version%3d${selectedAction.app_version}%26authentication_url%3d${authentication_url}%26scope%3d${resources}%26client_id%3d${client_id}%26client_secret%3d${client_secret}`
if (oauth_url !== undefined && oauth_url !== null && oauth_url.length > 0) {
state += `%26oauth_url%3d${oauth_url}`
console.log("ADDING OAUTH2 URL: ", state)
}
const url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&scope=${resources}&prompt=consent&state=${state}`
//const url = `https://accounts.zoho.com/oauth/v2/auth?response_type=code&client_id=${client_id}&scope=AaaServer.profile.Read&redirect_uri=${redirectUri}&prompt=consent`
console.log("Full URI: ", url)
console.log("Redirect Uri: ", redirectUri)
// &resource=https%3A%2F%2Fgraph.microsoft.com&
// FIXME: Awful, but works for prototyping
// How can we get a callback properly realtime?
// How can we properly try-catch without breaks on error?
try {
var newwin = window.open(url, "", "width=800,height=600")
//console.log(newwin)
var open = true
const timer = setInterval(() => {
if (newwin.closed) {
setButtonClicked(false)
clearInterval(timer);
//alert('"Secure Payment" window closed!');
getAppAuthentication(true, true)
}
}, 1000);
//do {
// setTimeout(() => {
// console.log(newwin)
// console.log("CLOSED", newwin.closed)
// if (newwin.closed) {
// open = false
// }
// }, 1000)
//}
//while(open === true)
} catch (e) {
alert.error("Failed authentication - probably bad credentials. Try again")
setButtonClicked(false)
}
return
//do {
//} while (
}
authenticationOption.app.actions = []
for (var key in selectedApp.authentication.parameters) {
if (authenticationOption.fields[selectedApp.authentication.parameters[key].name] === undefined) {
authenticationOption.fields[selectedApp.authentication.parameters[key].name] = ""
}
}
const handleSubmitCheck = () => {
console.log("NEW AUTH: ", authenticationOption)
if (authenticationOption.label.length === 0) {
authenticationOption.label = `Auth for ${selectedApp.name}`
//alert.info("Label can't be empty")
//return
}
// Automatically mapping fields that already exist (predefined).
// Warning if fields are NOT filled
for (var key in selectedApp.authentication.parameters) {
if (authenticationOption.fields[selectedApp.authentication.parameters[key].name].length === 0) {
if (selectedApp.authentication.parameters[key].value !== undefined && selectedApp.authentication.parameters[key].value !== null && selectedApp.authentication.parameters[key].value.length > 0) {
authenticationOption.fields[selectedApp.authentication.parameters[key].name] = selectedApp.authentication.parameters[key].value
} else {
if (selectedApp.authentication.parameters[key].schema.type === "bool") {
authenticationOption.fields[selectedApp.authentication.parameters[key].name] = "false"
} else {
alert.info("Field "+selectedApp.authentication.parameters[key].name+" can't be empty")
return
}
}
}
}
console.log("Action: ", selectedAction)
selectedAction.authentication_id = authenticationOption.id
selectedAction.selectedAuthentication = authenticationOption
if (selectedAction.authentication === undefined || selectedAction.authentication === null) {
selectedAction.authentication = [authenticationOption]
} else {
selectedAction.authentication.push(authenticationOption)
}
setSelectedAction(selectedAction)
var newAuthOption = JSON.parse(JSON.stringify(authenticationOption))
var newFields = []
for (const key in newAuthOption.fields) {
const value = newAuthOption.fields[key]
newFields.push({
key: key,
value: value,
})
}
console.log("FIELDS: ", newFields)
newAuthOption.fields = newFields
setNewAppAuth(newAuthOption)
//appAuthentication.push(newAuthOption)
//setAppAuthentication(appAuthentication)
//
//if (configureWorkflowModalOpen) {
// setSelectedAction({})
//}
//setUpdate(authenticationOption.id)
/*
{selectedAction.authentication.map(data => (
<MenuItem key={data.id} style={{backgroundColor: inputColor, color: "white"}} value={data}>
*/
}
const handleScopeChange = (event) => {
const {
target: { value },
} = event;
console.log("VALUE: ", value)
// On autofill we get a the stringified value.
setSelectedScopes(typeof value === 'string' ? value.split(',') : value)
}
if (authenticationOption.label === null || authenticationOption.label === undefined) {
authenticationOption.label = selectedApp.name+" authentication"
}
//console.log(
return (
<div>
<DialogTitle><div style={{color: "white"}}>Authentication for {selectedApp.name}</div></DialogTitle>
<DialogContent>
<span style={{}}>
<b>Oauth2 requires a client ID and secret to authenticate. This is usually made in the remote system.</b>
<a target="_blank" rel="norefferer" href="https://shuffler.io/docs/apps#authentication" style={{textDecoration: "none", color: "#f85a3e"}}> Learn more about Oauth2 with Shuffle</a><div/>
</span>
{/*<TextField
style={{backgroundColor: theme.palette.inputColor, borderRadius: theme.palette.borderRadius,}}
InputProps={{
style:{
color: "white",
marginLeft: "5px",
maxWidth: "95%",
height: 50,
fontSize: "1em",
},
}}
fullWidth
color="primary"
placeholder={"Auth july 2020"}
defaultValue={`Auth for ${selectedApp.name}`}
onChange={(event) => {
authenticationOption.label = event.target.value
}}
/>
<Divider style={{marginTop: 15, marginBottom: 15, backgroundColor: "rgb(91, 96, 100)"}}/>
*/}
{!manuallyConfigure ? null :
<span>
{selectedApp.authentication.parameters.map((data, index) => {
//console.log(data, index)
if (data.name === "client_id" || data.name === "client_secret") {
return null
}
if (data.name !== "url") {
return null
}
if (oauthUrl.length === 0) {
setOauthUrl(data.value)
}
return (
<div key={index} style={{marginTop: 10}}>
<LockOpenIcon style={{marginRight: 10}}/>
<b>{data.name}</b>
{data.schema !== undefined && data.schema !== null && data.schema.type === "bool" ?
<Select
SelectDisplayProps={{
style: {
marginLeft: 10,
}
}}
defaultValue={"false"}
fullWidth
onChange={(e) => {
console.log("Value: ", e.target.value)
authenticationOption.fields[data.name] = e.target.value
}}
style={{backgroundColor: theme.palette.surfaceColor, color: "white", height: 50}}
>
<MenuItem key={"false"} style={{backgroundColor: theme.palette.inputColor, color: "white"}} value={"false"}>
false
</MenuItem>
<MenuItem key={"true"} style={{backgroundColor: theme.palette.inputColor, color: "white"}} value={"true"}>
true
</MenuItem>
</Select>
:
<TextField
style={{backgroundColor: theme.palette.inputColor, borderRadius: theme.palette.borderRadius,}}
InputProps={{
style:{
color: "white",
marginLeft: "5px",
maxWidth: "95%",
height: 50,
fontSize: "1em",
},
}}
fullWidth
type={data.example !== undefined && data.example.includes("***") ? "password" : "text"}
color="primary"
defaultValue={data.value !== undefined && data.value !== null ? data.value : ""}
placeholder={data.example}
onChange={(event) => {
authenticationOption.fields[data.name] = event.target.value
console.log("Setting oauth url")
setOauthUrl(event.target.value)
//const [oauthUrl, setOauthUrl] = React.useState("")
}}
/>
}
</div>
)
})}
{allscopes.length === 0 ? null :
<Select
multiple
value={selectedScopes}
style={{backgroundColor: theme.palette.inputColor, color: "white", }}
onChange={(e) => {
handleScopeChange(e)
}}
fullWidth
input={<Input id="select-multiple-native" />}
renderValue={(selected) => selected.join(', ')}
MenuProps={MenuProps}
>
{allscopes.map((data, index) => {
return (
<MenuItem key={index} value={data}>
<Checkbox checked={selectedScopes.indexOf(data) > -1} />
<ListItemText primary={data} />
</MenuItem>
)
})}
</Select>
}
<TextField
style={{marginTop: 20, backgroundColor: theme.palette.inputColor, borderRadius: theme.palette.borderRadius,}}
InputProps={{
style:{
color: "white",
marginLeft: "5px",
maxWidth: "95%",
height: 50,
fontSize: "1em",
},
}}
fullWidth
color="primary"
placeholder={"Client ID"}
onChange={(event) => {
setClientId(event.target.value)
//authenticationOption.label = event.target.value
}}
/>
<TextField
style={{backgroundColor: theme.palette.inputColor, borderRadius: theme.palette.borderRadius,}}
InputProps={{
style:{
color: "white",
marginLeft: "5px",
maxWidth: "95%",
height: 50,
fontSize: "1em",
},
}}
fullWidth
color="primary"
placeholder={"Client Secret"}
onChange={(event) => {
setClientSecret(event.target.value)
//authenticationOption.label = event.target.value
}}
/>
</span>
}
<Button
style={{marginBottom: 40, marginTop: 20, borderRadius: theme.palette.borderRadius}}
disabled={clientSecret.length === 0 || clientId.length === 0 || buttonClicked}
variant="contained"
fullWidth
onClick={() => {
handleOauth2Request(clientId, clientSecret, oauthUrl, selectedScopes)
}}
color="primary"
>
{buttonClicked ?
<CircularProgress style={{color: "white", }} />
:
"Oauth2 request"
}
</Button>
{defaultConfigSet ?
<span style={{}}>
... or
<Button
style={{marginLeft: 10, borderRadius: theme.palette.borderRadius}}
disabled={clientSecret.length === 0 || clientId.length === 0}
variant="text"
onClick={() => {
setManuallyConfigure(!manuallyConfigure)
if (manuallyConfigure) {
setClientId(authenticationType.client_id)
setClientSecret(authenticationType.client_secret)
} else {
setClientId("")
setClientSecret("")
}
}}
color="primary"
>
{manuallyConfigure ? "Use auto-config" : "Manually configure Oauth2"}
</Button>
</span>
:
null
}
</DialogContent>
</div>
)
}
export default AuthenticationOauth2
frontend/src/components/OrgHeader.js
+89 -15
View File
File diff suppressed because one or more lines are too long
frontend/src/components/ParsedAction.jsx
+278 -47
View File
@@ -8,10 +8,14 @@ import { useTheme } from '@material-ui/core/styles';
import NestedMenuItem from "material-ui-nested-menu-item";
//import NestedMenuItem from "./NestedMenu.jsx";
import {Popper, TextField, Drawer, Button, Paper, Grid, Tabs, InputAdornment, Tab, ButtonBase, Tooltip, Select, MenuItem, Divider, Dialog, Modal, DialogActions, DialogTitle, InputLabel, DialogContent, FormControl, IconButton, Menu, Input, FormGroup, FormControlLabel, Typography, Checkbox, Breadcrumbs, CircularProgress, Switch, Fade} from '@material-ui/core';
import {GetApp as GetAppIcon, Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons';
import {Popper, TextField, TextareaAutosize, Drawer, Button, Paper, Grid, Tabs, InputAdornment, Tab, ButtonBase, Tooltip, Select, MenuItem, Divider, Dialog, Modal, DialogActions, DialogTitle, InputLabel, DialogContent, FormControl, IconButton, Menu, Input, FormGroup, FormControlLabel, Typography, Checkbox, Breadcrumbs, CircularProgress, Switch, Fade} from '@material-ui/core';
import {HelpOutline as HelpOutlineIcon, Description as DescriptionIcon, GetApp as GetAppIcon, Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons';
import Autocomplete from '@material-ui/lab/Autocomplete';
import CodeMirror from '@uiw/react-codemirror';
import 'codemirror/keymap/sublime';
import 'codemirror/theme/gruvbox-dark.css';
const useStyles = makeStyles({
notchedOutline: {
@@ -46,10 +50,13 @@ const useStyles = makeStyles({
//)
const ParsedAction = (props) => {
const {workflow, setWorkflow, setAction, setSelectedAction, setUpdate, appActionArguments, selectedApp, workflowExecutions, setSelectedResult, selectedAction, setSelectedApp, setSelectedTrigger, setSelectedEdge, setCurrentView, cy, setAuthenticationModalOpen,setVariablesModalOpen, setCodeModalOpen, selectedNameChange, rightsidebarStyle, showEnvironment, selectedActionEnvironment, environments, setNewSelectedAction, appApiViewStyle, globalUrl, setSelectedActionEnvironment, requiresAuthentication, hideExtraTypes, scrollConfig, setScrollConfig } = props
const {workflow, setWorkflow, setAction, setSelectedAction, setUpdate, appActionArguments, selectedApp, workflowExecutions, setSelectedResult, selectedAction, setSelectedApp, setSelectedTrigger, setSelectedEdge, setCurrentView, cy, setAuthenticationModalOpen,setVariablesModalOpen, setCodeModalOpen, selectedNameChange, rightsidebarStyle, showEnvironment, selectedActionEnvironment, environments, setNewSelectedAction, appApiViewStyle, globalUrl, setSelectedActionEnvironment, requiresAuthentication, hideExtraTypes, scrollConfig, setScrollConfig, authenticationType, appAuthentication, getAppAuthentication } = props
const theme = useTheme();
const classes = useStyles()
const [expansionModalOpen, setExpansionModalOpen] = React.useState(false);
const keywords = ["len(", "lower(", "upper(", "trim(", "split(", "length(", "number(", "parse(", "join("]
const getParents = (action) => {
if (cy === undefined) {
@@ -327,7 +334,8 @@ const ParsedAction = (props) => {
}
// 1. Take
const actionvalue = {"type": "action", "id": item.id, "name": item.label, "autocomplete": `${item.label.split(" ").join("_")}`, "example": exampledata}
const itemlabelComplete = item.label === null || item.label === undefined ? "" : item.label.split(" ").join("_")
const actionvalue = {"type": "action", "id": item.id, "name": item.label, "autocomplete": itemlabelComplete, "example": exampledata}
actionlist.push(actionvalue)
}
}
@@ -337,12 +345,13 @@ const ParsedAction = (props) => {
})
const changeActionParameter = (event, count, data) => {
//console.log(event)
if (data.name.startsWith("${") && data.name.endsWith("}")) {
// PARAM FIX - Gonna use the ID field, even though it's a hack
const paramcheck = selectedAction.parameters.find(param => param.name === "body")
if (paramcheck !== undefined) {
// Escapes all double quotes
const toReplace = event.target.value.trim().replaceAll("\\\"", "\"").replaceAll("\"", "\\\"")
const toReplace = event.target.value.trim().replaceAll("\\\"", "\"").replaceAll("\"", "\\\"");
console.log("REPLACE WITH: ", toReplace)
if (paramcheck["value_replace"] === undefined || paramcheck["value_replace"] === null) {
paramcheck["value_replace"] = [{
@@ -472,6 +481,7 @@ const ParsedAction = (props) => {
}
}
//console.log("CHANGING ACTION COUNT !")
selectedActionParameters[count].value = event.target.value
selectedAction.parameters[count].value = event.target.value
setSelectedAction(selectedAction)
@@ -479,6 +489,150 @@ const ParsedAction = (props) => {
//setUpdate(event.target.value)
}
const changeActionParameterCodemirror = (event, count, data) => {
console.log(event)
if (data.name.startsWith("${") && data.name.endsWith("}")) {
// PARAM FIX - Gonna use the ID field, even though it's a hack
const paramcheck = selectedAction.parameters.find(param => param.name === "body")
if (paramcheck !== undefined) {
// Escapes all double quotes
const toReplace = event.target.value.trim().replaceAll("\\\"", "\"").replaceAll("\"", "\\\"");
console.log("REPLACE WITH: ", toReplace)
if (paramcheck["value_replace"] === undefined || paramcheck["value_replace"] === null) {
paramcheck["value_replace"] = [{
"key": data.name,
"value": toReplace,
}]
console.log("IN IF: ", paramcheck)
} else {
const subparamindex = paramcheck["value_replace"].findIndex(param => param.key === data.name)
if (subparamindex === -1) {
paramcheck["value_replace"].push({
"key": data.name,
"value": toReplace,
})
} else {
paramcheck["value_replace"][subparamindex]["value"] = toReplace
}
console.log("IN ELSE: ", paramcheck)
}
//console.log("PARAM: ", paramcheck)
//if (paramcheck.id === undefined) {
// console.log("Normal paramcheck")
//} else {
// selectedActionParameters[count]["value_replace"] = paramcheck
// selectedAction.parameters[count]["value_replace"] = paramcheck
//}
if (paramcheck["value_replace"] === undefined) {
selectedActionParameters[count]["value_replace"] = paramcheck
selectedAction.parameters[count]["value_replace"] = paramcheck
} else {
selectedActionParameters[count]["value_replace"] = paramcheck["value_replace"]
selectedAction.parameters[count]["value_replace"] = paramcheck["value_replace"]
}
console.log("RESULT: ", selectedAction)
setSelectedAction(selectedAction)
//setUpdate(Math.random())
return
}
}
if (event.display.maxLine.text[event.display.maxLine.text.length-1] === "$") {
if (!showDropdown) {
setShowAutocomplete(false)
setShowDropdown(true)
setShowDropdownNumber(count)
}
} else {
if (showDropdown) {
setShowDropdown(false)
}
}
// bad detection mechanism probably
if (event.display.maxLine.text[event.display.maxLine.text.length-1] === "." && actionlist.length > 0) {
console.log("GET THE LAST ARGUMENT FOR NODE!")
// THIS IS AN EXAMPLE OF SHOWING IT
/*
const inputdata = {"data": "1.2.3.4", "dataType": "4.5.6.6"}
setJsonList(GetParsedPaths(inputdata, ""))
if (!showDropdown) {
setShowAutocomplete(false)
setShowDropdown(true)
setShowDropdownNumber(count)
}
console.log(jsonList)
*/
// Search for the item backwards
// 1. Reverse search backwards from . -> $
// 2. Search the actionlist for the item
// 3. Find the data for the specific item
var curstring = ""
var record = false
for (var key in selectedActionParameters[count].value) {
const item = selectedActionParameters[count].value[key]
if (record) {
curstring += item
}
if (item === "$") {
record = true
curstring = ""
}
}
//console.log("CURSTRING: ", curstring)
if (curstring.length > 0 && actionlist !== null) {
// Search back in the action list
curstring = curstring.split(" ").join("_").toLowerCase()
var actionItem = actionlist.find(data => data.autocomplete.split(" ").join("_").toLowerCase() === curstring)
if (actionItem !== undefined) {
console.log("Found item: ", actionItem)
//actionItem.example = actionItem.example.trim()
//actionItem.example = actionItem.example.split(" None").join(" \"None\"")
//actionItem.example = actionItem.example.split("\'").join("\"")
var jsonvalid = true
try {
const tmp = String(JSON.parse(actionItem.example))
if (!actionItem.example.includes("{") && !actionItem.example.includes("[")) {
jsonvalid = false
}
} catch (e) {
jsonvalid = false
}
if (jsonvalid) {
setJsonList(GetParsedPaths(JSON.parse(actionItem.example), ""))
if (!showDropdown) {
setShowAutocomplete(false)
setShowDropdown(true)
setShowDropdownNumber(count)
}
}
}
}
} else {
if (jsonList.length > 0) {
setJsonList([])
}
}
selectedActionParameters[count].value = event.display.maxLine.text
selectedAction.parameters[count].value = event.display.maxLine.text
setSelectedAction(selectedAction)
//setUpdate(Math.random())
//setUpdate(event.target.value)
}
const changeActionParameterVariable = (fieldvalue, count) => {
//console.log("CALLED THIS ONE WITH VALUE!", fieldvalue)
//if (selectedVariableParameter === fieldvalue) {
@@ -599,6 +753,10 @@ const ParsedAction = (props) => {
var placeholder = "Static value"
if (data.example !== undefined && data.example !== null && data.example.length > 0) {
placeholder = data.example
if (data.name === "url" && data.value.length === 0) {
data.value = data.example
}
}
if (data.name.startsWith("${") && data.name.endsWith("}")) {
@@ -684,10 +842,12 @@ const ParsedAction = (props) => {
}
const clickedFieldId = "rightside_field_"+count
//<TextareaAutosize
// <CodeMirror
var datafield =
<TextField
disabled={disabled}
style={{backgroundColor: theme.palette.inputColor, borderRadius: theme.palette.borderRadius, border: selectedActionParameters[count].required || selectedActionParameters[count].configuration ? "2px solid #f85a3e" : "",}}
style={{backgroundColor: theme.palette.inputColor, borderRadius: theme.palette.borderRadius, border: selectedActionParameters[count].required || selectedActionParameters[count].configuration ? "2px solid #f85a3e" : "", color: "white", width: "100%", fontSize: "1em", }}
InputProps={{
style:{
color: "white",
@@ -696,28 +856,29 @@ const ParsedAction = (props) => {
maxWidth: "95%",
fontSize: "1em",
},
endAdornment: (
hideExtraTypes ? null :
<InputAdornment position="end">
<Tooltip title="Autocomplete text" placement="top">
<AddCircleOutlineIcon style={{cursor: "pointer"}} onClick={(event) => {
setMenuPosition({
top: event.pageY+10,
left: event.pageX+10,
})
setShowDropdownNumber(count)
setShowDropdown(true)
setShowAutocomplete(true)
}}/>
</Tooltip>
</InputAdornment>
)
endAdornment: (
hideExtraTypes ? null :
<InputAdornment position="end">
<Tooltip title="Autocomplete the text" placement="top">
<AddCircleOutlineIcon style={{cursor: "pointer"}} onClick={(event) => {
setMenuPosition({
top: event.pageY+10,
left: event.pageX+10,
})
setShowDropdownNumber(count)
setShowDropdown(true)
setShowAutocomplete(true)
}}/>
</Tooltip>
</InputAdornment>
)
}}
fullWidth
multiline={multiline}
onClick={() => {
//console.log("Clicked field: ", clickedFieldId)
console.log("Clicked field: ", clickedFieldId)
setExpansionModalOpen(false)
if (setScrollConfig !== undefined && scrollConfig !== null && scrollConfig !== undefined && scrollConfig.selected !== clickedFieldId) {
scrollConfig.selected = clickedFieldId
setScrollConfig(scrollConfig)
@@ -728,11 +889,22 @@ const ParsedAction = (props) => {
rows={rows}
color="primary"
defaultValue={data.value}
//value={data.value}
//options={{
// theme: 'gruvbox-dark',
// keyMap: 'sublime',
// mode: 'python',
//}}
//height={multiline ? 50 : 150}
type={placeholder.includes("***") || (data.configuration && (data.name.toLowerCase().includes("api") || data.name.toLowerCase().includes("key") || data.name.toLowerCase().includes("pass"))) ? "password" : "text"}
placeholder={placeholder}
onChange={(event) => {
changeActionParameter(event, count, data)
//changeActionParameterCodemirror(event, count, data)
changeActionParameter(event, count, data)
}}
helperText={selectedApp.generated && selectedApp.activated && data.name === "body" ?
<span style={{color:"white", marginBottom: 5, marginleft: 5,}}>
{openApiHelperText}
@@ -790,7 +962,7 @@ const ParsedAction = (props) => {
endAdornment: (
hideExtraTypes ? null :
<InputAdornment position="end">
<Tooltip title="Autocomplete text" placement="top">
<Tooltip title="Autocomplete the text" placement="top">
<AddCircleOutlineIcon style={{cursor: "pointer"}} onClick={(event) => {
setMenuPosition({
top: event.pageY+10,
@@ -1102,6 +1274,13 @@ const ParsedAction = (props) => {
}
tmpitem = (tmpitem.charAt(0).toUpperCase()+tmpitem.substring(1)).replaceAll("_", " ")
if (tmpitem === "Username basic") {
tmpitem = "Username"
} else if (tmpitem === "Password basic") {
tmpitem = "Password"
}
const description = data.description === undefined ? "" : data.description
const tooltipDescription =
<span>
@@ -1163,11 +1342,12 @@ const ParsedAction = (props) => {
</Tooltip>
</div>
*/}
{(selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 && selectedActionParameters[count].required === true && selectedActionParameters[count].unique_toggled !== undefined) || hideExtraTypes ? null :
{/*(selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 && selectedActionParameters[count].required === true && selectedActionParameters[count].unique_toggled !== undefined) || hideExtraTypes ? null :
<div style={{display: "flex"}}>
<Tooltip color="secondary" title="Value must be unique" placement="top">
<div style={{cursor: "pointer", color: staticcolor}} onClick={(e) => {}}>
<Checkbox
tabIndex="-1"
checked={selectedActionParameters[count].unique_toggled}
style={{
color: theme.palette.primary.secondary,
@@ -1185,7 +1365,7 @@ const ParsedAction = (props) => {
</div>
</Tooltip>
</div>
}
*/}
</div>
{datafield}
{showDropdown && showDropdownNumber === count && data.variant === "STATIC_VALUE" && jsonList.length > 0 ?
@@ -1256,6 +1436,27 @@ const ParsedAction = (props) => {
return null
}
const expansionModal =
<Dialog modal
open={expansionModalOpen}
onClose={() => {
setExpansionModalOpen(false)
}}
PaperProps={{
style: {
backgroundColor: theme.palette.surfaceColor,
color: "white",
minWidth: 600,
padding: 50,
},
}}
>
<DialogTitle><span style={{color: "white"}}>Workflow Variable</span></DialogTitle>
<DialogContent>
Hello
</DialogContent>
</Dialog>
//const CustomPopper = function (props) {
// const classes = useStyles()
// return <Popper {...props} className={classes.root} placement="bottom" />
@@ -1264,12 +1465,13 @@ const ParsedAction = (props) => {
const baselabel = selectedAction.label
return (
<div style={appApiViewStyle} id="parsed_action_view">
{expansionModal}
{hideExtraTypes === true ? null :
<span>
<div style={{display: "flex", minHeight: 40, marginBottom: 30}}>
<div style={{flex: 1}}>
<h3 style={{marginBottom: 5}}>{(selectedAction.app_name.charAt(0).toUpperCase()+selectedAction.app_name.substring(1)).replaceAll("_", " ")}</h3>
<div style={{display: "flex",}}>
<div style={{display: "flex", marginTop: 10, }}>
<IconButton style={{marginTop: "auto", marginBottom: "auto", height: 30, paddingLeft: 0, paddingRight: 0}} onClick={() => {
console.log("FIND EXAMPLE RESULTS FOR ", selectedAction)
if (workflowExecutions.length > 0) {
@@ -1298,19 +1500,24 @@ const ParsedAction = (props) => {
<ArrowLeftIcon style={{color: "white"}}/>
</Tooltip>
</IconButton>
<span style={{}}>
<Typography style={{marginTop: 5,}}><a rel="norefferer" href="https://shuffler.io/docs/workflows#nodes" target="_blank" style={{textDecoration: "none", color: "#f85a3e"}}>What are actions?</a></Typography>
{selectedAction.errors !== undefined && selectedAction.errors !== null && selectedAction.errors.length > 0 ?
<div>
Errors: {selectedAction.errors.join("\n")}
</div>
: null
}
</span>
<IconButton style={{marginTop: "auto", marginBottom: "auto", height: 30, paddingLeft: 25, paddingRight: 0}} onClick={() => {
setAuthenticationModalOpen(true)
}}>
<Tooltip color="primary" title="Read app docs" placement="top">
<DescriptionIcon style={{color: "white"}} />
</Tooltip>
</IconButton>
<IconButton style={{marginTop: "auto", marginBottom: "auto", height: 30, paddingLeft: 25, paddingRight: 0}} onClick={() => {}}>
<a rel="norefferer" href="https://shuffler.io/docs/workflows#nodes" target="_blank" style={{textDecoration: "none", color: "#f85a3e"}}>
<Tooltip color="primary" title="What are actions?" placement="top">
<HelpOutlineIcon style={{color: "white"}}/>
</Tooltip>
</a>
</IconButton>
</div>
</div>
<div style={{display: "flex", flexDirection: "column",}}>
{selectedAction.id === workflow.start ? null :
{/*selectedAction.id === workflow.start ? null :
<Tooltip color="primary" title={"Make this node the start action"} placement="top">
<Button style={{zIndex: 5000, marginTop: 10,}} color="primary" variant="outlined" onClick={(e) => {
defineStartnode(e)
@@ -1318,7 +1525,7 @@ const ParsedAction = (props) => {
<KeyboardArrowRightIcon />
</Button>
</Tooltip>
}
*/}
{selectedApp.versions !== null && selectedApp.versions !== undefined && selectedApp.versions.length > 1 ?
<Select
defaultValue={selectedAction.app_version}
@@ -1360,6 +1567,7 @@ const ParsedAction = (props) => {
fullWidth
color="primary"
placeholder={selectedAction.label}
defaultValue={selectedAction.label}
onChange={selectedNameChange}
onBlur={(e) => {
const name = e.target.value
@@ -1387,6 +1595,11 @@ const ParsedAction = (props) => {
<Tooltip color="primary" title={"Add authentication option"} placement="top">
<span>
<Button color="primary" style={{}} fullWidth variant="contained" onClick={() => {
console.log(authenticationType)
//if (authenticationType.type === "oauth2" && authenticationType.redirect_uri !== undefined && authenticationType.redirect_uri !== null) {
// return null
//}
setAuthenticationModalOpen(true)
}}>
<AddIcon style={{marginRight: 10, }}/> Authenticate {selectedApp.name}
@@ -1401,7 +1614,7 @@ const ParsedAction = (props) => {
<div style={{display: "flex"}}>
<Select
labelId="select-app-auth"
value={selectedAction.selectedAuthentication}
value={Object.getOwnPropertyNames(selectedAction.selectedAuthentication).length === 0 ? "No selection" : selectedAction.selectedAuthentication}
SelectDisplayProps={{
style: {
marginLeft: 10,
@@ -1409,14 +1622,32 @@ const ParsedAction = (props) => {
}}
fullWidth
onChange={(e) => {
//console.log("CHOSE AN AUTHENTICATION OPTION: ", e.target.value)
selectedAction.selectedAuthentication = e.target.value
selectedAction.authentication_id = e.target.value.id
setSelectedAction(selectedAction)
setUpdate(Math.random())
if (e.target.value === "No selection") {
selectedAction.selectedAuthentication = {}
selectedAction.authentication_id = ""
for (var key in selectedAction.parameters) {
//console.log(selectedAction.parameters[key])
if (selectedAction.parameters[key].configuration) {
selectedAction.parameters[key].value = ""
}
}
setSelectedAction(selectedAction)
setUpdate(Math.random())
} else {
//console.log("CHOSE AN AUTHENTICATION OPTION: ", e.target.value)
selectedAction.selectedAuthentication = e.target.value
selectedAction.authentication_id = e.target.value.id
setSelectedAction(selectedAction)
setUpdate(Math.random())
}
}}
style={{backgroundColor: theme.palette.inputColor, color: "white", height: 50, maxWidth: rightsidebarStyle.maxWidth-80, borderRadius: theme.palette.borderRadius,}}
>
<MenuItem style={{backgroundColor: theme.palette.inputColor, color: "white"}} value="No selection">
<em>No selection</em>
</MenuItem>
{selectedAction.authentication.map(data => {
//console.log("AUTH DATA: ", data)
return(
@@ -1511,7 +1742,7 @@ const ParsedAction = (props) => {
<MenuItem style={{backgroundColor: theme.palette.inputColor, color: "white"}} value="No selection">
<em>No selection</em>
</MenuItem>
<Divider />
<Divider style={{backgroundColor: theme.palette.inputColor }} />
{workflow.execution_variables.map(data => (
<MenuItem style={{backgroundColor: theme.palette.inputColor, color: "white"}} value={data.name}>
{data.name}
frontend/src/components/ScrollToTop.jsx
+2 -9
View File
@@ -1,8 +1,7 @@
import { useEffect } from 'react';
import { withRouter } from 'react-router-dom';
import ReactGA from 'react-ga';
function ScrollToTop({setCurpath, history }) {
function ScrollToTop({getUserNotifications, setCurpath, history }) {
useEffect(() => {
const unlisten = history.listen(() => {
window.scroll({
@@ -11,14 +10,8 @@ function ScrollToTop({setCurpath, history }) {
behavior: "smooth",
});
//ReactGA.event({
// category: "referral",
// action: "new_user_referral",
// label: "",
//})
ReactGA.pageview(window.location.pathname)
setCurpath(window.location.pathname)
getUserNotifications()
});
return () => {
unlisten();
frontend/src/css/font1.woff2
BIN
View File
Binary file not shown.
frontend/src/css/font2.woff2
BIN
View File
Binary file not shown.
frontend/src/css/font3.woff2
BIN
View File
Binary file not shown.
frontend/src/css/font4.woff2
BIN
View File
Binary file not shown.
frontend/src/css/font5.woff2
BIN
View File
Binary file not shown.
frontend/src/css/nunito.css
+40
View File
@@ -0,0 +1,40 @@
/* cyrillic-ext */
@font-face {
font-family: 'Nunito Sans';
font-style: normal;
font-weight: 400;
src: url('./font1.woff2') format('woff2');
unicode-range: U+0460-052F, U+1C80-1C88, U+20B4, U+2DE0-2DFF, U+A640-A69F, U+FE2E-FE2F;
}
/* cyrillic */
@font-face {
font-family: 'Nunito Sans';
font-style: normal;
font-weight: 400;
src: url('./font2.woff2') format('woff2');
unicode-range: U+0400-045F, U+0490-0491, U+04B0-04B1, U+2116;
}
/* vietnamese */
@font-face {
font-family: 'Nunito Sans';
font-style: normal;
font-weight: 400;
src: url('./font3.woff2') format('woff2');
unicode-range: U+0102-0103, U+0110-0111, U+0128-0129, U+0168-0169, U+01A0-01A1, U+01AF-01B0, U+1EA0-1EF9, U+20AB;
}
/* latin-ext */
@font-face {
font-family: 'Nunito Sans';
font-style: normal;
font-weight: 400;
src: url('./font4.woff2') format('woff2');
unicode-range: U+0100-024F, U+0259, U+1E00-1EFF, U+2020, U+20A0-20AB, U+20AD-20CF, U+2113, U+2C60-2C7F, U+A720-A7FF;
}
/* latin */
@font-face {
font-family: 'Nunito Sans';
font-style: normal;
font-weight: 400;
src: url('./font5.woff2') format('woff2');
unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+2000-206F, U+2074, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD;
}
frontend/src/index.css
+1 -1
View File
@@ -1,4 +1,4 @@
@import url('https://fonts.googleapis.com/css?family=Nunito+Sans');
@import url('./css/nunito.css');
body {
margin: 0;
frontend/src/theme.js
+2
View File
File diff suppressed because one or more lines are too long
frontend/src/views/Admin.jsx
+335 -115
View File
@@ -4,7 +4,7 @@ import { makeStyles } from '@material-ui/styles';
import { useTheme } from '@material-ui/core/styles';
import {Link} from 'react-router-dom';
import {Paper, Card, Tooltip, FormControlLabel, Typography, Switch, Select, MenuItem, Divider, TextField, Button, Tabs, Tab, Grid, List, ListItem, ListItemText, ListItemAvatar, ListItemSecondaryAction, IconButton, Avatar, Zoom, Dialog, DialogTitle, DialogActions, DialogContent, CircularProgress } from '@material-ui/core';
import {FormControl, InputLabel, Paper, Card, Tooltip, FormControlLabel, Typography, Switch, Select, MenuItem, Divider, TextField, Button, Tabs, Tab, Grid, List, ListItem, ListItemText, ListItemAvatar, ListItemSecondaryAction, IconButton, Avatar, Zoom, Dialog, DialogTitle, DialogActions, DialogContent, CircularProgress } from '@material-ui/core';
import {Edit as EditIcon, FileCopy as FileCopyIcon, Publish as PublishIcon, SelectAll as SelectAllIcon, OpenInNew as OpenInNewIcon, CloudDownload as CloudDownloadIcon, Description as DescriptionIcon, Polymer as PolymerIcon, CheckCircle as CheckCircleIcon, Close as CloseIcon, Apps as AppsIcon, Image as ImageIcon, Delete as DeleteIcon, Cached as CachedIcon, AccessibilityNew as AccessibilityNewIcon, Lock as LockIcon, Eco as EcoIcon, Schedule as ScheduleIcon, Cloud as CloudIcon, Business as BusinessIcon} from '@material-ui/icons';
@@ -29,6 +29,7 @@ const Admin = (props) => {
const [firstRequest, setFirstRequest] = React.useState(true);
const [orgRequest, setOrgRequest] = React.useState(true);
const [modalUser, setModalUser] = React.useState({});
const [orgName, setOrgName] = React.useState("")
const [modalOpen, setModalOpen] = React.useState(false);
const [cloudSyncModalOpen, setCloudSyncModalOpen] = React.useState(false);
@@ -48,7 +49,10 @@ const Admin = (props) => {
const [authentication, setAuthentication] = React.useState([]);
const [schedules, setSchedules] = React.useState([])
const [files, setFiles] = React.useState([])
const [selectedNamespace, setSelectedNamespace] = React.useState("default")
const [fileNamespaces, setFileNamespaces] = React.useState([]);
const [selectedUser, setSelectedUser] = React.useState({})
const [newUsername, setNewUsername] = React.useState("");
const [newPassword, setNewPassword] = React.useState("");
const [selectedUserModalOpen, setSelectedUserModalOpen] = React.useState(false)
const [selectedAuthentication, setSelectedAuthentication] = React.useState({})
@@ -155,7 +159,7 @@ const Admin = (props) => {
setTimeout(() => {
getAppAuthentication()
}, 1000)
alert.success("Successfully deleted authentication!")
//alert.success("Successfully deleted authentication!")
}
}),
)
@@ -184,8 +188,10 @@ const Admin = (props) => {
if (responseJson["success"] === false) {
alert.error("Failed stopping schedule")
} else {
getSchedules()
alert.success("Successfully stopped schedule!")
setTimeout(() => {
getSchedules()
}, 1500)
//alert.success("Successfully stopped schedule!")
}
}),
)
@@ -365,6 +371,45 @@ const Admin = (props) => {
});
}
const createSubOrg = (currentOrgId, name) => {
const data = { "name": name, "org_id": currentOrgId}
console.log(data)
const url = globalUrl + `/api/v1/orgs/${currentOrgId}/create_sub_org`
fetch(url, {
mode: 'cors',
method: 'POST',
body: JSON.stringify(data),
credentials: 'include',
crossDomain: true,
withCredentials: true,
headers: {
'Content-Type': 'application/json; charset=utf-8',
},
})
.then(response =>
response.json().then(responseJson => {
if (responseJson["success"] === false) {
if (responseJson.reason !== undefined) {
alert.error(responseJson.reason)
} else {
alert.error("Failed creating suborg")
}
} else {
alert.success("Successfully created suborg!")
setSelectedUserModalOpen(false)
}
setOrgName("")
setModalOpen(false)
}),
)
.catch(error => {
alert.error("Err: " + error.toString())
});
}
const onPasswordChange = () => {
const data = { "username": selectedUser.username, "newpassword": newPassword }
const url = globalUrl + '/api/v1/users/passwordchange';
@@ -457,6 +502,10 @@ const Admin = (props) => {
if (responseJson["success"] === false) {
alert.error("Failed getting org: ", responseJson.readon)
} else {
if (responseJson.sync_features === undefined || responseJson.sync_features === null) {
responseJson.sync_features = {}
}
setSelectedOrganization(responseJson)
var lists = {
"active": {
@@ -559,19 +608,19 @@ const Admin = (props) => {
}
// Horrible frontend fix for environments
const setDefaultEnvironment = (name) => {
// FIXME - add some check here ROFL
alert.info("Setting default env to " + name)
const setDefaultEnvironment = (environment) => {
// FIXME - add more checks to this
alert.info("Setting default env to " + environment.name)
var newEnv = []
for (var key in environments) {
if (environments[key].Name == name) {
if (environments[key].id == environment.id) {
if (environments[key].archived) {
alert.error("Can't set archived to default")
return
}
environments[key].default = true
} else if (environments[key].default == true && environments[key].name !== name) {
} else if (environments[key].default == true && environments[key].id !== environment.id) {
environments[key].default = false
}
@@ -592,11 +641,15 @@ const Admin = (props) => {
response.json().then(responseJson => {
if (responseJson["success"] === false) {
alert.error(responseJson.reason)
getEnvironments()
setTimeout(() => {
getEnvironments()
}, 1500)
} else {
setLoginInfo("")
setModalOpen(false)
getEnvironments()
setTimeout(() => {
getEnvironments()
}, 1500)
}
}),
)
@@ -632,23 +685,46 @@ const Admin = (props) => {
})
}
const deleteEnvironment = (name) => {
const deleteEnvironment = (environment) => {
// FIXME - add some check here ROFL
alert.info("Deleting environment " + name)
//const name = environment.name
//alert.info("Modifying environment " + name)
//var newEnv = []
//for (var key in environments) {
// if (environments[key].Name == name) {
// if (environments[key].default) {
// alert.error("Can't modify the default environment")
// return
// }
// if (environments[key].type === "cloud" && !environments[key].archived) {
// alert.error("Can't modify cloud environments")
// return
// }
// environments[key].archived = !environments[key].archived
// }
// newEnv.push(environments[key])
//}
const id = environment.id
//alert.info("Modifying environment " + environment.Name)
var newEnv = []
for (var key in environments) {
if (environments[key].Name == name) {
if (environments[key].id == id) {
if (environments[key].default) {
alert.error("Can't delete the default environment")
alert.error("Can't modify the default environment")
return
}
if (environments[key].type === "cloud") {
alert.error("Can't delete the cloud environments")
if (environments[key].type === "cloud" && !environments[key].archived) {
alert.error("Can't modify cloud environments")
return
}
environments[key].archived = true
environments[key].archived = !environments[key].archived
}
newEnv.push(environments[key])
@@ -795,8 +871,16 @@ const Admin = (props) => {
return response.json()
})
.then((responseJson) => {
//console.log(responseJson)
setFiles(responseJson)
if (responseJson.files !== undefined && responseJson.files !== null) {
setFiles(responseJson.files)
} else {
setFiles([])
}
console.log("NAMESPACES: ", responseJson.namespaces)
if (responseJson.namespaces !== undefined && responseJson.namespaces !== null) {
setFileNamespaces(responseJson.namespaces)
}
})
.catch(error => {
alert.error(error.toString())
@@ -931,6 +1015,9 @@ const Admin = (props) => {
}
const getOrgs = () => {
// API no longer in use, as it's in handleInfo request
return
fetch(globalUrl + "/api/v1/orgs", {
method: 'GET',
headers: {
@@ -1118,6 +1205,7 @@ const Admin = (props) => {
alert.error("Failed setting user: " + responseJson.reason)
} else {
alert.success("Set the user field " + field + " to " + value)
setSelectedUserModalOpen(false)
}
})
.catch(error => {
@@ -1257,6 +1345,44 @@ const Admin = (props) => {
>
<DialogTitle><span style={{ color: "white" }}><EditIcon style={{marginTop: 5}}/> Editing {selectedUser.username}</span></DialogTitle>
<DialogContent>
{isCloud ?
null
:
<div style={{ display: "flex" }}>
<TextField
style={{ marginTop: 0, backgroundColor: theme.palette.inputColor, flex: 3 , marginRight: 10,}}
InputProps={{
style: {
height: 50,
color: "white",
},
}}
color="primary"
required
fullWidth={true}
placeholder="New username"
type="text"
id="standard-required"
autoComplete="username"
margin="normal"
variant="outlined"
defaultValue={selectedUser.username}
onChange={e => {
setNewUsername(e.target.value)
}}
/>
<Button
style={{ maxHeight: 50, flex: 1 }}
variant="outlined"
color="primary"
onClick={() => {
setUser(selectedUser.id, "username", newUsername)
}}
>
Submit
</Button>
</div>
}
{isCloud ?
null
:
@@ -1543,7 +1669,7 @@ const Admin = (props) => {
</IconButton>
</Tooltip>
{selectedOrganization.name.length > 0 ?
<OrgHeader setSelectedOrganization={setSelectedOrganization} globalUrl={globalUrl} selectedOrganization={selectedOrganization}/>
<OrgHeader userdata={userdata} setSelectedOrganization={setSelectedOrganization} globalUrl={globalUrl} selectedOrganization={selectedOrganization}/>
:
<div style={{paddingTop: 250, width: 250, margin: "auto", textAlign: "center"}}>
<CircularProgress />
@@ -1657,65 +1783,65 @@ const Admin = (props) => {
</div>
}
<Typography style={{marginTop: 40, marginLeft: 10, marginBottom: 5,}}>Cloud sync features</Typography>
<Grid container style={{width: "100%", marginBottom: 15, }}>
{Object.keys(selectedOrganization.sync_features).map(function(key, index) {
if (key === "schedule") {
return null
}
<Grid container style={{width: "100%", marginBottom: 15, }}>
{selectedOrganization.sync_features === undefined || selectedOrganization.sync_features === null ? null : Object.keys(selectedOrganization.sync_features).map(function(key, index) {
if (key === "schedule") {
return null
}
const item = selectedOrganization.sync_features[key]
const newkey = key.replaceAll("_", " ")
const griditem = {
"primary": newkey,
"secondary": item.description === undefined || item.description === null || item.description.length === 0 ? "Not defined yet" : item.description,
"limit": item.limit,
"usage": 0,
"data_collection": "None",
"active": item.active,
"icon": <PolymerIcon style={{color: itemColor}}/>,
}
const item = selectedOrganization.sync_features[key]
const newkey = key.replaceAll("_", " ")
const griditem = {
"primary": newkey,
"secondary": item.description === undefined || item.description === null || item.description.length === 0 ? "Not defined yet" : item.description,
"limit": item.limit,
"usage": 0,
"data_collection": "None",
"active": item.active,
"icon": <PolymerIcon style={{color: itemColor}}/>,
}
return (
<Zoom key={index} >
<GridItem data={griditem} />
</Zoom>
)
})}
</Grid>
<Divider style={{ marginTop: 20, marginBottom: 20, backgroundColor: theme.palette.inputColor }} />
{isCloud && selectedOrganization.subscriptions !== undefined && selectedOrganization.subscriptions !== null && selectedOrganization.subscriptions.length > 0 ?
<div style={{marginTop: 30, marginBottom: 20}}>
<Typography style={{marginTop: 40, marginLeft: 10, marginBottom: 5,}}>
Your subscription{selectedOrganization.subscriptions.length > 1 ? "s" : ""}
</Typography>
<Grid container spacing={3} style={{marginTop: 15}}>
{selectedOrganization.subscriptions.reverse().map((sub, index) => {
return (
<Grid item key={index} xs={4}>
<Card elevation={6} style={{backgroundColor: theme.palette.inputColor, color: "white", padding: 25, textAlign: "left",}}>
<b>Type</b>: {sub.level}<div/>
<b>Recurrence</b>: {sub.recurrence}<div/>
{sub.active ?
<div>
<b>Started</b>: {new Date(sub.startdate*1000).toISOString()}<div/>
<Button variant="outlined" color="primary" style={{marginTop: 15}} onClick={() => {
cancelSubscriptions(sub.reference)
}}>
Cancel subscription
</Button>
</div>
:
<div>
<b>Cancelled</b>: {new Date(sub.cancellationdate*1000).toISOString()}<div/>
<Typography color="textSecondary">
<b>Status</b>: Deactivated
</Typography>
</div>
}
</Card>
</Grid>
)
return (
<Zoom key={index} >
<GridItem data={griditem} />
</Zoom>
)
})}
</Grid>
<Divider style={{ marginTop: 20, marginBottom: 20, backgroundColor: theme.palette.inputColor }} />
{isCloud && selectedOrganization.subscriptions !== undefined && selectedOrganization.subscriptions !== null && selectedOrganization.subscriptions.length > 0 ?
<div style={{marginTop: 30, marginBottom: 20}}>
<Typography style={{marginTop: 40, marginLeft: 10, marginBottom: 5,}}>
Your subscription{selectedOrganization.subscriptions.length > 1 ? "s" : ""}
</Typography>
<Grid container spacing={3} style={{marginTop: 15}}>
{selectedOrganization.subscriptions.reverse().map((sub, index) => {
return (
<Grid item key={index} xs={4}>
<Card elevation={6} style={{backgroundColor: theme.palette.inputColor, color: "white", padding: 25, textAlign: "left",}}>
<b>Type</b>: {sub.level}<div/>
<b>Recurrence</b>: {sub.recurrence}<div/>
{sub.active ?
<div>
<b>Started</b>: {new Date(sub.startdate*1000).toISOString()}<div/>
<Button variant="outlined" color="primary" style={{marginTop: 15}} onClick={() => {
cancelSubscriptions(sub.reference)
}}>
Cancel subscription
</Button>
</div>
:
<div>
<b>Cancelled</b>: {new Date(sub.cancellationdate*1000).toISOString()}<div/>
<Typography color="textSecondary">
<b>Status</b>: Deactivated
</Typography>
</div>
}
</Card>
</Grid>
)
})}
</Grid>
<Divider style={{ marginTop: 20, backgroundColor: theme.palette.inputColor }} />
</div>
@@ -1744,14 +1870,20 @@ const Admin = (props) => {
}}
>
<DialogTitle><span style={{ color: "white" }}>
{curTab === 1 ? "Add user" : "Add environment"}
{curTab === 1 ? "Add user" : curTab === 6 ? "Add Sub-Organization" : "Add environment"}
</span></DialogTitle>
<DialogContent>
{curTab === 1 && isCloud ?
<Typography variant="body1" style={{marginBottom: 10}}>
We'll send an email to invite them to your organization.
</Typography>
: null}
:
curTab === 6 ?
<Typography variant="body1" style={{marginBottom: 10}}>
The organization created will become a child of your current organization, and be available to you.
</Typography>
:
null }
{curTab === 1 ?
<div>
Username
@@ -1801,6 +1933,31 @@ const Admin = (props) => {
</span>
}
</div>
: curTab === 6 ?
<div>
Name
<TextField
color="primary"
style={{ backgroundColor: theme.palette.inputColor }}
autoFocus
InputProps={{
style: {
height: "50px",
color: "white",
fontSize: "1em",
},
}}
required
fullWidth={true}
placeholder={`${selectedOrganization.name} Copycat Inc.`}
id="orgname"
margin="normal"
variant="outlined"
onChange={(event) => {
setOrgName(event.target.value)
}}
/>
</div>
: curTab === 5 ?
<div>
Environment Name
@@ -1838,6 +1995,8 @@ const Admin = (props) => {
} else {
submitUser(modalUser)
}
} else if (curTab === 6) {
createSubOrg(selectedOrganization.id, orgName)
} else if (curTab === 5) {
submitEnvironment(modalUser)
}
@@ -1889,7 +2048,11 @@ const Admin = (props) => {
/>
<ListItemText
primary="Active"
style={{ minWidth: 180, maxWidth: 180 }}
style={{ minWidth: 150, maxWidth: 150 }}
/>
<ListItemText
primary="Type"
style={{ minWidth: 150 , maxWidth: 150 }}
/>
<ListItemText
primary="Actions"
@@ -1949,10 +2112,9 @@ const Admin = (props) => {
value={data.role}
fullWidth
onChange={(e) => {
console.log("VALUE: ", e.target.value)
setUser(data.id, "role", e.target.value)
}}
console.log("VALUE: ", e.target.value)
setUser(data.id, "role", e.target.value)
}}
style={{ backgroundColor: theme.palette.surfaceColor, color: "white", height: "50px" }}
>
<MenuItem style={{ backgroundColor: theme.palette.inputColor, color: "white" }} value={"admin"}>
@@ -1965,10 +2127,14 @@ const Admin = (props) => {
}
style ={{ minWidth: 135, maxWidth: 135, marginRight: 15,}}
/>
<ListItemText
primary={data.active ? "True" : "False"}
style={{ minWidth: 180, maxWidth: 180 }}
/>
<ListItemText
primary={data.active ? "True" : "False"}
style={{ minWidth: 150, maxWidth: 150}}
/>
<ListItemText
primary={data.login_type === undefined || data.login_type === null || data.login_type.length === 0 ? "Normal" : data.login_type}
style={{ minWidth: 150, maxWidth: 150}}
/>
<ListItemText style={{ display: "flex" }}>
<IconButton
onClick={() => {
@@ -2021,7 +2187,9 @@ const Admin = (props) => {
}
}
getFiles()
setTimeout(() => {
getFiles()
}, 2500)
}
const uploadFile = (e) => {
@@ -2061,6 +2229,30 @@ const Admin = (props) => {
>
<CachedIcon />
</Button>
{fileNamespaces !== undefined && fileNamespaces !== null && fileNamespaces.length > 1 ?
<FormControl>
<InputLabel id="input-namespace-label">Namespace</InputLabel>
<Select
labelId="input-namespace-select-label"
id="input-namespace-select-id"
style={{color: "white", minWidth: 100, float: "right",}}
value={selectedNamespace}
onChange={(event) => {
console.log("CHANGE NAMESPACE: ", event.target)
setSelectedNamespace(event.target.value)
}}
>
{fileNamespaces.map((data, index) => {
return (
<MenuItem key={index} value={data} style={{color: "white"}}>{data}</MenuItem>
)
})}
</Select>
</FormControl>
: null}
<Divider style={{marginTop: 20, marginBottom: 20, backgroundColor: theme.palette.inputColor}}/>
<List>
<ListItem>
@@ -2095,7 +2287,15 @@ const Admin = (props) => {
primary="File ID"
/>
</ListItem>
{files === undefined || files === null ? null : files.map((file, index) => {
{files === undefined || files === null || files.length === 0 ? null : files.map((file, index) => {
if (file.namespace === "") {
file.namespace = "default"
}
if (file.namespace !== selectedNamespace) {
return null
}
var bgColor = "#27292d"
if (index % 2 === 0) {
bgColor = "#1f2023"
@@ -2303,13 +2503,13 @@ const Admin = (props) => {
style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
/>
</ListItem>
{categories.map(data => {
{categories.map((data, index) => {
if (data.apps.length === 0) {
return null
}
return (
<ListItem>
<ListItem key={index}>
<ListItemText
primary={data.name}
style={{minWidth: 150, maxWidth: 150}}
@@ -2409,6 +2609,7 @@ const Admin = (props) => {
bgColor = "#1f2023"
}
return (
<ListItem key={index} style={{backgroundColor: bgColor}}>
<ListItemText
@@ -2489,7 +2690,7 @@ const Admin = (props) => {
<div>
<div style={{marginTop: 20, marginBottom: 20,}}>
<h2 style={{display: "inline",}}>Environments</h2>
<span style={{marginLeft: 25}}>Decides what Orborus environment to execute an action in a workflow in.<a target="_blank" href="https://shuffler.io/docs/organizations#environments" style={{textDecoration: "none", color: "#f85a3e"}}>Learn more</a></span>
<span style={{marginLeft: 25}}>Decides what Orborus environment to execute an action in a workflow in. <a target="_blank" href="https://shuffler.io/docs/organizations#environments" style={{textDecoration: "none", color: "#f85a3e"}}>Learn more</a></span>
</div>
<Button
style={{}}
@@ -2572,13 +2773,13 @@ const Admin = (props) => {
{environment.default ?
null
:
<Button variant="outlined" style={{borderRadius: "0px"}} onClick={() => setDefaultEnvironment(environment.Name)} color="primary">Set default</Button>
<Button variant="outlined" style={{borderRadius: "0px"}} onClick={() => setDefaultEnvironment(environment)} color="primary">Set default</Button>
}
</ListItemText>
<ListItemText
style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
>
<Button disabled={environment.archived} variant="outlined" style={{borderRadius: "0px"}} onClick={() => deleteEnvironment(environment.Name)} color="primary">Archive</Button>
<Button variant={environment.archived ? "contained" : "outlined"} style={{borderRadius: "0px"}} onClick={() => deleteEnvironment(environment)} color="primary">{environment.archived ? "Activate" : "Disable"}</Button>
{/*<Button disabled={environment.archived} variant="outlined" style={{borderRadius: "0px"}} onClick={() => flushQueue(environment.Name)} color="primary">Flush Queue</Button>*/}
</ListItemText>
<ListItemText
@@ -2592,7 +2793,7 @@ const Admin = (props) => {
</div>
: null
const organizationsTab = curTab === 7 ?
const organizationsTab = curTab === 6 ?
<div>
<div style={{marginTop: 20, marginBottom: 20,}}>
<h2 style={{display: "inline",}}>Organizations</h2>
@@ -2602,28 +2803,31 @@ const Admin = (props) => {
style={{}}
variant="contained"
color="primary"
disabled
onClick={() => {
setModalOpen(true)
}}
>
Add organization
Add suborganization
</Button>
<Divider style={{marginTop: 20, marginBottom: 20, backgroundColor: theme.palette.inputColor}}/>
<List>
<ListItem>
<ListItemText
primary="Name"
style={{minWidth: 150, maxWidth: 150}}
primary="Logo"
style={{minWidth: 100, maxWidth: 100}}
/>
<ListItemText
primary="id"
style={{minWidth: 200, maxWidth: 200}}
primary="Name"
style={{minWidth: 250, maxWidth: 250}}
/>
<ListItemText
primary="Your role"
style={{minWidth: 150, maxWidth: 150}}
/>
<ListItemText
primary="id"
style={{minWidth: 400, maxWidth: 400}}
/>
<ListItemText
primary="Selected"
style={{minWidth: 150, maxWidth: 150}}
@@ -2633,25 +2837,41 @@ const Admin = (props) => {
style={{minWidth: 150, maxWidth: 150}}
/>
</ListItem>
{organizations !== undefined && organizations !== null && organizations.length > 0 ?
{userdata.orgs !== undefined && userdata.orgs !== null && userdata.orgs.length > 0 ?
<span>
{organizations.map((data, index) => {
{userdata.orgs.map((data, index) => {
const isSelected = props.userdata.active_org.id === undefined ? "False" : props.userdata.active_org.id === data.id ? "True" : "False"
const imagesize = 40
const imageStyle = {width: imagesize, height: imagesize, pointerEvents: "none", }
const image = data.image === "" ?
<img alt={data.name} src={theme.palette.defaultImage} style={imageStyle} />
:
<img alt={data.name} src={data.image} style={imageStyle} />
var bgColor = "#27292d"
if (index % 2 === 0) {
bgColor = "#1f2023"
}
return (
<ListItem key={index}>
<ListItem key={index} style={{backgroundColor: bgColor,}}>
<ListItemText
primary={data.name}
style={{minWidth: 150, maxWidth: 150}}
primary={image}
style={{minWidth: 100, maxWidth: 100}}
/>
<ListItemText
primary={data.id}
style={{minWidth: 200, maxWidth: 200}}
primary={data.name}
style={{minWidth: 250, maxWidth: 250}}
/>
<ListItemText
primary={data.role}
style={{minWidth: 150, maxWidth: 150}}
/>
<ListItemText
primary={data.id}
style={{minWidth: 400, maxWidth: 400}}
/>
<ListItemText
primary={isSelected}
style={{minWidth: 150, maxWidth: 150}}
@@ -2673,7 +2893,7 @@ const Admin = (props) => {
</div>
: null
const hybridTab = curTab === 6 ?
const hybridTab = curTab === 7 ?
<div>
<div style={{marginTop: 20, marginBottom: 20,}}>
<h2 style={{display: "inline",}}>Hybrid</h2>
@@ -2719,7 +2939,7 @@ const Admin = (props) => {
const iconStyle = {marginRight: 10}
const data =
<div style={{width: 1366, margin: "auto", overflowX: "hidden", marginTop: 25,}}>
<div style={{width: 1300, margin: "auto", overflowX: "hidden", marginTop: 25,}}>
<Paper style={paperStyle}>
<Tabs
value={curTab}
@@ -2733,21 +2953,21 @@ const Admin = (props) => {
<Tab label=<span><DescriptionIcon style={iconStyle} />Files</span> />
<Tab label=<span><ScheduleIcon style={iconStyle} />Schedules</span> />
{isCloud ? null : <Tab label=<span><EcoIcon style={iconStyle} />Environments</span>/>}
{window.location.protocol == "http:" && window.location.port === "3000" ? <Tab label=<span><CloudIcon style={iconStyle} /> Hybrid</span>/> : null}
{window.location.protocol == "http:" && window.location.port === "3000" ? <Tab label=<span><BusinessIcon style={iconStyle} /> Organizations</span>/> : null}
{window.location.protocol === "http:" && window.location.port === "3000" ? <Tab label=<span><LockIcon style={iconStyle} />Categories</span>/> : null}
{isCloud ? null : <Tab label=<span><BusinessIcon style={iconStyle} /> Organizations</span>/>}
{/*window.location.protocol == "http:" && window.location.port === "3000" ? <Tab label=<span><CloudIcon style={iconStyle} /> Hybrid</span>/> : null*/}
{/*window.location.protocol === "http:" && window.location.port === "3000" ? <Tab label=<span><LockIcon style={iconStyle} />Categories</span>/> : null*/}
</Tabs>
<Divider style={{marginTop: 0, marginBottom: 10, backgroundColor: "rgb(91, 96, 100)"}} />
<div style={{padding: 15}}>
{organizationView}
{authenticationView}
{appCategoryView}
{usersView}
{environmentView}
{schedulesView}
{filesView}
{hybridTab}
{organizationsTab}
{appCategoryView}
</div>
</Paper>
</div>
frontend/src/views/AngularWorkflow.jsx
+824 -275
View File
File diff suppressed because one or more lines are too long
frontend/src/views/AppCreator.jsx
+588 -75
View File
File diff suppressed because it is too large Load Diff
frontend/src/views/Apps.jsx
+50 -44
View File
@@ -9,7 +9,6 @@ import { useTheme } from '@material-ui/core/styles';
import YAML from 'yaml'
import {Link} from 'react-router-dom';
import ReactJson from 'react-json-view'
import { useAlert } from "react-alert";
import Dropzone from '../components/Dropzone';
@@ -130,6 +129,7 @@ const Apps = (props) => {
const upload = React.useRef(null);
const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" ? true : false
const borderRadius = 3
const viewWidth = 590
const { start, stop } = useInterval({
duration: 5000,
@@ -178,6 +178,7 @@ const Apps = (props) => {
color: "#ffffff",
width: "100%",
display: "flex",
margin: "auto",
}
const paperAppStyle = {
@@ -474,13 +475,14 @@ const Apps = (props) => {
const dividerColor = "rgb(225, 228, 232)"
const uploadViewPaperStyle = {
minWidth: 662.5,
maxWidth: 662.5,
minWidth: viewWidth,
maxWidth: viewWidth,
color: "white",
borderRadius: 5,
backgroundColor: surfaceColor,
display: "flex",
//display: "flex",
marginBottom: 10,
overflow: "hidden",
}
const UploadView = () => {
@@ -520,7 +522,7 @@ const Apps = (props) => {
<Link to={editUrl} style={{textDecoration: "none"}}>
<Tooltip title={"Edit OpenAPI app"}>
<Button
variant="outlined"
variant="contained"
component="label"
color="primary"
style={{marginTop: 10, marginRight: 10,}}
@@ -620,15 +622,6 @@ const Apps = (props) => {
</MenuItem>
)
})}
{/*
<ReactJson
src={JSON.parse(showResult)}
theme="solarized"
collapsed={false}
displayDataTypes={true}
name={"Example return value"}
/>
*/}
</div>
)
}
@@ -707,8 +700,8 @@ const Apps = (props) => {
{activateButton}
{(props.userdata !== undefined && (props.userdata.role === "admin" || props.userdata.id === selectedApp.owner) || !selectedApp.generated) ?
<div>
{downloadButton}
{editButton}
{downloadButton}
{deleteButton}
</div>
: null}
@@ -771,7 +764,7 @@ const Apps = (props) => {
{/*<p><b>Owner:</b> {selectedApp.owner}</p>*/}
{selectedApp.privateId !== undefined && selectedApp.privateId.length > 0 ? <p><b>PrivateID:</b> {selectedApp.privateId}</p> : null}
<Divider style={{marginBottom: 10, marginTop: 10, backgroundColor: dividerColor}}/>
<div style={{padding: 20}}>
<div style={{paddingTop: 20, paddingBottom: 20, }}>
{selectedApp.link.length > 0 ? <p><b>URL:</b> {selectedApp.link}</p> : null}
<div style={{marginTop: 15, marginBottom: 15}}>
<b>Actions</b>
@@ -852,7 +845,7 @@ const Apps = (props) => {
<h2>App Creator</h2>
<a rel="norefferer" href="https://shuffler.io/docs/apps" style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">How it works</a>
&nbsp;- <a href="https://github.com/frikky/security-openapis" style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">Security API's</a>
&nbsp;- <a href="https://apis.guru/browse-apis/" style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">OpenAPI directory</a>
&nbsp;- <a href="https://github.com/APIs-guru/openapi-directory/tree/main/APIs" style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">OpenAPI directory</a>
&nbsp;- <a href="https://editor.swagger.io/" style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">OpenAPI Validator</a>
<div/>
<Typography variant="body2" color="textSecondary">
@@ -932,7 +925,11 @@ const Apps = (props) => {
console.log("Error in dropzone: ", e)
}
reader.readAsText(files[0]);
try {
reader.readAsText(files[0]);
} catch(error) {
alert.error("Failed to read file")
}
};
useEffect(() => {
@@ -950,9 +947,9 @@ const Apps = (props) => {
}, [appValidation, isDropzone]);
const appView = isLoggedIn ?
<Dropzone style={{maxWidth: window.innerWidth > 1366 ? 1366 : 1200, margin: "auto", padding: 20 }} onDrop={uploadFile}>
<Dropzone style={{width: viewWidth*2+20, margin: "auto", padding: 20 }} onDrop={uploadFile}>
<div style={appViewStyle}>
<div style={{flex: 1, }}>
<div style={{flex: 1, maxWidth: viewWidth, marginRight: 10,}}>
<Breadcrumbs aria-label="breadcrumb" separator="›" style={{color: "white",}}>
<Link to="/apps" style={{textDecoration: "none", color: "inherit",}}>
<Typography variant="h6" style={{color: "rgba(255,255,255,0.5)"}}>
@@ -969,9 +966,9 @@ const Apps = (props) => {
: null}
</Breadcrumbs>
<div style={{marginTop: 15}} />
<UploadView/>
<UploadView />
</div>
<div style={{flex: 1, marginLeft: 10, marginRight: 10, }}>
<div style={{flex: 1, marginLeft: 10, maxWidth: viewWidth, }}>
<div style={{display: "flex",}}>
<div style={{flex: 1, marginBottom: 15, }}>
<Typography variant="h6">
@@ -980,35 +977,39 @@ const Apps = (props) => {
</div>
{isCloud ? null :
<span>
<Tooltip title={"Reload apps locally"} style={{marginTop: "28px", width: "100%"}} aria-label={"Upload"}>
<Button
variant="outlined"
component="label"
color="primary"
style={{margin: 5, maxHeight: 50, marginTop: 10}}
onClick={() => {
hotloadApps()
}}
>
<CachedIcon />
</Button>
</Tooltip>
{isLoading ? null :
<Tooltip title={"Reload apps locally"} style={{marginTop: "28px", width: "100%"}} aria-label={"Upload"}>
<Button
variant="outlined"
component="label"
color="primary"
style={{margin: 5, maxHeight: 50, marginTop: 10}}
disabled={isLoading}
onClick={() => {
hotloadApps()
}}
>
{isLoading ? <CircularProgress size={25} /> : <CachedIcon />}
</Button>
</Tooltip>
}
<Tooltip title={"Download from Github"} style={{marginTop: "28px", width: "100%"}} aria-label={"Upload"}>
<Button
variant="outlined"
component="label"
color="primary"
style={{margin: 5, maxHeight: 50, marginTop: 10}}
disabled={isLoading}
onClick={() => {
setOpenApi(baseRepository)
setLoadAppsModalOpen(true)
}}
>
<CloudDownloadIcon />
{isLoading ? <CircularProgress size={25} /> : <CloudDownloadIcon />}
</Button>
</Tooltip>
</span>
}
}
</div>
<div style={{height: 50}}>
<TextField
@@ -1072,9 +1073,12 @@ const Apps = (props) => {
<CircularProgress style={{width: 40, height: 40, margin: "auto"}}/>
:
<Paper square style={uploadViewPaperStyle}>
<h4 style={{margin: 10}}>
<Typography variant="body1" style={{margin: 10}}>
No apps have been created, uploaded or downloaded yet. Click "Load existing apps" above to get the baseline. This may take a while as its building docker images.
</h4>
</Typography>
<Typography variant="body1" style={{margin: 10}}>
If you're still not able to see any apps, please follow our <a href={"https://shuffler.io/docs/troubleshooting#load_all_apps_locally"} style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">troubleshooting guide for loading apps!</a>
</Typography>
</Paper>
}
</div>
@@ -1571,11 +1575,13 @@ const Apps = (props) => {
<Button style={{borderRadius: "0px"}} onClick={() => setLoadAppsModalOpen(false)} color="primary">
Cancel
</Button>
<Button style={{borderRadius: "0px"}} disabled={openApi.length === 0 || !openApi.includes("http")} onClick={() => {
handleGithubValidation(true)
}} color="primary">
Force update
</Button>
{isCloud ? null :
<Button style={{borderRadius: "0px"}} disabled={openApi.length === 0 || !openApi.includes("http")} onClick={() => {
handleGithubValidation(true)
}} color="primary">
Force update
</Button>
}
<Button variant="outlined" style={{float: "left", borderRadius: "0px"}} disabled={openApi.length === 0 || !openApi.includes("http")} onClick={() => {
handleGithubValidation(false)
}} color="primary">
frontend/src/views/Docs.jsx
+183 -21
View File
@@ -5,14 +5,15 @@ import ReactMarkdown from 'react-markdown';
import {BrowserView, MobileView} from "react-device-detect";
import {Link} from 'react-router-dom';
import {Divider, Button, Menu, MenuItem, Typography, Paper, List} from '@material-ui/core';
import {Tooltip, Divider, Button, Menu, MenuItem, Typography, Paper, List} from '@material-ui/core';
import {Link as LinkIcon, Edit as EditIcon} from '@material-ui/icons';
const Body = {
maxWidth: '1000px',
minWidth: '768px',
margin: 'auto',
display: "flex",
heigth: "100%",
height: "100%",
color: "white",
//textAlign: "center",
};
@@ -23,15 +24,24 @@ const hrefStyle = {
textDecoration: "none"
}
const innerHrefStyle = {
color: "rgba(255, 255, 255, 0.75)",
textDecoration: "none"
}
const Docs = (props) => {
const { globalUrl, selectedDoc, serverside, isMobile, } = props;
const theme = useTheme();
const [mobile, setMobile] = useState(isMobile === true ? true : false);
const [data, setData] = useState("");
const [firstrequest, setFirstrequest] = useState(true);
const [list, setList] = useState([]);
const [, setListLoaded] = useState(false);
const [anchorEl, setAnchorEl] = React.useState(null);
const [headingSet, setHeadingSet] = React.useState(false);
const [selectedMeta, setSelectedMeta] = React.useState({link: "hello", read_time: 2, });
const [tocLines, setTocLines] = React.useState([]);
const [baseUrl, setBaseUrl] = React.useState(serverside === true ? "" : window.location.href)
function handleClick(event) {
@@ -48,14 +58,14 @@ const Docs = (props) => {
position: "relative",
padding: 30,
paddingTop: 15,
borderRadius: 5,
height: "80vh",
marginTop: 15,
}
const SideBar = {
maxWidth: 250,
flex: "1",
position: "fixed",
marginTop: 35,
}
const fetchDocList = () => {
@@ -71,7 +81,7 @@ const Docs = (props) => {
if (responseJson.success) {
setList(responseJson.list)
} else {
setList(["error"])
setList(["# Error loading documentation. Please contact us if this persists."])
}
setListLoaded(true)
})
@@ -91,6 +101,58 @@ const Docs = (props) => {
if (responseJson.success) {
setData(responseJson.reason)
document.title = "Shuffle "+docId+" documentation"
if (responseJson.meta !== undefined) {
setSelectedMeta(responseJson.meta)
}
//console.log("TOC list: ", responseJson.reason)
if (responseJson.reason !== undefined && responseJson.reason !== null) {
const splitkey = responseJson.reason.split("\n")
var innerTocLines = []
var record = false
for (var key in splitkey) {
const line = splitkey[key]
//console.log("Line: ", line)
if (line.toLowerCase().includes("table of contents")) {
record = true
continue
}
if (record && line.length < 3) {
record = false
}
if (record) {
const parsedline = line.split("](")
if (parsedline.length > 1) {
parsedline[0] = parsedline[0].replaceAll("*", "")
parsedline[0] = parsedline[0].replaceAll("[", "")
parsedline[0] = parsedline[0].replaceAll("]", "")
parsedline[0] = parsedline[0].replaceAll("(", "")
parsedline[0] = parsedline[0].replaceAll(")", "")
parsedline[0] = parsedline[0].trim()
parsedline[1] = parsedline[1].replaceAll("*", "")
parsedline[1] = parsedline[1].replaceAll("[", "")
parsedline[1] = parsedline[1].replaceAll("]", "")
parsedline[1] = parsedline[1].replaceAll(")", "")
parsedline[1] = parsedline[1].replaceAll("(", "")
parsedline[1] = parsedline[1].trim()
//console.log(parsedline[0], parsedline[1])
innerTocLines.push({
"text": parsedline[0],
"link": parsedline[1]
})
} else {
console.log("Bad line for parsing: ", line)
}
}
}
setTocLines(innerTocLines)
}
} else {
setData("# Error\nThis page doesn't exist.")
}
@@ -100,14 +162,21 @@ const Docs = (props) => {
if (firstrequest) {
setFirstrequest(false)
if (!serverside) {
if (window.innerWidth < 768) {
setMobile(true)
}
}
if (selectedDoc !== undefined) {
setData(selectedDoc.reason)
setList(selectedDoc.list)
setListLoaded(true)
} else {
fetchDocList()
fetchDocs(props.match.params.key)
if (!serverside) {
fetchDocList()
fetchDocs(props.match.params.key)
}
}
}
@@ -118,6 +187,7 @@ const Docs = (props) => {
}
const parseElementScroll = () => {
const offset = 45
var parent = document.getElementById("markdown_wrapper_outer")
if (parent !== null) {
//console.log("IN PARENT")
@@ -135,7 +205,12 @@ const Docs = (props) => {
// Fix location..
if (element.innerHTML.toLowerCase() === name) {
//console.log(element.offsetTop)
element.scrollIntoView({behavior: "smooth"})
//element.scrollTo({
// top: element.offsetTop+offset,
// behavior: "smooth"
//})
found = true
//element.scrollTo({
// top: element.offsetTop-100,
@@ -147,7 +222,7 @@ const Docs = (props) => {
// H#
if (!found) {
elements = parent.getElementsByTagName('h3')
console.log(name)
//console.log("NAMe: ", name)
found = false
for (key in elements) {
const element = elements[key]
@@ -158,6 +233,10 @@ const Docs = (props) => {
// Fix location..
if (element.innerHTML.toLowerCase() === name) {
element.scrollIntoView({behavior: "smooth"})
//element.scrollTo({
// top: element.offsetTop-offset,
// behavior: "smooth"
//})
found = true
//element.scrollTo({
// top: element.offsetTop-100,
@@ -187,10 +266,10 @@ const Docs = (props) => {
const markdownStyle = {
color: "rgba(255, 255, 255, 0.65)",
flex: "1",
maxWidth: isMobile ? "100%" : 750,
maxWidth: mobile ? "100%" : 750,
overflow: "hidden",
paddingBottom: 200,
marginLeft: isMobile ? 0 : 275,
marginLeft: mobile ? 0 : 275,
}
function OuterLink(props) {
@@ -214,12 +293,65 @@ const Docs = (props) => {
)
}
function Heading(props) {
const element = React.createElement(`h${props.level}`, {style: {marginTop: 40}}, props.children)
const Heading = (props) => {
const element = React.createElement(`h${props.level}`, {style: {marginTop: props.level === 1 ? 20 : 50}}, props.children)
const [hover, setHover] = useState(false)
var extraInfo = ""
if (props.level === 1) {
extraInfo =
<div style={{backgroundColor: theme.palette.inputColor, padding: 15, borderRadius: theme.palette.borderRadius, marginBottom: 30, display: "flex",}}>
<div style={{flex: 3, display: "flex", vAlign: "center",}}>
{mobile ? null :
<Typography style={{display: "inline", marginTop: 6, }}>
<a rel="norefferer" target="_blank" href={selectedMeta.link} target="_blank" style={{textDecoration: "none", color: "#f85a3e"}}>
<Button style={{}} variant="outlined">
<EditIcon /> &nbsp;&nbsp;Edit
</Button>
</a>
</Typography>
}
{mobile ? null :
<div style={{height: "100%", width: 1, backgroundColor: "white", marginLeft: 50, marginRight: 50, }} />
}
<Typography style={{display: "inline", marginTop: 11, }}>
{selectedMeta.read_time} minute{selectedMeta.read_time === 1 ? "" : "s"} to read
</Typography>
</div>
<div style={{flex: 2}}>
{mobile || selectedMeta.contributors === undefined || selectedMeta.contributors === null ? "" :
<div style={{margin: 10, height: "100%", display: "inline",}}>
{selectedMeta.contributors.slice(0,7).map((data, index) => {
return (
<a rel="norefferer" target="_blank" href={data.url} target="_blank" style={{textDecoration: "none", color: "#f85a3e"}}>
<Tooltip title={data.url} placement="bottom">
<img alt={data.url} src={data.image} style={{marginTop: 5, marginRight: 10, height: 40, borderRadius: 40, }} />
</Tooltip>
</a>
)
})}
</div>
}
</div>
</div>
}
return (
<Typography>
<Typography
onMouseOver={() => {
setHover(true)
}} >
{props.level !== 1 ? <Divider style={{width: "90%", marginTop: 40, backgroundColor: theme.palette.inputColor}} /> : null}
{element}
{/*hover ? <LinkIcon onMouseOver={() => {setHover(true)}} style={{cursor: "pointer", display: "inline", }} onClick={() => {
window.location.href += "#hello"
console.log(window.location)
//window.history.pushState('page2', 'Title', '/page2.php');
//window.history.replaceState('page2', 'Title', '/page2.php');
}} />
: ""
*/}
{extraInfo}
</Typography>
)
}
@@ -234,19 +366,44 @@ const Docs = (props) => {
// );
//}
const postDataBrowser =
const postDataBrowser = list === undefined || list === null ? null :
<div style={Body}>
<div style={SideBar}>
<Paper style={SidebarPaperStyle}>
<List style={{listStyle: "none", paddingLeft: "0", }}>
{list.map((item, index) => {
{list.map((data, index) => {
const item = data.name
if (item === undefined) {
return null
}
const path = "/docs/"+item
const newname = item.charAt(0).toUpperCase()+item.substring(1).split("_").join(" ").split("-").join(" ")
const itemMatching = props.match.params.key.toLowerCase() === item.toLowerCase()
//const [tocLines, setTocLines] = React.useState([]);
return (
<li key={index} style={{marginTop: 15,}}>
<Link key={index} style={hrefStyle} to={path} onClick={() => {fetchDocs(item)}}>
<Typography variant="h6"><b>{newname}</b></Typography>
<li key={index} style={{marginTop: 10,}}>
<Link key={index} style={hrefStyle} to={path} onClick={() => {
setTocLines([])
fetchDocs(item)
}}>
<Typography style={{color: itemMatching ? "#f86a3e" : "inherit"}} variant="body1"><b>> {newname}</b></Typography>
</Link>
{itemMatching && tocLines !== null && tocLines !== undefined && tocLines.length > 0 ?
<div style={{marginLeft: 5}}>
{tocLines.map((data, index) => {
//console.log(data)
return (
<Link key={index} style={innerHrefStyle} to={data.link} onClick={() => {}}>
<Typography variant="body2" style={{cursor: "pointer"}}>
- {data.text}
</Typography>
</Link>
)
})}
</div>
: null}
</li>
)
})}
@@ -278,7 +435,7 @@ const Docs = (props) => {
flexDirection: "column",
}
const postDataMobile =
const postDataMobile = list === undefined || list === null ? null :
<div style={mobileStyle}>
<div>
<Button fullWidth aria-controls="simple-menu" aria-haspopup="true" variant="outlined" color="primary" onClick={handleClick}>
@@ -294,7 +451,12 @@ const Docs = (props) => {
open={Boolean(anchorEl)}
onClose={handleClose}
>
{list.map((item, index) => {
{list.map((data, index) => {
const item = data.name
if (item === undefined) {
return null
}
const path = "/docs/"+item
const newname = item.charAt(0).toUpperCase()+item.substring(1).split("_").join(" ").split("-").join(" ")
return (
@@ -344,7 +506,7 @@ const Docs = (props) => {
</div>
return (
<div>
<div style={{}}>
{loadedCheck}
</div>
)
frontend/src/views/LoginPage.jsx
+91 -4
View File
@@ -1,6 +1,7 @@
/* eslint-disable react/no-multi-comp */
import React, { useState } from 'react';
import { makeStyles } from '@material-ui/styles';
import { useInterval } from 'react-powerhooks';
import {CircularProgress, TextField, Button, Paper, Typography} from '@material-ui/core'
import { useTheme } from '@material-ui/core/styles';
@@ -27,11 +28,13 @@ const useStyles = makeStyles({
const LoginDialog = props => {
const theme = useTheme();
const { globalUrl, isLoaded, isLoggedIn, setIsLoggedIn, setCookie, register } = props;
const { globalUrl, isLoaded, isLoggedIn, setIsLoggedIn, setCookie, register, checkLogin } = props;
const [username, setUsername] = useState("");
const [password, setPassword] = useState("");
const [firstRequest, setFirstRequest] = useState(true);
const [loginLoading, setLoginLoading] = useState(false);
const [loginViewLoading, setLoginViewLoading] = useState(false);
const [ssoUrl, setSSOUrl] = useState("")
// Used to swap from login to register. True = login, false = register
@@ -47,7 +50,6 @@ const LoginDialog = props => {
window.location.pathname = "/workflows"
}
const checkAdmin = () => {
const url = globalUrl + '/api/v1/checkusers';
fetch(url, {
@@ -61,6 +63,20 @@ const LoginDialog = props => {
if (responseJson["success"] === false) {
setLoginInfo(responseJson["reason"])
} else {
if (responseJson.sso_url !== undefined && responseJson.sso_url !== null) {
setSSOUrl(responseJson.sso_url)
}
if (loginViewLoading) {
setLoginViewLoading(false)
checkLogin()
stop()
if (responseJson.reason !== undefined && responseJson.reason !== null) {
setLoginInfo(responseJson.reason)
}
}
if (responseJson.reason === "stay") {
window.location.pathname = "/adminsetup"
}
@@ -68,10 +84,21 @@ const LoginDialog = props => {
}),
)
.catch(error => {
setLoginInfo("Error logging in - please refresh in a minute ", error)
if (!loginViewLoading) {
setLoginViewLoading(true)
start()
}
})
}
const { start, stop } = useInterval({
duration: 3000,
startImmediate: false,
callback: () => {
checkAdmin()
}
})
if (firstRequest) {
setFirstRequest(false)
checkAdmin()
@@ -178,6 +205,50 @@ const LoginDialog = props => {
<div style={{position: "absolute", top: -imgsize/2-10, left: 250-imgsize/2, height: imgsize, width: imgsize, }}>
<img src="images/Shuffle_logo.png" style={{height: imgsize+10, width: imgsize+10, border: "2px solid rgba(255,255,255,0.6)", borderRadius: imgsize,}}/>
</div>
{loginViewLoading ?
<div style={{textAlign: "center", marginTop: 50, }}>
<Typography variant="body2" style={{marginBottom: 20, color: "white",}}>
Waiting for the Shuffle database to become available. This may take up to a minute.
</Typography>
{loginInfo === undefined || loginInfo === null || loginInfo.length === 0 ?
null
:
<div style={{ marginTop: "10px" }}>
Response: {loginInfo}
</div>
}
<CircularProgress color="secondary" style={{color: "white",}} />
<Paper style={{
paddingLeft: "30px",
paddingRight: "30px",
paddingBottom: "30px",
paddingTop: "30px",
position: "relative",
backgroundColor: theme.palette.inputColor,
textAlign: "left",
marginTop: 15,
}}>
<Typography variant="body2" style={{marginBottom: 20, color: "white",}}>
<b>Are you sure Shuffle is <a rel="norefferer" target="_blank" href="https://github.com/frikky/Shuffle/blob/master/.github/install-guide.md" style={{textDecoration: "none", color: "#f86a3e"}}>installed correctly</a>?</b>
</Typography>
<Typography variant="body2" style={{marginBottom: 20, color: "white",}}>
<b>1.</b> Make sure shuffle-database folder has correct access: <br/><br/>
sudo chown 1000:1000 -R shuffle-database
</Typography>
<Typography variant="body2" style={{marginBottom: 20, color: "white",}}>
<b>2</b>. Restart docker-compose:<br/><br/>
sudo docker-compose restart
</Typography>
</Paper>
<Typography variant="body2" style={{marginBottom: 10, color: "white", marginTop: 20, }}>
Need help? <a rel="norefferer" target="_blank" href="https://discord.gg/B2CBzUm" style={{textDecoration: "none", color: "#f86a3e"}}>Join the Discord!</a>
</Typography>
</div>
:
<form onSubmit={onSubmit} style={{ margin: "15px 15px 15px 15px", color: "white", }}>
<h2>{formtitle}</h2>
Username
@@ -233,14 +304,30 @@ const LoginDialog = props => {
/>
</div>
<div style={{ display: "flex", marginTop: "15px" }}>
<Button color="primary" variant="contained" type="submit" style={{ flex: "1", marginRight: "5px" }} disabled={!handleValidateForm() || loginLoading}>
<Button color="primary" variant="contained" type="submit" style={{ flex: "1", }} disabled={!handleValidateForm() || loginLoading}>
{loginLoading ? <CircularProgress color="secondary" style={{color: "white",}} /> : "SUBMIT"}
</Button>
</div>
<div style={{ marginTop: "10px" }}>
{loginInfo}
</div>
{ssoUrl !== undefined && ssoUrl !== null && ssoUrl.length > 0 ?
<div>
<Typography style={{textAlign: "center", }}>
Or
</Typography>
<div style={{textAlign: "center", margin: 10, }}>
<Button fullWidth color="secondary" variant="outlined" type="button" style={{ flex: "1", marginTop: 5}} onClick={() => {
console.log("CLICK")
window.location = ssoUrl
}}>
Use SSO
</Button>
</div>
</div>
: null}
</form>
}
</Paper>
</div>
frontend/src/views/SetAuthentication.jsx
+147
View File
@@ -0,0 +1,147 @@
import React, {useRef, useState, useEffect, useLayoutEffect} from 'react';
import { Typography, CircularProgress } from '@material-ui/core';
const SetAuthentication = (props) => {
const { globalUrl, isLoggedIn, isLoaded, userdata } = props;
const [firstRequest, setFirstRequest] = useState(true)
const [finished, setFinished] = useState(false)
const [response, setResponse] = useState("")
const [failed, setFailed] = useState(false)
if (firstRequest) {
setFirstRequest(false)
//code
//session_state
const urlSearchParams = new URLSearchParams(window.location.search);
const params = Object.fromEntries(urlSearchParams.entries());
const authenticationStore = []
var appAuthData = {
"label": "",
"app": {
"name": "",
"id": "",
"app_version": "",
},
"fields": [],
"type": "oauth2",
}
if (window !== undefined && window !== null) {
console.log(window.location)
appAuthData.fields.push({"key": "redirect_uri", "value": window.location.origin+window.location.pathname})
}
if (params.code !== undefined && params.code !== null) {
appAuthData.fields.push({"key": "code", "value": params.code})
}
if (params.session_state !== undefined && params.session_state !== null) {
appAuthData.fields.push({"key": "session_state", "value": params.session_state})
}
if (params.state !== undefined && params.state !== null) {
const paramsplit = params.state.split("&")
console.log(paramsplit)
for (var key in paramsplit) {
const query = paramsplit[key].split("=")
console.log(query)
if (query.length !== 2) {
console.log("INVALID QUERY: ", query)
continue
}
if (query[0] === "workflow_id") {
appAuthData.reference_workflow = query[1]
}
if (query[0] === "reference_action_id") {
//appAuthData.ReferenceWorkflow = query[1]
}
if (query[0] === "app_name") {
appAuthData.app.name = query[1]
appAuthData.label = "Oauth2 for "+query[1]
}
if (query[0] === "app_id") {
appAuthData.app.id = query[1]
}
if (query[0] === "app_version") {
appAuthData.app.app_version = query[1]
}
if (query[0] === "authentication_url") {
appAuthData.fields.push({"key": "authentication_url", "value": query[1]})
}
if (query[0] === "scope") {
appAuthData.fields.push({"key": "scope", "value": query[1]})
}
if (query[0] === "client_id") {
appAuthData.fields.push({"key": "client_id", "value": query[1]})
}
if (query[0] === "client_secret") {
appAuthData.fields.push({"key": "client_secret", "value": query[1]})
}
if (query[0] === "oauth_url") {
appAuthData.fields.push({"key": "oauth_url", "value": query[1]})
}
}
}
console.log(appAuthData)
fetch(globalUrl+"/api/v1/apps/authentication", {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
},
credentials: "include",
body: JSON.stringify(appAuthData),
})
.then((response) => {
if (response.status !== 200) {
console.log("Status not 200 for oauth2 authentication")
setFailed(true)
}
return response.json()
})
.then((responseJson) => {
//setUserSettings(responseJson)
console.log("Resp: ", responseJson)
setFinished(true)
setResponse(responseJson.reason)
setTimeout(() => {
window.close()
}, 1000)
})
.catch(error => {
console.log(error)
});
}
return (
<div style={{width: 1000, margin: "auto", itemAlign: "center",}}>
<Typography variant="h6" style={{marginLeft: "auto", marginRight: "auto", marginTop: 200, }}>
{!finished ? <CircularProgress /> : "DONE WITH AUTH - this will close soon!!"}
<div />
{failed ? "Failed setup. Error: " : ""} {response}
</Typography>
</div>
)
}
export default SetAuthentication;
frontend/src/views/SetAuthenticationSSO.jsx
+143
View File
@@ -0,0 +1,143 @@
import React, {useRef, useState, useEffect, useLayoutEffect} from 'react';
import { Typography, CircularProgress } from '@material-ui/core';
const SetAuthentication = (props) => {
const { globalUrl, isLoggedIn, isLoaded, userdata } = props;
const [firstRequest, setFirstRequest] = useState(true)
const [finished, setFinished] = useState(false)
const [response, setResponse] = useState("")
const [failed, setFailed] = useState(false)
if (firstRequest) {
setFirstRequest(false)
//code
//session_state
const urlSearchParams = new URLSearchParams(window.location.search);
const params = Object.fromEntries(urlSearchParams.entries());
const authenticationStore = []
var appAuthData = {
"label": "",
"app": {
"name": "",
"id": "",
"app_version": "",
},
"fields": [],
"type": "oauth2",
}
if (window !== undefined && window !== null) {
console.log(window.location)
appAuthData.fields.push({"key": "redirect_uri", "value": window.location.origin+window.location.pathname})
}
if (params.code !== undefined && params.code !== null) {
appAuthData.fields.push({"key": "code", "value": params.code})
}
if (params.session_state !== undefined && params.session_state !== null) {
appAuthData.fields.push({"key": "session_state", "value": params.session_state})
}
if (params.state !== undefined && params.state !== null) {
const paramsplit = params.state.split("&")
console.log(paramsplit)
for (var key in paramsplit) {
const query = paramsplit[key].split("=")
console.log(query)
if (query.length !== 2) {
console.log("INVALID QUERY: ", query)
continue
}
if (query[0] === "workflow_id") {
appAuthData.reference_workflow = query[1]
}
if (query[0] === "reference_action_id") {
//appAuthData.ReferenceWorkflow = query[1]
}
if (query[0] === "app_name") {
appAuthData.app.name = query[1]
appAuthData.label = "Oauth2 for "+query[1]
}
if (query[0] === "app_id") {
appAuthData.app.id = query[1]
}
if (query[0] === "app_version") {
appAuthData.app.app_version = query[1]
}
if (query[0] === "authentication_url") {
appAuthData.fields.push({"key": "authentication_url", "value": query[1]})
}
if (query[0] === "scope") {
appAuthData.fields.push({"key": "scope", "value": query[1]})
}
if (query[0] === "client_id") {
appAuthData.fields.push({"key": "client_id", "value": query[1]})
}
if (query[0] === "client_secret") {
appAuthData.fields.push({"key": "client_secret", "value": query[1]})
}
}
}
console.log(appAuthData)
fetch(globalUrl+"/api/v1/apps/authentication", {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
},
credentials: "include",
body: JSON.stringify(appAuthData),
})
.then((response) => {
if (response.status !== 200) {
console.log("Status not 200 for oauth2 authentication")
setFailed(true)
}
return response.json()
})
.then((responseJson) => {
//setUserSettings(responseJson)
console.log("Resp: ", responseJson)
setFinished(true)
setResponse(responseJson.reason)
setTimeout(() => {
window.close()
}, 1000)
})
.catch(error => {
console.log(error)
});
}
return (
<div style={{width: 1000, margin: "auto", itemAlign: "center",}}>
<Typography variant="h6" style={{marginLeft: "auto", marginRight: "auto", marginTop: 200, }}>
{!finished ? <CircularProgress /> : "DONE WITH AUTH - this will close soon!!"}
<div />
{failed ? "Failed setup. Error: " : ""} {response}
</Typography>
</div>
)
}
export default SetAuthentication;
frontend/src/views/Workflows.jsx
+403 -298
View File
File diff suppressed because it is too large Load Diff
functions/extensions/aws-lambda/deploy.sh
-9
View File
@@ -1,9 +0,0 @@
GOOS=linux go build main.go
zip function.zip main
aws lambda update-function-code \
--function-name shuffler-forwarder \
--runtime go1.* \
--zip-file fileb://function.zip \
--handler main \
--role arn:aws:iam::123456789012:role/execution_role
functions/extensions/aws-lambda/main.go
-66
View File
@@ -1,66 +0,0 @@
package main
import (
"context"
//"encoding/json"
//"fmt"
"github.com/aws/aws-lambda-go/lambda"
"net/http"
)
type LambdaPayload struct {
RequestContext struct {
Elb struct {
TargetGroupArn string `json:"targetGroupArn"`
} `json:"elb"`
} `json:"requestContext"`
HTTPMethod string `json:"httpMethod"`
Path string `json:"path"`
Headers map[string]string `json:"headers"`
QueryStringParameters map[string]string `json:"queryStringParameters"`
Body string `json:"body"`
IsBase64Encoded bool `json:"isBase64Encoded"`
}
type LambdaResponse struct {
IsBase64Encoded bool `json:"isBase64Encoded"`
StatusCode int `json:"statusCode"`
StatusDescription string `json:"statusDescription"`
Headers struct {
SetCookie string `json:"Set-cookie"`
ContentType string `json:"Content-Type"`
} `json:"headers"`
Body string `json:"body"`
}
func lambda_handler(ctx context.Context, payload LambdaPayload) (LambdaResponse, error) {
response := &LambdaResponse{}
response.Headers.ContentType = "text/html"
response.StatusCode = http.StatusBadRequest
response.StatusDescription = http.StatusText(http.StatusBadRequest)
if payload.HTTPMethod == http.MethodGet && payload.Path == "/myfavoritecar" {
res := "TEST"
//car := &Car{}
//car.Model = "Corvette"
//car.Color = "Red"
//car.Year = 1999
//res, err := json.Marshal(car)
//if err != nil {
// fmt.Println(err)
// response.StatusCode = http.StatusInternalServerError
// response.StatusDescription = http.StatusText(http.StatusInternalServerError)
// return *response, err
//}
response.Headers.ContentType = "application/json"
response.Body = string(res)
response.StatusCode = http.StatusOK
response.StatusDescription = http.StatusText(http.StatusOK)
return *response, nil
} else {
return *response, nil
}
}
func main() {
lambda.Start(lambda_handler)
}
functions/extensions/elasticsearch/run.sh
+19
View File
@@ -0,0 +1,19 @@
#docker run -d -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e ELASTICSEARCH_USERNAME=frikky -e ELASTICSEARCH_PASSWORD=likeme -e xpack.security.enabled=true docker.elastic.co/elasticsearch/elasticsearch:7.12.1
#docker run -d -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.12.1
#
#
#
#echo "\nWaiting for 1.5 minute, then adding data"
#sleep 90
#echo "\nSlept 90 seconds: ADDING DATA"
#curl -XPOST http://localhost:9200/_security/user/frikky -H "Content-Type: application/json" -d '{"enabled": true, "email": "frikky@shuffler.io"}'
#curl -XPOST -u frikky:likeme http://localhost:9200/samples/_doc -H "Content-Type: application/json" -d '{"src": "122.14.137.67", "dst": "103.35.191.16", "message": "alert", "md5": "CAEF973033E593C625FB2AA34F7026DC", "sha256": "DB1AEC5222075800EDA75D7205267569679B424E5C58A28102417F46D3B5790D", "hits": 0}'
#echo
#curl -XPOST -u frikky:likeme http://localhost:9200/samples/_doc -H "Content-Type: application/json" -d '{"src": "134.119.219.71", "dst": "103.35.191.41", "message": "alert", "md5": "9498FF82A64FF445398C8426ED63EA5B", "sha256": "8B2E701E91101955C73865589A4C72999AEABC11043F712E05FDB1C17C4AB19A", "hits": 0}'
#
#echo
#curl -XPOST -u frikky:likeme http://localhost:9200/samples2/_doc -H "Content-Type: application/json" -d '{"src": "122.14.137.67", "dst": "103.35.191.16", "message": "alert", "md5": "CAEF973033E593C625FB2AA34F7026DC", "sha256": "DB1AEC5222075800EDA75D7205267569679B424E5C58A28102417F46D3B5790D"}'
#echo
#curl -XPOST -u frikky:likeme http://localhost:9200/samples2/_doc -H "Content-Type: application/json" -d '{"src": "134.119.219.71", "dst": "103.35.191.41", "message": "alert", "md5": "9498FF82A64FF445398C8426ED63EA5B", "sha256": "8B2E701E91101955C73865589A4C72999AEABC11043F712E05FDB1C17C4AB19A"}'
#echo
functions/extensions/k8s/shuffle/.helmignore
+23
View File
@@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
functions/extensions/k8s/shuffle/Chart.yaml
+24
View File
@@ -0,0 +1,24 @@
apiVersion: v2
name: shuffle
description: A Helm chart for Kubernetes
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "1.16.0"
functions/extensions/k8s/shuffle/templates/NOTES.txt
+22
View File
@@ -0,0 +1,22 @@
1. Get the application URL by running these commands:
{{- if .Values.ingress.enabled }}
{{- range $host := .Values.ingress.hosts }}
{{- range .paths }}
http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
{{- end }}
{{- end }}
{{- else if contains "NodePort" .Values.service.type }}
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "shuffle.fullname" . }})
export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
echo http://$NODE_IP:$NODE_PORT
{{- else if contains "LoadBalancer" .Values.service.type }}
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "shuffle.fullname" . }}'
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "shuffle.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
echo http://$SERVICE_IP:{{ .Values.service.port }}
{{- else if contains "ClusterIP" .Values.service.type }}
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "shuffle.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
echo "Visit http://127.0.0.1:8080 to use your application"
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
{{- end }}
functions/extensions/k8s/shuffle/templates/_helpers.tpl
+62
View File
@@ -0,0 +1,62 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "shuffle.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "shuffle.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "shuffle.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "shuffle.labels" -}}
helm.sh/chart: {{ include "shuffle.chart" . }}
{{ include "shuffle.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "shuffle.selectorLabels" -}}
app.kubernetes.io/name: {{ include "shuffle.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "shuffle.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "shuffle.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}
functions/extensions/k8s/shuffle/templates/deployment.yaml
+253
View File
@@ -0,0 +1,253 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Values.name }}frontend
namespace: {{ .Values.namespace | quote }}
spec:
replicas: 1
selector:
matchLabels:
service: shuffle
app: shuffle-frontend
template:
metadata:
labels:
service: shuffle
app: shuffle-frontend
spec:
containers:
- name: shuffle-frontend
image: ghcr.io/frikky/shuffle-frontend:nightly
imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
env:
- name: BACKEND_HOSTNAME
value: backend-service
- name: TZ
value: Asia/Shanghai
ports:
- name: http
containerPort: 80
hostPort: 3001
- name: https
containerPort: 443
hostname: shuffle-frontend
restartPolicy: Always
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Values.name }}backend
namespace: {{ .Values.namespace | quote }}
spec:
replicas: 1
selector:
matchLabels:
service: shuffle
app: shuffle-backend
template:
metadata:
labels:
service: shuffle
app: shuffle-backend
spec:
containers:
- name: shuffle-backend
image: ghcr.io/frikky/shuffle-backend:nightly
env:
- name: BACKEND_HOSTNAME
value: "backend-service"
- name: BACKEND_PORT
value: "5001"
- name: ENVIRONMENT_NAME
value: "Shuffle"
- name: HTTPS_PROXY
- name: HTTP_PROXY
- name: ORG_ID
value: "Shuffle"
- name: OUTER_HOSTNAME
value: "backend-service"
- name: SHUFFLE_APP_FORCE_UPDATE
value: "false"
- name: SHUFFLE_APP_HOTLOAD_FOLDER
value: "/shuffle-apps"
- name: SHUFFLE_APP_HOTLOAD_LOCATION
value: "/shuffle-apps"
- name: SHUFFLE_CONTAINER_AUTO_CLEANUP
value: "false"
- name: SHUFFLE_DEFAULT_APIKEY
- name: SHUFFLE_DEFAULT_PASSWORD
- name: SHUFFLE_DEFAULT_USERNAME
- name: SHUFFLE_DOWNLOAD_AUTH_BRANCH
- name: SHUFFLE_DOWNLOAD_AUTH_PASSWORD
- name: SHUFFLE_DOWNLOAD_AUTH_USERNAME
- name: SHUFFLE_DOWNLOAD_WORKFLOW_BRANCH
- name: SHUFFLE_DOWNLOAD_WORKFLOW_LOCATION
- name: SHUFFLE_DOWNLOAD_WORKFLOW_PASSWORD
- name: SHUFFLE_DOWNLOAD_WORKFLOW_USERNAME
- name: SHUFFLE_ELASTIC
value: "true"
- name: SHUFFLE_OPENSEARCH_APIKEY
- name: SHUFFLE_OPENSEARCH_CERTIFICATE_FILE
- name: SHUFFLE_OPENSEARCH_CLOUDID
- name: SHUFFLE_OPENSEARCH_PASSWORD
- name: SHUFFLE_OPENSEARCH_PROXY
- name: SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY
value: "true"
- name: SHUFFLE_OPENSEARCH_URL
value: http://opensearch-service:9200
- name: SHUFFLE_OPENSEARCH_USERNAME
value: ""
- name: SHUFFLE_PASS_APP_PROXY
value: "FALSE"
- name: SHUFFLE_PASS_WORKER_PROXY
value: "FALSE"
volumeMounts:
- mountPath: /var/run/docker.sock
name: docker-sock
- mountPath: /shuffle-apps
name: shuffle-app-hotload-location
- mountPath: /shuffle-files
name: shuffle-file-location
hostname: shuffle-backend
volumes:
- name: docker-sock
hostPath:
path: /var/run/docker.sock
- name: shuffle-app-hotload-location
hostPath:
path: /data/kubernetes/shuffle-apps
type: DirectoryOrCreate
- name: shuffle-file-location
hostPath:
path: /data/kubernetes/shuffle-files
type: DirectoryOrCreate
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Values.name }}orborus
namespace: {{ .Values.namespace | quote }}
spec:
replicas: 1
selector:
matchLabels:
service: shuffle
app: shuffle-orborus
template:
metadata:
labels:
service: shuffle
app: shuffle-orborus
spec:
containers:
- name: shuffle-orborus
image: ghcr.io/frikky/shuffle-orborus:nightly
env:
- name: RUNNING_MODE
value: kubernetes
- name: BASE_URL
value: http://backend-service:5001
- name: CLEANUP
value: "false"
- name: DOCKER_API_VERSION
value: "1.40"
- name: ENVIRONMENT_NAME
value: Shuffle
- name: HTTPS_PROXY
- name: HTTP_PROXY
- name: ORG_ID
value: Shuffle
- name: SHUFFLE_APP_SDK_VERSION
value: 0.8.97
- name: SHUFFLE_BASE_IMAGE_NAME
value: frikky
- name: SHUFFLE_BASE_IMAGE_REGISTRY
value: ghcr.io
- name: SHUFFLE_BASE_IMAGE_TAG_SUFFIX
value: "-0.8.80"
- name: SHUFFLE_ORBORUS_EXECUTION_TIMEOUT
value: "600"
- name: SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY
value: "50"
- name: SHUFFLE_PASS_WORKER_PROXY
value: "TRUE"
- name: SHUFFLE_WORKER_VERSION
value: nightly
- name: TZ
value: Asia/Shanghai
volumeMounts:
- mountPath: /var/run/docker.sock
name: docker-sock
hostname: shuffle-orborus
volumes:
- name: docker-sock
hostPath:
path: /var/run/docker.sock
#---
#apiVersion: apps/v1
#kind: Deployment
#metadata:
# name: {{ .Values.name }}opensearch
# namespace: {{ .Values.namespace | quote }}
#spec:
# replicas: 1
# selector:
# matchLabels:
# service: shuffle
# app: shuffle-opensearch
# template:
# metadata:
# labels:
# service: shuffle
# app: shuffle-opensearch
# spec:
# nodeSelector:
# node.bdlab-venus.com/opensearch: available
# initContainers:
# - name: permissions-fix
# image: frikky/busybox
# #volumeMounts:
# # - name: opensearch-claim0
# # mountPath: /usr/share/elasticsearch/data
# command: [ 'chown' ]
# args: [ '1000:1000', '/usr/share/elasticsearch/data' ]
# containers:
# - name: shuffle-opensearch
# image: opensearchproject/opensearch:1.0.1
# env:
# - name: TZ
# value: Asia/Shanghai
# - name: bootstrap.memory_lock
# value: "false"
# - name: OPENSEARCH_JAVA_OPTS
# value: "-Xms1024m -Xmx1024m"
# - name: opendistro_security.disabled
# value: "true"
# - name: cluster.routing.allocation.disk.threshold_enabled
# value: "false"
# - name: cluster.name
# value: shuffle-cluster
# - name: node.name
# value: opensearch-service
# - name: discovery.seed_hosts
# value: opensearch-service
# - name: cluster.initial_master_nodes
# value: opensearch-service
# volumeMounts:
# - mountPath: /usr/share/opensearch/data
# name: opensearch-claim0
#volumes:
# - name: opensearch-claim0
# persistentVolumeClaim:
# claimName: opensearch-claim0
# volumeMounts:
# - mountPath: /usr/share/opensearch/data
# readOnly: true
# name: db-location
# volumes:
# - name: db-location
# hostPath:
# path: /data/kubernetes/shuffle-opensearch
# type: DirectoryOrCreate
functions/extensions/k8s/shuffle/templates/hpa.yaml
+28
View File
@@ -0,0 +1,28 @@
{{- if .Values.autoscaling.enabled }}
apiVersion: autoscaling/v2beta1
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "shuffle.fullname" . }}
labels:
{{- include "shuffle.labels" . | nindent 4 }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "shuffle.fullname" . }}
minReplicas: {{ .Values.autoscaling.minReplicas }}
maxReplicas: {{ .Values.autoscaling.maxReplicas }}
metrics:
{{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
- type: Resource
resource:
name: cpu
targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
{{- end }}
{{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
{{- end }}
{{- end }}
functions/extensions/k8s/shuffle/templates/ingress.yaml
+61
View File
@@ -0,0 +1,61 @@
{{- if .Values.ingress.enabled -}}
{{- $fullName := include "shuffle.fullname" . -}}
{{- $svcPort := .Values.service.port -}}
{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
{{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
{{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
{{- end }}
{{- end }}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1beta1
{{- else -}}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: {{ $fullName }}
labels:
{{- include "shuffle.labels" . | nindent 4 }}
{{- with .Values.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
ingressClassName: {{ .Values.ingress.className }}
{{- end }}
{{- if .Values.ingress.tls }}
tls:
{{- range .Values.ingress.tls }}
- hosts:
{{- range .hosts }}
- {{ . | quote }}
{{- end }}
secretName: {{ .secretName }}
{{- end }}
{{- end }}
rules:
{{- range .Values.ingress.hosts }}
- host: {{ .host | quote }}
http:
paths:
{{- range .paths }}
- path: {{ .path }}
{{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
pathType: {{ .pathType }}
{{- end }}
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ $fullName }}
port:
number: {{ $svcPort }}
{{- else }}
serviceName: {{ $fullName }}
servicePort: {{ $svcPort }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
functions/extensions/k8s/shuffle/templates/persistent-volume.yaml
+24
View File
@@ -0,0 +1,24 @@
#apiVersion: v1
#kind: PersistentVolume
#metadata:
# name: opensearch-claim0
# labels:
# app: opensearch-claim0
#spec:
# capacity:
# storage: "10G"
# volumeMode: Filesystem
# persistentVolumeReclaimPolicy: Retain
# storageClassName: local-storage
# accessModes:
# - "ReadWriteOnce"
# local:
# path: "/data/kubernetes/shuffle-opensearch"
# nodeAffinity:
# required:
# nodeSelectorTerms:
# - matchExpressions:
# - key: node.dollar.com/opensearch
# operator: In
# values:
# - available
functions/extensions/k8s/shuffle/templates/persistentvolumeclaim.yaml
+17
View File
@@ -0,0 +1,17 @@
#apiVersion: v1
#kind: PersistentVolumeClaim
#metadata:
# name: opensearch-claim0
# namespace: {{ .Values.namespace }}
# labels:
# app: opensearch-claim0
#spec:
# selector:
# matchLabels:
# app: opensearch-claim0
# accessModes:
# - ReadWriteOnce
# storageClassName: local-storage
# resources:
# requests:
# storage: 5Gi
functions/extensions/k8s/shuffle/templates/service.yaml
+52
View File
@@ -0,0 +1,52 @@
apiVersion: v1
kind: Service
metadata:
name: backend-service
namespace: {{ .Values.namespace | quote }}
spec:
ports:
- name: "5001"
port: 5001
targetPort: 5001
selector:
app: shuffle-backend
---
apiVersion: v1
kind: Service
metadata:
name: frontend-service
namespace: {{ .Values.namespace | quote }}
spec:
type: NodePort
externalTrafficPolicy: Local
ports:
- name: "3001"
port: 3001
nodePort: 3001
targetPort: 80
- name: "3443"
port: 3443
nodePort: 3443
targetPort: 443
protocol: TCP
selector:
app: shuffle-frontend
#---
#apiVersion: v1
#kind: Service
#metadata:
# name: opensearch-service
# namespace: {{ .Values.namespace | quote }}
#spec:
# type: NodePort
# externalTrafficPolicy: Local
# ports:
# - name: "9200"
# port: 9200
# targetPort: 9200
# nodePort: 9200
# protocol: TCP
# selector:
# app: shuffle-opensearch
functions/extensions/k8s/shuffle/templates/serviceaccount.yaml
+12
View File
@@ -0,0 +1,12 @@
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "shuffle.serviceAccountName" . }}
labels:
{{- include "shuffle.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
functions/extensions/k8s/shuffle/templates/tests/test-connection.yaml
+15
View File
@@ -0,0 +1,15 @@
apiVersion: v1
kind: Pod
metadata:
name: "{{ include "shuffle.fullname" . }}-test-connection"
labels:
{{- include "shuffle.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": test
spec:
containers:
- name: wget
image: busybox
command: ['wget']
args: ['{{ include "shuffle.fullname" . }}:{{ .Values.service.port }}']
restartPolicy: Never
functions/extensions/k8s/shuffle/values.yaml
+82
View File
@@ -0,0 +1,82 @@
# Default values for shuffle.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 1
image:
repository: nginx
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: ""
imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""
serviceAccount:
# Specifies whether a service account should be created
create: true
# Annotations to add to the service account
annotations: {}
# The name of the service account to use.
# If not set and create is true, a name is generated using the fullname template
name: ""
podAnnotations: {}
podSecurityContext: {}
# fsGroup: 2000
securityContext: {}
# capabilities:
# drop:
# - ALL
# readOnlyRootFilesystem: true
# runAsNonRoot: true
# runAsUser: 1000
service:
type: ClusterIP
port: 80
ingress:
enabled: false
className: ""
annotations: {}
# kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
hosts:
- host: chart-example.local
paths:
- path: /
pathType: ImplementationSpecific
tls: []
# - secretName: chart-example-tls
# hosts:
# - chart-example.local
resources: {}
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 100
targetCPUUtilizationPercentage: 80
# targetMemoryUtilizationPercentage: 80
nodeSelector: {}
tolerations: []
affinity: {}
functions/extensions/kafka/README.md
-4
View File
@@ -1,4 +0,0 @@
```
curl -sSL https://raw.githubusercontent.com/bitnami/bitnami-docker-kafka/master/docker-compose.yml > docker-compose.yml
docker-compose up -d
```
functions/extensions/kafka/docker-compose.yml
-28
View File
@@ -1,28 +0,0 @@
version: "2"
services:
zookeeper:
image: docker.io/bitnami/zookeeper:3
ports:
- "2181:2181"
volumes:
- "zookeeper_data:/bitnami"
environment:
- ALLOW_ANONYMOUS_LOGIN=yes
kafka:
image: docker.io/bitnami/kafka:2
ports:
- "9092:9092"
volumes:
- "kafka_data:/bitnami"
environment:
- KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
- ALLOW_PLAINTEXT_LISTENER=yes
depends_on:
- zookeeper
volumes:
zookeeper_data:
driver: local
kafka_data:
driver: local
functions/extensions/kafka/kafka_local.py
-54
View File
@@ -1,54 +0,0 @@
import json
import os
import requests
from time import sleep
from kafka import KafkaProducer, KafkaConsumer
import kafka
shuffle_url = os.getenv("SHUFFLE_URL")
shuffle_apikey = os.getenv("SHUFFLE_APIKEY")
shuffle_workflow = os.getenv("SHUFFLE_WORKFLOW")
headers = {"Authorization": "Bearer %s" % shuffle_apikey}
topic = "workflow_%s" % shuffle_workflow
server = "localhost:9092"
def produce():
print("Starting producer")
producer = KafkaProducer(
bootstrap_servers=[server],
value_serializer=lambda x:
json.dumps(x).encode('utf-8')
)
print("Adding data!")
for e in range(15):
data = {"some": e, "data": "luuuuul"}
try:
ret = producer.send(topic, value=data)
print(ret.get())
except kafka.errors.KafkaTimeoutError as e:
print("Kafka error: %s" % e)
continue
def consume():
print("Starting consumer")
consumer = KafkaConsumer(
topic,
bootstrap_servers=[server],
auto_offset_reset="earliest",
enable_auto_commit=True,
value_deserializer=lambda x: json.loads(x.decode('utf-8'))
)
print("Getting data")
for message in consumer:
message = message.value
print("MSG: ", message)
ret = requests.post("%s/api/v1/%s/execute" % shuffle_url, headers=headers, data=message)
print(ret.status_code)
print(ret.text)
if __name__ == "__main__":
produce()
#consume()
functions/extensions/kafka/requirements.txt
-2
View File
@@ -1,2 +0,0 @@
kafka
kafka-python
functions/extensions/misp/Dockerfile
+23
View File
@@ -0,0 +1,23 @@
FROM python:3.9.4-alpine as base
FROM base as builder
RUN mkdir /install
WORKDIR /install
FROM base
RUN apk add g++
COPY --from=builder /install /usr/local
COPY requirements.txt /requirements.txt
RUN pip3 install -r /requirements.txt
RUN mkdir /app
WORKDIR /app
COPY requirements.txt /app/requirements.txt
RUN python3 -m pip install -r /app/requirements.txt
COPY sub.py /app/sub.py
CMD ["python3", "sub.py"]
functions/extensions/misp/docker-compose.yml
+9
View File
@@ -0,0 +1,9 @@
version: '3'
services:
zmq:
image: ghcr.io/frikky/shuffle-zmq:latest
environment:
- ZMQ_HOSTNAME=localhost
- ZMQ_PORT=50000
- ZMQ_FORWARD_URL=https://shuffler.io/api/v1/hooks/webhook_e09bea36-9976-1421-82bc-b8764ca83c1e
restart: unless-stopped
functions/extensions/misp/requirements.txt
+2
View File
@@ -0,0 +1,2 @@
pyzmq
requests
functions/extensions/misp/sub.py
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
print("Running imports")
import sys
import zmq
import json
import time
import pprint
import os
import sys
import requests
forward_url = os.getenv("ZMQ_FORWARD_URL", "")
print("Checking forward url (ZMQ_FORWARD_URL): %s" % forward_url)
def handle_hook(data):
ret = requests.post(forward_url, json=data)
print(ret.text)
print(ret.status_code)
def main():
host = os.getenv("ZMQ_HOST", "localhost")
port = os.getenv("ZMQ_PORT", "50000")
if len(forward_url) == 0:
print("Failed to start - define ZMQ_FORWARD_URL for webhook forwarder")
exit(0)
print("Starting connection setup to %s:%s" % (host, port))
context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.connect ("tcp://%s:%s" % (host, port))
socket.setsockopt(zmq.SUBSCRIBE, b'')
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
print("Starting zmq check for %s:%s" % (host, port))
while True:
socks = dict(poller.poll(timeout=None))
if socket in socks and socks[socket] == zmq.POLLIN:
message = socket.recv()
#print(message)
topic, s, m = message.decode('utf-8').partition(" ")
d = json.loads(m)
try:
# print test if you want status (heartbeat)
test = d["status"]
except KeyError:
handle_hook(d)
time.sleep(1)
if __name__ == "__main__":
print("In init ")
main()
functions/onprem/orborus/Dockerfile
+3 -3
View File
@@ -1,4 +1,4 @@
FROM golang:1.16.0-buster as builder
FROM golang:1.17.2-buster as builder
RUN mkdir /app
WORKDIR /app
@@ -15,8 +15,8 @@ RUN go get github.com/docker/docker/api/types && \
RUN go build
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o orborus .
FROM alpine:3.12
RUN apk add --no-cache bash
FROM alpine:3.14.2
RUN apk add --no-cache bash tzdata
COPY --from=builder /app/ /
CMD ["./orborus"]
functions/onprem/orborus/build.sh
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=shuffle-orborus
VERSION=0.8.98
VERSION=0.9.23
echo "Running docker build with $NAME:$VERSION"
#docker rmi frikky/shuffle:$NAME --force
functions/onprem/orborus/go.mod
+31 -8
View File
@@ -3,21 +3,44 @@ module orborus
go 1.13
require (
cloud.google.com/go/datastore v1.6.0 // indirect
cloud.google.com/go/storage v1.18.1 // indirect
github.com/Masterminds/semver v1.5.0 // indirect
github.com/Microsoft/go-winio v0.4.16 // indirect
github.com/algolia/algoliasearch-client-go/v3 v3.18.1 // indirect
github.com/Microsoft/go-winio v0.5.0 // indirect
github.com/algolia/algoliasearch-client-go/v3 v3.21.0 // indirect
github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect
github.com/containerd/containerd v1.4.3 // indirect
github.com/containerd/containerd v1.5.7 // indirect
github.com/creack/pty v1.1.16 // indirect
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/docker v20.10.1+incompatible
github.com/docker/docker v20.10.9+incompatible
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/frikky/shuffle-shared v0.0.40
github.com/gogo/protobuf v1.3.1 // indirect
github.com/mackerelio/go-osstat v0.1.0
github.com/frikky/kin-openapi v0.40.0 // indirect
github.com/frikky/shuffle-shared v0.1.15
github.com/go-openapi/swag v0.19.15 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/kr/pretty v0.3.0 // indirect
github.com/mackerelio/go-osstat v0.2.1
github.com/mailru/easyjson v0.7.7 // indirect
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.1 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/rogpeppe/go-internal v1.8.0 // indirect
github.com/satori/go.uuid v1.2.0
github.com/sirupsen/logrus v1.7.0 // indirect
go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
golang.org/x/mod v0.5.1 // indirect
golang.org/x/net v0.0.0-20211014172544-2b766c08f1c0 // indirect
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
golang.org/x/tools v0.1.7 // indirect
google.golang.org/genproto v0.0.0-20211013025323-ce878158c4d4 // indirect
google.golang.org/grpc v1.41.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
)
functions/onprem/orborus/go.sum
+824
View File
File diff suppressed because it is too large Load Diff
functions/onprem/orborus/orborus.go
+23 -15
View File
@@ -51,11 +51,12 @@ var baseimagetagsuffix = os.Getenv("SHUFFLE_BASE_IMAGE_TAG_SUFFIX")
var orgId = os.Getenv("ORG_ID")
var baseUrl = os.Getenv("BASE_URL")
var environment = os.Getenv("ENVIRONMENT_NAME")
var dockerApiVersion = os.Getenv("DOCKER_API_VERSION")
var runningMode = strings.ToLower(os.Getenv("RUNNING_MODE"))
var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP"))
var timezone = os.Getenv("TZ")
var containerName = os.Getenv("ORBORUS_CONTAINER_NAME")
var executionIds = []string{}
var dockercli *dockerclient.Client
@@ -101,18 +102,25 @@ func getThisContainerId() {
if err == nil {
containerId = strings.TrimSpace(string(out))
// cgroup error. Hardcoding this.
// cgroup error. Use fallback strategy below.
// https://github.com/moby/moby/issues/7015
//log.Printf("Checking if %s is in %s", ".scope", string(out))
if strings.Contains(string(out), ".scope") {
containerId = "shuffle-orborus"
containerId = ""
//docker-76c537e9a4b7c7233011f5d70e6b7f2d600b6413ac58a96519b8dca7a3f7117a.scope
}
} else {
if fCol == "0" {
containerId = "shuffle-orborus"
log.Printf("[WARNING] Failed getting container ID: %s", err)
}
log.Printf("[WARNING] Failed getting container ID: %s", err)
}
}
if containerId == "" {
if containerName != "" {
containerId = containerName
log.Printf("[INFO] Falling back to CONTAINER_NAME as container ID")
} else {
containerId = "shuffle-orborus"
log.Printf(`[WARNING] CONTAINER_NAME is not set. Falling back to default name "%s" as container ID`, containerId)
}
}
@@ -137,14 +145,7 @@ func deployWorker(image string, identifier string, env []string) {
Binds: []string{
"/var/run/docker.sock:/var/run/docker.sock:rw",
},
}
// form container id and use it as network source if it's not empty
if containerId != "" {
//log.Printf("[INFO] Found container ID %s", containerId)
hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId))
} else {
//log.Printf("[INFO] Empty self container id, continue without NetworkMode")
NetworkMode: container.NetworkMode(fmt.Sprintf("container:%s", containerId)),
}
if cleanupEnv == "true" {
@@ -346,6 +347,12 @@ func main() {
os.Exit(3)
}
if timezone == "" {
timezone = "Europe/Amsterdam"
}
log.Printf("[INFO] Running with timezone %s", timezone)
workerTimeout := 600
if workerTimeoutEnv != "" {
tmpInt, err := strconv.Atoi(workerTimeoutEnv)
@@ -560,6 +567,7 @@ func main() {
fmt.Sprintf("ENVIRONMENT_NAME=%s", environment),
fmt.Sprintf("BASE_URL=%s", baseUrl),
fmt.Sprintf("CLEANUP=%s", cleanupEnv),
fmt.Sprintf("TZ=%s", timezone),
fmt.Sprintf("SHUFFLE_PASS_APP_PROXY=%s", os.Getenv("SHUFFLE_PASS_APP_PROXY")),
}
functions/onprem/worker/Dockerfile
+3 -3
View File
@@ -1,4 +1,4 @@
FROM golang:1.16.0-buster as builder
FROM golang:1.17.2-buster as builder
WORKDIR /app
#RUN go get github.com/docker/docker/api/types github.com/docker/docker/api/types/container github.com/docker/docker/client
@@ -25,13 +25,13 @@ RUN go build
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker .
## ALPINE IMAGE
FROM alpine:3.12
FROM alpine:3.14.2
ENV SHUFFLE_BASE_IMAGE_REGISTRY=docker.io
ENV SHUFFLE_BASE_IMAGE_NAME=frikky/shuffle
ENV SHUFFLE_BASE_IMAGE_TAG_SUFFIX=0.8.70
RUN apk add --no-cache bash
RUN apk add --no-cache bash tzdata
COPY --from=builder /app/ /
CMD ["./worker"]
functions/onprem/worker/build.sh
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=shuffle-worker
VERSION=0.8.101
VERSION=0.9.23
echo "Running docker build with $NAME:$VERSION"
#CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin .
functions/onprem/worker/go.mod
+29 -6
View File
@@ -3,24 +3,47 @@ module worker
go 1.15
require (
cloud.google.com/go/datastore v1.6.0 // indirect
cloud.google.com/go/storage v1.18.1 // indirect
github.com/Masterminds/semver v1.5.0 // indirect
github.com/Microsoft/go-winio v0.4.16 // indirect
github.com/algolia/algoliasearch-client-go/v3 v3.18.1 // indirect
github.com/Microsoft/go-winio v0.5.0 // indirect
github.com/algolia/algoliasearch-client-go/v3 v3.21.0 // indirect
github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect
github.com/containerd/containerd v1.4.4 // indirect
github.com/containerd/containerd v1.5.7 // indirect
github.com/creack/pty v1.1.16 // indirect
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/docker v20.10.5+incompatible
github.com/docker/docker v20.10.9+incompatible
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/frikky/shuffle-shared v0.0.63
github.com/frikky/kin-openapi v0.40.0 // indirect
github.com/frikky/shuffle-shared v0.1.15
github.com/fsouza/go-dockerclient v1.7.2
github.com/go-git/go-billy/v5 v5.3.1 // indirect
github.com/go-git/go-git/v5 v5.4.2 // indirect
github.com/go-openapi/swag v0.19.15 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-github/v28 v28.1.1 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/gorilla/mux v1.8.0
github.com/kr/pretty v0.3.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.1 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pkg/errors v0.9.1 // indirect
google.golang.org/grpc v1.37.1 // indirect
github.com/rogpeppe/go-internal v1.8.0 // indirect
go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
golang.org/x/mod v0.5.1 // indirect
golang.org/x/net v0.0.0-20211014172544-2b766c08f1c0 // indirect
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
golang.org/x/tools v0.1.7 // indirect
google.golang.org/genproto v0.0.0-20211013025323-ce878158c4d4 // indirect
google.golang.org/grpc v1.41.0 // indirect
gopkg.in/src-d/go-git.v4 v4.13.1 // indirect
)
functions/onprem/worker/go.sum
+1429
View File
File diff suppressed because it is too large Load Diff
functions/onprem/worker/worker.go
+92 -105
View File
@@ -16,7 +16,6 @@ import (
"net/http"
"net/url"
"os"
"os/exec"
"strings"
"time"
@@ -39,9 +38,9 @@ var environment = os.Getenv("ENVIRONMENT_NAME")
var baseUrl = os.Getenv("BASE_URL")
var appCallbackUrl = os.Getenv("BASE_URL")
var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP"))
var timezone = os.Getenv("TZ")
var baseimagename = "frikky/shuffle"
var registryName = "registry.hub.docker.com"
var fallbackName = "shuffle-orborus"
var sleepTime = 2
var requestCache *cache.Cache
var topClient *http.Client
@@ -60,38 +59,6 @@ var startAction string
var results []shuffle.ActionResult
var allLogs map[string]string
var containerId string
// form container id of current running container
func getThisContainerId() string {
if len(containerId) > 0 {
return containerId
}
id := ""
cmd := fmt.Sprintf("cat /proc/self/cgroup | grep memory | tail -1 | cut -d/ -f3 | grep -o -E '[0-9A-z]{64}'")
out, err := exec.Command("bash", "-c", cmd).Output()
if err == nil {
id = strings.TrimSpace(string(out))
//log.Printf("Checking if %s is in %s", ".scope", string(out))
if strings.Contains(string(out), ".scope") {
id = fallbackName
}
}
return id
}
func init() {
containerId = getThisContainerId()
if len(containerId) == 0 {
log.Printf("[WARNING] No container ID found. Not running containerized? This should only show during testing")
} else {
log.Printf("[INFO] Found container ID for this worker: %s", containerId)
}
}
// removes every container except itself (worker)
func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason string, handleResultSend bool) {
log.Printf("[INFO] Shutdown (%s) started with reason %#v. Result amount: %d. ResultsSent: %d, Send result: %#v", workflowExecution.Status, reason, len(workflowExecution.Results), requestsSent, handleResultSend)
@@ -134,62 +101,67 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason
log.Printf("[INFO] NOT cleaning up containers. IDS: %d, CLEANUP env: %s", len(containerIds), cleanupEnv)
}
abortUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowExecution.Workflow.ID, workflowExecution.ExecutionId)
if len(reason) > 0 && len(nodeId) > 0 {
log.Printf("[INFO] Running abort of workflow because it should be finished")
path := fmt.Sprintf("?reason=%s", url.QueryEscape(reason))
if len(nodeId) > 0 {
path += fmt.Sprintf("&node=%s", url.QueryEscape(nodeId))
}
if len(environment) > 0 {
path += fmt.Sprintf("&env=%s", url.QueryEscape(environment))
}
//fmt.Println(url.QueryEscape(query))
abortUrl += path
log.Printf("[INFO] Abort URL: %s", abortUrl)
req, err := http.NewRequest(
"GET",
abortUrl,
nil,
)
if err != nil {
log.Println("[INFO] Failed building request: %s", err)
}
// FIXME: Add an API call to the backend
authorization := os.Getenv("AUTHORIZATION")
if len(authorization) > 0 {
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", authorization))
} else {
log.Printf("[ERROR] No authorization specified for abort")
}
req.Header.Add("Content-Type", "application/json")
client := &http.Client{
Transport: &http.Transport{
Proxy: nil,
},
}
httpProxy := os.Getenv("HTTP_PROXY")
httpsProxy := os.Getenv("HTTPS_PROXY")
if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" {
client = &http.Client{}
} else {
if len(httpProxy) > 0 {
log.Printf("[INFO] Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
abortUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowExecution.Workflow.ID, workflowExecution.ExecutionId)
path := fmt.Sprintf("?reason=%s", url.QueryEscape(reason))
if len(nodeId) > 0 {
path += fmt.Sprintf("&node=%s", url.QueryEscape(nodeId))
}
if len(httpsProxy) > 0 {
log.Printf("[INFO] Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
if len(environment) > 0 {
path += fmt.Sprintf("&env=%s", url.QueryEscape(environment))
}
}
log.Printf("[INFO] All App Logs: %#v", allLogs)
_, err = client.Do(req)
if err != nil {
log.Printf("[WARNING] Failed abort request: %s", err)
//fmt.Println(url.QueryEscape(query))
abortUrl += path
log.Printf("[INFO] Abort URL: %s", abortUrl)
req, err := http.NewRequest(
"GET",
abortUrl,
nil,
)
if err != nil {
log.Println("[INFO] Failed building request: %s", err)
}
// FIXME: Add an API call to the backend
authorization := os.Getenv("AUTHORIZATION")
if len(authorization) > 0 {
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", authorization))
} else {
log.Printf("[ERROR] No authorization specified for abort")
}
req.Header.Add("Content-Type", "application/json")
client := &http.Client{
Transport: &http.Transport{
Proxy: nil,
},
}
httpProxy := os.Getenv("HTTP_PROXY")
httpsProxy := os.Getenv("HTTPS_PROXY")
if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" {
client = &http.Client{}
} else {
if len(httpProxy) > 0 {
log.Printf("[INFO] Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
}
if len(httpsProxy) > 0 {
log.Printf("[INFO] Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
}
}
log.Printf("[INFO] All App Logs: %#v", allLogs)
_, err = client.Do(req)
if err != nil {
log.Printf("[WARNING] Failed abort request: %s", err)
}
} else {
log.Printf("[INFO] NOT running abort during shutdown.")
}
log.Printf("[INFO] Finished shutdown (after %d seconds). ", sleepDuration)
@@ -214,15 +186,8 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
Type: "json-file",
Config: map[string]string{},
},
Resources: container.Resources{},
}
// form container id and use it as network source if it's not empty
containerId = getThisContainerId()
if containerId != "" {
hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId))
} else {
log.Printf("[WARNING] Empty self container id, continue without NetworkMode")
Resources: container.Resources{},
NetworkMode: container.NetworkMode(fmt.Sprintf("container:worker-%s", workflowExecution.ExecutionId)),
}
// Removing because log extraction should happen first
@@ -760,6 +725,16 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
Value: workflowExecution.ExecutionId,
})
action.Parameters = append(action.Parameters, shuffle.WorkflowAppActionParameter{
Name: "source_node",
Value: trigger.ID,
})
action.Parameters = append(action.Parameters, shuffle.WorkflowAppActionParameter{
Name: "source_auth",
Value: workflowExecution.Authorization,
})
//trigger.LargeImage = ""
//err = handleSubworkflowExecution(client, workflowExecution, trigger, action)
//if err != nil {
@@ -957,6 +932,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
fmt.Sprintf("AUTHORIZATION=%s", workflowExecution.Authorization),
fmt.Sprintf("CALLBACK_URL=%s", baseUrl),
fmt.Sprintf("BASE_URL=%s", appCallbackUrl),
fmt.Sprintf("TZ=%s", timezone),
}
if strings.ToLower(os.Getenv("SHUFFLE_PASS_APP_PROXY")) == "true" {
@@ -968,13 +944,19 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
// Fixes issue:
// standard_init_linux.go:185: exec user process caused "argument list too long"
// https://devblogs.microsoft.com/oldnewthing/20100203-00/?p=15083
maxSize := 32700 - len(string(actionData)) - 2000
if len(executionData) < maxSize {
log.Printf("[INFO] ADDING FULL_EXECUTION because size is smaller than %d", maxSize)
env = append(env, fmt.Sprintf("FULL_EXECUTION=%s", string(executionData)))
} else {
log.Printf("[WARNING] Skipping FULL_EXECUTION because size is larger than %d", maxSize)
}
// FIXME: Ensure to NEVER do this anymore
// This potentially breaks too much stuff. Better to have the app poll the data.
_ = executionData
/*
maxSize := 32700 - len(string(actionData)) - 2000
if len(executionData) < maxSize {
log.Printf("[INFO] ADDING FULL_EXECUTION because size is smaller than %d", maxSize)
env = append(env, fmt.Sprintf("FULL_EXECUTION=%s", string(executionData)))
} else {
log.Printf("[WARNING] Skipping FULL_EXECUTION because size is larger than %d", maxSize)
}
*/
// Uses a few ways of getting / checking if an app is available
// 1. Try original with lowercase
@@ -982,8 +964,8 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
// 3. Add remote repo location
images := []string{
image,
fmt.Sprintf("%s:%s_%s", baseimagename, strings.Replace(action.AppName, " ", "-", -1), action.AppVersion),
fmt.Sprintf("%s/%s:%s_%s", registryName, baseimagename, parsedAppname, action.AppVersion),
fmt.Sprintf("%s:%s_%s", baseimagename, strings.Replace(action.AppName, " ", "-", -1), action.AppVersion),
}
// If cleanup is set, it should run for efficiency
@@ -1131,7 +1113,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
if strings.Contains(buildBuf.String(), "errorDetail") {
log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image)
log.Printf("[DEBUG] Shutting down (14)")
shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
shutdown(workflowExecution, action.ID, fmt.Sprintf("Error deploying container: %s", buildBuf.String()), true)
}
log.Printf("[INFO] Successfully downloaded %s", image)
@@ -1958,7 +1940,6 @@ func downloadDockerImageBackend(client *http.Client, imageName string) error {
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", authorization))
} else {
log.Printf("[WARNING] No auth found - running backend download without it.")
//req.Header.Add("Authorization", fmt.Sprintf("Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"))
//return
}
@@ -2042,6 +2023,12 @@ func main() {
}
}
if timezone == "" {
timezone = "Europe/Amsterdam"
}
log.Printf("[INFO] Running with timezone %s", timezone)
//imageName := fmt.Sprintf("%s/%s:shuffle_openapi_1.0.0", registryName, baseimagename)
//downloadDockerImageBackend(client, imageName)