diff --git a/.env b/.env index 53204db2..0b1e8cee 100755 --- a/.env +++ b/.env @@ -1,5 +1,4 @@ # Default execution environment for workers -ORG_ID=Shuffle ENVIRONMENT_NAME=Shuffle # Sanitize liquid.py input diff --git a/.github/install-guide.md b/.github/install-guide.md index 5233ff47..cf2186e6 100755 --- a/.github/install-guide.md +++ b/.github/install-guide.md @@ -18,7 +18,7 @@ The Docker setup is the default setup, and is ran with docker compose. This is [ **PS: if you're setting up Shuffle on Windows, go to the next step (Windows Docker setup)** -1. Make sure you have [Docker](https://docs.docker.com/get-docker/) installed, and that you have a minimum of **2Gb of RAM** available. +1. Make sure you have [Docker](https://docs.docker.com/get-docker/) installed, and that you have a minimum of **4Gb of RAM** available. 2. Download Shuffle ```bash git clone https://github.com/Shuffle/Shuffle diff --git a/.github/workflows/helm-test.yml b/.github/workflows/helm-test.yml new file mode 100644 index 00000000..fb0e34aa --- /dev/null +++ b/.github/workflows/helm-test.yml @@ -0,0 +1,31 @@ +name: Helm Test + +on: + push: + paths: + - "functions/kubernetes/charts/**" + workflow_dispatch: + +permissions: + contents: read + +jobs: + main: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install apt dependencies + run: | + curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null + sudo apt-get install apt-transport-https -y --no-install-recommends + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list + sudo apt-get update + sudo apt-get install helm -y --no-install-recommends + + - name: Update helm dependencies + run: helm dependency update ./functions/kubernetes/charts/shuffle + + - name: Template helm chart using default values + run: helm template shuffle ./functions/kubernetes/charts/shuffle --debug --values ./functions/kubernetes/charts/shuffle/values.yaml diff --git a/.github/workflows/tagged-nightly-release.yaml b/.github/workflows/tagged-nightly-release.yaml new file mode 100644 index 00000000..c24b7e1d --- /dev/null +++ b/.github/workflows/tagged-nightly-release.yaml @@ -0,0 +1,92 @@ +name: Tagged Nightly Release +on: + release: + types: [published] + branches: + - nightly + +# there is a very clear point to this existing. +# we want to also release versions that look like this: +# v2.1.0-nightly-date, v2.1.0-nightly-date-1, v2.1.0-nightly-date-2 +# we NEVER want to send customers a "nightly" tag. We always want to send them +# a tagged nightly tag. So that when something breaks, They can always +# point to it. + +jobs: + main: + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + include: + - app: frontend + path: frontend + experimental: true + - app: backend + path: backend + experimental: true + - app: app_sdk + path: backend/app_sdk + experimental: true + - app: orborus + path: functions/onprem/orborus + experimental: true + - app: worker + path: functions/onprem/worker + experimental: true + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set version + id: set_version + run: | + if [[ ${{ github.event_name }} == 'release' ]]; then + echo "VERSION=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT + else + echo "VERSION=nightly-untagged-latest" >> $GITHUB_OUTPUT + fi + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: "amd64,arm64,arm" + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to Ghcr + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Ghcr Build and push + id: docker_build + uses: docker/build-push-action@v4 + env: + BUILDX_NO_DEFAULT_LOAD: true + with: + logout: false + context: ${{ matrix.path }}/ + file: ${{ matrix.path }}/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + tags: | + ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }} + ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }} + frikky/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }} + frikky/shuffle:${{ matrix.app }} + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file diff --git a/backend/Dockerfile b/backend/Dockerfile index 9e27ffbe..16ffeff0 100755 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -27,7 +27,7 @@ FROM alpine:latest as certs RUN apk add --update ca-certificates # Sets up the final image -FROM alpine:3.21.2 +FROM alpine:3.22.1 # FIXME: Install cgo because CGO_ENABLED=1 during build RUN apk add --no-cache libc6-compat diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 61a1a032..d978245c 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -4,8 +4,8 @@ go 1.24.0 toolchain go1.24.3 -//replace github.com/frikky/schemaless => ../../../schemaless //replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +//replace github.com/frikky/schemaless => ../../../schemaless //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi @@ -14,17 +14,19 @@ require ( cloud.google.com/go/storage v1.55.0 github.com/basgys/goxml2json v1.1.0 github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 - github.com/docker/docker v28.2.2+incompatible + github.com/docker/docker v28.3.3+incompatible github.com/frikky/kin-openapi v0.42.0 github.com/fsouza/go-dockerclient v1.12.1 github.com/ghodss/yaml v1.0.0 + github.com/go-co-op/gocron v1.37.0 github.com/go-git/go-billy/v5 v5.6.2 github.com/go-git/go-git/v5 v5.16.1 github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.84 - golang.org/x/crypto v0.39.0 + github.com/shuffle/shuffle-shared v0.9.14 + github.com/shuffle/singul v0.0.16 + golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 google.golang.org/grpc v1.72.2 gopkg.in/yaml.v3 v3.0.1 @@ -55,7 +57,6 @@ require ( github.com/bitly/go-simplejson v0.5.1 // indirect github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf // indirect github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect - github.com/cenkalti/backoff/v5 v5.0.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.6.1 // indirect github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect @@ -72,7 +73,7 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frikky/schemaless v0.0.16 // indirect + github.com/frikky/schemaless v0.0.20 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect @@ -114,18 +115,21 @@ require ( github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opensearch-project/opensearch-go v1.1.0 // indirect github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect + github.com/osteele/liquid v1.7.0 // indirect + github.com/osteele/tuesday v1.0.3 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect - github.com/sashabaranov/go-openai v1.40.1 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/sashabaranov/go-openai v1.40.5 // indirect github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/sendgrid/sendgrid-go v3.16.1+incompatible // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect @@ -135,19 +139,18 @@ require ( go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect go.opentelemetry.io/otel v1.36.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect go.opentelemetry.io/otel/metric v1.36.0 // indirect go.opentelemetry.io/otel/sdk v1.36.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect go.opentelemetry.io/otel/trace v1.36.0 // indirect - go.opentelemetry.io/proto/otlp v1.5.0 // indirect + go.uber.org/atomic v1.9.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/net v0.40.0 // indirect + golang.org/x/net v0.41.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.15.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/term v0.32.0 // indirect - golang.org/x/text v0.26.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/term v0.33.0 // indirect + golang.org/x/text v0.27.0 // indirect golang.org/x/time v0.11.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 34b04706..71508c8f 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -126,8 +126,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw= -github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -152,8 +152,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.16 h1:4d2ZktB9xGsAusbbKliOI8TuriSrdIMzD/6ToY3wkz8= -github.com/frikky/schemaless v0.0.16/go.mod h1:jT48kTcmr1q3o8i+8qe7g+eCsbwaz2Q9CjOJevQQzQs= +github.com/frikky/schemaless v0.0.20 h1:S/A2pQcRN9qa2RnufvxwCeM06trjG0JLTF3urt1tFQI= +github.com/frikky/schemaless v0.0.20/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= github.com/fsouza/go-dockerclient v1.12.1 h1:FMoLq+Zhv9Oz/rFmu6JWkImfr6CBgZOPcL+bHW4gS0o= github.com/fsouza/go-dockerclient v1.12.1/go.mod h1:OqsgJJcpCwqyM3JED7TdfM9QVWS5O7jSYwXxYKmOooY= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= @@ -162,6 +162,8 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= +github.com/go-co-op/gocron v1.37.0 h1:ZYDJGtQ4OMhTLKOKMIch+/CY70Brbb1dGdooLEhh7b0= +github.com/go-co-op/gocron v1.37.0/go.mod h1:3L/n6BkO7ABj+TrfSVXLRzsP26zmikL4ISkLQ0O8iNY= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM= @@ -243,6 +245,7 @@ github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAx github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= @@ -278,6 +281,7 @@ github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zt github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -325,10 +329,15 @@ github.com/opensearch-project/opensearch-go v1.1.0 h1:eG5sh3843bbU1itPRjA9QXbxcg github.com/opensearch-project/opensearch-go v1.1.0/go.mod h1:+6/XHCuTH+fwsMJikZEWsucZ4eZMma3zNSeLrTtVGbo= github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ= github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8= +github.com/osteele/liquid v1.7.0 h1:VsbPSchE5D5S5scylAIvERET4dnCxsO6IDri2oSJ5Dk= +github.com/osteele/liquid v1.7.0/go.mod h1:xU0Z2dn2hOQIEFEWNmeltOmCtfhtoW/2fCyiNQeNG+U= +github.com/osteele/tuesday v1.0.3 h1:SrCmo6sWwSgnvs1bivmXLvD7Ko9+aJvvkmDjB5G4FTU= +github.com/osteele/tuesday v1.0.3/go.mod h1:pREKpE+L03UFuR+hiznj3q7j3qB1rUZ4XfKejwWFF2M= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= @@ -336,12 +345,16 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= -github.com/sashabaranov/go-openai v1.40.1 h1:bJ08Iwct5mHBVkuvG6FEcb9MDTfsXdTYPGjYLRdeTEU= -github.com/sashabaranov/go-openai v1.40.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/sashabaranov/go-openai v1.40.5 h1:SwIlNdWflzR1Rxd1gv3pUg6pwPc6cQ2uMoHs8ai+/NY= +github.com/sashabaranov/go-openai v1.40.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0= @@ -350,8 +363,10 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.84 h1:ElIMQYjKBVOiadbiGkSzt/lPU5xqaQwRxQvk9wx/xYM= -github.com/shuffle/shuffle-shared v0.8.84/go.mod h1:RdfNxqCPI+zU4jQKy3E/p4Io2injm7LpSKQUCDHNtLk= +github.com/shuffle/shuffle-shared v0.9.14 h1:POkTHO+bByuv8HiKuCMSGgtpDKk86ISr6ooLG8vQfuE= +github.com/shuffle/shuffle-shared v0.9.14/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw= +github.com/shuffle/singul v0.0.16 h1:dW+0Mln9R1aUJ0fjikpWcxbjoQWqJHxe4kSxh2tQN5E= +github.com/shuffle/singul v0.0.16/go.mod h1:LYkp320A6gsoPlYbXUM+WvEPUVAuutlSsqnVKyRy4gs= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -359,8 +374,8 @@ github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnB github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -417,8 +432,10 @@ go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFw go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= -go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= -go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= +go.opentelemetry.io/proto/otlp v1.6.0 h1:jQjP+AQyTf+Fe7OKj/MfkDrmK4MNVtw2NpXsf9fefDI= +go.opentelemetry.io/proto/otlp v1.6.0/go.mod h1:cicgGehlFuNdgZkcALOCh3VE6K/u2tAjzlRhDwmVpZc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -428,8 +445,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= -golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -478,8 +495,8 @@ golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= -golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -495,8 +512,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -521,14 +538,14 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= -golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= +golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -539,8 +556,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= @@ -572,8 +589,8 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= -golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= +golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= +golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index cfbc768a..720ca901 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3,6 +3,7 @@ package main import ( uuid "github.com/satori/go.uuid" "github.com/shuffle/shuffle-shared" + "github.com/shuffle/singul/pkg" "archive/zip" "bufio" @@ -62,6 +63,7 @@ var registryName = "registry.hub.docker.com" var runningEnvironment = "onprem" var syncUrl = "https://shuffler.io" +var debug = false //var syncUrl = "http://localhost:5002" type retStruct struct { @@ -319,7 +321,7 @@ func checkError(cmdName string, cmdArgs []string) error { scanner := bufio.NewScanner(cmdReader) go func() { for scanner.Scan() { - fmt.Printf("Out: %s\n", scanner.Text()) + log.Printf("Out: %s\n", scanner.Text()) } }() @@ -638,7 +640,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { } else { log.Printf("[DEBUG] Successfully created the default org!") - defaultEnv := os.Getenv("ORG_ID") + defaultEnv := os.Getenv("ENVIRONMENT_NAME") if len(defaultEnv) == 0 { defaultEnv = "Shuffle" log.Printf("[DEBUG] Setting default environment for org to %s", defaultEnv) @@ -1068,6 +1070,74 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { activatedAppIds = append(activatedAppIds, app.ID) } + parsedStatus := []string{} + if len(org.ManagerOrgs) > 0 || userInfo.ActiveOrg.CreatorOrg != "" { + parsedStatus = append(parsedStatus, "sub_org") + + parentOrgId := userInfo.ActiveOrg.CreatorOrg + if len(userInfo.ActiveOrg.CreatorOrg) == 0 { + parentOrgId = org.ManagerOrgs[0].Id + } + + // Check for licensing/branding of parent and override + parentOrg, err := shuffle.GetOrg(ctx, parentOrgId) + if err == nil { + if parentOrg.LeadInfo.IntegrationPartner { + parsedStatus = append(parsedStatus, "integration_partner") + + // except theme take from parent org + org.Branding.EnableChat = parentOrg.Branding.EnableChat + org.Branding.HomeUrl = parentOrg.Branding.HomeUrl + org.Branding.DocumentationLink = parentOrg.Defaults.DocumentationReference + org.Branding.SupportEmail = parentOrg.Branding.SupportEmail + org.Branding.LogoutUrl = parentOrg.Branding.LogoutUrl + org.Branding.BrandColor = parentOrg.Branding.BrandColor + org.Branding.BrandName = parentOrg.Branding.BrandName + org.Branding.GlobalUser = parentOrg.Branding.GlobalUser + + if len(org.Branding.Theme) == 0 { + org.Branding.Theme = parentOrg.Branding.Theme + } + + userInfo.ActiveOrg.Branding = parentOrg.Branding + userInfo.ActiveOrg.Image = parentOrg.Image + userInfo.ActiveOrg.Branding.DocumentationLink = parentOrg.Defaults.DocumentationReference + userInfo.ActiveOrg.Branding.BrandColor = parentOrg.Branding.BrandColor + userInfo.ActiveOrg.Branding.SupportEmail = org.Branding.SupportEmail + userInfo.ActiveOrg.Branding.LogoutUrl = org.Branding.LogoutUrl + userInfo.ActiveOrg.Branding.BrandName = parentOrg.Branding.BrandName + + if len(org.Branding.Theme) == 0 { + userInfo.ActiveOrg.Branding.Theme = parentOrg.Branding.Theme + } else { + userInfo.ActiveOrg.Branding.Theme = org.Branding.Theme + } + + // check whether current is global user or not? means is user part of parent org or not + for _, user := range parentOrg.Users { + if user.Id == userInfo.Id && user.Role == "admin" { + userInfo.ActiveOrg.Branding.GlobalUser = true + break + } + } + } + } + } else { + // for parent org branding + if org.LeadInfo.IntegrationPartner { + userInfo.ActiveOrg.Branding.Theme = org.Branding.Theme + userInfo.ActiveOrg.Branding.DocumentationLink = org.Defaults.DocumentationReference + userInfo.ActiveOrg.Branding.SupportEmail = org.Branding.SupportEmail + userInfo.ActiveOrg.Branding.LogoutUrl = org.Branding.LogoutUrl + userInfo.ActiveOrg.Branding.BrandColor = org.Branding.BrandColor + userInfo.ActiveOrg.Branding.BrandName = org.Branding.BrandName + + parsedStatus = append(parsedStatus, "integration_partner") + } + } + + aiEnabled := os.Getenv("OPENAI_API_URL") != "" && os.Getenv("AI_MODEL") != "" + returnValue := shuffle.HandleInfo{ Success: true, Username: userInfo.Username, @@ -1091,6 +1161,8 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { Licensed: licensed, ActiveApps: activatedAppIds, Theme: userInfo.Theme, + OrgStatus: parsedStatus, + AIEnabled: aiEnabled, } returnData, err := json.Marshal(returnValue) @@ -1330,7 +1402,7 @@ func parseWorkflowParameters(resp http.ResponseWriter, request *http.Request) (m return t, err } - //fmt.Println(curjson.String()) + //log.Println(curjson.String()) //log.Printf("Parsing json a second time: %s", string(curjson.String())) err = json.Unmarshal(curjson.Bytes(), &t) @@ -2484,7 +2556,7 @@ func execSubprocess(cmdName string, cmdArgs []string) error { scanner := bufio.NewScanner(cmdReader) go func() { for scanner.Scan() { - fmt.Printf("Out: %s\n", scanner.Text()) + log.Printf("Out: %s\n", scanner.Text()) } }() @@ -3456,7 +3528,7 @@ func handleAppHotload(ctx context.Context, location string, forceUpdate bool) er return err } - _, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate) + _, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate, false) if err != nil { log.Printf("[WARNING] Githubfolders error: %s", err) return err @@ -3909,7 +3981,7 @@ func runInitCloudSetup() { if err != nil { log.Printf("[INFO] Failed initial setup: %s", err) } else { - log.Printf("[INFO] Ran initial setup!") + log.Printf("[INFO] Finished initial cloudsync setup!") } } @@ -3925,7 +3997,7 @@ func runInitEs(ctx context.Context) { log.Printf("[INFO] Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy) } - defaultEnv := os.Getenv("ORG_ID") + defaultEnv := os.Getenv("ENVIRONMENT_NAME") if len(defaultEnv) == 0 { defaultEnv = "Shuffle" log.Printf("[DEBUG] Setting default environment for org to %s", defaultEnv) @@ -4031,8 +4103,9 @@ func runInitEs(ctx context.Context) { time.Sleep(30 * time.Second) } - // FIXME: This should ONLY run on one backend instance + shuffle.InitOpensearchIndexes() + // FIXME: This should ONLY run on one backend instance. This may cause interference. schedules, err := shuffle.GetAllSchedules(ctx, "ALL") if err != nil { log.Printf("[WARNING] Failed getting schedules during service init: %s", err) @@ -4078,14 +4151,23 @@ func runInitEs(ctx context.Context) { //log.Printf("Schedule: %#v", schedule) //log.Printf("Schedule time: every %d seconds", schedule.Seconds) - jobret, err := newscheduler.Every(schedule.Seconds).Seconds().NotImmediately().Run(job(schedule)) - if err != nil { - log.Printf("[ERROR] Failed to start schedule for workflow %s: %s", schedule.WorkflowId, err) + if schedule.Seconds == 0 && len(schedule.Frequency) > 0 { + cronJob, err := CronScheduler.Cron(schedule.Frequency).Do(job(schedule)) + if err != nil { + log.Printf("[ERROR] Failed to start schedule for workflow %s: %s", schedule.WorkflowId, err) + } else { + log.Printf("[DEBUG] Successfully started schedule for workflow %s", schedule.WorkflowId) + } + cronJobs[schedule.Id] = cronJob } else { - log.Printf("[DEBUG] Successfully started schedule for workflow %s", schedule.WorkflowId) + jobret, err := newscheduler.Every(schedule.Seconds).Seconds().NotImmediately().Run(job(schedule)) + if err != nil { + log.Printf("[ERROR] Failed to start schedule for workflow %s: %s", schedule.WorkflowId, err) + } else { + log.Printf("[DEBUG] Successfully started schedule for workflow %s", schedule.WorkflowId) + } + scheduledJobs[schedule.Id] = jobret } - - scheduledJobs[schedule.Id] = jobret } } @@ -4337,20 +4419,8 @@ func runInitEs(ctx context.Context) { } // Getting apps to see if we should initialize a test - // FIXME: Isn't this a little backwards? workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000, 0) log.Printf("[INFO] Getting and validating workflowapps. Got %d with err %#v", len(workflowapps), err) - - // accept any certificate (might be useful for testing) - //customGitClient := &http.Client{ - // Transport: &http.Transport{ - // TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - // }, - // Timeout: 15 * time.Second, - //} - //client.InstallProtocol("http", githttp.NewClient(customGitClient)) - //client.InstallProtocol("https", githttp.NewClient(customGitClient)) - if err != nil && len(workflowapps) == 0 { log.Printf("[WARNING] Failed getting apps (runInit): %s", err) } else if err == nil && len(workflowapps) < 10 { @@ -4360,8 +4430,9 @@ func runInitEs(ctx context.Context) { url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION") if len(url) == 0 { - log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/shuffle/shuffle-apps") - url = "https://github.com/shuffle/shuffle-apps" + log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/shuffle/python-apps") + + url = "https://github.com/shuffle/python-apps" //url = "" //return } @@ -4401,7 +4472,7 @@ func runInitEs(ctx context.Context) { _ = r //iterateAppGithubFolders(fs, dir, "", "testing") - _, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate) + _, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate, true) if err != nil { log.Printf("[WARNING] Error from app load in init: %s", err) } @@ -4409,6 +4480,10 @@ func runInitEs(ctx context.Context) { // Hotloads locally location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER") + if len(location) == 0 { + location = "./shuffle-apps" + } + if len(location) != 0 { handleAppHotload(ctx, location, false) } @@ -5090,6 +5165,7 @@ func handleAppZipUpload(resp http.ResponseWriter, request *http.Request) { func initHandlers() { var err error ctx := context.Background() + CronScheduler.StartAsync() log.Printf("[DEBUG] Starting Shuffle backend - initializing database connection") //requestCache = cache.New(5*time.Minute, 10*time.Minute) @@ -5176,11 +5252,13 @@ func initHandlers() { r.HandleFunc("/api/v1/workflows/queue", handleGetWorkflowqueue).Methods("GET", "POST") r.HandleFunc("/api/v1/workflows/queue/confirm", handleGetWorkflowqueueConfirm).Methods("POST") - // App specific - // From here down isnt checked for org specific + // App specific. Partially Singul. + r.HandleFunc("/api/v1/apps/categories", shuffle.GetActiveCategories).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/apps/categories/run", singul.RunCategoryAction).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/apps/{key}/execute", executeSingleAction).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/{key}/run", executeSingleAction).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/apps/categories", shuffle.GetActiveCategories).Methods("GET", "OPTIONS") + //r.HandleFunc("/api/v1/apps/categories/run", shuffle.RunCategoryAction).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/upload", handleAppZipUpload).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}/activate", activateWorkflowAppDocker).Methods("GET", "OPTIONS") @@ -5248,6 +5326,8 @@ func initHandlers() { // First v2 API r.HandleFunc("/api/v2/workflows/{key}/executions", shuffle.GetWorkflowExecutionsV2).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v2/workflows/generate/llm", shuffle.HandleWorkflowGenerationResponse).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v2/workflows/edit/llm", shuffle.HandleEditWorkflowWithLLM).Methods("POST", "OPTIONS") // New for recommendations in Shuffle r.HandleFunc("/api/v1/recommendations/get_actions", shuffle.HandleActionRecommendation).Methods("POST", "OPTIONS") @@ -5308,8 +5388,8 @@ func initHandlers() { // EVERYTHING below here is NEW for 0.8.0 (written 25.05.2021) r.HandleFunc("/api/v1/workflows/{key}/publish", makeWorkflowPublic).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/cloud/setup", handleCloudSetup).Methods("POST", "OPTIONS") - //r.HandleFunc("/api/v1/orgs", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS") - //r.HandleFunc("/api/v1/orgs/", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/orgs", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/orgs/", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleGetOrg).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleEditOrg).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgid}/forms", shuffle.HandleGetOrgForms).Methods("GET", "OPTIONS") @@ -5404,6 +5484,10 @@ func initHandlers() { // Had to move away from mux, which means Method is fucked up right now. func main() { + if os.Getenv("DEBUG") == "true" { + debug = true + } + initHandlers() hostname, err := os.Hostname() if err != nil { diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index d33cee2c..3cf355ab 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -11,7 +11,7 @@ import ( "io" "io/ioutil" "log" - "math/rand" + //"math/rand" "net/http" "net/url" "os" @@ -26,6 +26,7 @@ import ( uuid "github.com/satori/go.uuid" newscheduler "github.com/carlescere/scheduler" + "github.com/go-co-op/gocron" "github.com/frikky/kin-openapi/openapi3" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" @@ -42,17 +43,22 @@ var baseEnvironment = "onprem" var cloudname = "cloud" var scheduledJobs = map[string]*newscheduler.Job{} +var cronJobs = map[string]*gocron.Job{} var scheduledOrgs = map[string]*newscheduler.Job{} +var CronScheduler = gocron.NewScheduler(time.UTC) + // Frequency = cronjob OR minutes between execution func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode, frequency, orgId string, body []byte) error { var err error testSplit := strings.Split(frequency, "*") cronJob := "" + isCron := false newfrequency := 0 if len(testSplit) > 5 { cronJob = frequency + isCron = true } else { newfrequency, err = strconv.Atoi(frequency) if err != nil { @@ -65,12 +71,7 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode //} else if int(newfrequency) < } - // Reverse. Can't handle CRON, only numbers - if len(cronJob) > 0 { - return errors.New("cronJob isn't formatted correctly") - } - - if newfrequency < 1 { + if newfrequency < 1 && !isCron { return errors.New("Frequency has to be more than 0") } @@ -96,17 +97,27 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode } } - log.Printf("[INFO] Starting frequency for execution: %d", newfrequency) + log.Printf("[INFO] Starting frequency for execution: %s", frequency) - //jobret, err := newscheduler.Every(newfrequency).Seconds().NotImmediately().Run(job) - jobret, err := newscheduler.Every(newfrequency).Seconds().Run(job) - if err != nil { - log.Printf("Failed to schedule workflow: %s", err) - return err + if isCron { + cronJob, err := CronScheduler.Cron(cronJob).Do(job) + if err != nil { + log.Printf("[ERROR] Failed to start schedule with cron(%s): %s", cronJob, err) + } + + cronJobs[scheduleId] = cronJob + } else { + //jobret, err := newscheduler.Every(newfrequency).Seconds().NotImmediately().Run(job) + jobret, err := newscheduler.Every(newfrequency).Seconds().Run(job) + if err != nil { + log.Printf("Failed to schedule workflow: %s", err) + return err + } + + scheduledJobs[scheduleId] = jobret } //scheduledJobs = append(scheduledJobs, jobret) - scheduledJobs[scheduleId] = jobret // Doesn't need running/not running. If stopped, we just delete it. timeNow := int64(time.Now().Unix()) @@ -117,6 +128,7 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode Argument: string(body), WrappedArgument: bodyWrapper, Seconds: newfrequency, + Frequency: frequency, CreationTime: timeNow, LastModificationtime: timeNow, LastRuntime: timeNow, @@ -245,7 +257,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { return } - // This is really the environment's name - NOT org-id + // This is really the environment's name - NOT OrgId environment := request.Header.Get("Org-Id") if len(environment) == 0 { log.Printf("[AUDIT] No org-id header set") @@ -254,7 +266,8 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { return } - orgId := request.Header.Get("org") + // Org => Org ID here + orgId := request.Header.Get("Org") if len(orgId) == 0 { //log.Printf("[AUDIT] No 'org' header set (get workflow queue). ") /* @@ -263,9 +276,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { return */ } - - orborusLabel := request.Header.Get("x-orborus-label") - + // This section is cloud custom for now auth := request.Header.Get("Authorization") if len(auth) == 0 { @@ -277,192 +288,46 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { */ } - //log.Printf("[AUDIT] Get workflow queue for org %s, env %s, orborus label %s", orgId, environment, orborusLabel) - ctx := shuffle.GetContext(request) - // Get all env and check the name? envs, err := shuffle.GetEnvironments(ctx, orgId) - if err != nil || len(envs) == 0 { - log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", environment, err) + //log.Printf("[WARNING] No env found for orgId %s during queue loading", orgId) } var env *shuffle.Environment - for _, e := range envs { - if e.Name == environment { - env = &e + found := false + for i := range envs { + if envs[i].Name == environment { + env = &envs[i] + found = true break } } - timeNow := time.Now().Unix() - err = shuffle.HandleOrborusFailover(ctx, request, resp, env) - if err != nil { - if !strings.Contains(err.Error(), "mismatch") { - log.Printf("[WARNING] Failed handling Orborus failover: %s", err) + // Only works onprem - shared queues across tenants + // without tenancy + if !found { + env, err = shuffle.GetEnvironment(ctx, environment, "") + if err != nil { + log.Printf("[WARNING] Failed to find the environment(%s) in org(%s). Could cause with Failover test", environment, orgId) } - - return } - //log.Printf("Found env: %#v", env) + // Handles failover control between Orborus' + // Further tracks checkin time to ensure this works properly + // across instances + err = shuffle.HandleOrborusFailover(ctx, request, resp, env) + if err != nil { + log.Printf("[WARNING] Failed handling Orborus failover: %s", err) + } + if len(env.OrgId) > 0 { orgId = env.OrgId } - // FIXME: Workflow stats disabled for now - // as it caused too many problems - // goal: track docker stuff once a minute and graph it - // For now: Disable this as it caused too many problems - if request.Method == "POST" && true == false { - //log.Printf("[DEBUG] POST to workflowqueue") - if rand.Intn(10) == 0 { - // Parse out body - body, err := ioutil.ReadAll(request.Body) - if err == nil { - - // Parse out CPU, memory and disk. - - var envData shuffle.OrborusStats - err = json.Unmarshal(body, &envData) - if err == nil && !envData.Swarm && !envData.Kubernetes && (envData.CPU > 0 || envData.Memory > 0 || envData.Disk > 0) { - - // Set the input in memory - envData.OrgId = orgId - envData.Environment = environment - envData.OrborusLabel = orborusLabel - envData.Timestamp = time.Now().Unix() - - if envData.CPU > 0 && envData.MaxCPU > 0 { - envData.CPUPercent = float64(envData.CPU) / float64(envData.MaxCPU) - } - - if envData.Memory > 0 && envData.MaxMemory > 0 { - envData.MemoryPercent = float64(envData.Memory) / float64(envData.MaxMemory) - } - - // Check if CPU percent constantly has stayed above X% for the last Y requests - percentageCheck := 90 - concurrentChecks := 2 - - //if int(envData.CPUPercent) > percentageCheck { - // Get cached data - percentages := []float64{} - cacheKey := fmt.Sprintf("%s_%s_percent", environment , strings.ToLower(orgId)) - - // Marshal float list into []byte - cacheData := []byte{} - cache, err := shuffle.GetCache(ctx, cacheKey) - if err == nil { - // Unmarshal into percentages - cacheData := []byte(cache.([]uint8)) - err = json.Unmarshal(cacheData, &percentages) - if err != nil { - log.Printf("[INFO] error in cache unmarshal for percentages: %s", err) - } - - if len(percentages) > concurrentChecks { - percentages = percentages[:concurrentChecks] - } - - percentages = append(percentages, envData.CPUPercent) - if len(percentages) > concurrentChecks { - //log.Printf("[INFO] Checking percentages: %v", percentages) - - // percentageCheck := 1 - sendAlert := true - for _, p := range percentages { - if int(p) < percentageCheck { - //log.Printf("[AUDIT] CPU percent is below %d: %d", percentageCheck, int(p)) - sendAlert = false - break - } - } - - if sendAlert { - log.Printf("[INFO] CPU percent has been above %d percent for the last 5 requests. Sending alert. Env: %s, org: %s", percentageCheck, environment, orgId) - - // Set notification + alert for organization - err = shuffle.CreateOrgNotification( - ctx, - fmt.Sprintf("CPU percent has been above %d percent", percentageCheck), - fmt.Sprintf("A environment %s has been using more than %d\\% CPU for the last 5 requests.", environment, percentageCheck), - fmt.Sprintf("/admin?tab=environments"), - environment, - true, - ) - - if err != nil { - log.Printf("[ERROR] error creating notification: %s", err) - } - - org, err := shuffle.GetOrg(ctx, environment) - if err == nil { - foundRecommendation := false - for _, recommendation := range org.Priorities { - if strings.Contains(recommendation.Name, "CPU") { - foundRecommendation = true - break - } - } - - if !foundRecommendation { - // Add to start of org.Priorities - org, _ = shuffle.AddPriority(*org, shuffle.Priority{ - Name: fmt.Sprintf("High CPU in environment %s", orgId), - Description: fmt.Sprintf("The environment %s has been using more than %d percent CPU. This indicates you may need to look at scaling.", orgId, percentageCheck), - Type: "scale", - Active: true, - URL: fmt.Sprintf("/admin?tab=environments"), - Severity: 1, - }, false) - - //Make last item the first item - org.Priorities = append([]shuffle.Priority{org.Priorities[len(org.Priorities)-1]}, org.Priorities[:len(org.Priorities)-1]...) - err = shuffle.SetOrg(ctx, *org, org.Id) - if err != nil { - log.Printf("[ERROR] Problem setting org: %s", err) - } - } - } - } - - if len(percentages) > 1 { - percentages = percentages[1:] - } - } - - // Marshal float list into []byte - } else { - //log.Printf("[ERROR] Failed getting cache: %s", err) - percentages = append(percentages, envData.CPUPercent) - } - - if len(percentages) > 0 { - //log.Printf("[DEBUG] Setting cache for %s: %#v", cacheKey, percentages) - cacheData, err = json.Marshal(percentages) - if err != nil { - log.Printf("[INFO] error in cache marshal: %s", err) - } - - // Add the new data - go shuffle.SetCache(ctx, cacheKey, cacheData, 5) - } - } - - //log.Printf("CPU percent: %f", envData.CPUPercent) - //log.Printf("Memory percent: %f", envData.MemoryPercent*100) - - go shuffle.SetenvStats(ctx, envData) - } - } - } - executionRequests, err := shuffle.GetWorkflowQueue(ctx, environment, 100) if err != nil { - // Skipping as this comes up over and over - //log.Printf("(2) Failed reading body for workflowqueue: %s", err) - resp.WriteHeader(401) + resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } @@ -471,10 +336,11 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { if len(executionRequests.Data) == 0 { executionRequests.Data = []shuffle.ExecutionRequest{} } else { - //log.Printf("In workflowqueue with %d", len(executionRequests.Data)) - // Try again :) + // Try again? I don't think this is necessary, and shouldn't really ever occur. + /* if len(env.Id) == 0 && len(env.Name) == 0 { + timeNow := int64(time.Now().Unix()) foundId := "" for _, requestData := range executionRequests.Data { execution, err := shuffle.GetWorkflowExecution(ctx, requestData.ExecutionId) @@ -487,9 +353,10 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } if len(environment) > 0 { - env, err := shuffle.GetEnvironment(ctx, foundId, environment) + + env, err := shuffle.GetEnvironment(ctx, environment, foundId) if err != nil { - log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", orgId, err) + log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", environment, err) //resp.WriteHeader(401) //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No env found matching %s"}`, id))) //return @@ -505,6 +372,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } } } + */ if len(executionRequests.Data) > 50 { executionRequests.Data = executionRequests.Data[0:49] @@ -1685,7 +1553,15 @@ func deleteSchedule(ctx context.Context, id string) error { value.Lock() } else { // FIXME - allow it to kind of stop anyway? - return errors.New("Can't find the schedule.") + if j, ok := cronJobs[id]; ok { + err := CronScheduler.RemoveByID(j) + if err != nil { + log.Printf("[ERROR] Failed to remove the scheduler %s", err) + return err + } + } else { + return errors.New("Can't find the schedule.") + } } } @@ -2141,14 +2017,14 @@ func validateAppInput(resp http.ResponseWriter, request *http.Request) { //fmt.Printf("File type: %s. MIME: %s\n", kind.Extension, kind.MIME.Value) if kind == filetype.Unknown { - fmt.Println("Unknown file type") + log.Println("Unknown file type") resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } if kind.MIME.Value != "application/zip" { - fmt.Println("Not zip, can't unzip") + log.Println("Not zip, can't unzip") resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -2307,17 +2183,24 @@ func handleSingleAppHotloadRequest(resp http.ResponseWriter, request *http.Reque resp.Write([]byte(`{"success": false}`)) return } + if user.Role != "admin" { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Must be admin to hotload apps"}`)) return } + location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER") + if len(location) == 0 { + location = "./shuffle-apps" + } + if len(location) == 0 { resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "SHUFFLE_APP_HOTLOAD_FOLDER not specified in .env"}`))) return } + requestUrlFields := strings.Split(request.URL.String(), "/") var appName string if requestUrlFields[1] == "api" { @@ -2381,6 +2264,10 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { } location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER") + if len(location) == 0 { + location = "./shuffle-apps" + } + if len(location) == 0 { resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "SHUFFLE_APP_HOTLOAD_FOLDER not specified in .env"}`))) @@ -2982,24 +2869,24 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { } location := strings.Split(request.URL.String(), "/") - var fileId string + var appId string if location[1] == "api" { if len(location) <= 4 { - resp.WriteHeader(401) + resp.WriteHeader(400) resp.Write([]byte(`{"success": false}`)) return } - fileId = location[4] + appId = location[4] } //log.Printf("[AUDIT] User Authentication failed in execute SINGLE action - CONTINUING ANYWAY: %s. Found OrgID: %#v", err, user.ActiveOrg.Id) - log.Printf("[AUDIT] User %s (%s) in org %s (%s) is running SINGLE App run for App ID '%s'", user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id, fileId) + log.Printf("[AUDIT] User %s (%s) in org %s (%s) is running SINGLE App run for App ID '%s'", user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id, appId) body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("[INFO] Failed single execution POST body read: %s", err) - resp.WriteHeader(401) + resp.WriteHeader(400) resp.Write([]byte(`{"success": false}`)) return } @@ -3024,7 +2911,15 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { decisionId = decision[0] } - workflowExecution, err := shuffle.PrepareSingleAction(ctx, user, fileId, body, runValidationAction, decisionId) + log.Printf("\n\nACTION TO RUN: %s. Body: %s. Source URL: %s\n\n", appId, string(body), request.URL.String()) + + workflowExecution, err := shuffle.PrepareSingleAction(ctx, user, appId, body, runValidationAction, decisionId) + if appId == "agent_starter" { + log.Printf("[INFO] Returning early for agent_starter single action execution: %s", workflowExecution.ExecutionId) + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization))) + return + } debugUrl := fmt.Sprintf("/workflows/%s?execution_id=%s", workflowExecution.Workflow.ID, workflowExecution.ExecutionId) resp.Header().Add("X-Debug-Url", debugUrl) @@ -3065,6 +2960,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { go shuffle.IncrementCache(ctx, workflowExecution.OrgId, "workflow_executions") executionRequest := shuffle.ExecutionRequest{ + Priority: 15, ExecutionId: workflowExecution.ExecutionId, WorkflowId: workflowExecution.Workflow.ID, Authorization: workflowExecution.Authorization, @@ -3110,7 +3006,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { } // Onlyname is used to -func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) ([]shuffle.BuildLaterStruct, []shuffle.BuildLaterStruct, error) { +func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate, duringStartup bool) ([]shuffle.BuildLaterStruct, []shuffle.BuildLaterStruct, error) { var err error allapps := []shuffle.WorkflowApp{} @@ -3122,7 +3018,14 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. "YARA", "ATTACK-PREDICTOR", } - //if strings.ToUpper(workflowapp.Name) == strings.ToUpper(appname) { + + startupNames := []string{ + "shuffle-tools", + "http", + "email", + "shuffle-ai", + "shuffle-subflow", + } // It's here to prevent getting them in every iteration buildLaterFirst := []shuffle.BuildLaterStruct{} @@ -3132,6 +3035,19 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. continue } + //duringStartup + if duringStartup { + // Look for names: shuffle tools, http, email, shuffle ai + if shuffle.ArrayContains(startupNames, strings.ToLower(file.Name())) { + // Allowed to build during startup + + //log.Printf("\n\n\nFOUND MATCHING APP: %s\n\n\n", file.Name()) + } else { + //log.Printf("\n\n\nWRONG APP (2): %s\n\n\n", file.Name()) + continue + } + } + // Folder? switch mode := file.Mode(); { case mode.IsDir(): @@ -3148,7 +3064,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. } // Go routine? Hmm, this can be super quick I guess - buildFirst, buildLast, err := IterateAppGithubFolders(ctx, fs, dir, tmpExtra, "", forceUpdate) + buildFirst, buildLast, err := IterateAppGithubFolders(ctx, fs, dir, tmpExtra, "", forceUpdate, false) for _, item := range buildFirst { buildLaterFirst = append(buildLaterFirst, item) @@ -3160,7 +3076,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. if err != nil { log.Printf("[WARNING] Error reading folder: %s", err) - //buildFirst, buildLast, err := IterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate) + //buildFirst, buildLast, err := IterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate, false) if !forceUpdate { continue @@ -3460,6 +3376,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. "http", "email", } + for _, buildLater := range buildLaterFirst { found := false for _, appname := range initApps { @@ -3478,12 +3395,19 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. } // Prepend newSortedList to buildLaterFirst + handledImages := []string{} buildLaterFirst = append(newSortedList, buildLaterFirst...) - if len(extra) == 0 { log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst)) for _, item := range buildLaterFirst { + + if len(item.Tags) > 0 && shuffle.ArrayContains(handledImages, item.Tags[0]) { + continue + } + + handledImages = append(handledImages, item.Tags[0]) err = buildImageMemory(fs, item.Tags, item.Extra, true) + if err != nil { orgId := "" @@ -3513,6 +3437,12 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. if len(buildLaterList) > 0 { log.Printf("[INFO] Starting build of %d skipped docker images", len(buildLaterList)) for _, item := range buildLaterList { + if len(item.Tags) > 0 && shuffle.ArrayContains(handledImages, item.Tags[0]) { + continue + } + + handledImages = append(handledImages, item.Tags[0]) + err = buildImageMemory(fs, item.Tags, item.Extra, true) if err != nil { log.Printf("[INFO] Failed image build memory: %s", err) @@ -3646,7 +3576,7 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { } } - IterateAppGithubFolders(ctx, fs, dir, "", "", tmpBody.ForceUpdate) + IterateAppGithubFolders(ctx, fs, dir, "", "", tmpBody.ForceUpdate, false) } else if strings.Contains(tmpBody.URL, "s3") { //https://docs.aws.amazon.com/sdk-for-go/api/service/s3/ diff --git a/docker-compose.yml b/docker-compose.yml index f6683a2a..980e0764 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,7 +61,7 @@ services: security_opt: - seccomp:unconfined opensearch: - image: opensearchproject/opensearch:3.0.0 + image: opensearchproject/opensearch:3.2.0 hostname: shuffle-opensearch container_name: shuffle-opensearch environment: diff --git a/frontend/public/aiGenerateWorkflowSteps.svg b/frontend/public/aiGenerateWorkflowSteps.svg new file mode 100644 index 00000000..3f294fcb --- /dev/null +++ b/frontend/public/aiGenerateWorkflowSteps.svg @@ -0,0 +1,85 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 13289352..9425a1c6 100755 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -92,7 +92,7 @@ const App = (message, props) => { const [dataset, setDataset] = useState(false) const [isLoaded, setIsLoaded] = useState(false) const [curpath, setCurpath] = useState(typeof window === "undefined" || window.location === undefined ? "" : window.location.pathname) - const { themeMode, handleThemeChange, setBrandColor, brandColor,setThemeMode} = useContext(Context); + const { themeMode, handleThemeChange, setBrandColor, brandColor,setThemeMode, setSupportEmail, setLogoutUrl, setBrandName} = useContext(Context); const currentTheme = getTheme(themeMode, brandColor); const mainColor = currentTheme?.palette?.backgroundColor const [isPreviousThemeLight, setIsPreviousThemeLight] = useState(false) @@ -192,11 +192,44 @@ const App = (message, props) => { { path: "/" } ); } - if (responseJson?.theme?.length > 0) { - handleThemeChange(responseJson.theme) - }else{ - handleThemeChange("dark") - } + if (responseJson?.org_status?.includes("integration_partner")) { + if (responseJson?.active_org?.branding?.enable_chat !== true) { + + // Find the drift chatbox and remove it + } + + if (responseJson?.active_org?.branding?.theme?.length > 0 ) { + handleThemeChange(responseJson?.active_org?.branding?.theme) + } + + if (responseJson?.active_org?.branding?.brand_color?.length > 0 ) { + setBrandColor(responseJson?.active_org?.branding?.brand_color) + localStorage.setItem("brandColor", responseJson?.active_org?.branding?.brand_color) + } + + if (responseJson?.active_org?.branding?.brand_name?.length > 0 ) { + setBrandName(responseJson?.active_org?.branding?.brand_name) + localStorage.setItem("brandName", responseJson?.active_org?.branding?.brand_name) + } + + if (responseJson?.active_org?.branding?.support_email?.length > 0 ) { + setSupportEmail(responseJson?.active_org?.branding?.support_email) + } + + if (responseJson?.active_org?.branding?.logout_url?.length > 0 ) { + setLogoutUrl(responseJson?.active_org?.branding?.logout_url) + } + } else { + localStorage.removeItem("brandColor") + setBrandColor("#ff8544") + setBrandName("Shuffle") + localStorage.removeItem("brandName") + if(responseJson?.theme?.length > 0){ + handleThemeChange(responseJson?.theme) + }else{ + handleThemeChange("dark") + } + } }else { handleThemeChange("dark") setThemeMode("dark") diff --git a/frontend/src/components/AdminNavBar.jsx b/frontend/src/components/AdminNavBar.jsx index 2a480c95..fbbc391d 100644 --- a/frontend/src/components/AdminNavBar.jsx +++ b/frontend/src/components/AdminNavBar.jsx @@ -92,9 +92,14 @@ const AdminNavBar = (props) => { const HandleVisibleTabs = () => { if (userdata?.id?.length > 0) { if (userdata?.active_org?.role === "admin" || userdata?.support) { - setVisibleItems(items); + if (isChildOrg) { + const filteredItems = items.filter(item => item.text !== "Partner"); + setVisibleItems(filteredItems); + }else { + setVisibleItems(items); + } }else { - const filteredItems = items.filter(item => item.text !== "Users" && item.text !== "Files" && item.text !== "Datastore" && item.text !== "Triggers" && item.text !== "Locations"); + const filteredItems = items.filter(item => item.text !== "Users" && item.text !== "Files" && item.text !== "Datastore" && item.text !== "Triggers" && item.text !== "Locations" && item.text !== "Partner"); setVisibleItems(filteredItems); } } @@ -105,12 +110,12 @@ const AdminNavBar = (props) => { // Filter out Users and Tenants tabs if (userdata?.active_org?.role === "admin" || userdata?.support) { const filteredItems = items.filter(item => - item.text !== "Users" && item.text !== "Tenants" + item.text !== "Users" && item.text !== "Tenants" && item.text !== "Partner" ); setVisibleItems(filteredItems); }else { const filteredItems = items.filter(item => - item.text !== "Users" && item.text !== "Tenants" && item.text !== "Files" && item.text !== "Datastore" && item.text !== "Triggers" && item.text !== "Locations" + item.text !== "Users" && item.text !== "Partner" && item.text !== "Tenants" && item.text !== "Files" && item.text !== "Datastore" && item.text !== "Triggers" && item.text !== "Locations" ); setVisibleItems(filteredItems); } @@ -191,7 +196,7 @@ const AdminNavBar = (props) => { const params = new URLSearchParams(location.search); const tab = params?.get('tab')?.toLowerCase(); - if (tab === "users" || tab === "tenants") { + if (tab === "users" || tab === "tenants" || tab === "partner") { toast.info("You are not allowed to access this tab. Please contact your admin for more information. Redirecting to Organization Configuration tab."); setTimeout(() => { setSelectedItem("Organization"); @@ -200,6 +205,7 @@ const AdminNavBar = (props) => { } , 3000); } + } else if (userdata && isOrgLoaded && isUserDataLoaded && userdata?.active_org?.role !== "admin" && !userdata?.support) { const queryParams = new URLSearchParams(location.search); const tabName = queryParams?.get('admin_tab')?.toLowerCase(); diff --git a/frontend/src/components/AnalyticsTab.jsx b/frontend/src/components/AnalyticsTab.jsx index c97560ed..c64dee56 100644 --- a/frontend/src/components/AnalyticsTab.jsx +++ b/frontend/src/components/AnalyticsTab.jsx @@ -2,7 +2,6 @@ // import Switch from '@mui/material/Switch'; // import { Typography, Button } from '@mui/material'; // import { useNavigate, Link, useParams } from "react-router-dom"; -// import { Bar } from 'react-chartjs-2'; // import Grid from '@mui/material/Grid'; // import SearchIcon from '@mui/icons-material/Search'; // import NewReleasesIcon from '@mui/icons-material/NewReleases'; diff --git a/frontend/src/components/Billing.jsx b/frontend/src/components/Billing.jsx index 0b9daa6f..8b14eaf5 100644 --- a/frontend/src/components/Billing.jsx +++ b/frontend/src/components/Billing.jsx @@ -91,18 +91,21 @@ const Billing = memo((props) => { const [currentTab, setCurrentTab] = useState(0) const [allChildOrgs, setAllChildOrgs] = useState([]) const [allChildOrgsStats, setAllChildOrgsStats] = useState([]) + const [statistics, setStatistics] = useState([]) + const [monthlyAppRunsParent, setMonthlyAppRunsParent] = useState(0) + const [monthlyAllSuborgExecutions, setMonthlyAllSuborgExecutions] = useState(0) useEffect(() => { - if (userdata.app_execution_limit !== undefined && userdata.app_execution_usage !== undefined) { - const percentage = (userdata.app_execution_usage / userdata.app_execution_limit) * 100; + if (monthlyAppRunsParent > 0 || monthlyAllSuborgExecutions > 0) { + const percentage = ((monthlyAppRunsParent + monthlyAllSuborgExecutions) / userdata.app_execution_limit) * 100; setCurrentAppRunsInPercentage(Math.round(percentage)); - setCurrentAppRunsInNumber(userdata.app_execution_limit - userdata.app_execution_usage); + setCurrentAppRunsInNumber(userdata.app_execution_limit - userdata.app_execution_usage - userdata.app_executions_suborgs); } if (userdata?.id?.length > 0 && isLoggedIn === false){ setIsLoggedIn(true) } - }, [userdata]); + }, [monthlyAppRunsParent, monthlyAllSuborgExecutions, userdata]); const [BillingEmail, setBillingEmail] = useState(selectedOrganization?.Billing?.Email); @@ -199,6 +202,48 @@ const Billing = memo((props) => { } }, []) + + const getStats = (orgid) => { + + if (orgid === undefined || orgid === null) { + return + } + + fetch(`${globalUrl}/api/v1/orgs/${orgid}/stats`, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for workflows :O!: ", response.status); + return; + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson["success"] === false) { + return + } + + setStatistics(responseJson); + }) + .catch((error) => { + console.log("error: ", error) + }); + } + + + useEffect(() => { + if (selectedOrganization && selectedOrganization?.id?.length > 0) { + getStats(selectedOrganization.id); + } + }, [selectedOrganization]); + const paperStyle = { padding: 20, // maxWidth: 400, @@ -1860,19 +1905,6 @@ const Billing = memo((props) => { const updateAlertThreshold = (index, field, value) => { - if (field === 'percentage') { - if (value > 100 || value < 0) { - value = 0 - toast("The percentage value should be between 0 and 100") - } - } else if (field === 'count') { - if (value < 0 || value >= userdata.app_execution_limit) { - value = 0 - toast("The count value should be greater than 0 and less than the total app execution limit") - } - } - - const totalValue = userdata.app_execution_limit; const newAlertThresholds = alertThresholds.map((threshold, i) => { if (i === index) { @@ -1986,7 +2018,13 @@ const Billing = memo((props) => { }; const isChildOrg = userdata?.active_org?.creator_org !== "" && userdata?.active_org?.creator_org !== undefined && userdata?.active_org?.creator_org !== null - + + useEffect(() => { + if (isChildOrg && currentTab === 0) { + setCurrentTab(1); + } + }, [isChildOrg, currentTab]); + return (
@@ -2386,8 +2424,8 @@ const Billing = memo((props) => { We offer priority support through consultations and training to help you make the most of our product. If you have any questions, please reach out to us at {supportEmail}. We offer priority support through consultations and training to help you make the most of our product. If you have any questions, please reach out to us at -
- {billingInfo.subscription !== undefined && billingInfo.subscription !== null ? ( +
+ {/* {billingInfo.subscription !== undefined && billingInfo.subscription !== null ? ( isChildOrg ? null : ( { selectedOrganization={selectedOrganization} /> ) - ) : null} + ) : null} */}
@@ -2423,9 +2461,21 @@ const Billing = memo((props) => { }} /> - You have used {currentAppRunsInPercentage}% of total app execution limit or {userdata.app_execution_usage} app runs out of {userdata.app_execution_limit} app runs. + You have used {currentAppRunsInPercentage}% of total app execution limit or {Number(monthlyAppRunsParent ?? 0) + Number(monthlyAllSuborgExecutions ?? 0)} app runs out of {userdata.app_execution_limit} app runs this month. - + + {userdata?.active_org?.creator_org?.length > 0 ? null : + ( + <> + + Parent Organization App Executions: {monthlyAppRunsParent} + + + Sub-Organization App Executions: {monthlyAllSuborgExecutions || "N/A"} + + + )} +
Set email alert thresholds for app runs @@ -2442,8 +2492,8 @@ const Billing = memo((props) => { : " " + 0 + " "} app runs. - - Please note: Once your app runs reach the set alert threshold, all admins in the organization will receive an email notification. + + Please note: Once your app runs reach the set alert threshold, all admins in the organization will receive an email notification. For Parent organizations, the alert will be sent base on the total app runs from both parent and sub-organizations. For Sub-organizations, the alert will be sent based on the app runs of the sub-organization only.
{alertThresholds.map((threshold, index) => ( @@ -2603,12 +2653,7 @@ const Billing = memo((props) => { { - setCurrentTab(-1) - - // Force re-render - setTimeout(() => { - setCurrentTab(newValue) - }, 100); + setCurrentTab(newValue) }} style={{ marginTop: 20 }} TabIndicatorProps={{ @@ -2620,52 +2665,66 @@ const Billing = memo((props) => { } }} > - } + - - {isCloud ? - - : null} - + {isChildOrg ? null : + />} + {isCloud ? + + : null}
- {currentTab === 0 ? -
- -
+ { + currentTab === 0 ? + : currentTab === 1 ? -
+
- : + : currentTab === 2 ? { setAllChildOrgs={setAllChildOrgs} allChildOrgsStats={allChildOrgsStats} setAllChildOrgsStats={setAllChildOrgsStats} + currentTab={currentTab} /> + : + }
@@ -2706,6 +2776,7 @@ const BillingStatsChildOrg = memo(({ userdata, globalUrl, selectedOrganization, const { themeMode, brandColor, supportEmail } = useContext(Context); const theme = getTheme(themeMode, brandColor); + // Handle page change const handleChangePage = (event, newPage) => { setPage(newPage); @@ -2830,6 +2901,7 @@ const BillingStatsChildOrg = memo(({ userdata, globalUrl, selectedOrganization, usage: stat?.monthly_app_executions || "N/A", workflows_usage: stat?.total_workflow_executions || "N/A", workflow_usage_limit: subOrg?.sync_features?.workflow_executions?.limit || "N/A", + app_runs_hard_limit: subOrg?.Billing?.app_runs_hard_limit || 0, } }) @@ -2890,6 +2962,33 @@ const BillingStatsChildOrg = memo(({ userdata, globalUrl, selectedOrganization, )} }, + { + field: "app_runs_hard_limit", headerName: "App Executions Hard Limit", width: 200, renderCell: (params) => { + console.log("params.row: ", params.row) + return ( + <> + + {params.row.app_runs_hard_limit} + + { + setOpen(true) + setEditingOrgId(params.row.orgId) + setEditing("app_executions_hard_limit") + if (params.row.app_runs_hard_limit === "N/A") { + setLimit("") + } else { + setLimit(params.row.app_runs_hard_limit) + } + }} + > + + + + ) + } + } ] setSubOrgStatsColumns(columns) @@ -2913,7 +3012,7 @@ const BillingStatsChildOrg = memo(({ userdata, globalUrl, selectedOrganization, return } - if (selectedOrganization.sync_features.app_executions.limit <= 10000) { + if (selectedOrganization.sync_features.app_executions.limit <= 10000 && editing === "app_executions") { toast.error("Insufficient app execution limit to increase child org limit") return } @@ -2939,7 +3038,10 @@ const BillingStatsChildOrg = memo(({ userdata, globalUrl, selectedOrganization, const org = subOrgs[orgIndex] - org.sync_features[editing].limit = limit + if (editing !== "app_executions_hard_limit") { + org.sync_features[editing].limit = limit + } + org.sync_features.editing = true const sync_features = org.sync_features const data = { @@ -2947,6 +3049,13 @@ const BillingStatsChildOrg = memo(({ userdata, globalUrl, selectedOrganization, sync_features: sync_features, } + if (editing === "app_executions_hard_limit") { + data.editing = "app_runs_hard_limit"; + data.billing = { + app_runs_hard_limit: limit || 0 + }; + } + const url = `${globalUrl}/api/v1/orgs/${orgId}`; fetch(url, { method: "POST", @@ -2974,6 +3083,12 @@ const BillingStatsChildOrg = memo(({ userdata, globalUrl, selectedOrganization, newRows[orgIndex].workflow_usage_limit = limit; return newRows; }); + }else if (editing === "app_executions_hard_limit") { + setSubOrgStatsRows((prevRows) => { + const newRows = [...prevRows]; + newRows[orgIndex].app_runs_hard_limit = limit; + return newRows; + }); } } }) @@ -3087,10 +3202,21 @@ const IncreaseLimitPopUp = memo(({ open, onClose, limit, setLimit, HandleEditLim } }} > - + {editing === "app_executions_hard_limit" ? ( + + Add {editing.replaceAll("_", " ").replace(/\b\w/g, c => c.toUpperCase())} + + ) : ( + Increase {editing.replaceAll("_", " ").replace(/\b\w/g, c => c.toUpperCase())} Limit + )} + { editing === "app_executions_hard_limit" ? ( + + Please note that once you set a hard limit for app runs workflows will not be able to run if the limit is reached. You will be notified by email when you reach the limit. + + ) : null} setCurrentLimit(e.target.value)} diff --git a/frontend/src/components/BillingStats.jsx b/frontend/src/components/BillingStats.jsx index 5bb09ddb..8e011d42 100644 --- a/frontend/src/components/BillingStats.jsx +++ b/frontend/src/components/BillingStats.jsx @@ -31,57 +31,10 @@ import { Box, } from "@mui/material"; -import { - BarChart, - BarSeries, - Bar, - BarLabel, - - GridlineSeries, - Gridline, - TooltipArea, - ChartTooltip, - TooltipTemplate, -} from 'reaviz'; - import { typecost, typecost_single, } from "../views/HandlePaymentNew.jsx"; +import LineChartWrapper from '../components/LineChartWrapper.jsx'; import { Context } from '../context/ContextApi.jsx'; -const LineChartWrapper = ({keys, inputname, height, width}) => { - const [hovered, setHovered] = useState(""); - const inputdata = keys.data === undefined ? keys : keys.data - const {themeMode} = useContext(Context) - const theme = getTheme(themeMode) - - - return ( -
- - {inputname} - - - - } - /> - } - gridlines={ - } /> - } - /> - -
- ) -} - const AppStats = (defaultprops) => { const { @@ -92,6 +45,12 @@ const AppStats = (defaultprops) => { inputWorkflows, clickedFromOrgTab, syncStats, + statistics, + monthlyAppRunsParent, + setMonthlyAppRunsParent, + monthlyAllSuborgExecutions, + setMonthlyAllSuborgExecutions, + currentTab } = defaultprops; const [keys, setKeys] = useState([]) @@ -104,7 +63,6 @@ const AppStats = (defaultprops) => { const [endTime, setEndTime] = useState("") const [startTime, setStartTime] = useState("") - const [statistics, setStatistics] = useState(undefined); const [filteredStatistics, setFilteredStatistics] = useState(undefined); const [apprunCost, setApprunCost] = useState(0) @@ -125,6 +83,11 @@ const AppStats = (defaultprops) => { } }, []) + useEffect(() => { + if (statistics && statistics?.org_id?.length > 0) { + handleDataSetting(statistics, "day") + } +}, [statistics]) const getWorkflowStats = async (workflow, startTime, endTime) => { @@ -266,6 +229,7 @@ const AppStats = (defaultprops) => { const statKey = syncStats === true ? "onprem_stats" : "daily_statistics" if (statistics[statKey] === undefined || statistics[statKey] === null) { setFilteredStatistics(statistics) + setMonthlyAppRunsParent(statistics["monthly_app_executions"] ?? 0) return } @@ -312,20 +276,27 @@ const AppStats = (defaultprops) => { } } - // Make a date at the 1st of the current month + // Make a date at the 1st of the current month - only when no start time is selected var foundstarttime = (new Date()) - foundstarttime.setDate(1) if (startTime !== "" && startTime !== undefined && startTime !== null) { - foundstarttime = startTime + foundstarttime = new Date(startTime) + // Set to start of day to include the entire start date + foundstarttime.setHours(0, 0, 0, 0) + } else { + // Default to 1st of current month when no start time is selected + foundstarttime.setDate(1) + foundstarttime.setHours(0, 0, 0, 0) } - // Set to tomorrow by default + // Set end time properly var foundendtime = (new Date()) - foundendtime.setDate(foundendtime.getDate() + 1) - - // Check if endtime is after the daily statistics["date"] string if (endTime !== "" && endTime !== undefined && endTime !== null) { - foundendtime = endTime + foundendtime = new Date(endTime) + // Set to end of day to include the entire end date + foundendtime.setHours(23, 59, 59, 999) + } else { + // Default to current date when no end time is selected + foundendtime.setHours(23, 59, 59, 999) } // Check if start time is before the daily statistics["date"] string @@ -337,8 +308,20 @@ const AppStats = (defaultprops) => { } const date = new Date(item["date"]) - if (date >= foundstarttime) { - if (date <= foundendtime) { + // Normalize the date to start of day for comparison + const normalizedDate = new Date(date) + normalizedDate.setHours(0, 0, 0, 0) + + // Normalize foundstarttime for comparison + const normalizedStartTime = new Date(foundstarttime) + normalizedStartTime.setHours(0, 0, 0, 0) + + // Normalize foundendtime for comparison + const normalizedEndTime = new Date(foundendtime) + normalizedEndTime.setHours(0, 0, 0, 0) + + if (normalizedDate >= normalizedStartTime) { + if (normalizedDate <= normalizedEndTime) { newlist.push(item) } } @@ -373,6 +356,10 @@ const AppStats = (defaultprops) => { workflowexecutions += item["workflow_executions"] appexecutions += item["app_executions"] + if (currentTab === 0) { + appexecutions += (item["child_app_executions"] ?? 0) + } + estimatedcost += (item["app_executions"] * invocationCost) } @@ -390,7 +377,16 @@ const AppStats = (defaultprops) => { } setFilteredStatistics(tmpstats) - handleDataSetting(tmpstats, "day") + handleDataSetting(tmpstats, "day") + // if we have done monthly reset than only show monthly app runs as current month app run + const currentMonth = new Date().getMonth() + 1 + if (!monthlyAppRunsParent && statistics["monthly_app_executions"] > 0 && currentMonth === statistics["last_monthly_reset_month"]) { + setMonthlyAppRunsParent(statistics["monthly_app_executions"]) + } + + if (!monthlyAllSuborgExecutions && statistics["monthly_child_app_executions"]> 0 && currentMonth === statistics["last_monthly_reset_month"]) { + setMonthlyAllSuborgExecutions(statistics["monthly_child_app_executions"]) + } if (workflows !== undefined && workflows !== null && workflows.length > 0) { @@ -476,7 +472,7 @@ const AppStats = (defaultprops) => { if (item["child_app_executions"] !== undefined && item["child_app_executions"] !== null) { childorgappRuns["data"].push({ key: new Date(item["date"]), - data: inputdata["child_app_executions"] + data: item["child_app_executions"] }) } @@ -496,40 +492,45 @@ const AppStats = (defaultprops) => { } } - // Adds data for today - if (inputdata["daily_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) { - appRuns["data"].push({ - key: new Date(), - data: inputdata["daily_app_executions"] - }) + // Only add today's data if endTime is not set or if today falls within the selected date range + const today = new Date() + const shouldAddTodayData = endTime === "" || endTime === undefined || endTime === null || + (new Date(endTime) >= today.setHours(0, 0, 0, 0)) - appcostRuns["data"].push({ - key: new Date(), - data: (inputdata["daily_app_executions"] * invocationCost).toFixed(2) - }) - } + if (shouldAddTodayData) { + // Adds data for today + if (inputdata["daily_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) { + appRuns["data"].push({ + key: new Date(), + data: inputdata["daily_app_executions"] + }) - if (inputdata["daily_child_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) { - childorgappRuns["data"].push({ - key: new Date(), - data: inputdata["daily_child_app_executions"] - }) + appcostRuns["data"].push({ + key: new Date(), + data: (inputdata["daily_app_executions"] * invocationCost).toFixed(2) + }) + } - //setApprunCosts(appcostRuns) - } + if (inputdata["daily_child_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) { + childorgappRuns["data"].push({ + key: new Date(), + data: inputdata["daily_child_app_executions"] + }) + } - if (inputdata["daily_workflow_executions"] !== undefined && inputdata["daily_workflow_executions"] !== null) { - workflowRuns["data"].push({ - key: new Date(), - data: inputdata["daily_workflow_executions"] - }) - } + if (inputdata["daily_workflow_executions"] !== undefined && inputdata["daily_workflow_executions"] !== null) { + workflowRuns["data"].push({ + key: new Date(), + data: inputdata["daily_workflow_executions"] + }) + } - if (inputdata["daily_subflow_executions"] !== undefined && inputdata["daily_subflow_executions"] !== null) { - subflowRuns["data"].push({ - key: new Date(), - data: inputdata["daily_subflow_executions"] - }) + if (inputdata["daily_subflow_executions"] !== undefined && inputdata["daily_subflow_executions"] !== null) { + subflowRuns["data"].push({ + key: new Date(), + data: inputdata["daily_subflow_executions"] + }) + } } // Only for parent orgs @@ -541,49 +542,8 @@ const AppStats = (defaultprops) => { setWorkflowRuns(workflowRuns) setAppruns(appRuns) setApprunCosts(appcostRuns) - } - - const getStats = (orgid) => { - - if (orgid === undefined || orgid === null) { - return - } - - fetch(`${globalUrl}/api/v1/orgs/${orgid}/stats`, { - method: "GET", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - credentials: "include", - }) - .then((response) => { - if (response.status !== 200) { - console.log("Status not 200 for workflows :O!: ", response.status); - return; - } - - return response.json(); - }) - .then((responseJson) => { - if (responseJson["success"] === false) { - return - } - - setStatistics(responseJson) - handleDataSetting(responseJson, "day") - }) - .catch((error) => { - console.log("error: ", error) - }); } - useEffect(() => { - if(selectedOrganization?.id?.length > 0) { - getStats(selectedOrganization.id) - } - }, [selectedOrganization]) - const paperStyle = { textAlign: "center", padding: "40px", @@ -703,15 +663,19 @@ const AppStats = (defaultprops) => { ] const data = ( -
+
All shown statistics are gathered from Your Organisation Statistics. - It exists to give you more insight into your workflows, and to understand your utilization of the Shuffle platform. The billing tracker is in Beta, and is always calculated manually before being invoiced. + {currentTab === 0 ? + + All Organization app runs are calculated base on addition of parent org app runs + all child org app runs. + : It exists to give you more insight into your workflows, and to + understand your utilization of the Shuffle platform.{" "}}
{syncStats !== true ? null : "PS: You are currently looking at data from your onprem synced org"} @@ -722,7 +686,7 @@ const AppStats = (defaultprops) => { {filteredStatistics !== undefined ?
- {syncStats == true ? null : + {/* {syncStats == true ? null : The cost of app runs in the selected period based on {filteredStatistics.monthly_app_executions} App Runs. These numbers do not exclude your included 10.000/month or {includedExecutions} App Runs per month. App Run cost: ${invocationCost}. @@ -742,7 +706,7 @@ const AppStats = (defaultprops) => { - } + } */} {syncStats === true ? null : { } - {syncStats === true ? null : + {syncStats === true || currentTab === 0 ? null : Workflow runs in the selected period @@ -778,7 +742,7 @@ const AppStats = (defaultprops) => { } - {syncStats === true ? null : + {/* {syncStats === true ? null : Estimated cost to be billed at the end of the current month. Subtracted contractually included app runs. Actual cost month to date: ${monthToDateCost}. App Run cost: ${invocationCost}. @@ -793,15 +757,9 @@ const AppStats = (defaultprops) => { - } -
- : null} -
- - - {clickedFromOrgTab? ( - -
+ } */} + +
{
- ):( - -
- } - /> - } - />
-
- )} + : null} +
{appRuns === undefined ? null : - + } - {childOrgsAppRuns === undefined ? + {childOrgsAppRuns === undefined || currentTab === 1 ? null : - + } - {workflowRuns === undefined ? + {workflowRuns === undefined || currentTab === 0? null : - + } - {subflowRuns === undefined ? + {subflowRuns === undefined || currentTab === 0 ? null : - + } {/*appRunCosts === undefined ? @@ -991,7 +918,7 @@ const AppStats = (defaultprops) => { */} - {syncStats === true ? null : + {syncStats === true || currentTab === 0 ? null :
{resultLoading ?
@@ -1047,7 +974,7 @@ const AppStats = (defaultprops) => { ) const dataWrapper = ( -
{data}
+
{data}
); return dataWrapper; diff --git a/frontend/src/components/CacheView.jsx b/frontend/src/components/CacheView.jsx index 1e94d131..b4aac34b 100644 --- a/frontend/src/components/CacheView.jsx +++ b/frontend/src/components/CacheView.jsx @@ -4,7 +4,8 @@ import { getTheme } from "../theme.jsx"; import { toast } from 'react-toastify'; import ReactJson from "react-json-view-ssr"; -import { GetIconInfo } from "../views/Workflows2.jsx"; +import { GetIconInfo, } from "../views/Workflows2.jsx"; +import { validateJson, handleReactJsonClipboard, } from "../views/Workflows.jsx"; import { red } from "../views/AngularWorkflow.jsx"; import CollectIngestModal from "../components/CollectIngestModal.jsx"; import { @@ -34,6 +35,7 @@ import { InputLabel, Pagination, PaginationItem, + Avatar, } from "@mui/material"; import { @@ -76,8 +78,8 @@ import { SmartToy as SmartToyIcon, Settings as SettingsIcon, FilterAlt as FilterAltIcon, + CompareArrows as CompareArrowsIcon, } from "@mui/icons-material"; -import { validateJson, } from "../views/Workflows.jsx"; import { Context } from "../context/ContextApi.jsx"; const scrollStyle1 = { @@ -130,21 +132,22 @@ const CacheView = memo((props) => { }) const [_, setUpdate] = useState(Math.random()) const [selectedRows, setSelectedRows] = useState([]); - // Direct category migration from ../components/Files.jsx const [selectAllChecked, setSelectAllChecked] = React.useState(false) const [renderTextBox, setRenderTextBox] = React.useState(false); - const [datastoreCategories, setDatastoreCategories] = React.useState(["default"]); + const [datastoreCategories, setDatastoreCategories] = React.useState(["default", "protected"]); const [selectedCategory, setSelectedCategory] = React.useState("default"); const [selectedFileId, setSelectedFileId] = React.useState(""); const [updateToThisCategory, setUpdateToThisCategory] = useState("") const [workflows, setWorkflows] = useState([]); + const [apps, setApps] = useState([]); const [selectedFiles, setSelectedFiles] = useState([]); const [showAutomationMenu, setShowAutomationMenu] = useState(false); const [showSettingsMenu, setShowSettingsMenu] = useState(false); const [showCollectIngestMenu, setShowCollectIngestMenu] = useState(false); + var to_be_copied = ""; const defaultAutomation = [ { "name": "Run workflow", @@ -156,6 +159,42 @@ const CacheView = memo((props) => { "icon": , "enabled": false, }, + { + "name": "Correlate Categories", + "description": "", + "type": "singul", + "options": [{ + "key": "datastore_categories", + "value": "", + }], + "icon": , + "enabled": false, + "disabled": false, + }, + { + "name": "Run AI Agent", + "description": "", + "options": [{ + "key": "", + "value": "", + }], + "icon": , + "enabled": false, + "disabled": true, + }, + { + "name": "Send webhook", + "description": "Sends the updated value to a specified webhook URL.", + "options": [{ + "key": "webhook_url", + "value": "", + }], + "icon": , + "enabled": false, + }, + + + { "name": "Send message", "description": "", @@ -180,27 +219,6 @@ const CacheView = memo((props) => { "enabled": false, "disabled": true, }, - { - "name": "Run AI Agent", - "description": "", - "options": [{ - "key": "", - "value": "", - }], - "icon": , - "enabled": false, - "disabled": true, - }, - { - "name": "Send webhook", - "description": "Sends the updated value to a specified webhook URL.", - "options": [{ - "key": "webhook_url", - "value": "", - }], - "icon": , - "enabled": false, - }, ] const [categoryAutomations, setCategoryAutomations] = useState(defaultAutomation) @@ -210,6 +228,35 @@ const CacheView = memo((props) => { const theme = getTheme(themeMode, brandColor); const classes = useStyles(); + const getApps = () => { + const url = `${globalUrl}/api/v1/apps` + fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for workflows :O!"); + return; + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson?.success === false) { + toast.warn("Failed to load apps. Please try again or contact support@shuffler if this persists.") + } else { + setApps(responseJson) + } + }) + .catch((error) => { + toast(error.toString()); + }); + } const getWorkflows = () => { const url = `${globalUrl}/api/v1/workflows` @@ -251,6 +298,7 @@ const CacheView = memo((props) => { useEffect(() => { getWorkflows() + getApps() listOrgCache(orgId, selectedCategory, 0, pageSize, page) }, []) @@ -275,7 +323,6 @@ const CacheView = memo((props) => { } var url = `${globalUrl}/api/v1/orgs/${orgId}/list_cache` - if (category !== undefined && category !== null && category !== "default" && category !== "") { url += "?category=" + category.replaceAll(" ", "_") } else { @@ -322,7 +369,7 @@ const CacheView = memo((props) => { .then((responseJson) => { setCachedLoaded(true); if (responseJson?.success === true) { - setListCache(responseJson.keys) + setListCache(responseJson.keys); if (responseJson.total_amount !== undefined && responseJson.total_amount !== null && responseJson.total_amount > 0) { setTotalAmount(responseJson.total_amount) @@ -351,8 +398,8 @@ const CacheView = memo((props) => { } } - if ((category === undefined || category === "default" || category === "") && datastoreCategories.length === 1 && datastoreCategories[0] === "default") { - var newcategories = ["default"] + if ((category === undefined || category === "default" || category === "") && datastoreCategories.length === 2 && datastoreCategories[0] === "default") { + var newcategories = ["default", "protected"] for (var key in responseJson.keys) { var foundcategory = responseJson.keys[key].category if (foundcategory !== undefined && foundcategory !== null && foundcategory !== ""){ @@ -538,7 +585,7 @@ const CacheView = memo((props) => { }) .then((responseJson) => { setAddCache(responseJson); - toast("New key added Successfully!"); + toast.success("New key added!"); listOrgCache(orgId, selectedCategory, 0, pageSize, page); setModalOpen(false); }) @@ -563,34 +610,6 @@ const CacheView = memo((props) => { } } - const handleReactJsonClipboard = (copy) => { - const elementName = "copy_element_shuffle"; - let copyText = document.getElementById(elementName); - - if (copyText) { - if (copy.namespace && copy.name && copy.src) { - copy = copy.src; - } - - const clipboard = navigator.clipboard; - if (!clipboard) { - toast("Can only copy over HTTPS (port 3443)"); - return; - } - - let stringified = JSON.stringify(copy); - if (stringified.startsWith('"') && stringified.endsWith('"')) { - stringified = stringified.slice(1, -1); - } - - navigator.clipboard.writeText(stringified); - toast("Copied value to clipboard, NOT json path."); - } else { - console.log("Failed to copy from " + elementName + ": ", copyText); - } - }; - - const timestamp = (timestamp) => { if (timestamp === undefined || timestamp === null || timestamp === "") { return null @@ -680,6 +699,7 @@ const CacheView = memo((props) => {
+ { {showOptions && ( updatedAutomation.options.map((option, optionIndex) => { - if (option?.key === "workflow_id") { + if (option?.key === "datastore_categories") { + if (datastoreCategories === undefined || datastoreCategories === null || datastoreCategories.length <= 1) { + return ( + + No categories available. Please add categories in the settings. + + ) + } + + return ( + option?.value.includes(c)) || []} + classes={{ inputRoot: classes.inputRoot }} + ListboxProps={{ + style: { + backgroundColor: theme.palette.surfaceColor, + color: theme.palette.text.primary, + borderRadius: theme.palette.borderRadius, + }, + }} + onChange={(event, newValue) => { + console.log("New Value: ", newValue) + + option.value = "" + for (var i = 0; i < newValue.length; i++) { + option.value += newValue[i] + "," + } + + if (newValue.length > 0) { + updatedAutomation.enabled = true + } else { + updatedAutomation.enabled = false + } + + updatedAutomation.options[optionIndex] = option + setUpdatedAutomation(updatedAutomation) + setUpdated(true) + + setUpdate(Math.random()) // Force re-render + }} + + getOptionLabel={(option) => { + if (option === undefined || option === null) { + return "No Categories Selected"; + } + + return option + }} + options={datastoreCategories} + fullWidth + style={{ + backgroundColor: theme.palette.textFieldStyle.backgroundColor, + borderRadius: theme.palette.textFieldStyle.borderRadius, + color: theme.palette.textFieldStyle.color, + height: 35, + marginBottom: 40, + }} + renderOption={(props, data, state) => { + const fixedname = data?.charAt(0)?.toUpperCase() + data?.slice(1)?.replaceAll("_", " ") + const iconDetails = GetIconInfo({ + "app_name": fixedname, + "name": fixedname, + }) + + const keyfound = option?.value.includes(data) + + return ( + + {data.image !== undefined && data.image !== null && data.image.length > 0 ? + {data.name} + : null} + + Choose {data} + + + } > + + +
+ {iconDetails?.originalIcon && ( + iconDetails?.originalIcon + )} +
+ + {fixedname} +
+
+
+ ) + }} + renderInput={(params) => { + return ( + + ) + }} + /> + ) + } else if (option?.key === "workflow_id") { return ( { {...props} style={{ backgroundColor: theme.palette.surfaceColor, - color: data.id === option?.value ? "red" : theme.palette.text.primary, - borderBottom: data.id === "parent" ? "2px solid rgba(255,255,255,0.5)" : null + color: data.id === option?.value ? red : theme.palette.text.primary, }} value={data} > @@ -1297,12 +1462,25 @@ const CacheView = memo((props) => { sortable: true, }, { - width: 600, + width: 540, field: 'value', filterable: true, headerName: 'Value', renderCell: (props) => { const data = props.row + + if (data?.category?.toLowerCase() === "protected") { + return ( + + *************** + + ) + } + const validate = validateJson(data.value) return ( @@ -1321,22 +1499,17 @@ const CacheView = memo((props) => { backgroundColor: theme.palette.platformColor, border: theme.palette.defaultBorder, padding: 5, - minWidth: 600, - maxHeight: 600, + minWidth: 500, + maxHeight: 500, overflowY: "auto", }} collapsed={true} enableClipboard={(copy) => { - // handleReactJsonClipboard(copy); + handleReactJsonClipboard(copy) }} collapseStringsAfterLength={theme.palette.jsonCollapseStringsAfterLength} iconStyle={theme.palette.jsonIconStyle} displayDataTypes={false} - onSelect={(select) => { - - // HandleJsonCopy(showResult, select, data.action.label); - console.log("SELECTED!: ", select); - }} name={null} /> : @@ -1348,6 +1521,74 @@ const CacheView = memo((props) => { ) } }, + { + field: 'category', + headerName: 'Category', + description: 'Category for this key.', + width: 75, + filterable: false, + sortable: true, + renderCell: (props) => { + // Return avatar with hover for the category + const data = props.row + const clickCategory = (e) => { + setCategoryConfig(undefined) + setCategoryAutomations(defaultAutomation) + + if (selectAllChecked || selectedFiles.length > 0) { + setUpdateToThisCategory(data.category) + return + } + + setSelectedCategory(data.category) + if (data.category === "all" || data.category === "default") { + listOrgCache(orgId, "", 0, pageSize, page) + } else { + listOrgCache(orgId, data.category, 0, pageSize, page) + } + + // Add it to the url as a query + if (window.location.search.includes("category=")) { + const newurl = window.location.href.replace(/category=[^&]+/, `category=${data.category}`) + window.history.pushState({ path: newurl }, "", newurl) + } else { + window.history.pushState({ path: window.location.href }, "", `${window.location.href}&category=${data.category}`) + } + } + + + const iconDetails = GetIconInfo({ + "app_name": data.category, + "name": data.category, + }) + + const avatarLetter = (data.category === "" || data.category === "default" ? " " : data.category.charAt(0).toUpperCase())[0] + return ( + + { + clickCategory(e) + }} + style={{ + color: "white", + backgroundColor: iconDetails?.iconBackgroundColor || theme.palette.primary.secondary, + marginLeft: 15, + height: 30, + width: 30, + cursor: data.category !== "" && data.category !== "default" ? "pointer" : "default", + }} + variant="rounded" + > + {iconDetails?.originalIcon ? + iconDetails?.originalIcon + : + avatarLetter + } + + + ) + } + }, { field: 'actions', headerName: 'Actions', @@ -1361,7 +1602,7 @@ const CacheView = memo((props) => { return ( - {data?.workflow_id === "" || data?.workflow_id === null || data?.workflow_id === undefined ? + {data?.workflow_id === "" || data?.workflow_id === null || data?.workflow_id === undefined || data?.workflow_id?.length !== 36 ? { { title={"Go to workflow"} style={{}} aria-label={"Download"} + placement="left" > { > { { + onClick={(e) => { + e.preventDefault() + e.stopPropagation() // Try to make the value JSON indented const valid = validateJson(data.value) var newvalue = data.value @@ -1469,7 +1713,9 @@ const CacheView = memo((props) => { { + onClick={(e) => { + e.preventDefault() + e.stopPropagation() window.open(`${globalUrl}/api/v1/orgs/${orgId}/cache/${data.key}?type=text&authorization=${data.public_authorization}`, "_blank"); }} > @@ -1486,7 +1732,9 @@ const CacheView = memo((props) => { { + onClick={(e) => { + e.preventDefault() + e.stopPropagation() deleteEntry(orgId, data.key, data.category) }} > @@ -1609,6 +1857,8 @@ const CacheView = memo((props) => { workflows={workflows} getWorkflows={getWorkflows} + + apps={apps} /> {cacheDistributionModal} @@ -1658,6 +1908,13 @@ const CacheView = memo((props) => { > Learn more + + {selectedCategory === "protected" ? +
+ Protected keys are encrypted, only available to admins, and will be masked when used in workflows. This is a basic protection, and is NOT bulletproof. +
+ : null} +
@@ -2067,15 +2324,17 @@ const CacheView = memo((props) => { { + setSelectedRows(newSelection); + }} onRowSelectionModelChange={(newSelection) => { - setSelectedRows(newSelection) + setSelectedRows(newSelection); }} keepNonExistentRowsSelected={false} - getRowId={(row) => row.key} + getRowId={(row) => `${row?.key}_${row?.category}`} autoHeight={true} sx={{ @@ -2245,6 +2504,11 @@ const CacheView = memo((props) => {
+
); }) diff --git a/frontend/src/components/CollectIngestModal.jsx b/frontend/src/components/CollectIngestModal.jsx index afba4afa..d00faffd 100644 --- a/frontend/src/components/CollectIngestModal.jsx +++ b/frontend/src/components/CollectIngestModal.jsx @@ -10,19 +10,27 @@ import { DialogTitle, DialogContent, Typography, + IconButton, Paper, LinearProgress, Grid, Button, + Tooltip, + Autocomplete, + TextField, } from '@mui/material'; import { Rocket as RocketIcon, FilterAlt as FilterAltIcon, + Add as AddIcon, } from '@mui/icons-material'; +import algoliasearch from 'algoliasearch/lite'; +const searchClient = algoliasearch("JNSS5CFDZZ", "c8f882473ff42d41158430be09ec2b4e") + const CollectIngestModal = (props) => { - const { globalUrl, open, setOpen } = props; + const { globalUrl, open, setOpen, workflows, getWorkflows, apps, } = props; const { themeMode, brandColor } = useContext(Context); const theme = getTheme(themeMode, brandColor); @@ -37,12 +45,17 @@ const CollectIngestModal = (props) => { return null } - const startIngestion = (appname, index) => { - console.log("APPNAME:", appname, "INDEX:", index) + const startIngestion = (bundleName, appnames, category, index) => { - const body = { - "app_name": appname, - "label": appname, + var body = { + "label": bundleName, + "app_name": appnames, + + "category": "", + } + + if (category !== undefined && category !== null) { + body.category = category } const url = `${globalUrl}/api/v2/workflows/generate` @@ -61,21 +74,29 @@ const CollectIngestModal = (props) => { return response.json(); }) .then((data) => { + + if (getWorkflows !== undefined) { + getWorkflows() + } + console.log("Ingestion started successfully:", data); - toast.success(`Ingestion for ${appname} started successfully!`); + toast.success(`Ingestion for ${bundleName} started successfully!`); }) .catch((error) => { console.error("Error starting ingestion:", error); - toast.error(`Failed to start ingestion for ${appname}. Please try again.`); + toast.error(`Failed to start ingestion for ${bundleName}. Please try again or contact support@shuffler.io if this persists.`); }); } const IngestItem = (props) => { - const { type, index } = props + const { type, appCategory, index, webhook } = props const [hovering, setHovering] = useState(false); - const [isFinished, setIsFinished] = useState(false); + const [selectedApps, setSelectedApps] = useState([]); + + const [showAppsearch, setShowAppsearch] = useState(false); + const [algoliaOptions, setAlgoliaOptions] = useState([]); const appname = type const ingestedAmount = 20 @@ -85,49 +106,222 @@ const CollectIngestModal = (props) => { "name": appname, }) + var foundMatchingWorkflow = null + if (workflows !== undefined && workflows !== null && workflows.length > 0) { + const parsedName = type.toLowerCase().replaceAll(" ", "_"); + const foundWorkflow = workflows.find((workflow) => { + return workflow?.name?.toLowerCase().replaceAll(" ", "_") === parsedName + }) + + if (foundWorkflow !== undefined && foundWorkflow !== null) { + foundMatchingWorkflow = foundWorkflow + + + // Find relevant apps and maps them + if (apps.length > 0 && selectedApps.length === 0 && foundWorkflow?.actions !== undefined && foundWorkflow?.actions !== null && foundWorkflow?.actions.length > 0) { + var newSelectedApps = [] + for (var actionkey in foundWorkflow.actions) { + const action = foundWorkflow.actions[actionkey] + + if (action?.app_name !== "Singul" && action?.app_id !== "integration") { + continue + } + + for (var paramkey in action.parameters) { + const param = action.parameters[paramkey] + if (!((param.name === "app_name" || param.name === "appName") && param.value !== undefined && param.value !== null && param.value.length > 0)) { + continue + } + + // Find the app in available apps + const parsedname = param.value.replaceAll(" ", "_").toLowerCase() + for (var appkey in apps) { + const appname = apps[appkey].name.replaceAll(" ", "_").toLowerCase() + if (appname === parsedname) { + newSelectedApps.push(apps[appkey]) + break + } + } + + break + } + } + + if (newSelectedApps.length > 0) { + setSelectedApps(newSelectedApps) + } + } + + } + } + + var matchingapps = [] + if (appCategory !== undefined && appCategory !== null && appCategory.length > 0 && apps !== undefined && apps !== null && apps.length > 0) { + for (var appkey in apps) { + const app = apps[appkey] + + for (var categorykey in app.categories) { + const category = app.categories[categorykey] + if (category.toLowerCase() === appCategory.toLowerCase()) { + matchingapps.push(app) + } + } + } + } + return ( // setHovering(true)} + onMouseEnter={() => { + + //if (foundMatchingWorkflow !== null) { + //} else { + setHovering(true) + //} + }} onMouseLeave={() => setHovering(false)} > -
- {iconDetails?.originalIcon && ( - iconDetails?.originalIcon - )} +
- +
- {appname} - +
+ {selectedApps.map((app, index) => { + // Show image of each one + return ( +
+ + + +
+ ) + })} + + + { + setShowAppsearch(!showAppsearch) + }} + > + + + +
+ + {showAppsearch ? + { + setSelectedApps(value) + }} + + getOptionLabel={(option) => { + const parsedname = option.name.replaceAll("_", " ") + + return ( +
+ {option.name} + + {parsedname} + +
+ ) + }} + renderInput={(params) => { + return ( + + ) + }} + /> + : + + } +
+ +
+ {iconDetails?.originalIcon && ( + iconDetails?.originalIcon + )} + + + + {appname} + +
- + {foundMatchingWorkflow !== null ? + + + + + + + + : null} {hovering ?
: null} - {isFinished ? + {foundMatchingWorkflow !== null ?
{ }}> {ingestedAmount} / X + + {/* { variant="determinate" fullWidth value={{ingestedAmount}} /> + */}
: null} @@ -158,7 +355,7 @@ const CollectIngestModal = (props) => { sx: { borderRadius: theme?.palette?.DialogStyle?.borderRadius, border: theme?.palette?.DialogStyle?.border, - minWidth: 500, + minWidth: 850, minHeight: 700, fontFamily: theme?.typography?.fontFamily, backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, @@ -189,9 +386,10 @@ const CollectIngestModal = (props) => { - + - + + diff --git a/frontend/src/components/DashboardBarchart.jsx b/frontend/src/components/DashboardBarchart.jsx index 74970549..fbeab07e 100644 --- a/frontend/src/components/DashboardBarchart.jsx +++ b/frontend/src/components/DashboardBarchart.jsx @@ -1,7 +1,8 @@ import React from 'react'; -import { Bar } from 'react-chartjs-2'; import { toast } from "react-toastify"; +import LineChartWrapper from '../components/LineChartWrapper.jsx'; + export const LoadStats = (globalUrl, cachekey) => { if (globalUrl === undefined) { console.log("Error: Global URL is undefined") @@ -64,84 +65,13 @@ export const LoadStats = (globalUrl, cachekey) => { }) } +// This wrapper is a waste lol const DashboardBarchart = (props) => { - // this is clearly unfinished and not worth my time. - // refer to health page to see how i made it work there w/o using chartjs + const { timelineData, title, height, } = props; - // const { timelineData, title, height, } = props; - // var inputHeight = 15 - // if (height !== undefined && height !== null) { - // inputHeight = height - // } - - // const barOptions = { - // plugins: { - // tooltip: { - // enabled: true, // Ensure tooltips are enabled - // }, - // }, - // tooltips: { - // mode: 'index', - // intersect: false, - // }, - // legend: { - // display: false - // }, - // layout: { - // padding: { - // top: 0, // Adjust the top padding as needed - // bottom: -10, // Adjust the bottom padding as needed - // left: 0, // Adjust the left padding as needed - // right: 0, // Adjust the right padding as needed - // }, - // }, - // scales: { - // y: { - // beginAtZero: false, - // }, - // yAxes: [{ - // ticks: { - // display: false - // }, - // beginAtZero: false, - // }], - // xAxes: [{ - // ticks: { - // display: false - // }, - // beginAtZero: false, - // }] - // }, - // tooltips: { - // callbacks: { - // label: function (tooltipItem, data) { - // const label = data.labels[tooltipItem.index] - // return label.split('\n')[0] - // }, - // afterLabel: function (tooltipItem, data) { - // const amount = tooltipItem.value === undefined || tooltipItem.value === null ? 0 : tooltipItem.value - // return `Amount: ${amount}` - // }, - // title: function () { - // return title === undefined ? '' : title - // } - // } - // } - // } - - // return ( - // { - // if (elements && elements.length > 0) { - // //toast("Click event") - // console.log("Clicked: ", elements) - // } - // }} - // /> - // ) + return ( + + ) } export default DashboardBarchart; diff --git a/frontend/src/components/EditOrgTab.jsx b/frontend/src/components/EditOrgTab.jsx index 0e2b5176..b1646e68 100644 --- a/frontend/src/components/EditOrgTab.jsx +++ b/frontend/src/components/EditOrgTab.jsx @@ -25,6 +25,7 @@ const EditOrgTab = (props) => { handleGetOrg, selectedStatus, setSelectedStatus, handleEditOrg, + handleStatusChange } = props; const [organizationFeatures, setOrganizationFeatures] = React.useState({}); const [users, setUsers] = React.useState([]); @@ -39,43 +40,6 @@ const EditOrgTab = (props) => { const { themeMode, brandColor } = useContext(Context); const theme = getTheme(themeMode, brandColor); - const handleStatusChange = (event) => { - const { value } = event.target; - setSelectedStatus(value); - - handleEditOrg( - selectedOrganization?.name, - selectedOrganization?.description, - selectedOrganization.id, - selectedOrganization.image, - { - app_download_repo: selectedOrganization?.defaults?.app_download_repo, - app_download_branch: selectedOrganization?.defaults?.app_download_branch, - workflow_download_repo: selectedOrganization?.defaults?.workflow_download_repo, - workflow_download_branch: selectedOrganization?.defaults?.workflow_download_branch, - notification_workflow: selectedOrganization?.defaults?.notification_workflow, - documentation_reference: selectedOrganization?.defaults?.documentation_reference, - workflow_upload_repo: selectedOrganization?.defaults?.workflow_upload_repo, - workflow_upload_branch: selectedOrganization?.defaults?.workflow_upload_branch, - workflow_upload_username: selectedOrganization?.defaults?.workflow_upload_username, - workflow_upload_token: selectedOrganization?.defaults?.workflow_upload_token, - newsletter: selectedOrganization?.defaults?.newsletter, - weekly_recommendations: selectedOrganization?.defaults?.weekly_recommendations, - }, - { - sso_entrypoint: selectedOrganization?.sso_config?.sso_entrypoint, - sso_certificate: selectedOrganization?.sso_config?.sso_certificate, - client_id: selectedOrganization?.sso_config?.client_id, - client_secret: selectedOrganization?.sso_config?.client_secret, - openid_authorization: selectedOrganization?.sso_config?.openid_authorization, - openid_token: selectedOrganization?.sso_config?.openid_token, - SSORequired: selectedOrganization?.sso_config?.SSORequired, - auto_provision: selectedOrganization?.sso_config?.auto_provision, - }, - value.length === 0 ? ["none"] : value, - ); - }; - const getUsers = () => { fetch(globalUrl + "/api/v1/getusers", { @@ -443,6 +407,7 @@ If you're interested, please let me know a time that works for you, or set up a isEditOrgTab={true} handleGetOrg={handleGetOrg} serverside={serverside} + handleStatusChange={handleStatusChange} />
diff --git a/frontend/src/components/EditWorkflow.jsx b/frontend/src/components/EditWorkflow.jsx index dbd8d3ea..9e83cc34 100644 --- a/frontend/src/components/EditWorkflow.jsx +++ b/frontend/src/components/EditWorkflow.jsx @@ -3,6 +3,7 @@ import { getTheme } from '../theme.jsx'; import { isMobile } from "react-device-detect" import { MuiChipsInput } from "mui-chips-input"; import { toast } from "react-toastify" +import ReactGA from 'react-ga4'; import UsecaseSearch from "../components/UsecaseSearch.jsx" import WorkflowGrid from "../components/WorkflowGrid.jsx" import dayjs from 'dayjs'; @@ -63,6 +64,10 @@ import { Add as AddIcon, Remove as RemoveIcon, EditNote as EditNoteIcon, + AutoAwesome as AutoAwesomeIcon, + CloudUpload as CloudUploadIcon, + CheckCircle as CheckCircleIcon, + Close as CloseIcon } from "@mui/icons-material"; const EditWorkflow = (props) => { @@ -72,6 +77,7 @@ const EditWorkflow = (props) => { const {themeMode, brandColor} = useContext(Context) const theme = getTheme(themeMode, brandColor) const [submitLoading, setSubmitLoading] = React.useState(false); + const [aiGenerateLoading, setAiGenerateLoading] = React.useState(false); const [showMoreClicked, setShowMoreClicked] = React.useState(isEditing !== false ? true : false); const [innerWorkflow, setInnerWorkflow] = React.useState(workflow) @@ -92,6 +98,11 @@ const EditWorkflow = (props) => { const [selectedCleanupActions, setSelectedCleanupActions] = React.useState(workflow?.form_control?.cleanup_actions !== undefined && workflow?.form_control?.cleanup_actions !== null ? JSON.parse(JSON.stringify(workflow?.form_control?.cleanup_actions)) : []) const [formWidth, setFormWidth] = React.useState(boxWidth === undefined || boxWidth === null ? 500 : boxWidth) + + // Flowchart upload states + const [uploadedImage, setUploadedImage] = React.useState(null) + const [imageBase64, setImageBase64] = React.useState("") + const [imageUploading, setImageUploading] = React.useState(false) const classes = useStyles(); @@ -101,6 +112,59 @@ const EditWorkflow = (props) => { } }, [formWidth]) + // Handle file upload and base64 conversion + const handleImageUpload = (file) => { + const allowedTypes = ['image/png', 'image/jpeg', 'image/jpg'] + if (!allowedTypes.includes(file.type)) { + toast.error("Please upload a PNG, JPG, or JPEG image") + return + } + + const maxSize = 5 * 1024 * 1024 // 5MB in bytes + if (file.size > maxSize) { + toast.error("Image must be less than 5MB") + return + } + + setImageUploading(true) + + const reader = new FileReader() + reader.onload = (e) => { + const base64 = e.target.result + setImageBase64(base64) + setUploadedImage({ + name: file.name, + size: file.size, + type: file.type + }) + setImageUploading(false) + + // Disable "Create from scratch" when image is uploaded + if (newWorkflow) { + setWorkflowAsCode(false) + } + } + reader.onerror = () => { + toast.error("Failed to read image file") + setImageUploading(false) + } + reader.readAsDataURL(file) + } + + const removeUploadedImage = () => { + setUploadedImage(null) + setImageBase64("") + setImageUploading(false) + } + + const formatFileSize = (bytes) => { + if (bytes === 0) return '0 Bytes' + const k = 1024 + const sizes = ['Bytes', 'KB', 'MB', 'GB'] + const i = Math.floor(Math.log(bytes) / Math.log(k)) + return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i] + } + if (scrollTo !== undefined && scrollTo !== null && scrollTo.length > 0 && scrollDone === false) { setTimeout(() => { const foundScroll = document.getElementById(scrollTo) @@ -220,7 +284,7 @@ const EditWorkflow = (props) => {
- + {newWorkflow ? "New" : "Editing"} Workflow @@ -298,93 +362,371 @@ const EditWorkflow = (props) => { paddingLeft: 30, backgroundColor: themeMode === "dark" ? "#262626" : theme.palette.DialogStyle.backgroundColor, }}> - + + + Generate using the name, description, usecases and tags provided. Required: Name + (Description OR Flowchart Image) + + } + > + + + + +
+ ) : ( + + // If new workflow, don't close it + if (isEditing) { + setModalOpen(false) + } + }} + color="primary" + > + {submitLoading ? : "Save Changes"} + + )}
@@ -409,6 +751,30 @@ const EditWorkflow = (props) => { id="Enter-Workflow-Name" /> + {newWorkflow === true ? + { + innerWorkflow.default_return_value = event.target.value + setInnerWorkflow(innerWorkflow) + setUpdate(Math.random()) + }} + InputProps={{ + style: { + color: "white", + }, + }} + color="primary" + defaultValue={innerWorkflow.default_return_value} + placeholder="Please describe your workflow below so the AI can generate it." + rows="3" + multiline + label="Description" + margin="dense" + fullWidth + /> + : null} +
{usecases !== null && usecases !== undefined && usecases.length > 0 ? @@ -505,6 +871,98 @@ const EditWorkflow = (props) => { />
+ {/* Flowchart Upload Section - Only for new workflows */} + {newWorkflow === true ? ( +
+ {!uploadedImage ? ( +
{ + const input = document.createElement('input') + input.type = 'file' + input.accept = 'image/png,image/jpeg,image/jpg' + input.onchange = (e) => { + if (e.target.files.length > 0) { + handleImageUpload(e.target.files[0]) + } + } + input.click() + }} + > + {imageUploading ? ( +
+ + + Processing image... + +
+ ) : ( +
+ + + Generate Workflow from Flowchart + + + Click to upload your flowchart - AI will convert it to a workflow + + + PNG, JPG, JPEG • Max 5MB + +
+ )} +
+ ) : ( +
+
+ +
+ + {uploadedImage.name} + + + {formatFileSize(uploadedImage.size)} + +
+
+ + + +
+ )} +
+ ) : null} + {showMoreClicked === true ?
{ - Beta Feature: When a workflow run is done, the data from the selected actions will be removed by replacing it with a default value. This is useful for cleaning up sensitive data, or data that is no longer needed. This is done after a workflow run is finished or aborted, and is not reversible. Data will remain in the workflow run result (last node value) even if the action result itself is cleaned up. + When a workflow run is done, the data from the selected actions will be removed by replacing it with a default value. This is useful for cleaning up sensitive data, or data that is no longer needed. This is done after a workflow run is finished or aborted, and is not reversible. Data will remain in the workflow run result (last node value) even if the action result itself is cleaned up. @@ -1285,7 +1743,7 @@ const EditWorkflow = (props) => { - {newWorkflow === true ? + {/*newWorkflow === true ? Relevant Workflows @@ -1311,9 +1769,8 @@ const EditWorkflow = (props) => { } - : null} - - {/*newWorkflow === true && name.length > 2 ? + : null*/} + {/*newWorkflow === true && name.length > 2 ?
{ //toast("No Orborus necessary for environment cloud. Create and use a different environment to run executions on-premises.",) return } - + if ( props.userdata.active_org === undefined || props.userdata.active_org === null ) { return; } - + const elementName = "copy_element_shuffle"; var auth = environment.auth === "" ? "cb5st3d3Z!3X3zaJ*Pc" : environment.auth - - // Escape exclamation marks for copying - auth = auth.replace("\\!", "!").replace(/!/g, "\\!") - + // Escape exclamation marks for copying + auth = auth.replace("\\!", "!").replace(/!/g, "\\!") const newUrl = globalUrl === "https://shuffler.io" ? "https://shuffle-backend-stbuwivzoq-nw.a.run.app" : globalUrl; - + var skipPipeline = false if (commandController.pipelines === true) { skipPipeline = true } - + var addProxy = false if (commandController.proxies === true) { addProxy = true } - + if (installationTab === 1) { return (`docker run -d \\ --restart=always \\ @@ -428,14 +426,32 @@ const EnvironmentTab = memo((props) => { -e SHUFFLE_SWARM_CONFIG=run \\ -e SHUFFLE_LOGS_DISABLED=true \\ -e BASE_URL="${newUrl}" \\${addProxy ? ` - -e HTTPS_PROXY=IP:PORT \\` : ""}${skipPipeline ? ` - -e SHUFFLE_SKIP_PIPELINES=true \\` : ""} + -e HTTPS_PROXY=IP:PORT \\` : ""}${skipPipeline ? ` + -e SHUFFLE_SKIP_PIPELINES=true \\` : ""} ghcr.io/shuffle/shuffle-orborus:latest `) } else if (installationTab === 2) { - return `https://shuffler.io/docs/configuration#kubernetes` + return `helm install shuffle-orborus oci://ghcr.io/shuffle/charts/shuffle \\ + --namespace shuffle --create-namespace \\ + --set opensearch.enabled=false \\ + --set backend.replicaCount=0 \\ + --set frontend.replicaCount=0 \\ + --set 'orborus.extraEnvVars[0].name=AUTH' \\ + --set 'orborus.extraEnvVars[0].value=${auth}' \\ + --set 'orborus.extraEnvVars[1].name=BASE_URL' \\ + --set 'orborus.extraEnvVars[1].value=${newUrl}' \\ + --set 'orborus.extraEnvVars[2].name=ENVIRONMENT_NAME' \\ + --set 'orborus.extraEnvVars[2].value=${environment.Name}' \\ + --set 'orborus.extraEnvVars[3].name=ORG' \\ + --set 'orborus.extraEnvVars[3].value=${environment.org_id}' \\ + --set shuffle.org="${environment.org_id}"${addProxy ? ` \\ + --set 'orborus.extraEnvVars[4].name=HTTPS_PROXY' \\ + --set 'orborus.extraEnvVars[4].value=IP:PORT'` : ""}${skipPipeline ? ` \\ + --set 'orborus.extraEnvVars[${addProxy ? 5 : 4}].name=SHUFFLE_SKIP_PIPELINES' \\ + --set 'orborus.extraEnvVars[${addProxy ? 5 : 4}].value=true'` : ""} \\ + --set persistence.storageClass=standard` } - + const commandData = `docker rm shuffle-orborus --force; \\\ndocker run -d \\ --restart=always \\ --name="shuffle-orborus" \\ @@ -445,12 +461,13 @@ const EnvironmentTab = memo((props) => { -e ENVIRONMENT_NAME="${environment.Name}" \\ -e ORG="${props.userdata.active_org.id}" \\ -e BASE_URL="${newUrl}" \\${addProxy ? ` - -e HTTPS_PROXY=IP:PORT \\` : ""}${skipPipeline ? ` - -e SHUFFLE_SKIP_PIPELINES=true \\` : ""} + -e HTTPS_PROXY=IP:PORT \\` : ""}${skipPipeline ? ` + -e SHUFFLE_SKIP_PIPELINES=true \\` : ""} ghcr.io/shuffle/shuffle-orborus:latest` - + return commandData - }; + }; + const submitEnvironment = (data) => { // FIXME - add some check here ROFL @@ -1475,7 +1492,7 @@ const EnvironmentTab = memo((props) => { {installationTab === 2 ? - Check our Kubernetes documentation for more information on how to run Shuffle on Kubernetes. The status of the node will change when connected. + Simply connect to your Kubernetes cluster and run the following command: : @@ -1489,7 +1506,6 @@ const EnvironmentTab = memo((props) => { "2. Run this command on the server you want to run workflows or store Pipeline data on"} - {installationTab === 2 ? null :
{ }} />
-
- } +
{installationTab === 2 ? null : diff --git a/frontend/src/components/Files.jsx b/frontend/src/components/Files.jsx index 71c3af28..50872ae9 100644 --- a/frontend/src/components/Files.jsx +++ b/frontend/src/components/Files.jsx @@ -25,8 +25,12 @@ import { Checkbox, Chip, Menu, + Pagination, + PaginationItem, } from "@mui/material"; +import { DataGrid } from "@mui/x-data-grid"; + import { Link as LinkIcon, OpenInNew as OpenInNewIcon, @@ -45,6 +49,7 @@ import Dropzone from "../components/Dropzone.jsx"; import ShuffleCodeEditor from "../components/ShuffleCodeEditor1.jsx"; import {getTheme} from "../theme.jsx"; import { Context } from "../context/ContextApi.jsx"; +import { red } from "../views/AngularWorkflow.jsx"; const Files = memo((props) => { const { globalUrl, userdata, serverside, selectedOrganization, isCloud,isSelectedFiles } = props; @@ -75,9 +80,370 @@ const Files = memo((props) => { const [showDistributionPopup, setShowDistributionPopup] = useState(false) const [selectedSubOrg, setSelectedSubOrg] = useState([]) const [fileIdSelectedForDistribution, setFileIdSelectedForDistribution] = useState("") + const [totalAmount, setTotalAmount] = useState(0); +const [page, setPage] = useState(0); +const [pageSize, setPageSize] = useState(50) +const [selectedRows, setSelectedRows] = useState([]); +const [filesLoaded, setFilesLoaded] = useState(false); //const alert = useAlert(); const allowedFileTypes = ["txt", "py", "yaml", "yml","json", "html", "js", "csv", "log", "eml", "msg", "md", "xml", "sh", "bat", "ps1", "psm1", "psd1", "ps1xml", "pssc", "psc1", "response"] var upload = ""; + const paginatedRows = files.slice(page * pageSize, (page + 1) * pageSize); + + const columns = [ + { + field : 'filename', + headerName: 'Name', + filterable: true, + sortable: true, + width: 250, + renderCell: (params) => { + if (params.row.filename === undefined || params.row.filename === null || params.row.filename.length < 1) { + return ( + + No name + + ) + } + + return ( + + + {params.row.filename} + + + ); + } + }, + { + field: 'Workflow', + headerName: 'Workflow', + renderCell: (params) => { + const file = params.row; + return ( + file.workflow_id === "global" || !file.workflow_id ? ( + + + + ) : ( + + + + + + + + + + ) + ); + }, +}, +{ + field: 'md5_sum', + headerName: 'MD5', + width: 100, + }, + { + field: "Status", + headerName: "Status", + renderCell: (params) => { + const file = params.row; + return ( + + {file.status.charAt(0).toUpperCase() + file.status.slice(1)} + + ); + } + }, + { + field: "filesize", + headerName: "Filesize", + + }, + { + field: "actions", + headerName: "Actions", + width: 200, + renderCell: (params) => { + const file = params.row; + const filenamesplit = file.filename.split(".") + const iseditable = file.filesize < 2000000 && file.status === "active" && (allowedFileTypes.includes(filenamesplit[filenamesplit.length-1]) || !file?.filename.includes(".")) + return ( + + + + { + e.stopPropagation(); + e.preventDefault(); + setOpenEditor(true) + setOpenFileId(file.id) + readFileData(file) + }} + > + + + + + + + {/* + + + { + // Open the file, without downloading it + window.open(`${globalUrl}/api/v1/files/${file.id}/content?type=text&authorization=${file.public_authorization}`, "_blank noreferrer noopener") + }} + > + + + + + */} + + + { + e.stopPropagation(); + e.preventDefault(); + downloadFile(file); + }} + > + + + + + + + + + + + + { + e.stopPropagation(); + e.preventDefault(); + navigator.clipboard.writeText(file.id); + document.execCommand("copy"); + + toast(file.id + " copied to clipboard"); + }} + > + + + + + + + + + + { + e.stopPropagation(); + e.preventDefault(); + deleteFile(file.id, true); + }} + > + + + + + + + + + ) + } + }, + { + field: "distribution", + headerName: "Distribution", + renderCell: (params) => { + const file = params.row; + const isDistributed = file?.suborg_distribution?.length > 0 ? true : false; + + return ( + <> + {selectedOrganization.id !== undefined && file?.org_id !== selectedOrganization.id ? + + + + : + + { + e.stopPropagation(); + e.preventDefault(); + setShowDistributionPopup(true) + if(file?.suborg_distribution?.length > 0){ + setSelectedSubOrg(file.suborg_distribution) + }else{ + setSelectedSubOrg([]) + } + setFileIdSelectedForDistribution(file.id) + }} + /> + + } + + ) + } + } + ] + const handleKeyDown = (event) => { if (event.key === 'Enter') { @@ -97,7 +463,7 @@ const Files = memo((props) => { editFileConfig(id, "suborg_distribute", [...new Set(selectedSubOrg)]) } - + const editFileConfig = (id, parentAction, selectedSubOrg) => { const data = { id: id, @@ -121,9 +487,9 @@ const Files = memo((props) => { .then((response) => response.json().then((responseJson) => { if (responseJson["success"] === false) { - toast("Failed overwriting files"); + toast.error("Failed overwriting files"); } else { - toast("Successfully updated file!"); + toast.success("File updated!"); setTimeout(() => { getFiles(); }, 1000); @@ -170,6 +536,7 @@ const Files = memo((props) => { } const getFiles = (namespace) => { + setFilesLoaded(false) var parsedurl = `${globalUrl}/api/v1/files` if (namespace === undefined || namespace === null || namespace === "default") { @@ -199,6 +566,11 @@ const Files = memo((props) => { .then((responseJson) => { if (responseJson.files !== undefined && responseJson.files !== null) { setFiles(responseJson.files); + if (responseJson.total_amount !== undefined && responseJson.total_amount !== null && responseJson.total_amount > 0) { + setTotalAmount(responseJson.total_amount) + } else { + setTotalAmount(responseJson.files.length) + } setShowLoader(false) setShowDistributionPopup(false) } else if (responseJson.list !== undefined && responseJson.list !== null) { @@ -227,7 +599,9 @@ const Files = memo((props) => { }) .catch((error) => { toast(error.toString()); - }); + }).finally(() => { + setFilesLoaded(true) + }); }; useEffect(() => { @@ -585,8 +959,12 @@ const Files = memo((props) => { ): null - const deleteFile = (file) => { - fetch(globalUrl + "/api/v1/files/" + file.id, { + const deleteFile = (fileId, showSinglDeleteToast) => { + + console.log("Deleting file with ID: ", fileId) + console.log("showSinglDeleteToast: ", showSinglDeleteToast) + + fetch(globalUrl + "/api/v1/files/" + fileId, { method: "DELETE", headers: { "Content-Type": "application/json", @@ -602,18 +980,20 @@ const Files = memo((props) => { return response.json(); }) .then((responseJson) => { - if (responseJson.success) { - toast("Successfully deleted file") + if (responseJson.success && showSinglDeleteToast === true) { + toast.success("Deleted file") } else if ( responseJson.reason !== undefined && responseJson.reason !== null ) { - toast("Failed to delete file: " + responseJson.reason); + toast.error("Failed to delete file: " + responseJson.reason); } - setTimeout(() => { - getFiles(selectedCategory) - }, 1500); + if (showSinglDeleteToast === true) { + setTimeout(() => { + getFiles(selectedCategory) + }, 1500); + } }) .catch((error) => { toast(error.toString()); @@ -909,7 +1289,7 @@ const Files = memo((props) => { {fileDistributionModal}
-
+
@@ -1131,529 +1511,136 @@ const Files = memo((props) => { backgroundColor: theme.palette.textFieldStyle.backgroundColor, }} />} -
- - + { + setSelectedRows(newSelection); + }} + sx={{ + marginTop: 1, + height: files.length*52, + width: "100%", + '.MuiTablePagination-selectLabel, .MuiTablePagination-select, .MuiTablePagination-selectIcon': { + display: 'none', + }, + marginBottom: 20, + }} + hideFooterSelectedRowCount={true} + hideFooter={true} + pagination + autoHeight={true} + getRowId={(row) => row.id} + keepNonExistentRowsSelected={false} + loading={filesLoaded === false} + /> +
- {[ - - { - setSelectAllChecked((prev) => !prev); - setSelectedFiles((prev) => { - if (prev.length === files.length) { - return [] - } else { - return files.map((_, index) => !prev.includes(index)) - } - }) - if (selectAllChecked) { - setSelectedFileId([]) - } else { - setSelectedFileId( - files - .filter((file) => file.namespace === selectedCategory) - .map((file) => file.id) - ); - } - }} - /> - , - "Name", - "Workflow", - "Md5", - "Status", - "Filesize", - "Actions", - "Distribution" - ] - .filter(Boolean) - .map((header, index) => ( - - ))} - - {showLoader ? - [...Array(6)].map((_, rowIndex) => ( - - {Array(8) - .fill() - .map((_, colIndex) => ( - - - - ))} - - )): - files.length === 0 ? ( -
- - No files found - -
- ):( - files?.map((file, index) => { - if (file.namespace === "") { - file.namespace = "default"; - } - - if (file.namespace !== selectedCategory) { - return null; - } - - var bgColor = themeMode === "dark" ? "#212121" : "#FFFFFF"; - if (index % 2 === 0) { - bgColor = themeMode === "dark" ? "#1A1A1A" : "#EAEAEA"; - } - const isDistributed = file?.suborg_distribution?.length > 0 ? true : false; - const filenamesplit = file.filename.split(".") - const iseditable = file.filesize < 2000000 && file.status === "active" && (allowedFileTypes.includes(filenamesplit[filenamesplit.length-1]) || !file?.filename.includes(".")) - return ( - - {/* - - */} - - {handleFileCheckboxChange(index); setSelectedFileId(prev => { - if (prev.includes(file.id)) { - return prev.filter((item) => item !== file.id) - } else { - return [...prev, file.id] - } - })}} - /> - - - - - - : ( - - - - - - - - - - ) - } - style={{ - display: 'table-cell', - overflow: "hidden", - }} - /> - - {file.md5_sum} - - )} - primaryTypographyProps={{ - style:{ - display: 'table-cell', - marginLeft:isSelectedFiles? 15:null, - overflow: "hidden", - whiteSpace: 'nowrap', - textOverflow: 'ellipsis', - maxWidth: 200, - } - }} - /> - - - - - - { - setOpenEditor(true) - setOpenFileId(file.id) - readFileData(file) - }} - > - - - - - - - {/* - - - { - // Open the file, without downloading it - window.open(`${globalUrl}/api/v1/files/${file.id}/content?type=text&authorization=${file.public_authorization}`, "_blank noreferrer noopener") - }} - > - - - - - */} - - - { - downloadFile(file); - }} - > - - - - - - +
- - - - { - navigator.clipboard.writeText(file.id); - document.execCommand("copy"); - - toast(file.id + " copied to clipboard"); - }} - > - - - - - - - - - - { - deleteFile(file); - }} - > - - - - - - - - + display: "flex", + textAlign: "center", + }}> + + {page * pageSize + 1} - {Math.min((page + 1) * pageSize, totalAmount)} of {totalAmount} + + + { + + return ( + + ) + + }} + onChange={(e, value) => { + if (value < 1) { + return + } + + const newPage = value-1 + console.log("New page: ", value) + // handleChangePage() + + setPage(newPage) + }} + /> + + {selectedRows.length > 0 ? + + : null} +
+
-
+
) }) diff --git a/frontend/src/components/HealthBarChart.jsx b/frontend/src/components/HealthBarChart.jsx index a4f05273..a5d1dec4 100644 --- a/frontend/src/components/HealthBarChart.jsx +++ b/frontend/src/components/HealthBarChart.jsx @@ -1,22 +1,93 @@ -import React from 'react'; -import { Bar } from 'react-chartjs-2'; +import React, { useState } from 'react'; const HealthBarChart = (props) => { - const { globalUrl, filteredData, options, onBarClick } = props; - - return ( - { - if (elements && elements.length > 0) { - onBarClick(elements); - } + const { filteredData, onBarClick } = props; + const [hoveredBar, setHoveredBar] = useState(null); + + if (!filteredData || !Array.isArray(filteredData)) { + console.error('Invalid chart data format'); + return null; + } + + const formatData = () => { + return filteredData.map((item) => ({ + label: item.date, + value: item.avgRunFinished, + color: item.color, + executionIds: item.executionIds, + })); + }; + + const handleBarClick = (data) => { + if (onBarClick) { + onBarClick(data); + } + }; + + const chartData = formatData(); + + const barWidth = `calc((100% - ${(chartData.length - 1) * 5}px) / ${chartData.length})`; + const barGap = '5px'; + + return ( +
+ {/* Chart Container */} +
+ {chartData.map((item, index) => ( +
- ); + onMouseEnter={() => setHoveredBar(item)} + onMouseLeave={() => setHoveredBar(null)} + onClick={() => handleBarClick(item)} + /> + ))} +
+ + {/* Tooltip */} + {hoveredBar && ( +
+
Date: {hoveredBar.label}
+
Uptime: {hoveredBar.value}%
+
Executions: {hoveredBar.executionIds.length}
+
+ )} +
+ ); }; export default HealthBarChart; diff --git a/frontend/src/components/HealthPage.jsx b/frontend/src/components/HealthPage.jsx index fcadb776..660719e0 100644 --- a/frontend/src/components/HealthPage.jsx +++ b/frontend/src/components/HealthPage.jsx @@ -1,31 +1,35 @@ import React, { useEffect, useState, useCallback } from 'react'; -import { Bar } from 'react-chartjs-2'; -//import healthData1 from '../healthstats1.json'; -import { toast } from "react-toastify" -import CheckOutlinedIcon from '@mui/icons-material/CheckOutlined'; +import { toast } from "react-toastify"; +import { + CheckOutlined as CheckOutlinedIcon, +} from '@mui/icons-material'; + import { Button, ButtonGroup, Typography, + LinearProgress, } from "@mui/material"; + import HealthBarChart from '../components/HealthBarChart.jsx'; +import LiveExecutionsChart from '../components/LiveExecutionsGraph.jsx'; const HealthPage = (props) => { - const { globalUrl, userdata } = props; + const { userdata, globalUrl } = props; const [healthData, setHealthData] = useState(null); const [selectedRange, setSelectedRange] = useState('30d'); + const [liveExecutionsData, setLiveExecutionsData] = useState([]); const [filteredData, setFilteredData] = useState([]); const [averageUptime, setAverageUptime] = useState(0); + const [liveExecutionsRange, setLiveExecutionsRange] = useState('1h'); // Default to 1h + const [isHealthLoading, setIsHealthLoading] = useState(false); // Loading state for HealthBarChart + const [isLiveExecutionsLoading, setIsLiveExecutionsLoading] = useState(false); // Loading state for LiveExecutionsChart - const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; - //const globalUrl = `https://shuffler.io` - - console.log("HEALTHPAGE 1") + const isCloud = (window.location.host === "localhost:3002" || window.location.host === "shuffler.io") ? true : (process.env.IS_SSR === "true"); const fetchHealthStats = useCallback(async () => { - //const after = new Date().getTime() - 90 * 24 * 60 * 60 + setIsHealthLoading(true); // Start loading for HealthBarChart try { - //const response = await fetch(`${globalUrl}/api/v1/health/stats?after=${after}`, { const response = await fetch(`${globalUrl}/api/v1/health/stats`, { method: "GET", headers: { @@ -42,31 +46,118 @@ const HealthPage = (props) => { } catch (error) { console.error("Error fetching health stats:", error); toast.error("Failed loading health stats"); + } finally { + setIsHealthLoading(false); // Stop loading for HealthBarChart + } + }, [globalUrl]); + + const fetchLiveExecutions = useCallback(async (range = '1h') => { + setIsLiveExecutionsLoading(true); // Start loading for LiveExecutionsChart + try { + const fetchOptions = { + method: "GET", + credentials: "include", + }; + + if (window.location.host !== "localhost:3002") { + fetchOptions.headers = { + "Content-Type": "application/json", + Accept: "application/json", + }; + } + + const now = Math.floor(Date.now() / 1000); + // let after = now - 3600; // Default to 1h + let mode = "" + + switch (range) { + case '1h': + mode = "1h" + break; + case '7h': + mode = "7h" + break; + case '1d': + mode = "1d" + break; + case '7d': + mode = "7d" + break; + case 'month': + mode = "month" + break; + default: + mode = "1h" + } + + if (!userdata.support_access) { + return + } + + const response = await fetch( + `${globalUrl}/api/v1/health/executions/live?mode=${mode}`, + fetchOptions + ); + + if (!response.ok) { + throw new Error("Failed to fetch live executions"); + } + + const data = await response.json(); + console.log("Raw live executions data:", data); + + if (Array.isArray(data)) { + const formattedData = data + .map(item => ({ + ...item, + // failed: Number(item.failed) || 0, + executing: Number(item.executing) || 0, + finished: Number(item.finished) || 0, + aborted: Number(item.aborted) || 0, + created_at: Number(item.created_at) || 0 + })) + .sort((a, b) => a.created_at - b.created_at); + + console.log("Formatted live executions data:", formattedData); + setLiveExecutionsData(formattedData); + } else { + console.error("Received invalid data format:", data); + setLiveExecutionsData([]); + } + } catch (error) { + console.error("Error fetching live executions:", error); + toast.error("Failed loading live executions data"); + } finally { + setIsLiveExecutionsLoading(false); // Stop loading for LiveExecutionsChart } }, [globalUrl]); useEffect(() => { fetchHealthStats(); - }, [fetchHealthStats]); + fetchLiveExecutions(liveExecutionsRange); + + const interval = setInterval(() => fetchLiveExecutions(liveExecutionsRange), 60000); + return () => clearInterval(interval); + }, [fetchHealthStats, fetchLiveExecutions, liveExecutionsRange]); const extractRunFinished = (data, range) => { if (!data || !Array.isArray(data)) return []; - + const currentDate = new Date().getTime(); const rangeInMillis = { '24hr': 24 * 60 * 60 * 1000, '7day': 7 * 24 * 60 * 60 * 1000, '30d': 30 * 24 * 60 * 60 * 1000, '90d': 90 * 24 * 60 * 60 * 1000 - } - const filteredData = data.filter(item => currentDate - item.updated * 1000 <= rangeInMillis[range]) - - const aggregatedData = new Map() - + }; + const filteredData = data.filter(item => currentDate - item.updated * 1000 <= rangeInMillis[range]); + + const aggregatedData = new Map(); + filteredData.forEach(item => { const timestamp = item.updated * 1000; // Convert Unix timestamp to milliseconds let key; - + switch (range) { case '24hr': const date = new Date(timestamp); @@ -83,7 +174,7 @@ const HealthPage = (props) => { default: key = new Date(timestamp).toLocaleDateString(); } - + // Check if date already exists in the map if (aggregatedData.has(key)) { // Update aggregated values @@ -100,13 +191,13 @@ const HealthPage = (props) => { }); } }); - + // Calculate averages and assign colors const result = Array.from(aggregatedData.entries()).map(([key, { totalEntries, totalRunFinished, executionIds }]) => { const avg = totalEntries > 0 ? totalRunFinished / totalEntries : 0; const FinalAvg = avg * 100; let color; - + if (FinalAvg >= 100) { color = '#00F670'; } else if (FinalAvg >= 98.50 && FinalAvg <= 99.99) { @@ -114,7 +205,7 @@ const HealthPage = (props) => { } else if (FinalAvg <= 98.49) { color = '#FF354C'; } - + return { date: range === '24hr' ? `${key}:00` : key, avgRunFinished: FinalAvg, @@ -122,11 +213,10 @@ const HealthPage = (props) => { executionIds }; }); - + return result; }; - - + useEffect(() => { if (healthData) { const newData = extractRunFinished(healthData, selectedRange); @@ -160,19 +250,19 @@ const HealthPage = (props) => { const labels = filteredData.map((value, i) => { if (selectedRange === '24hr') { const [datePart, hourPart] = value.date.split(' '); - const [day, month, year] = datePart.split('/'); + const [month, day, year] = datePart.split('/'); const monthIndex = parseInt(month, 10) - 1; const date = new Date(year, monthIndex, day); let formattedDate = `${date.toLocaleString('en-US', { month: 'short', day: '2-digit' })}`; - // Add hour part if available if (hourPart) { formattedDate += `, ${hourPart.split(':').slice(0, 2).join(':')}`; } + return `${formattedDate} \nUptime: ${value.avgRunFinished.toFixed(2)}%`; } else if (selectedRange === '7day') { const [datePart, hourPart] = value.date.split(' '); - const [day, month, year] = datePart.split('/'); + const [month, day, year] = datePart.split('/'); const monthIndex = parseInt(month, 10) - 1; const date = new Date(year, monthIndex, day); let formattedDate = `${date.toLocaleString('en-US', { month: 'short', day: '2-digit' })}`; @@ -185,12 +275,12 @@ const HealthPage = (props) => { } else { const dateParts = value.date.split('/'); // Assuming the date format is "DD/MM/YYYY" - const date = new Date(`${dateParts[2]}-${dateParts[1]}-${dateParts[0]}`); // Reformat the date string to "YYYY-MM-DD" + const date = new Date(`${dateParts[2]}-${dateParts[0]}-${dateParts[1]}`); // Reformat the date string to "YYYY-MM-DD" + return `${date.toLocaleDateString('en-US', { month: 'short', day: '2-digit', year: 'numeric' })} \nUptime: ${value.avgRunFinished.toFixed(2)}%`; } }); - if (selectedRange === '24hr') { barThickness = 35; } @@ -200,12 +290,6 @@ const HealthPage = (props) => { else if (selectedRange === '30d') { barThickness = 25; } - // let width = 800; // Default chart width - // if (selectedRange === '7day') { - // width = 400; // Adjust chart width for 7 days - // } else if (selectedRange === '30d') { - // width = 600; // Adjust chart width for 30 days - // } const datasets = [{ label: "", @@ -213,14 +297,11 @@ const HealthPage = (props) => { backgroundColor: filteredData.map(item => item.color), borderWidth: 1, barThickness: barThickness, - // barPercentage: barPercentage, }]; return { labels, datasets }; }; - console.log("HEALTHPAGE 2") - const options = { legend: { display: false @@ -253,12 +334,11 @@ const HealthPage = (props) => { }, afterLabel: function (tooltipItem, data) { const label = data.labels[tooltipItem.index]; - // console.log(label) const uptime = label.match(/Uptime:\s*(\d+(?:\.\d+)?)/)[1]; // Extract uptime value using regex - return `Success Rate: ${uptime}%`; // Customize the uptime display + return `Test-Workflow Health: ${uptime}%`; // Customize the uptime display }, title: function () { - return 'Fully Oprational'; // Hide the tooltip title + return 'Fully Operational'; // Hide the tooltip title } } } @@ -284,32 +364,118 @@ const HealthPage = (props) => { } }; + const healthBarData = updateChartData() + return ( -
- +
+ {/* Health Bar Chart Section */} - - - - - + + + + + -
-
- -
- Workflow Health - Operational + {/* Loading Bar for HealthBarChart */} + {isHealthLoading && ( + + )} + +
+
+
+ +
+ Workflow Health + Operational +
-
- {averageUptime.toFixed(2)}% - Success Rate +
+ {averageUptime.toFixed(2)}% + Success Rate
- +
+ + {userdata.support_access && ( +
+
+ Live Executions + + + + + + {/* */} + +
+ {/* Loading Bar for LiveExecutionsChart */} + {isLiveExecutionsLoading && ( + + )} + +
+ )} + +
); }; diff --git a/frontend/src/components/HighlightedValueInSearch.jsx b/frontend/src/components/HighlightedValueInSearch.jsx new file mode 100644 index 00000000..f707a6fc --- /dev/null +++ b/frontend/src/components/HighlightedValueInSearch.jsx @@ -0,0 +1,257 @@ +import React, { useEffect, useRef, useCallback, useMemo } from 'react'; +import ReactJson from 'react-json-view-ssr'; +import CodeMirror from '@uiw/react-codemirror'; +import { python } from '@codemirror/lang-python'; +import { vscodeDark } from '@uiw/codemirror-theme-vscode'; + +const HighlightedValueInSearch = ({ value, searchTerm, theme }) => { + const containerRef = useRef(null); + const preRef = useRef(null); + const scrollTimeoutRef = useRef(null); + + const isJson = useCallback((str) => { + if (typeof str === 'object' && str !== null) return true; + if (typeof str !== 'string') return false; + + const trimmed = str.trim(); + if (!trimmed.startsWith('{') && !trimmed.startsWith('[')) return false; + + try { + JSON.parse(trimmed); + return true; + } catch { + return false; + } + }, []); + + const getJsonValue = useCallback((val) => { + try { + return typeof val === 'object' ? val : JSON.parse(val.trim()); + } catch { + return null; + } + }, []); + + const highlightInJson = useCallback((jsonStr, term) => { + if (!term) return jsonStr; + + try { + const escapedTerm = term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const regex = new RegExp(`(${escapedTerm})`, 'gi'); + const highlightId = `highlight-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + + return jsonStr.replace(regex, `$1`); + } catch { + return jsonStr; + } + }, []); + + const isPythonCode = useCallback((str) => { + if (typeof str !== 'string') return false; + + const pythonPatterns = [ + 'import ', 'from ', 'def ', 'class ', 'if __name__', + 'print(', 'return ', 'elif ', 'except:', 'try:', 'with ', + 'lambda ', 'yield ', 'async def', 'await ' + ]; + + const trimmed = str.trim(); + return pythonPatterns.some(pattern => trimmed.includes(pattern)) && + (trimmed.includes('def') || trimmed.includes('class')); + }, []); + + useEffect(() => { + if (!searchTerm || !containerRef.current || !preRef.current) return; + + if (scrollTimeoutRef.current) { + clearTimeout(scrollTimeoutRef.current); + } + + scrollTimeoutRef.current = setTimeout(() => { + try { + const container = containerRef.current; + const firstHighlight = preRef.current?.querySelector('mark'); + + if (firstHighlight && container) { + const containerRect = container.getBoundingClientRect(); + const highlightRect = firstHighlight.getBoundingClientRect(); + const scrollTop = highlightRect.top - containerRect.top + container.scrollTop - (container.clientHeight / 2); + + container.scrollTo({ top: Math.max(0, scrollTop), behavior: 'smooth' }); + } + } catch (error) { + console.warn('Auto-scroll failed:', error); + } + }, 100); + + return () => { + if (scrollTimeoutRef.current) { + clearTimeout(scrollTimeoutRef.current); + } + }; + }, [searchTerm, value]); + + const RegularText = useCallback(({ value: textValue, searchTerm: term }) => { + if (!textValue) return No content; + + const stringValue = String(textValue); + + // Check if it's Python code + if (isPythonCode(stringValue)) { + const lines = stringValue.split('\n'); + const isNodeView = lines[0]?.startsWith('Node:'); + const nodeTitle = isNodeView ? lines[0] : null; + const codeContent = isNodeView ? lines.slice(1).join('\n') : stringValue; + + return ( +
+ {nodeTitle && ( +
+ {nodeTitle} +
+ )} + +
+ +
+
+ ); + } + + if (!term) { + return ( + + {stringValue} + + ); + } + + try { + const parts = stringValue.split(term); + return ( + + {parts.map((part, index) => ( + + {part} + {index < parts.length - 1 && ( + + {term} + + )} + + ))} + + ); + } catch { + return {stringValue}; + } + }, [isPythonCode]); + + if (value == null) { + return null; + } + + try { + if (isJson(value)) { + const jsonValue = getJsonValue(value); + + if (jsonValue === null) { + return ; + } + + const jsonString = JSON.stringify(jsonValue, null, 2); + const hasMatch = searchTerm && jsonString.toLowerCase().includes(searchTerm.toLowerCase()); + + return ( +
+ {hasMatch ? ( +
+          ) : (
+            
+          )}
+        
+ ); + } + + return ; + + } catch (error) { + console.error('HighlightedValueInSearch error:', error); + return ; + } +}; + +export default React.memo(HighlightedValueInSearch); \ No newline at end of file diff --git a/frontend/src/components/LeftSideBar.jsx b/frontend/src/components/LeftSideBar.jsx index eeae20bc..cb851d43 100644 --- a/frontend/src/components/LeftSideBar.jsx +++ b/frontend/src/components/LeftSideBar.jsx @@ -578,7 +578,7 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => { }, }} > - {userdata && (userdata?.org_status?.includes("integration_partner") && userdata?.org_status?.includes("sub_org")) ? null : ( + {userdata && (userdata?.org_status?.includes("integration_partner") && userdata?.active_org?.role !== "admin") ? null : ( <> { - Version: 2.1.0-rc2 + Version: + 2.1.0 + diff --git a/frontend/src/components/LicencePopup.jsx b/frontend/src/components/LicencePopup.jsx index 0267267d..70e4cc6e 100644 --- a/frontend/src/components/LicencePopup.jsx +++ b/frontend/src/components/LicencePopup.jsx @@ -418,6 +418,10 @@ const LicencePopup = (props) => { showSupport = true } + if (userdata?.app_execution_limit >= 300000) { + top_text = "Enterprise Plan" + } + if (subscription.name.includes("Open Source")) { top_text = "Open Source" showSupport = true @@ -761,7 +765,7 @@ const LicencePopup = (props) => { { isCloud ? userdata?.app_execution_limit && userdata?.app_execution_limit !== 10000 ? - "You have already subscribed to the Scale plan, which includes " + (userdata?.app_execution_limit/1000) + "K app runs/month. You can increase the limit by upgrading current plan. Contact support@shuffler.io for more information." : + `You have already subscribed to the ${top_text}, which includes ${userdata?.app_execution_limit/1000}K app runs/month. You can increase the limit by upgrading current plan. Contact support@shuffler.io for more information.` : `You are using free Starter plan with max ${userdata?.app_execution_limit === 10000 ? "10,000" : "2,000"} runs per month. Upgrade to increase this limit.` : diff --git a/frontend/src/components/LineChartWrapper.jsx b/frontend/src/components/LineChartWrapper.jsx new file mode 100644 index 00000000..95aa8024 --- /dev/null +++ b/frontend/src/components/LineChartWrapper.jsx @@ -0,0 +1,107 @@ +import React, { useState, useEffect, useContext, memo, useMemo } from 'react' +import {getTheme} from '../theme.jsx'; +import { Context } from '../context/ContextApi.jsx'; + +import { + Typography, +} from '@mui/material'; + +import { + BarChart, + BarSeries, + Bar, + BarLabel, + + GridlineSeries, + Gridline, + TooltipArea, + ChartTooltip, + TooltipTemplate, +} from 'reaviz'; + +const LineChartWrapper = (props) => { + const {keys, inputname, height, width, border} = props + + const [hovered, setHovered] = useState(""); + const {themeMode} = useContext(Context) + const theme = getTheme(themeMode) + + var inputdata = keys.data === undefined ? keys : keys.data + + var newname = inputname === undefined || inputname === null ? "" : inputname.trim().replaceAll("_", " ") + newname = newname.charAt(0).toUpperCase() + newname.slice(1) + + if (inputdata?.key !== undefined && inputdata?.key !== null && inputdata?.key !== "" && inputdata?.datasets !== undefined && inputdata?.datasets !== null && inputdata?.datasets.length > 0 && inputdata?.labels !== undefined && inputdata?.labels !== null && inputdata?.labels.length > 0) { + + var tmpdata = inputdata?.datasets[0] + if (tmpdata?.data !== undefined && tmpdata?.data !== null && tmpdata?.data.length > 0 && inputdata?.labels?.length === tmpdata?.data?.length) { + console.log("Fix it!") + var newarray = [] + for (var key in tmpdata.data) { + var entry = { + "key": inputdata.labels[key] !== undefined ? inputdata.labels[key] : key, + "data": tmpdata.data[key], + } + + newarray.push(entry) + } + + inputdata = newarray + } + + } + + if (inputdata === undefined || inputdata === null) { + return ( + + Invalid linegraph data format + + ) + } + + var defaultStyle = { + color: "white", + padding: 30, + marginTop: 15, + overflow: "hidden", + + border: "1px solid rgba(255,255,255,0.3)", + borderRadius: theme.palette?.borderRadius, + backgroundColor: theme.palette.platformColor, + } + + if (border === false) { + defaultStyle.border = "none" + defaultStyle.borderRadius = 0 + defaultStyle.backgroundColor = "transparent" + } + + return ( +
+ + {newname} + + + + } + /> + } + gridlines={ + } /> + } + /> + +
+ ) +} + +export default LineChartWrapper; diff --git a/frontend/src/components/LiveExecutionsGraph.jsx b/frontend/src/components/LiveExecutionsGraph.jsx new file mode 100644 index 00000000..ac572e84 --- /dev/null +++ b/frontend/src/components/LiveExecutionsGraph.jsx @@ -0,0 +1,7 @@ +import React from 'react'; + +const LiveExecutionsGraph = ({ executions }) => { + return null +} + +export default LiveExecutionsGraph diff --git a/frontend/src/components/Navbar.jsx b/frontend/src/components/Navbar.jsx index 07be2cb8..63efe7a9 100644 --- a/frontend/src/components/Navbar.jsx +++ b/frontend/src/components/Navbar.jsx @@ -49,7 +49,7 @@ const menuData = { { title: "Shuffle", description: - "The most versatile automation engine with focus on security.", + "The most versatile automation engine. Focused on cybersecurity.", icon: "/images/icons/shuffleLogo.svg", path: "/docs/about", gaData: { @@ -61,7 +61,7 @@ const menuData = { { title: "Singul", description: - "Connect your favorite services with a singul line of code.", + "Connect to your favorite services with a singul line of code.", icon: "/images/logos/singul.svg", path: "https://singul.io", gaData: { @@ -70,6 +70,7 @@ const menuData = { label: "singul_click" } }, + /* { title: "API Explorer", description: @@ -82,18 +83,19 @@ const menuData = { label: "api_explorer_click" } }, + */ ], Services: [ { - title: "Professional Services", + title: "Proof of Concept", description: - "Professional Services help you solve problems at your convenience.", + "Register for POC to test the best of Shuffle for free for 30 days.", icon: "/images/ProfessionalServices.svg", - path: "/professional-services", + path: "/poc", gaData: { category: "navbar", action: "services_click", - label: "professional_services_click" + label: "proof_of_concept_click" } }, { @@ -101,7 +103,7 @@ const menuData = { description: "Support to help you build automations with confidence.", icon: "/images/Support.svg", - path: "/contact?category=support", + path: "/support", gaData: { category: "navbar", action: "services_click", @@ -1245,23 +1247,6 @@ const Navbar = (props) => { > {menuItem.title} - {menuItem.title === "Singul" && ( - - Beta: Coming Soon - - )} { } }} > - Become a partner + Become a Partner @@ -1676,7 +1661,7 @@ const Navbar = (props) => { const topbarHeight = showTopbar ? 40 : 0 const topbar = !isCloud || !showTopbar ? null : - curpath === "/" || curpath.includes("/docs") || curpath === "/pricing" || curpath === "/contact" || curpath === "/search" || curpath === "/usecases" || curpath === "/training" || curpath === "/professional-services" ? + curpath === "/" || curpath.includes("/docs") || curpath === "/pricing" || curpath === "/contact" || curpath === "/search" || curpath === "/usecases" || curpath === "/training" || curpath === "/poc" || curpath === "/professional-services" ? {/* uncommit this to show topbar for release */} {/*
diff --git a/frontend/src/components/OrgHeaderexpandedNew.jsx b/frontend/src/components/OrgHeaderexpandedNew.jsx index 525491e6..d0cd4952 100644 --- a/frontend/src/components/OrgHeaderexpandedNew.jsx +++ b/frontend/src/components/OrgHeaderexpandedNew.jsx @@ -56,7 +56,8 @@ const OrgHeaderexpandedNew = (props) => { adminTab, selectedStatus, setSelectedStatus, - isEditOrgTab + isEditOrgTab, + handleStatusChange } = props; const classes = useStyles(); @@ -68,7 +69,7 @@ const OrgHeaderexpandedNew = (props) => { style: { maxHeight: ITEM_HEIGHT * 4.5 + ITEM_PADDING_TOP, width: 300, - borderRadius: 20, + borderRadius: 4, overflowY: "scroll", }, }, @@ -93,40 +94,6 @@ const OrgHeaderexpandedNew = (props) => { const { themeMode, supportEmail, brandColor } = useContext(Context); const theme = getTheme(themeMode, brandColor); - const handleStatusChange = (event) => { - const { value } = event.target; - handleEditOrg( - orgName, - orgDescription, - selectedOrganization.id, - selectedOrganization?.image, - { - app_download_repo: selectedOrganization?.defaults?.app_download_repo, - app_download_branch: selectedOrganization?.defaults?.app_download_branch, - workflow_download_repo: selectedOrganization?.defaults?.workflow_download_repo, - workflow_download_branch: selectedOrganization?.defaults?.workflow_download_branch, - notification_workflow: selectedOrganization?.defaults?.notification_workflow, - documentation_reference: selectedOrganization?.defaults?.documentation_reference, - workflow_upload_repo: selectedOrganization?.defaults?.workflow_upload_repo, - workflow_upload_branch: selectedOrganization?.defaults?.workflow_upload_branch, - workflow_upload_username: selectedOrganization?.defaults?.workflow_upload_username, - workflow_upload_token: selectedOrganization?.defaults?.workflow_upload_token, - newsletter: selectedOrganization?.defaults?.newsletter, - weekly_recommendations: selectedOrganization?.defaults?.weekly_recommendations, - }, - { - sso_entrypoint: selectedOrganization?.sso_config?.sso_entrypoint, - sso_certificate: selectedOrganization?.sso_config?.sso_certificate, - client_id: selectedOrganization?.sso_config?.client_id, - client_secret: selectedOrganization?.sso_config?.client_secret, - openid_authorization: selectedOrganization?.sso_config?.openid_authorization, - openid_token: selectedOrganization?.sso_config?.openid_token, - SSORequired: selectedOrganization?.sso_config?.SSORequired, - auto_provision: selectedOrganization?.sso_config?.auto_provision, - }, - value.length === 0 ? ["none"] : value, - ) - } const [appDownloadBranch, setAppDownloadBranch] = React.useState( selectedOrganization.defaults === undefined ? defaultBranch @@ -452,7 +419,7 @@ const OrgHeaderexpandedNew = (props) => {
- Name + Name {
{userdata?.support ? (
-
Status
+
Status
{ credentials: "include", }).then((response) => { if (response.status !== 200) { - toast(`Failed getting config for ${item.id}: `, response.reason); + //toast.error(`Failed getting config for ${item.id}: `, response.reason) console.log("Status not 200 for app config :O!"); - return; + return } - return response.json(); + + return response.json() }).then((responseJson) => { if (!responseJson.success) { console.log("Could not get app config") @@ -209,7 +210,7 @@ const Priorities = memo((props) => { }).catch((error) => { console.log("Error getting app config: " + error); - toast("Error getting app config: " + error); + //toast.error("Error getting app config: " + error); }) }) }) diff --git a/frontend/src/components/RuntimeDebugger.jsx b/frontend/src/components/RuntimeDebugger.jsx index 9ac474d5..8b99f4b2 100644 --- a/frontend/src/components/RuntimeDebugger.jsx +++ b/frontend/src/components/RuntimeDebugger.jsx @@ -79,9 +79,14 @@ const RuntimeDebugger = (props) => { const [searchLoading, setSearchLoading] = useState(false) const [rowCursor, setCursor] = useState("") const [rowsPerPage, setRowsPerPage] = useState(10) + const [maxExecutionCount, setMaxExecutionCount] = useState(50) const [resultRows, setResultRows] = useState([]) const [selectedWorkflowExecutions, setSelectedWorkflowExecutions] = useState([]) const [suborgWorkflowRuns, setSuborgWorkflowRuns] = useState(false) + const [paginationModel, setPaginationModel] = useState({ + page: 0, + pageSize: 10, + }) const [openWorkflowMenu, setOpenWorkflowMenu] = useState(false) const [workflows, setWorkflows] = useState([ {"id": "", "name": "All Workflows",} @@ -665,6 +670,13 @@ const RuntimeDebugger = (props) => { }, ] + useEffect(() => { + setPaginationModel(prev => ({ + ...prev, + pageSize: rowsPerPage + })) + }, [rowsPerPage]) + useEffect(() => { // Check if the user is currently focusing a texxtfield or not // If they are, don't submit the search @@ -672,7 +684,7 @@ const RuntimeDebugger = (props) => { return } - submitSearch(workflowId, status, startTime, endTime, rowCursor, rowsPerPage, suborgWorkflowRuns) + submitSearch(workflowId, status, startTime, endTime, rowCursor, maxExecutionCount, suborgWorkflowRuns) }, [workflowId, status, startTime, endTime]) const textfieldStyle = { @@ -723,7 +735,7 @@ const RuntimeDebugger = (props) => { setWorkflowId(e.target.value.id) setSuborgWorkflowRuns(false) - submitSearch(e.target.value.id, status, startTime, endTime, rowCursor, rowsPerPage, false) + submitSearch(e.target.value.id, status, startTime, endTime, rowCursor, maxExecutionCount, false) } const executeWorkflow = (execution) => { @@ -819,9 +831,12 @@ const RuntimeDebugger = (props) => {
- Workflow Run Debugger {totalCount !== 0 ? ` (~${totalCount})` : ""} + + Workflow Run Debugger {totalCount !== 0 ? ` (~${totalCount})` : ""} + + {selectedWorkflowExecutions.length > 0 ? - +
{userdata?.active_org?.creator_org?.length === 0 ? (
@@ -956,7 +999,8 @@ const RuntimeDebugger = (props) => { setStartTime("") setEndTime("") setSearchQuery("") - submitSearch("", "", "", "", rowCursor, rowsPerPage, !suborgWorkflowRuns)} + setMaxExecutionCount(50) + submitSearch("", "", "", "", rowCursor, 50, !suborgWorkflowRuns)} } color="secondary" /> @@ -967,7 +1011,7 @@ const RuntimeDebugger = (props) => {
{ - submitSearch(workflowId, status, startTime, endTime, rowCursor, rowsPerPage, suborgWorkflowRuns) + submitSearch(workflowId, status, startTime, endTime, rowCursor, maxExecutionCount, suborgWorkflowRuns) }} style={{display: "flex", justifyContent: "center", alignItems: "center", }}> Status @@ -1158,7 +1202,8 @@ const RuntimeDebugger = (props) => { setEndTime("") setSearchQuery("") setSuborgWorkflowRuns(false) - submitSearch("", "", "", "", rowCursor, rowsPerPage, false) + setMaxExecutionCount(50) + submitSearch("", "", "", "", rowCursor, 50, false) }} > @@ -1169,7 +1214,7 @@ const RuntimeDebugger = (props) => { variant="outlined" color="primary" onClick={() => { - submitSearch(workflowId, status, startTime, endTime, rowCursor, rowsPerPage, suborgWorkflowRuns) + submitSearch(workflowId, status, startTime, endTime, rowCursor, maxExecutionCount, suborgWorkflowRuns) }} disabled={searchLoading} style={{height: 50, minWidth: 100, marginTop: 15, }} @@ -1181,20 +1226,18 @@ const RuntimeDebugger = (props) => { { - setRowsPerPage(newPageSize) - submitSearch(workflowId, status, startTime, endTime, rowCursor, newPageSize, suborgWorkflowRuns) + + onPaginationModelChange={(newPaginationModel) => { + setPaginationModel(newPaginationModel) + setRowsPerPage(newPaginationModel.pageSize) + // No API call needed - this is just for client-side pagination }} - // event for when clicking next page - // Hide page changer - onPageChange={(params) => { - console.log("params: ", params) - }} - onSelectionModelChange={(newSelection) => { + + onRowSelectionModelChange={(newSelection) => { //console.log("newSelection: ", newSelection) //setSelectedWorkflowExecutionsIndexes(newSelection) var found = [] diff --git a/frontend/src/components/ShuffleCodeEditor1.jsx b/frontend/src/components/ShuffleCodeEditor1.jsx index 043c5de2..3424d3d2 100644 --- a/frontend/src/components/ShuffleCodeEditor1.jsx +++ b/frontend/src/components/ShuffleCodeEditor1.jsx @@ -47,6 +47,8 @@ import { RestartAlt as RestartAltIcon, ArrowForward as ArrowForwardIcon, KeyboardReturn as KeyboardReturnIcon, + FormatIndentIncrease as FormatIndentIncreaseIcon, + Fullscreen as FullscreenIcon, } from '@mui/icons-material'; @@ -143,6 +145,34 @@ const CodeEditor = (props) => { handleConditionFieldChange, } = props + // Auto-indent JSON-like content (with safety hehe) + const autoIndentContent = React.useCallback((content) => { + // Safety checks :) + if (!content || typeof content !== 'string' || content.trim().length === 0) { + return content; + } + + try { + // Check if content looks like JSON (starts with { or [) + const trimmedContent = content.trim(); + if (trimmedContent.startsWith('{') || trimmedContent.startsWith('[')) { + try { + const parsed = JSON.parse(trimmedContent); + return IndentJsonLikeString(JSON.stringify(parsed), 2); + } catch (parseError) { + return IndentJsonLikeString(content, 2); + } + } + + // Return original content if it doesn't look like JSON + return content; + } catch (error) { + // If anything goes wrong, return original content + console.warn('Auto-indent failed, using original content:', error); + return content; + } + }, []); + const [localcodedata, setlocalcodedata] = React.useState(codedata === undefined || codedata === null || codedata.length === 0 ? "" : codedata); // const {codelang, setcodelang} = props @@ -184,6 +214,7 @@ const CodeEditor = (props) => { "result": baseResult, }) const [executing, setExecuting] = useState(false) + const [fullScreenModeEnabled, setFullScreenModeEnabled] = useState(fullScreenMode === true || fullScreenMode === "true" || localStorage.getItem("codeEditorFullScreen") === "true") const liquidOpen = Boolean(anchorEl); const mathOpen = Boolean(anchorEl2); @@ -200,6 +231,22 @@ const CodeEditor = (props) => { expectedOutput(localcodedata) }, [localcodedata]) + // Auto-indent when codedata prop changes + useEffect(() => { + if (codedata && codedata !== localcodedata && typeof codedata === 'string') { + try { + const indentedContent = autoIndentContent(codedata); + if (indentedContent !== undefined && indentedContent !== null) { + setlocalcodedata(indentedContent); + } + } catch (error) { + console.warn('Failed to auto-indent codedata:', error); + // Fallback to original codedata + setlocalcodedata(codedata); + } + } + }, [codedata, autoIndentContent]) + let navigate = useNavigate(); const [searchParams] = useSearchParams(); const actionId = searchParams.get('action_id'); @@ -217,7 +264,13 @@ const CodeEditor = (props) => { } const action = workflow?.actions?.find(action => action.id === actionId); - setlocalcodedata(editorData?.value); + try { + const indentedContent = autoIndentContent(editorData?.value); + setlocalcodedata(indentedContent || editorData?.value); + } catch (error) { + console.warn('Failed to auto-indent action data:', error); + setlocalcodedata(editorData?.value); + } setSelectedAction(action); // Update available variables when action changes @@ -230,7 +283,13 @@ const CodeEditor = (props) => { } const trigger = workflow?.triggers?.find(trigger => trigger.id === triggerId); - setlocalcodedata(editorData?.value); + try { + const indentedContent = autoIndentContent(editorData?.value); + setlocalcodedata(indentedContent || editorData?.value); + } catch (error) { + console.warn('Failed to auto-indent trigger data:', error); + setlocalcodedata(editorData?.value); + } setSelectedTrigger(trigger); // Update available variables when trigger changes @@ -244,7 +303,13 @@ const CodeEditor = (props) => { } const condition = selectedEdge?.conditions?.find(condition => condition.id === conditionId); - setlocalcodedata(editorData?.value); + try { + const indentedContent = autoIndentContent(editorData?.value); + setlocalcodedata(indentedContent || editorData?.value); + } catch (error) { + console.warn('Failed to auto-indent condition data:', error); + setlocalcodedata(editorData?.value); + } setSelectedCondition(condition); // Update available variables when condition changes updateAvailableVariables(actionlist); @@ -963,7 +1028,6 @@ const CodeEditor = (props) => { // vs // $variable.#.subvalue // if you put both of those lines in the same editor, then it will replace both (somehow). Make sure $variable.#.subvalue exists while testing. - console.log("FOUNDLOC: ", fixedVariable, foundlocation) for (var j = 0; j < actionlist.length; j++) { if (fixedVariable.slice(1,).toLowerCase() !== actionlist[j].autocomplete.toLowerCase()) { continue @@ -990,7 +1054,6 @@ const CodeEditor = (props) => { } try { - console.log("REPLACE: ", foundlocation, fixedVariable, newvalue) if (newvalue !== "") { if (foundlocation === -1) { input = input.replace(fixedVariable, newvalue, 1) @@ -1303,7 +1366,7 @@ const CodeEditor = (props) => { editor.completers = [customCompleter] } - if (fullScreenMode) { + if (fullScreenMode && fullScreenModeEnabled === true) { return ( { ) } + const IndentJsonLikeString = (input, indentSize = 2) => { + const indent = ' '.repeat(indentSize); + let level = 0; + let inString = false; + let escapeNext = false; + let result = ''; + + for (let i = 0; i < input.length; i++) { + let char = input[i]; + + if (escapeNext) { + result += char; + escapeNext = false; + continue; + } + + if (char === '\\') { + escapeNext = true; + result += char; + continue; + } + + if (char === '"') { + inString = !inString; + result += char; + continue; + } + + if (!inString) { + if (char === '{' || char === '[') { + result += char + '\n' + indent.repeat(++level); + continue; + } else if (char === '}' || char === ']') { + result += '\n' + indent.repeat(--level) + char; + continue; + } else if (char === ',') { + result += char + '\n' + indent.repeat(level); + continue; + } else if (char === ':') { + result += ': '; + continue; + } else if (char === ' ' || char === '\t' || char === '\n' || char === '\r') { + // Skip whitespace characters when not in string + continue; + } + } + + result += char; + } + + return result; + } + + const SourceDataOption = (option) => { const { innerdata, parsedPaths, defaultExpanded } = option @@ -1526,15 +1643,15 @@ const CodeEditor = (props) => { // zIndex: 12501, pointerEvents: "auto", color: theme.palette.DialogStyle.color, - minWidth: isMobile || isWorkflowEditor ? "100%" : isFileEditor ? "650px" : "80%", - maxWidth: isMobile || isWorkflowEditor ? "100%" : isFileEditor ? "650px" : "1100px", - minHeight: isMobile || isWorkflowEditor ? "100%" : "auto", - maxHeight: isMobile || isWorkflowEditor ? "100%" : "700px", + minWidth: isMobile || isWorkflowEditor || fullScreenModeEnabled ? "100%" : isFileEditor ? "650px" : "80%", + maxWidth: isMobile || isWorkflowEditor || fullScreenModeEnabled ? "100%" : isFileEditor ? "650px" : "1100px", + minHeight: isMobile || isWorkflowEditor || fullScreenModeEnabled ? "100%" : "auto", + maxHeight: isMobile || isWorkflowEditor || fullScreenModeEnabled ? "100%" : "700px", border: "3px solid rgba(255,255,255,0.3)", - padding: isMobile ? "25px 10px 25px 10px" : isWorkflowEditor ? "25px 10px 25px 200px" : "25px", + padding: fullScreenModeEnabled ? "50px 0px 20px 200px" : isMobile ? "25px 10px 25px 10px" : isWorkflowEditor ? "25px 10px 25px 200px" : "25px", backgroundColor: themeMode === "dark" ? "black" : theme.palette.DialogStyle.backgroundColor, - opacity: isWorkflowEditor ? 0.93 : 1, + opacity: isWorkflowEditor || fullScreenModeEnabled ? 0.93 : 1, }, }} > @@ -1549,39 +1666,66 @@ const CodeEditor = (props) => { : null} + {fullScreenModeEnabled ? null : + + { + }} + > + + + + } { + setFullScreenModeEnabled(!fullScreenModeEnabled) + localStorage.setItem("codeEditorFullScreen", !fullScreenModeEnabled) }} > - + {!fullScreenModeEnabled ? + + : + + } { @@ -2127,6 +2271,30 @@ const CodeEditor = (props) => { marginLeft: 100, }} disabled={editorData === undefined || editorData.example === undefined || editorData.example === null || editorData.example.length === 0} + onClick={() => { + const indentedText = IndentJsonLikeString(localcodedata, 2) + if (indentedText !== undefined && indentedText !== null) { + setlocalcodedata(indentedText) + } else { + toast.warn("Could not indent the text. Please check the input format.", { autoClose: 5000 }) + } + }} + color="secondary" + > + + + + + + { if (fixExample !== undefined) { const newExample = fixExample(editorData.example) @@ -2196,14 +2364,15 @@ const CodeEditor = (props) => { value={localcodedata} mode={isWorkflowEditor ? "yaml" : selectedAction === undefined ? "json" : selectedAction.name === "execute_python" ? "python" : selectedAction.name === "execute_bash" ? "bash" : "json"} theme="gruvbox" - height={isFileEditor ? 450 : isWorkflowEditor ? "90vh" : 550} - width={isFileEditor ? 650 : isWorkflowEditor ? "90vw" : "100%"} + height={fullScreenModeEnabled ? "84vh" : isFileEditor ? 450 : isWorkflowEditor ? "90vh" : 550} + width={isFileEditor ? 650 : fullScreenModeEnabled ? "50vw" : isWorkflowEditor ? "90vw" : "100%"} markers={markers} highlightActiveLine={false} enableBasicAutocompletion={true} completers={[customCompleter]} + showPrintMargin={false} style={{ wordBreak: "break-word", @@ -2351,8 +2520,9 @@ const CodeEditor = (props) => { style={{ border: `1px solid rgba(255, 255, 255, 0.15)`, position: "absolute", - top: 20, - right: 100, + top: fullScreenModeEnabled ? 50 : 20, + right: fullScreenModeEnabled ? 240 : 120, + maxHeight: 35, minWidth: 70, zIndex: 1200, @@ -2466,8 +2636,8 @@ const CodeEditor = (props) => { borderRadius: 5, border: `2px solid ${theme.palette.inputColor}`, padding: 10, - maxHeight: 190, - minheight: 190, + maxHeight: fullScreenModeEnabled ? 300 : 190, + minheight: fullScreenModeEnabled ? 300 : 190, overflow: "auto", }} collapsed={false} @@ -2476,9 +2646,14 @@ const CodeEditor = (props) => { }} displayDataTypes={false} onSelect={(select) => { - //HandleJsonCopy(executionResult.result, select, "exec"); + var basename = "exec" + if (selectedAction !== undefined && selectedAction !== null && Object.keys(selectedAction).length !== 0) { + basename = selectedAction.label.toLowerCase().replaceAll(" ", "_") + } + + HandleJsonCopy(executionResult.result, select, basename) }} - name={"Test result"} + name={"Run Output"} /> : @@ -2523,7 +2698,7 @@ const CodeEditor = (props) => {
-
+
+ + + + : + + setWorkflowDescription(e.target.value)} + disabled={isAiEditing} + onFocus={() => setIsFocused(true)} + onBlur={() => setIsFocused(false)} + fullWidth + + InputProps={{ + endAdornment: ( + + ) + }} + /> + } + + + AI Edits require you to manually review and accept changes.
You can discard unwanted edits. Uses your configured LLM or shuffler.io AI credits. Alpha feature. +
+ +
+ ); +}; + +export default WorkflowGenerationModal; diff --git a/frontend/src/components/ssoTab.jsx b/frontend/src/components/ssoTab.jsx index e220ea6f..d9339ef2 100644 --- a/frontend/src/components/ssoTab.jsx +++ b/frontend/src/components/ssoTab.jsx @@ -53,14 +53,14 @@ const SSOTab = ({selectedOrganization, userdata, isEditOrgTab, globalUrl, handle : selectedOrganization.sso_config.SSORequired); const [ssoCertificate, setSsoCertificate] = React.useState( - selectedOrganization.sso_config === undefined - ? "" - : selectedOrganization.sso_config.sso_certificate === undefined || - selectedOrganization.sso_config.sso_certificate.length === 0 - ? "" - : selectedOrganization.sso_config.sso_certificate - ); - + selectedOrganization.sso_config === undefined + ? "" + : selectedOrganization.sso_config.sso_certificate && + selectedOrganization.sso_config.sso_certificate.length > 0 + ? selectedOrganization.sso_config.sso_certificate + : selectedOrganization.sso_config.sso_long_certificate || "" + ); + const [openidClientId, setOpenidClientId] = React.useState( selectedOrganization.sso_config === undefined ? "" @@ -115,8 +115,8 @@ const SSOTab = ({selectedOrganization, userdata, isEditOrgTab, globalUrl, handle setOpenidToken(selectedOrganization?.sso_config?.openid_token) } - if (ssoCertificate !== selectedOrganization?.sso_config?.sso_certificate) { - setSsoCertificate(selectedOrganization?.sso_config?.sso_certificate) + if (ssoCertificate !== selectedOrganization?.sso_config?.sso_certificate || ssoCertificate !== selectedOrganization?.sso_config?.sso_long_certificate) { + setSsoCertificate(selectedOrganization?.sso_config?.sso_certificate || selectedOrganization?.sso_config?.sso_long_certificate) } if (ssoEntrypoint !== selectedOrganization?.sso_config?.sso_entrypoint) { diff --git a/frontend/src/defaultCytoscapeStyle.jsx b/frontend/src/defaultCytoscapeStyle.jsx index 79d35e35..5f95af0c 100644 --- a/frontend/src/defaultCytoscapeStyle.jsx +++ b/frontend/src/defaultCytoscapeStyle.jsx @@ -392,8 +392,8 @@ export default function defaultCytoscapeStyle(theme) { { selector: ".success-highlight", css: { - "background-color": "#41dcab", - "border-color": "#41dcab", + "background-color": "#02CB70", + "border-color": "#02CB70", "border-width": "5px", "transition-property": "background-color", "transition-duration": "0.5s", @@ -412,8 +412,8 @@ export default function defaultCytoscapeStyle(theme) { { selector: ".failure-highlight", css: { - "background-color": "#8e3530", - "border-color": "#8e3530", + "background-color": "#F53434", + "border-color": "#F53434", "border-width": "5px", "transition-property": "background-color", "transition-duration": "0.5s", @@ -433,7 +433,7 @@ export default function defaultCytoscapeStyle(theme) { selector: ".executing-highlight", css: { //"background-color": "#ffef47", - "border-color": "#ffef47", + "border-color": "#FECC00", "border-width": "8px", "transition-property": "border-width", "transition-duration": "0.25s", @@ -475,8 +475,8 @@ export default function defaultCytoscapeStyle(theme) { selector: "edge.executing-highlight", css: { width: "5px", - "target-arrow-color": "#ffef47", - "line-color": "#ffef47", + "target-arrow-color": "#FECC00", + "line-color": "#FECC00", "transition-property": "line-color, width", "transition-duration": "0.25s", }, @@ -495,9 +495,9 @@ export default function defaultCytoscapeStyle(theme) { { selector: "edge.success-highlight", css: { - width: "3px", - "target-arrow-color": "#41dcab", - "line-color": "#41dcab", + width: "4px", + "target-arrow-color": "#02CB70", + "line-color": "#02CB70", "transition-property": "line-color, width", "transition-duration": "0.5s", "line-fill": "linear-gradient", diff --git a/frontend/src/theme.jsx b/frontend/src/theme.jsx index 4835e4f1..8772ff23 100644 --- a/frontend/src/theme.jsx +++ b/frontend/src/theme.jsx @@ -391,6 +391,78 @@ export const getTheme = (themeMode, brandColor) => }, }, }, + { + props: { variant: 'aiButton' }, + style: { + background: 'linear-gradient(90deg, #ff8544 0%, #ec517c 50%, #9c5af2 100%)', + color: '#ffffff', + borderRadius: '4px', + whiteSpace: "nowrap", + textWrap: "normal", + border: 'none', + boxShadow: 'none', + transition: 'all 0.2s ease-in-out', + '&:hover': { + color: '#ffffff', + textShadow: '0 0 1px currentColor', + }, + '&:active': { + background: 'linear-gradient(90deg, #e6743a 0%, #d4456e 50%, #8a4de8 100%)', + }, + '&:disabled': { + background: themeMode === "dark" ? '#494949' : '#C9C9C9', + color: themeMode === "dark" ? '#9E9E9E' : '#616161', + }, + }, + }, + { + props: { variant: 'aiButtonGhost' }, + style: { + background: 'transparent', + border: 'none', + borderRadius: '4px', + color: '#ffffff', + whiteSpace: "nowrap", + textWrap: "normal", + boxShadow: 'none', + transition: 'all 0.2s ease-in-out', + position: 'relative', + '&::before': { + content: '""', + position: 'absolute', + top: 0, + left: 0, + right: 0, + bottom: 0, + borderRadius: '4px', + padding: '2px', + background: 'linear-gradient(90deg, #ff8544 0%, #ec517c 50%, #9c5af2 100%)', + mask: 'linear-gradient(#fff 0 0) content-box, linear-gradient(#fff 0 0)', + WebkitMask: 'linear-gradient(#fff 0 0) content-box, linear-gradient(#fff 0 0)', + maskComposite: 'exclude', + WebkitMaskComposite: 'xor', + transition: 'opacity 0.2s ease-in-out', + zIndex: -1, + }, + '&:hover': { + background: 'linear-gradient(90deg, #ff8544 0%, #ec517c 50%, #9c5af2 100%)', + color: '#ffffff', + '&::before': { + opacity: 0, + }, + }, + '&:active': { + background: 'linear-gradient(90deg, #e6743a 0%, #d4456e 50%, #8a4de8 100%)', + }, + '&:disabled': { + background: 'transparent', + color: themeMode === "dark" ? '#9E9E9E' : '#616161', + '&::before': { + background: themeMode === "dark" ? '#494949' : '#C9C9C9', + }, + }, + }, + }, ], }, MuiTab: { @@ -412,39 +484,39 @@ export const getTheme = (themeMode, brandColor) => MuiCssBaseline: { styleOverrides: ` @font-face { - font-family: 'Roboto'; + font-family: 'Inter'; font-style: normal; font-display: swap; font-weight: 300; - src: local('Roboto Light'), local('Roboto-Light'); + src: local('Inter Light'), local('Inter-Light'); } @font-face { - font-family: 'Roboto'; + font-family: 'Inter'; font-style: normal; font-display: swap; font-weight: 400; - src: local('Roboto'), local('Roboto-Regular'); + src: local('Inter Regular'), local('Inter-Regular'); } @font-face { - font-family: 'Roboto'; + font-family: 'Inter'; font-style: normal; font-display: swap; font-weight: 500; - src: local('Roboto Medium'), local('Roboto-Medium'); + src: local('Inter Medium'), local('Inter-Medium'); } @font-face { - font-family: 'Roboto'; + font-family: 'Inter'; font-style: normal; font-display: swap; font-weight: 600; - src: local('Roboto SemiBold'), local('Roboto-SemiBold'); + src: local('Inter SemiBold'), local('Inter-SemiBold'); } @font-face { - font-family: 'Roboto'; + font-family: 'Inter'; font-style: normal; font-display: swap; font-weight: 700; - src: local('Roboto Bold'), local('Roboto-Bold'); + src: local('Inter Bold'), local('Inter-Bold'); } `, }, diff --git a/frontend/src/views/Admin2.jsx b/frontend/src/views/Admin2.jsx index f94c3441..fcb675bc 100644 --- a/frontend/src/views/Admin2.jsx +++ b/frontend/src/views/Admin2.jsx @@ -87,6 +87,10 @@ const Admin2 = (props) => { leads.push("distribution partner"); } + if (responseJson.lead_info.channel_partner) { + leads.push("channel partner"); + } + if (responseJson.lead_info.service_partner) { leads.push("service partner"); } diff --git a/frontend/src/views/AgentUI.jsx b/frontend/src/views/AgentUI.jsx index ebc5a558..fab6d012 100644 --- a/frontend/src/views/AgentUI.jsx +++ b/frontend/src/views/AgentUI.jsx @@ -1,11 +1,14 @@ import React, { useState, useEffect, useContext, memo } from "react"; import { Context } from "../context/ContextApi.jsx"; +import { useNavigate, Link, useLocation } from "react-router-dom"; import { getTheme } from "../theme.jsx"; import { toast } from "react-toastify" import ReactJson from "react-json-view-ssr"; +import { v4 as uuidv4} from "uuid"; import { validateJson, collapseField, handleReactJsonClipboard, HandleJsonCopy } from "../views/Workflows.jsx"; import { + Box, Button, ButtonGroup, Typography, @@ -13,6 +16,7 @@ import { CircularProgress, Tooltip, IconButton, + TextField, } from '@mui/material' import { @@ -21,6 +25,8 @@ import { RestartAlt as RestartAltIcon, ExpandMore as ExpandMoreIcon, ExpandLess as ExpandLessIcon, + Send as SendIcon, + Error as ErrorIcon, } from '@mui/icons-material' import { @@ -33,15 +39,19 @@ const AgentUI = (props) => { const [buttonState, setButtonState] = useState("timeline") const [execution, setExecution] = useState(null) const [agentActionResult, setAgentActionResult] = useState(null) + const [agentRequestLoading, setAgentRequestLoading] = useState(false) const [data, setData] = useState({}) const [openIndexes, setOpenIndexes] = useState([]) const [disableButtons, setDisableButtons] = useState(false) const [originalStartTime, setOriginalStartTime] = useState(0) const [latestEndTime, setLatestEndTime] = useState(0) + const [showAgentStarter, setShowAgentStarter] = useState(false) + const [actionInput, setActionInput] = useState("") const {themeMode} = useContext(Context) const theme = getTheme(themeMode) + const navigate = useNavigate(); const agentWrapperStyle = { width: 1000, @@ -64,6 +74,10 @@ const AgentUI = (props) => { return } + if (node_id === undefined || node_id === null || node_id === "") { + return + } + var found = false for (var key in execution_data.results) { const item = execution_data.results[key] @@ -86,6 +100,16 @@ const AgentUI = (props) => { if (found === false) { toast.warn("Failed to find the relevant AI Agent result") + + if (execution_data?.results?.length === 1) { + setAgentActionResult(execution_data.results[0]) + const validatedData = validateJson(execution_data.results[0].result) + if (validatedData.valid) { + setData(validatedData.result) + } else { + toast.warn("Action output result is not valid JSON!") + } + } } } @@ -95,10 +119,10 @@ const AgentUI = (props) => { return } - if (node_id === undefined || node_id === null) { - toast.error("No node ID provided. Please provide node_id in the URL.") - return - } + //if (node_id === undefined || node_id === null || node_id === "") { + // toast.error("No node ID provided. Please provide node_id in the URL.") + // return + //} if (authorization === undefined || authorization === null) { toast.error("No authorization provided. Please provide authorization in the URL.") @@ -166,6 +190,7 @@ const AgentUI = (props) => { const url = `${globalUrl}/api/v1/apps/agent/run?rerun=true&decision_id=${decision?.run_details?.id}` var body = agentActionResult.action + console.log("BODY: ", body) body.source_execution = execution.execution_id body.source_workflow = execution.workflow.id @@ -202,10 +227,11 @@ const AgentUI = (props) => { const executionId = params.get("execution_id") const nodeId = params.get("node_id") const authorization = params.get("authorization") - if (executionId !== undefined && executionId !== null && nodeId !== undefined && nodeId !== null && authorization !== undefined && authorization !== null) { + if (executionId !== undefined && executionId !== null && authorization !== undefined && authorization !== null) { GetExecution(executionId, nodeId, authorization) } else { - toast.warn("No execution ID or node ID provided. Please provide execution_id and node_id in the URL.") + setShowAgentStarter(true) + //toast.warn("No execution ID or node ID provided. Please provide execution_id and node_id in the URL.") } }, []) @@ -221,6 +247,11 @@ const AgentUI = (props) => { + : + item.status === "ABORTED" || item.status === "FAILURE" ? + + + : @@ -246,11 +277,13 @@ const AgentUI = (props) => { const validate = validateJson(item.details) const itemStartTime = item.start_time var itemEndTime = item.end_time - if (itemStartTime !== undefined && (itemStartTime < originalStartTime || originalStartTime === 0)) { - setOriginalStartTime(itemStartTime) + if (itemStartTime !== undefined && itemStartTime !== originalStartTime && (itemStartTime < originalStartTime || originalStartTime === 0)) { + console.log("Rerender 1") + //setOriginalStartTime(itemStartTime) } if (itemEndTime !== undefined && itemEndTime > latestEndTime) { + console.log("Rerender 2") setLatestEndTime(itemEndTime) } @@ -280,23 +313,47 @@ const AgentUI = (props) => { borderTop: "1px solid " + theme.palette.surfaceColor, borderRadius: theme.palette.borderRadius, }} + onMouseEnter={() => { + if (!hovered) { + console.log("HOVER") + setHovered(true) + } + }} + onMouseLeave={() => { + if (hovered) { + setHovered(false) + } + }} >
setHovered(true)} - onMouseLeave={() => setHovered(false)} onClick={(e) => { if (item.details === undefined || item.details === null || item.details === "") { - toast("No details to open") + + if (item?.category === "agent" && item?.type === "agent") { + // Show all the data + if (openIndexes.includes(index)) { + console.log("Rerender 3") + setOpenIndexes(openIndexes.filter((i) => i !== index)) + } else { + console.log("Rerender 4") + setOpenIndexes([...openIndexes, index]) + } + } else { + toast.warn("No details to open") + } + return } if (openIndexes.includes(index)) { + console.log("Rerender 5") setOpenIndexes(openIndexes.filter((i) => i !== index)) } else { + console.log("Rerender 6") setOpenIndexes([...openIndexes, index]) } }} @@ -421,18 +478,31 @@ const AgentUI = (props) => { const TimelineRender = (props) => { const { agent_data } = props; + + const actionResult = execution?.results?.length > 0 ? execution.results[0] : execution var timelineItems = [ { "label": "AI Agent 2", "type": "agent", "category": "agent", + "details": actionResult?.result, - "status": agent_data.status, - "start_time": agent_data.started_at, - "end_time": agent_data.completed_at, + "status": agent_data?.status, + "start_time": agent_data?.started_at, + "end_time": agent_data?.completed_at, }, ] + // Autofixer for result lol + if ((agent_data?.decisions === undefined || agent_data?.decisions === null)) { + const verifiedInput = validateJson(actionResult?.result) + if (verifiedInput.valid === true && verifiedInput.result?.decisions !== undefined && verifiedInput.result?.decisions !== null) { + agent_data.decisions = verifiedInput.result?.decisions + + setAgentActionResult(actionResult) + } + } + var sortedTimelineItems = [] for (var key in agent_data?.decisions) { const item = agent_data.decisions[key] @@ -501,39 +571,145 @@ const AgentUI = (props) => { ) } + const submitInput = (inputText) => { + //toast.info("Submitting AI Agent input: " + inputText); + + setAgentRequestLoading(true) + //setShowAgentStarter(false); + //GetExecution(execution?.execution_id, execution?.node_id, execution?.authorization); + + if (inputText === undefined || inputText === null || inputText === "") { + toast.error("Please provide a valid input for the AI Agent.") + return + } + + // 1. Run the execution. Can this be a single-action run? + // 2. Get the execution ID and node ID from the response. + const uuid = uuidv4() + const data = { + "id": uuid, + "name":"agent", + //"app_name":"Shuffle AI", + "app_name":"AI Agent", // Failover for rerun + "app_id":"shuffle_agent", + "app_version":"1.0.0", + + "environment":"cloud", + "parameters":[ + { + "name":"app_name", + "value":"openai" + }, + { + "name":"input", + "value": inputText + }, + { + "name":"action", + "value":"list_tickets" + } + ]} + + const url = `${globalUrl}/api/v1/apps/agent_starter/run` + fetch(url, { + method: "POST", + body: JSON.stringify(data), + credentials: "include", + }) + .then((response) => { + setAgentRequestLoading(false) + return response.json() + }) + .then((responseJson) => { + //toast.success("Got response!") + console.log("Agent run response: ", responseJson) + + if (responseJson.success === true && responseJson.authorization !== undefined && responseJson.execution_id !== undefined) { + navigate("?execution_id=" + responseJson.execution_id + "&authorization=" + responseJson.authorization) + setShowAgentStarter(false) + GetExecution(responseJson.execution_id, "", responseJson.authorization) + } + }) + .catch((error) => { + setAgentRequestLoading(false) + toast.error("Error: " + error) + }) + + } + return (
- {/* - - Agent Input: {data.input} - - */} - - - - + {showAgentStarter ? + { + e.preventDefault(); + submitInput(actionInput); + }}> + +
- {buttonState === "timeline" ? - - : - null + + Shuffle AI Agents + + { + setActionInput(e.target.value) + }} + InputProps={{ + endAdornment: ( + agentRequestLoading ? + + : + + + + + + ), + }} + /> + + : +
+ + + + + + {buttonState === "timeline" ? + + : + null + } +
}
) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 0466fb85..e1adeb09 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -7,7 +7,6 @@ import { useInterval } from "react-powerhooks"; import { makeStyles, } from "@mui/styles"; import YAML from "yaml"; -import WorkflowTemplatePopup from "../components/WorkflowTemplatePopup.jsx" import { v4 as uuidv4, v5 as uuidv5, validate as isUUID, } from "uuid"; import { useNavigate, Link, useParams } from "react-router-dom"; import { useBeforeunload } from "react-beforeunload" @@ -71,6 +70,7 @@ import { Autocomplete, Radio, ButtonGroup, + Box, } from "@mui/material"; @@ -146,7 +146,9 @@ import CytoscapeComponent from "react-cytoscapejs"; import Draggable from "react-draggable"; import defaultCytoscapeStyle from "../defaultCytoscapeStyle.jsx"; +import WorkflowTemplatePopup from "../components/WorkflowTemplatePopup.jsx" import ShuffleCodeEditor from "../components/ShuffleCodeEditor1.jsx"; +import LineChartWrapper from "../components/LineChartWrapper.jsx"; import WorkflowValidationTimeline from "../components/WorkflowValidationTimeline.jsx" import { validateJson, collapseField, GetIconInfo, handleReactJsonClipboard, HandleJsonCopy, } from "../views/Workflows.jsx"; @@ -157,8 +159,13 @@ import ParsedAction from "../components/ParsedAction.jsx"; import PaperComponent from "../components/PaperComponent.jsx" import ExtraApps from "../components/ExtraApps.jsx" import EditWorkflow from "../components/EditWorkflow.jsx" +import HighlightedValueInSearch from "../components/HighlightedValueInSearch.jsx" import { act } from "react"; import { Context } from "../context/ContextApi.jsx"; +import WorkflowGenerationModal from "../components/WorkflowGenerationModal.jsx"; +import CodeMirror from '@uiw/react-codemirror'; +import { python } from '@codemirror/lang-python'; +import { vscodeDark } from '@uiw/codemirror-theme-vscode'; cytoscape.use(edgehandles); @@ -399,9 +406,8 @@ export function SetJsonDotnotation(jsonInput, inputKey) { //export const green = "#86c142"; -export const green = "#02CB70" +export const green = "#2BC07E" export const yellow = "#FECC00"; -//export const red = "#ff3632"; export const red = "#F53434"; export const grey = "#b0b0b0"; @@ -531,6 +537,8 @@ const AngularWorkflow = (defaultprops) => { const [appAuthentication, setAppAuthentication] = React.useState(undefined); const [variablesModalOpen, setVariablesModalOpen] = React.useState(false); const [aiQueryModalOpen, setAiQueryModalOpen] = React.useState(false) + const [workflowGenerationModalOpen, setWorkflowGenerationModalOpen] = React.useState(false); + const [workflowDescription, setWorkflowDescription] = React.useState(""); const [executionVariablesModalOpen, setExecutionVariablesModalOpen] = React.useState(false); const [authenticationModalOpen, setAuthenticationModalOpen] = React.useState(false); @@ -541,8 +549,7 @@ const AngularWorkflow = (defaultprops) => { const [workflowDone, setWorkflowDone] = React.useState(false); const [localFirstrequest, setLocalFirstrequest] = React.useState(true); - const [requiresAuthentication, setRequiresAuthentication] = - React.useState(false); + const [requiresAuthentication, setRequiresAuthentication] = React.useState(false); const [rightSideBarOpen, setRightSideBarOpen] = React.useState(false); const [showSkippedActions, setShowSkippedActions] = React.useState(false); const [lastExecution, setLastExecution] = React.useState(""); @@ -625,6 +632,7 @@ const AngularWorkflow = (defaultprops) => { const [executionFilter, setExecutionFilter] = React.useState("ALL") const [workflowExecutions, setWorkflowExecutions] = React.useState([]); + const [executionTimeline, setExecutionTimeline] = React.useState([]); const [workflowExecutionCount, setWorkflowExecutionCount] = React.useState(0); const [defaultEnvironmentIndex, setDefaultEnvironmentIndex] = React.useState(0); const [workflowRecommendations, setWorkflowRecommendations] = React.useState(undefined); @@ -649,6 +657,22 @@ const AngularWorkflow = (defaultprops) => { "attachedTo": "", }) + // For code editor + const [codeEditorModalOpen, setCodeEditorModalOpen] = React.useState(false); + const [codedata, setcodedata] = React.useState(""); + const [editorData, setEditorData] = React.useState({ + "name": "", + "value": "", + "field_number": -1, + "actionlist": [], + "field_id": "", + + "example": "", + }) + const [executionArgumentModalOpen, setExecutionArgumentModalOpen] = React.useState(false); + const [searchModalOpen, setSearchModalOpen] = React.useState(false); + + useEffect(() => { if (!firstrequest && isLoaded && isLoggedIn && editWorkflowModalOpen === false) { saveWorkflow(workflow) @@ -678,6 +702,271 @@ const AngularWorkflow = (defaultprops) => { const dragRef = React.useRef(false); + // Add this function to handle search + const searchWorkflow = (workflowData, term) => { + if (!term) return []; + + const results = []; + const searchTermLower = term.toLowerCase(); + + // Helper function to search through nodes + const searchNode = (node) => { + // Search node name/label + if (node.label?.toLowerCase()?.replaceAll("_", " ").includes(searchTermLower) || + node.app_name?.toLowerCase().includes(searchTermLower)) { + results.push({ + nodeId: node.id, + matchType: 'nodeName', + matchedValue: node.label || node.app_name, + path: `${node.label?.replaceAll("_", " ")}`, + image: node.large_image + }); + } + + // Search parameters + if (node.parameters) { + node.parameters.forEach(param => { + // Search parameter names + if (param.name?.toLowerCase().includes(searchTermLower)) { + results.push({ + nodeId: node.id, + matchType: 'fieldName', + matchedValue: param.name, + path: `${node.label?.replaceAll("_", " ")} > ${param.name}`, + image: node.large_image + }); + } + + // Search parameter values + if (param.value && typeof param.value === 'string' && + param.value?.toLowerCase().includes(searchTermLower)) { + results.push({ + nodeId: node.id, + matchType: 'fieldValue', + matchedValue: param.value, + path: `${node.label?.replace("_", " ")} > ${param.name}`, + image: node.large_image + }); + } + }); + } + }; + + // Search through all actions + workflowData.actions?.forEach(searchNode); + // Search through all triggers if they exist + workflowData.triggers?.forEach(searchNode); + + return results; + }; + + const SearchModal = memo(({ open, onClose, workflow }) => { + const [searchTerm, setSearchTerm] = useState(""); + const [searchResults, setSearchResults] = useState([]); + const searchTimeoutRef = useRef(null); + + // Custom debounce implementation + const handleSearchChange = (event) => { + const newTerm = event.target.value; + setSearchTerm(newTerm); + + // Clear any existing timeout + if (searchTimeoutRef.current) { + clearTimeout(searchTimeoutRef.current); + } + + // Set new timeout for search + searchTimeoutRef.current = setTimeout(() => { + if (!newTerm.trim()) { + setSearchResults([]); + return; + } + const results = searchWorkflow(workflow, newTerm); + setSearchResults(results); + }, 300); // 300ms delay + }; + + // Cleanup timeout on unmount + useEffect(() => { + return () => { + if (searchTimeoutRef.current) { + clearTimeout(searchTimeoutRef.current); + } + }; + }, []); + + return ( + + + Search Workflow (Beta version) + + + + + +
+ +
+ {searchResults.map((result, index) => ( + { + // Find the action in workflow.actions that matches the node ID + const action = workflow.actions.find(action => action.id === result.nodeId); + if (action) { + console.log("Selected action", action) + setSelectedAction(action); + setRightSideBarOpen(true) + } + + // Navigate to the node using the utility function + const navigationSuccess = navigateToNode(result.nodeId); + + if (!navigationSuccess) { + console.warn(`Failed to navigate to node: ${result.nodeId}`); + // Fallback: try to trigger node selection event manually + if (cy && result.nodeId) { + const cyNode = cy.getElementById(result.nodeId); + if (cyNode && cyNode.length > 0) { + cyNode.trigger('select'); + } + } + } + + // Scroll to specific field if it's a field search result + if ((result.matchType === 'fieldName' || result.matchType === 'fieldValue') && result.path) { + const paramName = result.path.split(' > ').pop(); + setTimeout(() => { + const field = document.querySelector(`[data-parameter="${paramName}"], [name="${paramName}"], #param_${paramName.replace(/[^a-zA-Z0-9_-]/g, '_')}`); + if (field) { + field.scrollIntoView({ behavior: 'smooth', block: 'center' }); + field.style.border = '3px solid #2BC07E'; + setTimeout(() => field.style.border = '', 2000); + } + }, 800); + } + + onClose(); + }} + > + + Node + + {result.path} + + + + + ))} + {searchTerm && searchResults.length === 0 && ( + + No results found + + )} +
+
+
+
+ ); + }); + + // Add keyboard shortcut handler in your main component + useEffect(() => { + const handleKeyPress = (event) => { + + if (((event.metaKey || event.ctrlKey) && event.key === 'f')) { + + // Check if any modal is currently open + const isAnyModalOpen = codeEditorModalOpen || executionModalOpen || + editWorkflowModalOpen || executionArgumentModalOpen || authenticationModalOpen || codeModalOpen || + authgroupModalOpen; + + if (isAnyModalOpen) { + return; + } + + event.preventDefault(); + setSearchModalOpen(true); + } + }; + + // Always add the event listener, but the handler will check permissions internally + document.addEventListener('keydown', handleKeyPress); + + return () => { + document.removeEventListener('keydown', handleKeyPress); + }; + }, [ + userdata?.support, + codeEditorModalOpen, + executionModalOpen, + editWorkflowModalOpen, + executionArgumentModalOpen, + authenticationModalOpen, + codeModalOpen, + authgroupModalOpen + ]); // New for generated stuff const releaseToConnectLabel = "Release to Connect" @@ -729,6 +1018,8 @@ const AngularWorkflow = (defaultprops) => { "List tickets", "Send Email", "Get specific ticket", + "Update ticket", + "Add ticket comment", ], "multiselect": true, }, @@ -764,6 +1055,7 @@ const AngularWorkflow = (defaultprops) => { { "id": "integration", "name": "Singul", + "is_valid": true, "large_image": theme.palette.singulGreen, "type": "ACTION", "app_version": "1.0.0", @@ -771,7 +1063,7 @@ const AngularWorkflow = (defaultprops) => { "authentication": { "type": "", }, - "description": "Support-use only", + "description": "Build & integrate tools easily with standard input and standard output. Built by Shuffle. https://singul.io", "actions": [{ "name": "Cases", "description": "Available actions for case management", @@ -967,21 +1259,30 @@ const AngularWorkflow = (defaultprops) => { "multiline": true, }] }, - ] - }] + { + "name": "Translate standard", + "description": "Translates your JSON data into a standard formats, then stores it in the Shuffle Datastore", + "label": "Translate standard", + "example": "{\"source_data\": \"{\\\"event\\\": \\\"login\\\", \\\"user\\\": \\\"john_doe\\\", \\\"timestamp\\\": \\\"2023-10-01T12:00:00Z\\\"}\", \"standard\": \"OCSF\"}", + "parameters": [{ + "name": "source_data", + "value": "", + "required": true, + "multiline": true, + }, + { + "name": "standard", + "value": "OCSF", + "description": "The standard to use from https://github.com/Shuffle/standards/tree/main", + "options": [ + "OCSF" + ], + "required": true, + "multiline": false, + }] + }, + ]}] - // For code editor - const [codeEditorModalOpen, setCodeEditorModalOpen] = React.useState(false); - const [codedata, setcodedata] = React.useState(""); - const [editorData, setEditorData] = React.useState({ - "name": "", - "value": "", - "field_number": -1, - "actionlist": [], - "field_id": "", - - "example": "", - }) const [loadedApps, setLoadedApps] = React.useState([]) @@ -1245,8 +1546,6 @@ const AngularWorkflow = (defaultprops) => { }, [selectedApp]) - const [executionArgumentModalOpen, setExecutionArgumentModalOpen] = React.useState(false); - // This should all be set once, not on every iteration // Use states and don't update lol const cloudSyncEnabled = @@ -1986,6 +2285,10 @@ const AngularWorkflow = (defaultprops) => { responseJson.executions = responseJson.runs } + if (responseJson !== undefined && responseJson !== null && responseJson.timeline !== undefined && responseJson.timeline !== null && responseJson.timeline.length > 0) { + setExecutionTimeline(responseJson.timeline) + } + if (responseJson !== undefined && responseJson !== null && responseJson.executions !== undefined && responseJson.executions !== null) { // - means it's opposite @@ -2085,7 +2388,7 @@ const AngularWorkflow = (defaultprops) => { .then((response) => { if (response.status !== 200) { stop(); - setExecutionModalView(0); + //setExecutionModalView(0); //toast("Failed loading the workflow run") console.log("Status not 200 for stream results :O!"); @@ -2195,6 +2498,63 @@ const AngularWorkflow = (defaultprops) => { }; + // Utility function to navigate to a specific node in the workflow + const navigateToNode = (nodeId) => { + if (!cy || !nodeId) { + console.warn('Cytoscape instance not available or nodeId missing'); + return false; + } + + try { + // Find the node in Cytoscape + const cyNode = cy.getElementById(nodeId); + + if (!cyNode || cyNode.length === 0) { + console.warn(`Node with ID ${nodeId} not found in Cytoscape`); + return false; + } + + // Default options + const defaultOptions = { + select: true, + unselectOthers: true, + center: true, + zoom: Math.max(cy.zoom(), 1), + animationDuration: 800, + easing: 'ease-in-out' + }; + + const config = { ...defaultOptions }; + + // Unselect all currently selected nodes if requested + if (config.unselectOthers) { + cy.$(':selected').unselect(); + } + + // Select the target node if requested + if (config.select) { + cyNode.select(); + } + + // Center and zoom to the node with smooth animation if requested + if (config.center) { + cy.animate({ + center: { + eles: cyNode + }, + zoom: config.zoom, + }, { + duration: config.animationDuration, + easing: config.easing + }); + } + + return true; + } catch (error) { + console.error('Error navigating to node:', error); + return false; + } + }; const handleColoring = (actionId, status, label) => { if (cy === undefined) { @@ -2258,7 +2618,7 @@ const AngularWorkflow = (defaultprops) => { currentnode.removeClass("shuffle-hover-highlight"); currentnode.removeClass("awaiting-data-highlight"); currentnode.addClass("success-highlight"); - incomingEdges.addClass("success-highlight"); + outgoingEdges.addClass("success-highlight"); if (visited !== undefined && visited !== null && !visited.includes(label)) { @@ -5368,7 +5728,8 @@ const AngularWorkflow = (defaultprops) => { if (!branchFound) { var relevantNodes = [] - const minDistance = 185 + //const minDistance = 185 + const minDistance = 85 const draggedNode = event.target const allnodes = cy.nodes().jsons() for (var nodekey in allnodes) { @@ -5398,7 +5759,7 @@ const AngularWorkflow = (defaultprops) => { if (decoratorNodeIds.includes(node.data.id)) { // Drag a little farther to remove it - if (distance > minDistance + 75) { + if (distance > minDistance + 125) { // Remove the branch? Why? const edgeToRemove = cy.getElementById(branches[branchkey].data.id) if (edgeToRemove !== null && edgeToRemove !== undefined) { @@ -6527,8 +6888,31 @@ const AngularWorkflow = (defaultprops) => { ) } - const requiresAuth = curapp.authentication.required; //&& ((curapp.authentication.parameters !== undefined && curapp.authentication.parameters !== null) || (curapp.authentication.type === "oauth2" && curapp.authentication.redirect_uri !== undefined && curapp.authentication.redirect_uri !== null)) - setRequiresAuthentication(requiresAuth); + var requiresAuth = curapp?.authentication?.required + if (curaction.app_id === "integration" || curaction.app_id === "shuffle_agent") { + + requiresAuth = false + for (var paramkey in curaction.parameters) { + const param = curaction.parameters[paramkey] + if (param.name === "app_name" && param?.value?.length > 0) { + + const foundapp = apps.find((a) => a.name === param.value) + if (foundapp !== undefined && foundapp !== null && foundapp?.authentication?.required === true) { + requiresAuth = true + for (var key in appAuthentication) { + if (appAuthentication[key]?.app?.name === foundapp?.name) { + requiresAuth = false + break + } + } + } + + break + } + } + } + + setRequiresAuthentication(requiresAuth) if (curapp.authentication.required) { //console.log("App requires auth.") // Setup auth here :) @@ -11586,6 +11970,71 @@ const AngularWorkflow = (defaultprops) => { } } + const generateAIWorkflow = () => { + const envToSend = selectedActionEnvironment?.Name || (isCloud ? "Cloud" : "Shuffle"); + + const data = { + query: workflowDescription, + workflow_id: props.match.params.key, + environment: envToSend + }; + + fetch(globalUrl + "/api/v2/workflows/generate/llm", { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(data), + credentials: "include", + }) + .then(async (response) => { + const json = await response.json(); + + if (response.status !== 200) { + if (json.reason !== undefined && json.reason !== null && json.reason.length > 0) { + toast.error("Workflow generation failed: " + json.reason) + } + + if (!isCloud) { + toast.info("Click here to set up a local LLM!", { + autoClose: 10000, + onClick: () => { + window.open("/docs/AI#self-hosting-models", "_blank") + } + }) + } + + return null; + } + + // AI “rejection” message (success: true + message) + if (json.success === true && typeof json.message === "string") { + toast(json.message); + return null; + } + + if (json.success === false) { + toast(json.message || "Operation failed"); + return null; + } + + if (!json || Object.keys(json).length === 0) { + toast("Workflow generation failed: empty response"); + return null; + } + + toast("Workflow generation successful"); + setWorkflow(json); + return json; + }) + .catch((error) => { + console.error("AI Workflow Generation Error:", error); + toast("Workflow generation failed due to network error"); + }); + }; + + const AppView = (props) => { const { allApps, prioritizedApps, filteredApps, extraApps } = props; // console.log("AppView Rendered!") @@ -11595,7 +12044,6 @@ const AngularWorkflow = (defaultprops) => { prioritizedApps, Array.prototype.concat.apply( filteredApps.filter((innerapp) => !internalIds.includes(innerapp.id.toLowerCase())), - triggers ) ) ) @@ -12311,18 +12759,15 @@ const AngularWorkflow = (defaultprops) => { } if (app.trigger_type === "PIPELINE" && userdata.support !== true) { - return null + return null } - if ((app.id === "integration" || app.id === "shuffle_agent") && userdata.support !== true) { - if (isCloud === false && app.id === "integration") { - } else { - return null - } - } + if ((app.id === "shuffle_agent") && userdata.support !== true) { + return null + } if (viewedApps.includes(app.id)) { - return null + return null } if (app.trigger_type !== undefined && app.trigger_type !== null && app.trigger_type.length > 0) { @@ -12374,9 +12819,9 @@ const AngularWorkflow = (defaultprops) => {
: -
+
- Apps need to be activated before they can be used. Search from our 2500+ apps to activate them for your organisation. + Apps need to be activated before they can be used. Search in the search bar from our 2500+ apps to activate them for.
} @@ -13709,6 +14154,10 @@ const AngularWorkflow = (defaultprops) => { continue } + if (execution.execution_argument === "{}" || execution.execution_argument === "[]") { + continue + } + if (availableArguments.includes(execution.execution_argument)) { continue } @@ -13966,7 +14415,7 @@ const AngularWorkflow = (defaultprops) => { :
- At least one node in this workflow requires an execution argument ($exec). Please select one below, or provide a custom one in the text field next to the run button. + At least one node in this workflow requires an execution argument ($exec). Please click one below, or provide a custom argument in the text field next to the run button. @@ -13976,8 +14425,45 @@ const AngularWorkflow = (defaultprops) => { Previously used arguments: {availableArguments.map((data) => { + + var defaultoutput = + + {data} + + + const validate = validateJson(data, true) + if (validate.valid === true) { + defaultoutput = +
+ + { + return collapseField(jsonField) + }} + iconStyle={theme.palette.jsonIconStyle} + collapseStringsAfterLength={theme.palette.jsonCollapseStringsAfterLength} + displayArrayKey={false} + displayDataTypes={false} + name={false} + /> +
+ } + return ( - { setExecutionText(data) executeWorkflow(data, workflow.start, lastSaved); @@ -13986,9 +14472,7 @@ const AngularWorkflow = (defaultprops) => { }} >
- - {data} - + {defaultoutput} ) })} @@ -13997,14 +14481,14 @@ const AngularWorkflow = (defaultprops) => {
} @@ -18187,9 +18671,10 @@ const AngularWorkflow = (defaultprops) => {
- const defaultEnvironment = environments && environments?.find( - (env) => env?.default && env?.Name?.toLowerCase() !== "cloud" - ); + + const defaultEnvironment = environments?.find( + (env) => env.default && env.Name.toLowerCase() !== "cloud" + ) if (selectedTrigger.trigger_type === "PIPELINE" && selectedTrigger.environment === "onprem" && defaultEnvironment !== undefined) { selectedTrigger.environment = defaultEnvironment.Name @@ -18795,15 +19280,15 @@ const AngularWorkflow = (defaultprops) => { const TopCytoscapeBar = (props) => { const [hovered, setHovered] = useState(false) - if (workflow.public === true) { + if (workflow?.public === true) { return null } - if (userdata.active_org === undefined || userdata.active_org === null) { + if (userdata?.active_org === undefined || userdata?.active_org === null) { return null } - const isCorrectOrg = workflow.public === true || userdata.active_org.id === undefined || userdata.active_org.id === null || workflow.org_id === null || workflow.org_id === undefined || workflow.org_id.length === 0 || userdata.active_org.id === workflow.org_id + const isCorrectOrg = workflow?.public === true || userdata?.active_org.id === undefined || userdata?.active_org.id === null || workflow?.org_id === null || workflow?.org_id === undefined || workflow?.org_id?.length === 0 || userdata?.active_org?.id === workflow?.org_id return (
@@ -18831,7 +19316,7 @@ const AngularWorkflow = (defaultprops) => { setLastSaved(false) }} > - {workflow.name !== undefined && workflow.name !== null && workflow.name.length > 0 ? + {workflow?.name !== undefined && workflow?.name !== null && workflow?.name?.length > 0 ? : null @@ -18887,7 +19372,7 @@ const AngularWorkflow = (defaultprops) => { {originalWorkflow?.suborg_distribution === undefined || originalWorkflow?.suborg_distribution === null || originalWorkflow?.suborg_distribution?.length === 0 || originalWorkflow?.suborg_distribution.includes("none") ?
- {originalWorkflow?.parentorg_workflow !== undefined && originalWorkflow?.parentorg_workflow !== null && originalWorkflow?.parentorg_workflow.length > 0 || workflow?.parentorg_workflow !== undefined && workflow?.parentorg_workflow !== null && workflow?.parentorg_workflow.length > 0 ? + {originalWorkflow?.parentorg_workflow !== undefined && originalWorkflow?.parentorg_workflow !== null && originalWorkflow?.parentorg_workflow?.length > 0 || workflow?.parentorg_workflow !== undefined && workflow?.parentorg_workflow !== null && workflow?.parentorg_workflow?.length > 0 ? null @@ -18917,7 +19402,7 @@ const AngularWorkflow = (defaultprops) => { : null} - {userdata !== undefined && userdata !== null && userdata.orgs !== undefined && userdata.orgs !== null && userdata.orgs.length > 1 && workflow?.id !== undefined && workflow?.id && workflow?.id?.length > 0 && userdata?.active_org?.creator_org?.length === 0 && userdata?.active_org?.id == workflow?.org_id ? + {userdata !== undefined && userdata !== null && userdata?.orgs !== undefined && userdata?.orgs !== null && userdata?.orgs?.length > 1 && workflow?.id !== undefined && workflow?.id && workflow?.id?.length > 0 && userdata?.active_org?.creator_org?.length === 0 && userdata?.active_org?.id == workflow?.org_id ?
- {showEnvironment === true && environments.length > 0 && selectedActionEnvironment !== undefined && selectedActionEnvironment !== null && selectedActionEnvironment.Name !== undefined && selectedActionEnvironment.Name !== null ? + {showEnvironment === true && environments?.length > 0 && selectedActionEnvironment !== undefined && selectedActionEnvironment !== null && selectedActionEnvironment.Name !== undefined && selectedActionEnvironment.Name !== null ? { */} + + + + : null} + +
@@ -20997,7 +21507,10 @@ const AngularWorkflow = (defaultprops) => {
*/} - {userdata.support === true || (userdata.avatar !== undefined && (userdata.avatar === creatorProfile.github_avatar || allowList.includes(userdata.public_username))) ? + {userdata.support || + (userdata.avatar !== undefined && (userdata.avatar === creatorProfile.github_avatar || allowList.includes(userdata.public_username))) || + (workflow?.owner?.length > 0 && workflow.owner === userdata?.active_org?.id && userdata?.active_org.role === "admin") || + (workflow?.owner?.length > 0 && workflow.owner === userdata?.id)?
You can see these buttons because you may have the correct access rights as a creator to help modify this workflow. @@ -21115,20 +21628,6 @@ const AngularWorkflow = (defaultprops) => {
); - const executionPaperStyle = { - minWidth: "95%", - maxWidth: "95%", - marginTop: 5, - color: theme.palette.text.primary, - marginBottom: 10, - padding: 5, - backgroundColor: theme.palette.platformColor, - borderRadius: theme.palette.borderRadius, - cursor: "pointer", - display: "flex", - minHeight: 45, - maxHeight: 45, - }; const parsedExecutionArgument = () => { var showResult = executionData.execution_argument.trim(); @@ -21624,7 +22123,8 @@ const AngularWorkflow = (defaultprops) => { : null} {executionModalView === 0 ? ( -
+ +
{
+ + {executionTimeline?.length > 0 && +
+ +
+ } + +
+ {workflowExecutions.length > 0 ? (
{workflowExecutions.map((data, index) => { @@ -21762,6 +22279,21 @@ const AngularWorkflow = (defaultprops) => { const foundnotifications = data.notifications_created === undefined || data.notifications_created === null ? 0 : data.notifications_created + const executionPaperStyle = { + padding: 5, + marginTop: 5, + minHeight: 45, + maxHeight: 45, + minWidth: "95%", + maxWidth: "95%", + display: "flex", + marginBottom: index === workflowExecutions.length-1 ? 200 : 10, + cursor: "pointer", + color: theme.palette.text.primary, + borderRadius: theme.palette.borderRadius, + backgroundColor: theme.palette.platformColor, + }; + return ( { {foundnotifications > 0 ? { e.preventDefault() e.stopPropagation() @@ -22738,7 +23270,7 @@ const AngularWorkflow = (defaultprops) => { style={{ zIndex: 50000 }} > 0 ? yellow : theme.palette.textColor, + color: relevant_errors.length > 0 ? red : theme.palette.textColor, }} /> @@ -22783,6 +23315,26 @@ const AngularWorkflow = (defaultprops) => { : null} + {data?.action?.name === "run_schemaless" || data?.action?.name === "run_singul" || data?.action?.name === "singul" && data?.action?.parameters?.length > 4 ? + + : null} + {data.action.app_name === "shuffle-subflow" && validate.result.success !== undefined && validate.result.success === true ? ( @@ -22944,7 +23496,7 @@ const AngularWorkflow = (defaultprops) => { > Action Logs - + More log details for this action are not available without an onprem environment with the SHUFFLE_LOGS_DISABLED environment variable set to false: SHUFFLE_LOGS_DISABLED=false. Logs are enabled by default, except in scale mode.
@@ -22966,14 +23518,15 @@ const AngularWorkflow = (defaultprops) => { return (
{data.value.length > 60 || checked.valid ? + { if (!showVariable) { @@ -22997,12 +23550,45 @@ const AngularWorkflow = (defaultprops) => { {showVariable ? data.value : null} + { + var copyText = document.getElementById("copy_element_shuffle"); + if (copyText !== undefined && copyText !== null) { + const clipboard = navigator.clipboard; + if (clipboard === undefined) { + toast("Can only copy over HTTPS (port 3443)"); + return; + } + + navigator.clipboard.writeText(data.value); + copyText.select(); + copyText.setSelectionRange( + 0, + 99999 + ); /* For mobile devices */ + + /* Copy the text inside the text field */ + document.execCommand("copy"); + toast("Copied to clipboard"); + } else { + console.log("Couldn't find the copy field: ", copyText); + } + }} + edge="end" + > + + + : - {data.name}: {showVariable ? data.value : null} + {data.name}: + {showVariable ? data.value : null} + } {open ? @@ -23024,8 +23610,8 @@ const AngularWorkflow = (defaultprops) => { { @@ -23035,7 +23621,7 @@ const AngularWorkflow = (defaultprops) => { window.open(data.value, "_blank") } }} - color={showlink ? "inherit" : "textSecondary"} + color={showlink ? "#ff8544" : "textSecondary"} > {data.value} @@ -23150,6 +23736,10 @@ const AngularWorkflow = (defaultprops) => { return `KMS authentication most likely failed. Check your notifications for more details on this page: /admin?admin_tab=notifications. If you need help with KMS, please contact ${supportEmail}` } + if (stringjson.includes("string indices must be integers")) { + return `String indices must be integers typically means you are getting a list, while you expected a dictionary. Check the Variable & Debug for more information.` + } + if (stringjson.includes("invalidurl")) { // IF count of "http" is more than one, 1, it's prolly invalid var additionalinfo = "" @@ -23199,8 +23789,9 @@ const AngularWorkflow = (defaultprops) => { sx: { pointerEvents: "auto", color: theme.palette.text.primary, - minWidth: isMobile ? "90%" : "750px", - maxHeight: "550px", + minWidth: isMobile ? "90%" : 900, + minHeight: 500, + maxHeight: 650, overflowY: "auto", overflowX: "hidden", border: theme.palette.defaultBorder, @@ -23418,7 +24009,9 @@ const AngularWorkflow = (defaultprops) => { > {selectedResult.action.label.replaceAll("_", " ")}
-
{selectedResult.action.name}
+ + {selectedResult.action.name} +
@@ -23658,6 +24251,7 @@ const AngularWorkflow = (defaultprops) => { globalUrl={globalUrl} setSelectedActionEnvironment={setSelectedActionEnvironment} requiresAuthentication={requiresAuthentication} + setRequiresAuthentication={setRequiresAuthentication} setLastSaved={setLastSaved} lastSaved={lastSaved} aiSubmit={aiSubmit} @@ -24385,6 +24979,7 @@ const AngularWorkflow = (defaultprops) => { }, }} fullWidth + multiline={!!data.multiline} type={ data.example !== undefined && data.example.includes("**") ? "password" @@ -24480,7 +25075,10 @@ const AngularWorkflow = (defaultprops) => { ) : null; - // This whole part is redundant. Made it part of Arguments instead. + // This whole part is redundant. Made it part of Arguments instead? + const foundIntegrationApp = selectedAction.app_id === "integration" || selectedAction.app_id === "shuffle_agent" ? selectedAction?.parameters?.find(param => param.name === "app_name") : undefined + const authApp = !authenticationModalOpen ? undefined : foundIntegrationApp === undefined || (selectedApp.id !== "integration" && selectedApp.id !== "shuffle_agent") ? selectedApp : apps.find(app => app.name === foundIntegrationApp?.value) || selectedApp; + const authenticationModal = authenticationModalOpen ? ( { backgroundColor: theme.palette.DialogStyle.backgroundColor, }} > - {selectedApp.reference_info === undefined || - selectedApp.reference_info === null || - selectedApp.reference_info.github_url === undefined || - selectedApp.reference_info.github_url === null || - selectedApp.reference_info.github_url.length === 0 ? ( + { authApp.reference_info === undefined || + authApp.reference_info === null || + authApp.reference_info.github_url === undefined || + authApp.reference_info.github_url === null || + authApp.reference_info.github_url.length === 0 ? ( { style={{ textDecoration: "none", color: theme.palette.linkColor }} > {`Documentation { {`Documentation {
{ {authenticationType.type === "oauth2" || authenticationType.type === "oauth2-app" ? { isCloud={isCloud} /> : - + }
{ */ }} > - {selectedApp.documentation === undefined || - selectedApp.documentation === null || - selectedApp.documentation.length === 0 ? ( + { authApp.documentation === undefined || + authApp.documentation === null || + authApp.documentation.length === 0 ? ( @@ -24708,7 +25306,7 @@ const AngularWorkflow = (defaultprops) => { textAlign: "left", }} > - {selectedApp.description} + {authApp.description}
{ toast.success("Opening remote Github documentation link. Thanks for contributing!") setTimeout(() => { - window.open(`https://github.com/Shuffle/openapi-apps/new/master/docs?filename=${selectedApp.name.toLowerCase()}.md`, "_blank") + window.open(`https://github.com/Shuffle/openapi-apps/new/master/docs?filename=${authApp.name.toLowerCase()}.md`, "_blank") }, 2500) }} > @@ -24761,11 +25359,11 @@ const AngularWorkflow = (defaultprops) => { Want to help change this app directly? - {selectedApp.reference_info === undefined || - selectedApp.reference_info === null || - selectedApp.reference_info.github_url === undefined || - selectedApp.reference_info.github_url === null || - selectedApp.reference_info.github_url.length === 0 ? ( + { authApp.reference_info === undefined || + authApp.reference_info === null || + authApp.reference_info.github_url === undefined || + authApp.reference_info.github_url === null || + authApp.reference_info.github_url.length === 0 ? (
{ Check it out on Github! @@ -24891,7 +25489,7 @@ const AngularWorkflow = (defaultprops) => { maxWidth: "100%", minWidth: "100%", }} > - {selectedApp.documentation} + {authApp.documentation}
)} @@ -24972,7 +25570,7 @@ const AngularWorkflow = (defaultprops) => { const [responseMsg, setResponseMsg] = useState(""); if (suggestionBox === undefined || suggestionBox.open === false) { - return false + return } return ( @@ -25444,7 +26042,7 @@ const AngularWorkflow = (defaultprops) => { }; // Check if event.target.value is an array. If it is, split with comma - if (parametername !== undefined && parametername.startsWith("${") && parametername.endsWith("}")) { + if (parametername !== undefined && parametername !== undefined && parametername?.startsWith("${") && parametername?.endsWith("}")) { var paramcheckIndex = selectedAction.parameters.findIndex(param => param.name === parametername) if (paramcheckIndex !== -1) { // Replace the value in the field @@ -25588,12 +26186,14 @@ const AngularWorkflow = (defaultprops) => { const actionIndex = workflow?.actions.findIndex(action => action.id === actionId); if (actionIndex >= 0) { // Find the parameter with matching name - console.log("fieldName", fieldName) const paramIndex = workflow.actions[actionIndex].parameters.findIndex(param => param.name === fieldName); if (paramIndex >= 0) { // Update the parameter value workflow.actions[actionIndex].parameters[paramIndex].value = newData; - + if(selectedAction !== undefined && selectedAction !== null && selectedAction.id === actionId) { + selectedAction.parameters[paramIndex].value = newData; + setSelectedAction(selectedAction) + } // Update workflow state to trigger re-render setWorkflow({...workflow}); setLastSaved(false); @@ -25676,6 +26276,20 @@ const AngularWorkflow = (defaultprops) => { {aiQueryModal} + + + {conditionsModal} {codePopoutModal} {workflowRevisions} @@ -25854,7 +26468,11 @@ const AngularWorkflow = (defaultprops) => { */} {loadedCheck} - + setSearchModalOpen(false)} + workflow={workflow} + />
); }; diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 1eaf5fcb..414a7af0 100755 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -54,7 +54,7 @@ import YAML from "yaml"; import { MuiChipsInput } from "mui-chips-input"; //import { useAlert import { ToastContainer, toast } from "react-toastify" -import words from "shellwords"; +import { split } from "shellwords"; import AvatarEditor from "react-avatar-editor"; import { Context } from "../context/ContextApi.jsx"; @@ -119,7 +119,7 @@ const parseCurl = (s) => { } try { - var args = rewrite(words.split(s)); + var args = rewrite(split(s)); } catch (e) { return s; } diff --git a/frontend/src/views/AppExplorer.jsx b/frontend/src/views/AppExplorer.jsx index f0069e4f..1f434a83 100644 --- a/frontend/src/views/AppExplorer.jsx +++ b/frontend/src/views/AppExplorer.jsx @@ -233,6 +233,7 @@ const buttonBackground = "linear-gradient(to right, #f86a3e, #f34079)"; const [secondaryApp, setSecondaryApp] = useState({}); const [firstRequest, setFirstRequest] = useState(true); const [publishModalOpen, setPublishModalOpen] = React.useState(false); + const [showDistributionPopup, setShowDistributionPopup] = React.useState(false); const [categories, setCategories] = useState(appCategories) const [newWorkflowCategories, setNewWorkflowCategories] = React.useState([]); @@ -743,7 +744,7 @@ const buttonBackground = "linear-gradient(to right, #f86a3e, #f34079)"; } }; - const activateApp = (action) => { + const activateApp = (action, org_id, multiple_request = false) => { if (serverside === true) { return } @@ -753,17 +754,20 @@ const buttonBackground = "linear-gradient(to right, #f86a3e, #f34079)"; if (action !== undefined && action !== null) { url = `${globalUrl}/api/v1/apps/${appId}/${action}` } - fetch(url, { + + + fetch(url, { method: "GET", headers: { "Content-Type": "application/json", - Accept: "application/json", + "Accept": "application/json", + "Org-Id": org_id !== undefined && org_id !== null && org_id?.length > 0 ? org_id : userdata.active_org.id }, credentials: "include", }) .then((response) => { if (response.status !== 200) { - console.log("Failed to activate"); + console.log("Failed to activate: " + response.statusText); } return response.json(); @@ -772,34 +776,37 @@ const buttonBackground = "linear-gradient(to right, #f86a3e, #f34079)"; if (responseJson.success === false) { if (action === undefined || action === null) { if (responseJson.reason !== undefined) { - toast("Failed to activate the app: "+responseJson.reason); + toast.warn("Failed to activate the app: "+responseJson.reason); } else { - toast("Failed to activate the app"); + toast.warn("Failed to activate the app"); } } else { if (responseJson.reason !== undefined) { - toast("Failed to perform action: "+responseJson.reason); + toast.warn("Failed to perform action: "+responseJson.reason); } else { - toast(`Failed to perform action. Please try again or contact ${supportEmail}`); + toast.warn(`Failed to perform action. Please try again or contact ${supportEmail}`); } } } else { - if (checkLogin !== undefined && checkLogin !== null) { + if (!showDistributionPopup && (checkLogin !== undefined && checkLogin !== null) && !multiple_request) { checkLogin() } if (action === undefined || action === null) { if (appExists) { - toast("App deactivated for your organization! Existing workflows with the app will continue to work.") + toast.success("App deactivated for your organization! Existing workflows with the app will continue to work.") } else { - toast("App activated for your organization!") + toast.success("App activated for your organization!") } } else { + if (responseJson.success && !multiple_request &&(responseJson.reason !== undefined || responseJson.reason !== null)) { + toast.success(`${responseJson.reason}`); + } } } }) .catch((error) => { - toast(error.toString()); + toast.error(error.toString()); }); }; @@ -1443,7 +1450,12 @@ const buttonBackground = "linear-gradient(to right, #f86a3e, #f34079)"; const index = searchClient.initIndex("appsearch"); console.log("Running appsearch for: ", appname); - + if (appname === "integration") { + // Redirect to https://singul.io + window.location.href = "https://singul.io" + } else if (appname === "shuffle_agent") { + navigate("/agents") + } index .search(appname) @@ -3909,6 +3921,220 @@ const buttonBackground = "linear-gradient(to right, #f86a3e, #f34079)"; ) : null; + + const handleActivateApp = (id, action) => { + if (action === "activate_all") { + const childOrgs = userdata.orgs.filter( + (data) => data.creator_org === userdata.active_org.id + ) + + // run app activation request for each org + const orgIds = childOrgs.map((data) => data.id); + orgIds.forEach((orgId) => { + activateApp("activate", orgId, true) + }) + + setTimeout(() => { + toast.success("App activated for all sub-orgs"); + }, 5000) + } else if (action === "deactivate_all") { + const childOrgs = userdata.orgs.filter( + (data) => data.creator_org === userdata.active_org.id + ) + + const orgIds = childOrgs.map((data) => data.id); + // run app deactivation request for each org + orgIds.forEach((orgId) => { + activateApp("deactivate", orgId, true) + }); + + setTimeout(() => { + toast.success("App deactivated for all sub-orgs"); + }, 5000); + + } else if (action === "activate_single") { + if (id === null) { + toast.error("Please select a sub-org to activate the app for."); + return; + } + + activateApp("activate", id); + } else if (action === "deactivate_single") { + if (id === null) { + toast.error("Please select a sub-org to deactivate the app for."); + return; + } + + activateApp("deactivate", id); + } + }; + + + + + const appDistributinModal = showDistributionPopup ? ( + setShowDistributionPopup(false)} + PaperProps={{ + sx: { + borderRadius: theme?.palette?.DialogStyle?.borderRadius || 3, + border: theme?.palette?.DialogStyle?.border, + fontFamily: theme?.typography?.fontFamily, + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + zIndex: 1000, + minWidth: 600, + minHeight: 320, + overflow: "auto", + '& .MuiDialogContent-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + }, + '& .MuiDialogTitle-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + p: 2, + }, + '& .MuiDialogActions-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + }, + }, + }} + > + + + Suborg App Distribution + + setShowDistributionPopup(false)} + sx={{ + color: theme.palette.text.primary, + }} + > + + + + + + handleActivateApp(null, "deactivate_all")} + sx={{ + borderRadius: 1, + px: 2, + '&:hover': { + backgroundColor: 'rgba(255,255,255,0.08)', + }, + }} + > + Deactivate for all suborgs + + + handleActivateApp(null, "activate_all")} + sx={{ + borderRadius: 1, + px: 2, + '&:hover': { + backgroundColor: 'rgba(255,255,255,0.08)', + }, + }} + > + Activate for all suborgs + + + {userdata.orgs.map((data, index) => { + if (data.creator_org !== userdata.active_org.id) return null; + + const imageSize = 28; + const imageStyle = { + width: imageSize, + height: imageSize, + borderRadius: '50%', + objectFit: 'cover', + marginRight: 12, + }; + + const imageSrc = data.image || theme.palette.defaultImage; + + return ( + + + {data.name} + + {data.name} + + + + + + + + ); + })} + + +) : null; + + const landingpageDataBrowser = (
{publishModal} + {appDistributinModal}
{isMobile ? null : ( } - {isMobile || app?.reference_org === userdata?.active_org?.id ? null : ( + {isMobile || app?.reference_org === userdata?.active_org?.id || (app?.suborg_distribution?.includes(userdata?.active_org?.id)) ? null : ( + )}
) : ( { if (!Array.isArray(apps)) return []; + const normalizedSearchQuery = (searchQuery || "").toLowerCase(); + return apps.filter((app) => { + if (!app) return false; + const matchesSearchQuery = ( - searchQuery === "" || // If searchQuery is empty, match all apps - app.name.toLowerCase().includes(searchQuery.toLowerCase()) || - (app.tags && app.tags.some(tag => - tag.toLowerCase().includes(searchQuery.toLowerCase()) + normalizedSearchQuery === "" || // If searchQuery is empty, match all apps + (app.name && app.name.toLowerCase().includes(normalizedSearchQuery)) || + (app.tags && Array.isArray(app.tags) && app.tags.some(tag => + tag && typeof tag === 'string' && tag.toLowerCase().includes(normalizedSearchQuery) )) || - (app.categories && app.categories.some((category) => - category.toLowerCase().includes(searchQuery.toLowerCase()) + (app.categories && Array.isArray(app.categories) && app.categories.some((category) => + category && typeof category === 'string' && category.toLowerCase().includes(normalizedSearchQuery) )) ); const matchesSelectedCategories = ( - selectedCategory.length === 0 || // If no category is selected, match all apps - (app.categories && app.categories.some(category => - selectedCategory.includes(category) + !Array.isArray(selectedCategory) || selectedCategory.length === 0 || // If no category is selected, match all apps + (app.categories && Array.isArray(app.categories) && app.categories.some(category => + category && selectedCategory.includes(category) )) ); const matchesSelectedTags = ( - selectedLabel.length === 0 || // If no label is selected, match all apps - (app.tags && app.tags.some(tag => - selectedLabel.includes(tag) + !Array.isArray(selectedLabel) || selectedLabel.length === 0 || // If no label is selected, match all apps + (app.tags && Array.isArray(app.tags) && app.tags.some(tag => + tag && selectedLabel.includes(tag) )) ); diff --git a/frontend/src/views/DashboardViews.jsx b/frontend/src/views/DashboardViews.jsx index 7905a21a..0748e80a 100644 --- a/frontend/src/views/DashboardViews.jsx +++ b/frontend/src/views/DashboardViews.jsx @@ -315,7 +315,7 @@ const RadialChart = ({keys, setSelectedCategory}) => { // What data do we fill in here? Idk const Dashboard = (props) => { - const { globalUrl, isLoggedIn } = props; + const { globalUrl, userdata, isLoggedIn } = props; //const alert = useAlert(); const [bigChartData, setBgChartData] = useState("data1"); const [dayAmount, setDayAmount] = useState(7); @@ -323,10 +323,10 @@ const Dashboard = (props) => { const [stats, setStats] = useState({}); const [changeme, setChangeme] = useState(""); const [statsRan, setStatsRan] = useState(false); - const [keys, setKeys] = useState([]) - const [treeKeys, setTreeKeys] = useState([]) + const [keys, setKeys] = useState([]) + const [treeKeys, setTreeKeys] = useState([]) - const [selectedUsecaseCategory, setSelectedUsecaseCategory] = useState(""); + const [selectedUsecaseCategory, setSelectedUsecaseCategory] = useState("") const [selectedUsecases, setSelectedUsecases] = useState([]); const [usecases, setUsecases] = useState([]); const [workflows, setWorkflows] = useState([]); @@ -750,8 +750,6 @@ const Dashboard = (props) => { const newname = data.key !== undefined ? data.key.replaceAll("_", " ") : "" - console.log("KEYDATA: ", data) - const loadNewStats = (newkey) => { const resp = LoadStats(globalUrl, newkey) if (resp !== undefined) { @@ -807,7 +805,7 @@ const Dashboard = (props) => { backgroundColor: theme.palette.inputColor, color: "white", height: 40, - maxWidth: 150, + maxWidth: 200, borderRadius: theme.palette?.borderRadius, }} > @@ -829,10 +827,12 @@ const Dashboard = (props) => { }
+ + ) @@ -848,14 +848,6 @@ const Dashboard = (props) => { : null}
- {/*widgetData === undefined || widgetData === null || widgetData === [] || widgetData.length === 0 ? null : - - - - - - */} - {newWidgetData === undefined || newWidgetData === null || newWidgetData === [] ? null : newWidgetData.map((data, index) => { @@ -872,7 +864,9 @@ const Dashboard = (props) => { ); const dataWrapper = ( -
{data}
+
+ {data} +
); return dataWrapper; diff --git a/frontend/src/views/Docs.jsx b/frontend/src/views/Docs.jsx index 2456f63d..8b4772c6 100755 --- a/frontend/src/views/Docs.jsx +++ b/frontend/src/views/Docs.jsx @@ -113,19 +113,29 @@ export const CopyToClipboard = (props) => { } export const Paragraph = (props) => { + // Filter out stray HTML artifacts like '>' or '/>' caused by HTML parsing edge-cases + const cleanedChildren = React.Children.toArray(props.children).filter((child) => { + if (typeof child === 'string') { + const trimmed = child.trim(); + // Remove single '>' or '/>' leftovers + if (trimmed === '>' || trimmed === '/>') return false; + } + return true; + }); + const element = React.createElement( `p`, {}, - props.children, + cleanedChildren, ) - if (props.children[0] != undefined) { - if(typeof props.children[0] === "string") { - if (props.children[0].includes('.mp4')) { + if (cleanedChildren[0] !== undefined) { + if(typeof cleanedChildren[0] === "string") { + if (cleanedChildren[0].includes('.mp4')) { return (
) @@ -169,6 +179,7 @@ export const Img = (props) => { // Find parent container and check width const isArticlePage = window.location.pathname.includes("/articles/") const isFormPage = window.location.pathname.includes("/forms/") + const isWorkflowPage = window.location.pathname.includes("/workflows/") var height = "auto" var width = isArticlePage ? 1000 : isFormPage ? 400: 750 @@ -176,7 +187,7 @@ export const Img = (props) => { const theme = getTheme(themeMode) const docsImageStyle = { - border: isFormPage ? null : "1px solid rgba(255,255,255,0.3)", + border: isFormPage || isWorkflowPage ? null : "1px solid rgba(255,255,255,0.3)", borderRadius: theme.palette?.borderRadius, width: width, maxWidth: width, diff --git a/frontend/src/views/LoginPage.jsx b/frontend/src/views/LoginPage.jsx index b717b9f6..e470255d 100755 --- a/frontend/src/views/LoginPage.jsx +++ b/frontend/src/views/LoginPage.jsx @@ -5,6 +5,7 @@ import { useNavigate, Link, useParams } from "react-router-dom"; import { isMobile } from "react-device-detect"; import { toast } from 'react-toastify'; import { useInterval } from "react-powerhooks"; +import ReactGA from 'react-ga4'; import VisibilityOutlinedIcon from '@mui/icons-material/VisibilityOutlined'; import VisibilityOffOutlinedIcon from '@mui/icons-material/VisibilityOffOutlined'; import { @@ -157,7 +158,7 @@ const FreePlanCard = ({ classes }) => { ); }; -const MarketplaceCard = ({ classes }) => { +const MarketplaceCard = ({ classes, isCloud }) => { const marketplaceOptions = [ { name: "Open Source Install", @@ -216,6 +217,15 @@ const MarketplaceCard = ({ classes }) => { >
@@ -1025,7 +1103,7 @@ const LoginPage = props => {
OR
- + )}
diff --git a/frontend/src/views/LoginPageOld.jsx b/frontend/src/views/LoginPageOld.jsx index aaee73d6..c0996436 100755 --- a/frontend/src/views/LoginPageOld.jsx +++ b/frontend/src/views/LoginPageOld.jsx @@ -344,7 +344,7 @@ const LoginDialog = (props) => { variant="body2" style={{ marginBottom: 20, color: "white" }} > - 1. Make sure shuffle-database folder has correct access, and that you have a minimum of 2Gb of RAM available:{" "} + 1. Make sure shuffle-database folder has correct access, and that you have a minimum of 4Gb of RAM available:{" "}

sudo chown -R 1000:1000 shuffle-database diff --git a/frontend/src/views/RunWorkflow.jsx b/frontend/src/views/RunWorkflow.jsx index a9a6cf5c..21cc9bb8 100644 --- a/frontend/src/views/RunWorkflow.jsx +++ b/frontend/src/views/RunWorkflow.jsx @@ -13,7 +13,7 @@ import { makeStyles } from '@mui/material/styles'; import { useInterval } from "react-powerhooks"; import { isMobile } from "react-device-detect"; import Markdown from "react-markdown"; -import theme from '../theme.jsx'; +import {getTheme} from '../theme.jsx'; import rehypeRaw from "rehype-raw"; import RecentWorkflow from "../components/RecentWorkflow.jsx"; @@ -58,6 +58,8 @@ const hrefStyle = { const RunWorkflow = (defaultprops) => { const { globalUrl, userdata, isLoaded, isLoggedIn, setIsLoggedIn, setCookie, register, serverside } = defaultprops; + const { themeMode, brandColor } = useContext(Context); + const theme = getTheme(themeMode, brandColor); const { supportEmail } = useContext(Context); let navigate = useNavigate(); diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 3fe551f4..32859549 100755 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -558,7 +558,7 @@ export const handleReactJsonClipboard = (copy) => { document.execCommand("copy"); console.log("COPYING!"); - toast("Copied value to clipboard, NOT json path.") + toast.success("Copied Value, NOT json path.") } else { console.log("Failed to copy from " + elementName + ": ", copyText); } diff --git a/frontend/src/views/Workflows2.jsx b/frontend/src/views/Workflows2.jsx index d702f24f..86c28e18 100644 --- a/frontend/src/views/Workflows2.jsx +++ b/frontend/src/views/Workflows2.jsx @@ -17,11 +17,14 @@ import GetAppIcon from '@mui/icons-material/GetApp'; // Material UI & Components import { makeStyles } from "@mui/styles"; import { Navigate } from "react-router-dom"; +import { isMobile } from "react-device-detect" +import ReactGA from 'react-ga4'; + +import LineChartWrapper from "../components/LineChartWrapper.jsx"; import SecurityFramework from '../components/SecurityFramework.jsx'; import EditWorkflow from "../components/EditWorkflow.jsx" import Priority from "../components/Priority.jsx"; import { Context } from "../context/ContextApi.jsx"; -import { isMobile } from "react-device-detect" // Material UI Components import { @@ -99,6 +102,9 @@ import { Psychology as PsychologyIcon, Wifi as WifiIcon, Devices as DevicesIcon, + AutoAwesome as AutoAwesomeIcon, + BarChart as BarChartIcon, + Lock as LockIcon, } from "@mui/icons-material"; // Additional Components @@ -115,14 +121,26 @@ import { InstantSearch, Configure, connectHits, connectSearchBox, connectRefinem import { debounce } from "lodash"; import { removeQuery } from "../components/ScrollToTop.jsx"; -import {green, yellow, red, grey } from "../views/AngularWorkflow.jsx" +import {green, yellow, red, grey, triggers as wfTriggers, } from "../views/AngularWorkflow.jsx" const searchClient = algoliasearch("JNSS5CFDZZ", "c8f882473ff42d41158430be09ec2b4e"); const svgSize = 24; -const imagesize = 22; +const imagesize = 23; +// Session-based modal visibility helper +const AI_ANNOUNCEMENT_SESSION_KEY = "ai_announcement_session"; + +const getCookie = (name) => { + if (typeof document === "undefined") return ""; + const pattern = `; ${document.cookie}`; + const parts = pattern.split(`; ${name}=`); + if (parts.length === 2) { + return decodeURIComponent(parts.pop().split(";").shift() || ""); + } + return ""; +}; @@ -212,8 +230,14 @@ export const GetIconInfo = (action) => { "release", ], }, - - + { + key: "secret", + values: [ + "api", + "password", + "protect", + ], + } ]; var selectedKey = "" @@ -402,6 +426,12 @@ export const GetIconInfo = (action) => { iconBackgroundColor: "green", originalIcon: , }, + secret: { + icon: "", + iconColor: "white", + iconBackgroundColor: "green", + originalIcon: , + } } /* @@ -670,6 +700,10 @@ const Workflows2 = (props) => { const [isLoadingWorkflow, setIsLoadingWorkflow] = useState(false); const [isLoadingPublicWorkflow, setIsLoadingPublicWorkflow] = useState(false); const [view, setView] = useState(localStorage?.getItem("workflowView") || "grid"); + + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + const [showExecutionStats, setShowExecutionStats] = React.useState(localStorage?.getItem("showExecutionStats") === "true" || isCloud) + const imgSize = 60; const { themeMode, brandColor, brandName } = useContext(Context); @@ -728,6 +762,8 @@ const Workflows2 = (props) => { var upload = ""; const [workflows, setWorkflows] = React.useState([]); + const [workflowTimelines, setWorkflowTimelines] = React.useState([]); + const [backgroundWorkflows, setBackgroundWorkflows] = React.useState([]); const [backupWorkflows, setBackupWorkflows] = React.useState([]); const [_, setUpdate] = React.useState(""); // Used for rendering, don't remove const [selectedUsecases, setSelectedUsecases] = React.useState([]); @@ -767,6 +803,8 @@ const Workflows2 = (props) => { const [actionImageList, setActionImageList] = React.useState([{ "large_image": "" }]) const [firstLoad, setFirstLoad] = React.useState(true); + const [aiAnnouncementModalOpen, setAiAnnouncementModalOpen] = React.useState(false); + const sessionRef = useRef(""); const [showMoreClicked, setShowMoreClicked] = React.useState(false); const [usecases, setUsecases] = React.useState([]); const [allUsecases, setAllUsecases] = React.useState({ @@ -875,10 +913,167 @@ const Workflows2 = (props) => { } - //const isCloud = - // window.location.host === "localhost:3002" || - // window.location.host === "shuffler.io"; - const isCloud = false + React.useEffect(() => { + if (!isLoggedIn) return; + + const bannerID = "banner_ai_announcement"; + const cookieSession = getCookie("__session") || ""; + sessionRef.current = cookieSession; + + try { + const storedSession = localStorage.getItem(AI_ANNOUNCEMENT_SESSION_KEY) || ""; + + // If backend says it's dismissed, sync storage and exit + if (userdata && Array.isArray(userdata.tutorials)) { + const alreadyDismissed = userdata.tutorials.some((t) => t?.name === bannerID); + if (alreadyDismissed) { + if (cookieSession && storedSession !== cookieSession) { + localStorage.setItem(AI_ANNOUNCEMENT_SESSION_KEY, cookieSession); + } + return; + } + } else if (!userdata) { + // Wait for userdata to load + return; + } + + // Open only if not dismissed in this session + if (cookieSession && storedSession !== cookieSession) { + setAiAnnouncementModalOpen(true); + } + } catch { + // If storage is unavailable, fallback to a single open per mount + setAiAnnouncementModalOpen((open) => open || true); + } + }, [isLoggedIn, userdata]); + + const handleCloseAiAnnouncement = React.useCallback(() => { + try { + const cookieSession = sessionRef.current || getCookie("__session") || ""; + if (cookieSession) { + localStorage.setItem(AI_ANNOUNCEMENT_SESSION_KEY, cookieSession); + } + } catch { + // ignore storage access issues + } + setAiAnnouncementModalOpen(false); + }, []); + + + const dismissAiAnnouncement = () => { + const bannerID = "banner_ai_announcement"; + + if (isCloud) { + ReactGA.event({ + category: "AIGeneratedNewWorkflow", + action: "dismiss_announcement", + label: userdata?.active_org?.id || userdata?.id || "", + }); + } + + handleCloseAiAnnouncement(); + + // Open Create Workflow modal (EditWorkflow) and temporarily highlight inputs/buttons + try { + setModalOpen(true) + setIsEditing(false) + setNewWorkflowName("") + setNewWorkflowDescription("") + setDefaultReturnValue("") + setEditingWorkflow({}) + setNewWorkflowTags([]) + setSelectedUsecases([]) + + // Wait a moment for the drawer to mount, then highlight + setTimeout(() => { + const BORDER = '#4CAF50'; + const DURATION = 1500; + + const highlightTextField = (inputEl) => { + if (!inputEl) return; + const formControl = inputEl.closest('.MuiFormControl-root') || inputEl.closest('.MuiInputBase-root') || inputEl.parentElement; + const inputRoot = formControl?.querySelector('.MuiOutlinedInput-root') || formControl?.querySelector('.MuiInputBase-root') || formControl; + const notch = formControl?.querySelector('fieldset'); + + const prevBoxShadow = inputRoot?.style?.boxShadow; + const prevBorderColor = notch?.style?.borderColor; + const prevBorderWidth = notch?.style?.borderWidth; + + if (inputRoot) inputRoot.style.boxShadow = '0 0 0 3px rgba(76,175,80,0.45)'; + if (notch) { + notch.style.borderColor = BORDER; + notch.style.borderWidth = '2px'; + } + + setTimeout(() => { + try { + if (inputRoot) inputRoot.style.boxShadow = prevBoxShadow || ''; + if (notch) { + notch.style.borderColor = prevBorderColor || ''; + notch.style.borderWidth = prevBorderWidth || ''; + } + } catch (_) {} + }, DURATION); + }; + + const highlightButton = (btnEl) => { + if (!btnEl) return; + const prevBoxShadow = btnEl.style.boxShadow; + const prevBorder = btnEl.style.border; + const prevRadius = btnEl.style.borderRadius; + btnEl.style.boxShadow = '0 0 0 3px rgba(76,175,80,0.45)'; + btnEl.style.border = '2px solid ' + BORDER; + btnEl.style.borderRadius = '6px'; + setTimeout(() => { + try { + btnEl.style.boxShadow = prevBoxShadow || ''; + btnEl.style.border = prevBorder || ''; + btnEl.style.borderRadius = prevRadius || ''; + } catch (_) {} + }, DURATION); + }; + + const nameEl = document.getElementById('Enter-Workflow-Name'); + highlightTextField(nameEl); + + const descEl = document.getElementById('Workflow-Description'); + highlightTextField(descEl); + + const aiBtn = document.getElementById('ai-generate-button'); + highlightButton(aiBtn); + }, 250); + } catch (e) { + console.debug('Failed to open and highlight Create Workflow modal:', e); + } + + fetch(globalUrl + '/api/v1/users/updateuser', { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + body: JSON.stringify({ + tutorial: bannerID, + user_id: userdata.id + }), + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Failed to dismiss AI announcement banner"); + } + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success) { + console.log("AI announcement banner dismissed successfully"); + } + }) + .catch((error) => { + console.log("Error dismissing AI announcement:", error); + }); + }; + const findWorkflow = (filters) => { console.log("Using filters: ", filters) @@ -1154,6 +1349,199 @@ const Workflows2 = (props) => { ) : null; + const aiAnnouncementModal = aiAnnouncementModalOpen ? ( + + + { + if (isCloud) { + ReactGA.event({ + category: "AIGeneratedNewWorkflow", + action: "close_announcement", + label: userdata?.active_org?.id || userdata?.id || "", + }); + } + handleCloseAiAnnouncement(); + } + } + aria-label="Close" + > + + + + + {/* Main two-column layout */} + + {/* Left: steps image (38%) */} + + + + + {/* Right: content (62%) */} + + {/* NEW badge */} + + + + NEW + + + + {/* Title */} + + Introducing AI Workflow Generation + + + {/* Body text */} + + Simply describe what you want your workflow to do, and our AI + will automatically generate the workflow for you. + + + + Quick start: Create Workflow → Describe → AI + Generate → Done. + + + + For self-hosted setups, see the{" "} + + setup docs + + + + {/* CTA button */} + + + + + + + + ) : null; + const deleteModal = deleteModalOpen ? ( { backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, zIndex: 1000, '& .MuiDialogContent-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, }, '& .MuiDialogTitle-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, }, } - }} + }} >
@@ -1351,6 +1739,53 @@ const Workflows2 = (props) => { window.location.reload(); } }, []); + + useEffect(() => { + if (workflows?.length === 0) { + return + } + + if (workflowTimelines?.length > 0) { + return + } + + //if (isLoggedIn !== true) { + // return + //} + + const results = Promise.all( + workflows.slice(0,16).map((workflow, index) => { + + return fetch(`${globalUrl}/api/v2/workflows/${workflow.id}/executions`, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }).then((response) => response.json()) + }) + ) + + results.then((res) => { + var newarray = [] + for (var resKey in res) { + const result = res[resKey] + if (result?.timeline === undefined || result?.timeline === null) { + continue + } + + newarray.push({ + "id": result.id, + "timeline": result.timeline, + }) + } + + setWorkflowTimelines(newarray) + }) + + + }, [workflows]) const getAvailableWorkflows = (amount) => { var storageWorkflows = [] @@ -1407,18 +1842,29 @@ const Workflows2 = (props) => { toast.info("No workflows found in this org. Feel free to look into our public workflows!" , { timeout: 7500, }) + setCurrTab(2) } + + localStorage.setItem("workflows", "[]") + setWorkflows([]) + setFilteredWorkflows([]) } var newarray = [] var backupWf = [] + var backgroundWf = [] for (var wfkey in responseJson) { const wf = responseJson[wfkey] - if (wf.public === true || wf.hidden === true) { + if (wf?.public === true || wf?.hidden === true) { continue } + if (wf?.background_processing === true) { + backgroundWf.push(wf) + continue + } + if (wf?.backup_config?.onprem_backup === true) { backupWf.push(wf) continue @@ -1427,6 +1873,10 @@ const Workflows2 = (props) => { newarray.push(wf) } + if (backgroundWf.length > 0) { + setBackgroundWorkflows(backgroundWf) + } + if (backupWf.length > 0) { setBackupWorkflows(backupWf) } @@ -1659,11 +2109,9 @@ const Workflows2 = (props) => { const paperAppStyle = { minHeight: 146, - maxHeight: 146, overflow: "hidden", width: "100%", color: "white", - display: "flex", fontFamily: theme.typography?.fontFamily, boxSizing: "border-box", position: "relative", @@ -2449,26 +2897,25 @@ const Workflows2 = (props) => { var orgName = ""; var orgId = ""; + + var imageStyle = { + width: imagesize, + height: imagesize, + pointerEvents: "none", + marginLeft: + data.creator_org !== undefined && data.creator_org.length > 0 + ? 20 + : 0, + borderRadius: 10, + cursor: "pointer", + marginRight: 10, + } + if (userdata.orgs !== undefined) { const foundOrg = userdata.orgs.find((org) => org.id === data["org_id"]); if (foundOrg !== undefined && foundOrg !== null) { //position: "absolute", bottom: 5, right: -5, - const imageStyle = { - width: imagesize, - height: imagesize, - pointerEvents: "none", - marginLeft: - data.creator_org !== undefined && data.creator_org.length > 0 - ? 20 - : 0, - borderRadius: 10, - border: - foundOrg.id === userdata.active_org.id - ? `3px solid ${boxColor}` - : null, - cursor: "pointer", - marginRight: 10, - }; + imageStyle.border = foundOrg.id === userdata.active_org.id ? `3px solid ${boxColor}` : null image = @@ -2492,6 +2939,44 @@ const Workflows2 = (props) => { } } + var triggerfound = false + var triggerstarted = false + var relevantTrigger = {} + for (var triggerkey in data?.triggers) { + + const trigger = data?.triggers[triggerkey] + if (trigger?.trigger_type === "WEBHOOK") { + triggerfound = true + image = wfTriggers[0].large_image + + relevantTrigger = trigger + if (trigger?.status === "running") { + imageStyle.border = `3px solid ${green}` + break + } else { + imageStyle.border = `3px solid ${red}` + } + + + } else if (trigger?.trigger_type === "SCHEDULE") { + triggerfound = true + image = wfTriggers[1].large_image + + relevantTrigger = trigger + if (trigger?.status === "running") { + imageStyle.border = `3px solid ${green}` + break + } else { + imageStyle.border = `3px solid ${red}` + } + + } + } + + if (!triggerfound) { + image = "" + } + var selectedCategory = "" if (data.usecase_ids !== undefined && data.usecase_ids !== null && data.usecase_ids.length > 0 && usecases !== null && usecases !== undefined && usecases.length > 0) { const oldcolor = boxColor.valueOf() @@ -2534,6 +3019,7 @@ const Workflows2 = (props) => { } image = data.creator_info !== undefined && data.creator_info !== null && data.creator_info.image !== undefined && data.creator_info.image !== null && data.creator_info.image.length > 0 ? : + const creatorname = data.creator_info !== undefined && data.creator_info !== null && data.creator_info.username !== undefined && data.creator_info.username !== null && data.creator_info.username.length > 0 ? data.creator_info.username : "Shuffle" if ((data.objectID === undefined || data.objectID === null) && data.id !== undefined && data.id !== null) { data.objectID = data.id @@ -2546,10 +3032,12 @@ const Workflows2 = (props) => { } } - + const foundTimeline = workflowTimelines.find((timeline) => timeline.id === data.id) return (
+ + {selectedCategory !== "" ?
{ position: "absolute", top: 0, left: 0, - height: paperAppStyle.minHeight, width: 3, backgroundColor: boxColor, borderRadius: "0 100px 0 0", fontFamily: theme.typography?.fontFamily, + + height: "100%", }} onClick={() => { addFilter(selectedCategory) @@ -2577,17 +3066,27 @@ const Workflows2 = (props) => { > {currTab === 2 ? null : - + +
{ - navigate("/admin") + //navigate("/admin") }} > - {image} + {image?.includes("data:image") ? + {orgName} + : + image + }
} + { /* @@ -2695,7 +3194,7 @@ const Workflows2 = (props) => { style={{ height: 24, width: 24, - filter: themeMode === "dark" ? "brightness(0.6)" : "brightness(0.9)", + filter: themeMode === "dark" ? "brightness(0.6)" : "brightness(1)", cursor: "pointer", }} onClick={() => { @@ -2716,7 +3215,7 @@ const Workflows2 = (props) => { style={{ height: 24, width: 24, - filter: "brightness(0.6)", + filter: themeMode === "dark" ? "brightness(0.6)" : "brightness(1)", cursor: "pointer", }} onClick={() => { @@ -2874,7 +3373,8 @@ const Workflows2 = (props) => { }) : null}
- {data.actions !== undefined && data.actions !== null && type !== "public" ? ( + + {type !== "public" ? (
{ {(data.sharing !== undefined && data.sharing !== null && data.sharing === "form") || (data?.form_control?.input_markdown !== undefined && data?.form_control?.input_markdown !== null && data?.form_control?.input_markdown !== "") && type !== "public" ? -
+
{ : null} {(data?.validation?.validation_ran === true && data?.validation?.valid === false && data?.validation?.errors?.length > 0 ) ? - -
+ +
{ }} style={{ padding: "0px", - color: "#979797", + transparency: 0.5, }} + color="primary" > - +
: null} + + {showExecutionStats === true && foundTimeline !== undefined && foundTimeline?.timeline?.length > 0 && +
+ +
+ } +
) } @@ -4414,16 +4933,33 @@ const Workflows2 = (props) => { {backupWorkflows.length > 0 && } + {backgroundWorkflows.length > 0 && + + } + { navigate("/forms") }} @@ -4431,7 +4967,7 @@ const Workflows2 = (props) => { ...tabStyle, marginRight: 0, marginLeft: 25, - ...(currTab === 4 ? tabActive : {}) + ...(currTab === 5 ? tabActive : {}) }} /> @@ -4664,7 +5200,23 @@ const Workflows2 = (props) => { paddingRight: 1, gap: 4 }}> - + + + { + + const newView = !showExecutionStats + localStorage.setItem("showExecutionStats", newView) + setShowExecutionStats(!showExecutionStats) + }} + disabled={currTab === 2} + > + + + + + navigate("/workflows/debug")} @@ -4817,6 +5369,28 @@ const Workflows2 = (props) => { ) })} + {currTab === 4 && backgroundWorkflows.map((data, index) => { + // Shouldn't be a part of this list + if (data.public === true) { + return null + } + + // if (firstLoad) { + // workflowDelay += 75 + // } else { + // return + // } + + return ( + + {/**/} + + {/**/} + + ) + })} + + { currTab !== 1 ? null : myWorkflows.length === 0 ? @@ -5212,10 +5786,10 @@ const Workflows2 = (props) => { */} - {/* {modalView} */} {deleteModal} {exportVerifyModal} {publishModal} + {aiAnnouncementModal} {workflowDownloadModalOpen} {/*!drawerOpen ? @@ -5279,8 +5853,10 @@ const Workflows2 = (props) => { width: '100%', height: '100%', } + // Maybe use gridview or something, idk return
{loadedCheck}
; + // return
{loadedCheck}
; }; diff --git a/functions/extensions/aws-s3-lambda/README.md b/functions/extensions/aws-s3-lambda/README.md deleted file mode 100644 index f665851a..00000000 --- a/functions/extensions/aws-s3-lambda/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# AWS Lambda forwarder to Shuffle -This function is made to forward S3 notifications to Shuffle to run a workflow when an object is made or updated. diff --git a/functions/extensions/aws-s3-lambda/s3_function.py b/functions/extensions/aws-s3-lambda/s3_function.py deleted file mode 100644 index d83d6154..00000000 --- a/functions/extensions/aws-s3-lambda/s3_function.py +++ /dev/null @@ -1,22 +0,0 @@ -import json -import urllib.parse -import urllib3 -import os - -print('Loading function') - -def lambda_handler(event, context): - # Get the object from the event and show its content type - bucket = event['Records'][0]['s3']['bucket']['name'] - - webhook = os.environ.get("SHUFFLE_WEBHOOK") - if not webhook: - return "No webhook environment defined: SHUFFLE_WEBHOOK" - - http = urllib3.PoolManager() - ret = http.request('POST', webhook, body=json.dumps(event["Records"][0]).encode("utf-8")) - if ret.status != 200: - return "Bad status code for webhook: %d" % ret.status - - print("Status code: %d\nData: %s" % (ret.status, ret.data)) - return "Successfully started with data %s" % ret.data diff --git a/functions/extensions/elasticsearch/run.sh b/functions/extensions/elasticsearch/run.sh deleted file mode 100755 index 666b1d95..00000000 --- a/functions/extensions/elasticsearch/run.sh +++ /dev/null @@ -1,19 +0,0 @@ -#docker run -d -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e ELASTICSEARCH_USERNAME=frikky -e ELASTICSEARCH_PASSWORD=likeme -e xpack.security.enabled=true docker.elastic.co/elasticsearch/elasticsearch:7.12.1 -#docker run -d -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.12.1 -# -# -# -#echo "\nWaiting for 1.5 minute, then adding data" -#sleep 90 -#echo "\nSlept 90 seconds: ADDING DATA" -#curl -XPOST http://localhost:9200/_security/user/frikky -H "Content-Type: application/json" -d '{"enabled": true, "email": "frikky@shuffler.io"}' - -#curl -XPOST -u frikky:likeme http://localhost:9200/samples/_doc -H "Content-Type: application/json" -d '{"src": "122.14.137.67", "dst": "103.35.191.16", "message": "alert", "md5": "CAEF973033E593C625FB2AA34F7026DC", "sha256": "DB1AEC5222075800EDA75D7205267569679B424E5C58A28102417F46D3B5790D", "hits": 0}' -#echo -#curl -XPOST -u frikky:likeme http://localhost:9200/samples/_doc -H "Content-Type: application/json" -d '{"src": "134.119.219.71", "dst": "103.35.191.41", "message": "alert", "md5": "9498FF82A64FF445398C8426ED63EA5B", "sha256": "8B2E701E91101955C73865589A4C72999AEABC11043F712E05FDB1C17C4AB19A", "hits": 0}' -# -#echo -#curl -XPOST -u frikky:likeme http://localhost:9200/samples2/_doc -H "Content-Type: application/json" -d '{"src": "122.14.137.67", "dst": "103.35.191.16", "message": "alert", "md5": "CAEF973033E593C625FB2AA34F7026DC", "sha256": "DB1AEC5222075800EDA75D7205267569679B424E5C58A28102417F46D3B5790D"}' -#echo -#curl -XPOST -u frikky:likeme http://localhost:9200/samples2/_doc -H "Content-Type: application/json" -d '{"src": "134.119.219.71", "dst": "103.35.191.41", "message": "alert", "md5": "9498FF82A64FF445398C8426ED63EA5B", "sha256": "8B2E701E91101955C73865589A4C72999AEABC11043F712E05FDB1C17C4AB19A"}' -#echo diff --git a/functions/extensions/k8s/shuffle/.helmignore b/functions/extensions/k8s/shuffle/.helmignore deleted file mode 100755 index 0e8a0eb3..00000000 --- a/functions/extensions/k8s/shuffle/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/functions/extensions/k8s/shuffle/Chart.yaml b/functions/extensions/k8s/shuffle/Chart.yaml deleted file mode 100755 index eb53a0b5..00000000 --- a/functions/extensions/k8s/shuffle/Chart.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v2 -name: shuffle -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "2.0.0" diff --git a/functions/extensions/k8s/shuffle/templates/NOTES.txt b/functions/extensions/k8s/shuffle/templates/NOTES.txt deleted file mode 100755 index eaba4460..00000000 --- a/functions/extensions/k8s/shuffle/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "shuffle.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "shuffle.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "shuffle.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "shuffle.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/functions/extensions/k8s/shuffle/templates/_helpers.tpl b/functions/extensions/k8s/shuffle/templates/_helpers.tpl deleted file mode 100755 index 66efef07..00000000 --- a/functions/extensions/k8s/shuffle/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "shuffle.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "shuffle.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "shuffle.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "shuffle.labels" -}} -helm.sh/chart: {{ include "shuffle.chart" . }} -{{ include "shuffle.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "shuffle.selectorLabels" -}} -app.kubernetes.io/name: {{ include "shuffle.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "shuffle.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "shuffle.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/functions/extensions/k8s/shuffle/templates/deployment.yaml b/functions/extensions/k8s/shuffle/templates/deployment.yaml deleted file mode 100755 index f2050d82..00000000 --- a/functions/extensions/k8s/shuffle/templates/deployment.yaml +++ /dev/null @@ -1,253 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.name }}frontend - namespace: {{ .Values.namespace | quote }} -spec: - replicas: 1 - selector: - matchLabels: - service: shuffle - app: shuffle-frontend - template: - metadata: - labels: - service: shuffle - app: shuffle-frontend - spec: - containers: - - name: shuffle-frontend - image: ghcr.io/frikky/shuffle-frontend:nightly - imagePullPolicy: {{ .Values.image.pullPolicy | quote }} - env: - - name: BACKEND_HOSTNAME - value: backend-service - - name: TZ - value: Asia/Shanghai - ports: - - name: http - containerPort: 80 - hostPort: 3001 - - name: https - containerPort: 443 - hostname: shuffle-frontend - restartPolicy: Always - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.name }}backend - namespace: {{ .Values.namespace | quote }} -spec: - replicas: 1 - selector: - matchLabels: - service: shuffle - app: shuffle-backend - template: - metadata: - labels: - service: shuffle - app: shuffle-backend - spec: - containers: - - name: shuffle-backend - image: ghcr.io/frikky/shuffle-backend:nightly - env: - - name: BACKEND_HOSTNAME - value: "backend-service" - - name: BACKEND_PORT - value: "5001" - - name: ENVIRONMENT_NAME - value: "Shuffle" - - name: HTTPS_PROXY - - name: HTTP_PROXY - - name: ORG_ID - value: "Shuffle" - - name: OUTER_HOSTNAME - value: "backend-service" - - name: SHUFFLE_APP_FORCE_UPDATE - value: "false" - - name: SHUFFLE_APP_HOTLOAD_FOLDER - value: "/shuffle-apps" - - name: SHUFFLE_APP_HOTLOAD_LOCATION - value: "/shuffle-apps" - - name: SHUFFLE_CONTAINER_AUTO_CLEANUP - value: "false" - - name: SHUFFLE_DEFAULT_APIKEY - - name: SHUFFLE_DEFAULT_PASSWORD - - name: SHUFFLE_DEFAULT_USERNAME - - name: SHUFFLE_DOWNLOAD_AUTH_BRANCH - - name: SHUFFLE_DOWNLOAD_AUTH_PASSWORD - - name: SHUFFLE_DOWNLOAD_AUTH_USERNAME - - name: SHUFFLE_DOWNLOAD_WORKFLOW_BRANCH - - name: SHUFFLE_DOWNLOAD_WORKFLOW_LOCATION - - name: SHUFFLE_DOWNLOAD_WORKFLOW_PASSWORD - - name: SHUFFLE_DOWNLOAD_WORKFLOW_USERNAME - - name: SHUFFLE_ELASTIC - value: "true" - - name: SHUFFLE_OPENSEARCH_APIKEY - - name: SHUFFLE_OPENSEARCH_CERTIFICATE_FILE - - name: SHUFFLE_OPENSEARCH_CLOUDID - - name: SHUFFLE_OPENSEARCH_PASSWORD - - name: SHUFFLE_OPENSEARCH_PROXY - - name: SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY - value: "true" - - name: SHUFFLE_OPENSEARCH_URL - value: http://opensearch-service:9200 - - name: SHUFFLE_OPENSEARCH_USERNAME - value: "" - - name: SHUFFLE_PASS_APP_PROXY - value: "FALSE" - - name: SHUFFLE_PASS_WORKER_PROXY - value: "FALSE" - volumeMounts: - - mountPath: /var/run/docker.sock - name: docker-sock - - mountPath: /shuffle-apps - name: shuffle-app-hotload-location - - mountPath: /shuffle-files - name: shuffle-file-location - hostname: shuffle-backend - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock - - name: shuffle-app-hotload-location - hostPath: - path: /data/kubernetes/shuffle-apps - type: DirectoryOrCreate - - name: shuffle-file-location - hostPath: - path: /data/kubernetes/shuffle-files - type: DirectoryOrCreate - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.name }}orborus - namespace: {{ .Values.namespace | quote }} -spec: - replicas: 1 - selector: - matchLabels: - service: shuffle - app: shuffle-orborus - template: - metadata: - labels: - service: shuffle - app: shuffle-orborus - spec: - containers: - - name: shuffle-orborus - image: ghcr.io/frikky/shuffle-orborus:nightly - env: - - name: RUNNING_MODE - value: kubernetes - - name: BASE_URL - value: http://backend-service:5001 - - name: CLEANUP - value: "false" - - name: DOCKER_API_VERSION - value: "1.40" - - name: ENVIRONMENT_NAME - value: Shuffle - - name: HTTPS_PROXY - - name: HTTP_PROXY - - name: ORG_ID - value: Shuffle - - name: SHUFFLE_APP_SDK_VERSION - value: 0.8.97 - - name: SHUFFLE_BASE_IMAGE_NAME - value: frikky - - name: SHUFFLE_BASE_IMAGE_REGISTRY - value: ghcr.io - - name: SHUFFLE_BASE_IMAGE_TAG_SUFFIX - value: "-0.8.80" - - name: SHUFFLE_ORBORUS_EXECUTION_TIMEOUT - value: "600" - - name: SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY - value: "50" - - name: SHUFFLE_PASS_WORKER_PROXY - value: "TRUE" - - name: SHUFFLE_WORKER_VERSION - value: nightly - - name: TZ - value: Asia/Shanghai - volumeMounts: - - mountPath: /var/run/docker.sock - name: docker-sock - hostname: shuffle-orborus - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock -#--- -#apiVersion: apps/v1 -#kind: Deployment -#metadata: -# name: {{ .Values.name }}opensearch -# namespace: {{ .Values.namespace | quote }} -#spec: -# replicas: 1 -# selector: -# matchLabels: -# service: shuffle -# app: shuffle-opensearch -# template: -# metadata: -# labels: -# service: shuffle -# app: shuffle-opensearch -# spec: -# nodeSelector: -# node.bdlab-venus.com/opensearch: available -# initContainers: -# - name: permissions-fix -# image: frikky/busybox -# #volumeMounts: -# # - name: opensearch-claim0 -# # mountPath: /usr/share/elasticsearch/data -# command: [ 'chown' ] -# args: [ '1000:1000', '/usr/share/elasticsearch/data' ] -# containers: -# - name: shuffle-opensearch -# image: opensearchproject/opensearch:1.0.1 -# env: -# - name: TZ -# value: Asia/Shanghai -# - name: bootstrap.memory_lock -# value: "false" -# - name: OPENSEARCH_JAVA_OPTS -# value: "-Xms1024m -Xmx1024m" -# - name: opendistro_security.disabled -# value: "true" -# - name: cluster.routing.allocation.disk.threshold_enabled -# value: "false" -# - name: cluster.name -# value: shuffle-cluster -# - name: node.name -# value: opensearch-service -# - name: discovery.seed_hosts -# value: opensearch-service -# - name: cluster.initial_master_nodes -# value: opensearch-service - # volumeMounts: - # - mountPath: /usr/share/opensearch/data - # name: opensearch-claim0 - #volumes: - # - name: opensearch-claim0 - # persistentVolumeClaim: - # claimName: opensearch-claim0 -# volumeMounts: -# - mountPath: /usr/share/opensearch/data -# readOnly: true -# name: db-location -# volumes: -# - name: db-location -# hostPath: -# path: /data/kubernetes/shuffle-opensearch -# type: DirectoryOrCreate diff --git a/functions/extensions/k8s/shuffle/templates/hpa.yaml b/functions/extensions/k8s/shuffle/templates/hpa.yaml deleted file mode 100755 index 49f84ca2..00000000 --- a/functions/extensions/k8s/shuffle/templates/hpa.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "shuffle.fullname" . }} - labels: - {{- include "shuffle.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "shuffle.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/functions/extensions/k8s/shuffle/templates/ingress.yaml b/functions/extensions/k8s/shuffle/templates/ingress.yaml deleted file mode 100755 index e35d2ae0..00000000 --- a/functions/extensions/k8s/shuffle/templates/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "shuffle.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "shuffle.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/functions/extensions/k8s/shuffle/templates/persistent-volume.yaml b/functions/extensions/k8s/shuffle/templates/persistent-volume.yaml deleted file mode 100755 index 934c68e2..00000000 --- a/functions/extensions/k8s/shuffle/templates/persistent-volume.yaml +++ /dev/null @@ -1,24 +0,0 @@ -#apiVersion: v1 -#kind: PersistentVolume -#metadata: -# name: opensearch-claim0 -# labels: -# app: opensearch-claim0 -#spec: -# capacity: -# storage: "10G" -# volumeMode: Filesystem -# persistentVolumeReclaimPolicy: Retain -# storageClassName: local-storage -# accessModes: -# - "ReadWriteOnce" -# local: -# path: "/data/kubernetes/shuffle-opensearch" -# nodeAffinity: -# required: -# nodeSelectorTerms: -# - matchExpressions: -# - key: node.dollar.com/opensearch -# operator: In -# values: -# - available diff --git a/functions/extensions/k8s/shuffle/templates/persistentvolumeclaim.yaml b/functions/extensions/k8s/shuffle/templates/persistentvolumeclaim.yaml deleted file mode 100755 index 7d1736a4..00000000 --- a/functions/extensions/k8s/shuffle/templates/persistentvolumeclaim.yaml +++ /dev/null @@ -1,17 +0,0 @@ -#apiVersion: v1 -#kind: PersistentVolumeClaim -#metadata: -# name: opensearch-claim0 -# namespace: {{ .Values.namespace }} -# labels: -# app: opensearch-claim0 -#spec: -# selector: -# matchLabels: -# app: opensearch-claim0 -# accessModes: -# - ReadWriteOnce -# storageClassName: local-storage -# resources: -# requests: -# storage: 5Gi diff --git a/functions/extensions/k8s/shuffle/templates/service.yaml b/functions/extensions/k8s/shuffle/templates/service.yaml deleted file mode 100755 index 8fc78dfa..00000000 --- a/functions/extensions/k8s/shuffle/templates/service.yaml +++ /dev/null @@ -1,52 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: backend-service - namespace: {{ .Values.namespace | quote }} -spec: - ports: - - name: "5001" - port: 5001 - targetPort: 5001 - selector: - app: shuffle-backend - ---- -apiVersion: v1 -kind: Service -metadata: - name: frontend-service - namespace: {{ .Values.namespace | quote }} -spec: - type: NodePort - externalTrafficPolicy: Local - ports: - - name: "3001" - port: 3001 - nodePort: 3001 - targetPort: 80 - - name: "3443" - port: 3443 - nodePort: 3443 - targetPort: 443 - protocol: TCP - selector: - app: shuffle-frontend - -#--- -#apiVersion: v1 -#kind: Service -#metadata: -# name: opensearch-service -# namespace: {{ .Values.namespace | quote }} -#spec: -# type: NodePort -# externalTrafficPolicy: Local -# ports: -# - name: "9200" -# port: 9200 -# targetPort: 9200 -# nodePort: 9200 -# protocol: TCP -# selector: -# app: shuffle-opensearch diff --git a/functions/extensions/k8s/shuffle/templates/serviceaccount.yaml b/functions/extensions/k8s/shuffle/templates/serviceaccount.yaml deleted file mode 100755 index 11e9bdb7..00000000 --- a/functions/extensions/k8s/shuffle/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "shuffle.serviceAccountName" . }} - labels: - {{- include "shuffle.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/functions/extensions/k8s/shuffle/templates/tests/test-connection.yaml b/functions/extensions/k8s/shuffle/templates/tests/test-connection.yaml deleted file mode 100755 index e68f3142..00000000 --- a/functions/extensions/k8s/shuffle/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "shuffle.fullname" . }}-test-connection" - labels: - {{- include "shuffle.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "shuffle.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/functions/extensions/k8s/shuffle/values.yaml b/functions/extensions/k8s/shuffle/values.yaml deleted file mode 100755 index 07e935c6..00000000 --- a/functions/extensions/k8s/shuffle/values.yaml +++ /dev/null @@ -1,82 +0,0 @@ -# Default values for shuffle. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: nginx - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/functions/extensions/scripts/disable_disk_check.sh b/functions/extensions/scripts/disable_disk_check.sh deleted file mode 100755 index cadda899..00000000 --- a/functions/extensions/scripts/disable_disk_check.sh +++ /dev/null @@ -1,11 +0,0 @@ -curl -XPUT -u admin:admin https://localhost:9200/_cluster/settings -H "Content-Type:application/json" -k -d \ -'{ - "transient": { - "cluster.routing.allocation.disk.threshold_enabled": false - } -}' - -curl -XPUT -u admin:admin https://localhost:9200/_all/_settings -H "Content-Type: application/json" -k -d \ -'{ - "index.blocks.read_only_allow_delete": null -}' diff --git a/functions/extensions/swarm/.gitignore b/functions/extensions/swarm/.gitignore deleted file mode 100644 index 9ce4a537..00000000 --- a/functions/extensions/swarm/.gitignore +++ /dev/null @@ -1 +0,0 @@ -shuffle-database/nodes diff --git a/functions/extensions/swarm/docker-compose.yml b/functions/extensions/swarm/docker-compose.yml deleted file mode 100644 index 57ea9b49..00000000 --- a/functions/extensions/swarm/docker-compose.yml +++ /dev/null @@ -1,131 +0,0 @@ -version: '3.4' -services: - backend: - image: ghcr.io/frikky/shuffle-backend:nightly - #hostname: shuffle-backend - environment: - BACKEND_HOSTNAME: backend - OUTER_HOSTNAME: backend - BACKEND_PORT: '5001' - HTTPS_PROXY: '' - HTTP_PROXY: '' - SHUFFLE_APP_DOWNLOAD_LOCATION: https://github.com/frikky/shuffle-apps - SHUFFLE_APP_FORCE_UPDATE: 'false' - SHUFFLE_APP_HOTLOAD_FOLDER: /shuffle-apps - SHUFFLE_APP_HOTLOAD_LOCATION: ./shuffle-apps - DATASTORE_EMULATOR_HOST: "shuffle-database:8000" - DOCKER_API_VERSION: '1.40' - SHUFFLE_BASE_IMAGE_NAME: frikky - SHUFFLE_BASE_IMAGE_REGISTRY: ghcr.io - SHUFFLE_BASE_IMAGE_TAG_SUFFIX: '-0.9.30' - SHUFFLE_CONTAINER_AUTO_CLEANUP: 'true' - SHUFFLE_DEFAULT_APIKEY: '' - SHUFFLE_FILE_LOCATION: /shuffle-files - SHUFFLE_OPENSEARCH_APIKEY: '' - SHUFFLE_OPENSEARCH_CERTIFICATE_FILE: '' - SHUFFLE_OPENSEARCH_CLOUDID: '' - SHUFFLE_OPENSEARCH_PROXY: '' - SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY: 'true' - SHUFFLE_OPENSEARCH_URL: http://opensearch:9200 - SHUFFLE_PASS_APP_PROXY: 'FALSE' - SHUFFLE_PASS_WORKER_PROXY: 'TRUE' - SHUFFLE_ELASTIC: 'true' - #SHUFFLE_ENCRYPTION_MODIFIER: - ports: - - "5001:5001" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - ./shuffle-apps:/shuffle-apps - - ./shuffle-files:/shuffle-files - networks: - - shuffle_prod - #- reverseproxy - depends_on: - - opensearch - logging: - driver: json-file - frontend: - image: ghcr.io/frikky/shuffle-frontend:nightly - healthcheck: - test: curl -fs http://localhost:80 || exit 1 - interval: 30s - timeout: 5s - retries: 3 - ports: - - "3001:80" - - "3443:443" - networks: - - shuffle_prod - #- reverseproxy - environment: - - "BACKEND_HOSTNAME=backend" - depends_on: - - backend - deploy: - update_config: - order: start-first - opensearch: - image: opensearchproject/opensearch:1.1.0 - healthcheck: - test: curl -fs http://localhost:9200/_cat/health || exit 1 - interval: 30s - timeout: 5s - retries: 3 - environment: - - bootstrap.memory_lock=false - - "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - - plugins.security.disabled=true - - cluster.routing.allocation.disk.threshold_enabled=false - - cluster.name=shuffle-cluster - - node.name=opensearch - - discovery.seed_hosts=opensearch - - cluster.initial_master_nodes=opensearch - - node.store.allow_mmap=false - volumes: - - ./shuffle-database:/usr/share/opensearch/data:rw - networks: - - shuffle_prod - #- reverseproxy - logging: - driver: json-file - - orborus: - image: ghcr.io/frikky/shuffle-orborus:nightly - #hostname: shuffle-orborus - environment: - #SHUFFLE_WORKER_VERSION: nightly - SHUFFLE_APP_SDK_VERSION: 0.8.97 - SHUFFLE_WORKER_VERSION: nightly - BASE_URL: http://backend:5001 - #BASE_URL: http://192.168.86.37:5001 - CLEANUP: 'true' - DOCKER_API_VERSION: '1.40' - ENVIRONMENT_NAME: Shuffle - HTTPS_PROXY: '' - HTTP_PROXY: '' - ORG_ID: Shuffle - SHUFFLE_BASE_IMAGE_NAME: frikky - SHUFFLE_BASE_IMAGE_REGISTRY: ghcr.io - SHUFFLE_BASE_IMAGE_TAG_SUFFIX: -0.8.80 - SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY: '50' - SHUFFLE_ORBORUS_EXECUTION_TIMEOUT: '800' - SHUFFLE_PASS_APP_PROXY: 'FALSE' - SHUFFLE_PASS_WORKER_PROXY: 'TRUE' - SHUFFLE_SCALE_REPLICAS: 5 - SHUFFLE_SWARM_NETWORK_NAME: shuffle_prod - SHUFFLE_SWARM_CONFIG: "run" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - networks: - - shuffle_prod - #- reverseproxy - logging: - driver: json-file - -networks: - shuffle_prod: - driver: overlay - external: true - #reverseproxy: - # driver: overlay - # #external: true diff --git a/functions/extensions/swarm/network.sh b/functions/extensions/swarm/network.sh deleted file mode 100644 index f3a323ed..00000000 --- a/functions/extensions/swarm/network.sh +++ /dev/null @@ -1 +0,0 @@ -docker network create -d overlay shuffle_prod diff --git a/functions/extensions/swarm/orborus.yml b/functions/extensions/swarm/orborus.yml deleted file mode 100644 index fb90b10f..00000000 --- a/functions/extensions/swarm/orborus.yml +++ /dev/null @@ -1,39 +0,0 @@ -version: '3.4' -services: - orborus: - image: ghcr.io/frikky/shuffle-orborus:nightly - #hostname: shuffle-orborus - environment: - #SHUFFLE_WORKER_VERSION: nightly - SHUFFLE_APP_SDK_VERSION: 0.8.97 - SHUFFLE_WORKER_VERSION: nightly - BASE_URL: http://:5001 - #BASE_URL: http://192.168.86.37:5001 - CLEANUP: 'true' - DOCKER_API_VERSION: '1.40' - ENVIRONMENT_NAME: Shuffle - HTTPS_PROXY: '' - HTTP_PROXY: '' - ORG_ID: Shuffle - SHUFFLE_BASE_IMAGE_NAME: frikky - SHUFFLE_BASE_IMAGE_REGISTRY: ghcr.io - SHUFFLE_BASE_IMAGE_TAG_SUFFIX: -0.8.80 - SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY: '50' - SHUFFLE_ORBORUS_EXECUTION_TIMEOUT: '800' - SHUFFLE_PASS_APP_PROXY: 'FALSE' - SHUFFLE_PASS_WORKER_PROXY: 'TRUE' - SHUFFLE_SCALE_REPLICAS: 5 - SHUFFLE_SWARM_NETWORK_NAME: shuffle_prod - SHUFFLE_SWARM_CONFIG: "run" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - networks: - - shuffle_prod - #- reverseproxy - logging: - driver: json-file - -networks: - shuffle_prod: - driver: overlay - external: true diff --git a/functions/extensions/swarm/run.sh b/functions/extensions/swarm/run.sh deleted file mode 100644 index faeadab8..00000000 --- a/functions/extensions/swarm/run.sh +++ /dev/null @@ -1,4 +0,0 @@ -docker swarm init -chown 1000:1000 -R shuffle-database/ -docker network create -d overlay shuffle_prod -docker stack deploy --compose-file=docker-compose.yml shuffle_swarm diff --git a/functions/extensions/swarm/run_orborus.sh b/functions/extensions/swarm/run_orborus.sh deleted file mode 100644 index bcbd626f..00000000 --- a/functions/extensions/swarm/run_orborus.sh +++ /dev/null @@ -1,4 +0,0 @@ -docker swarm init -chown 1000:1000 -R shuffle-database/ -docker network create -d overlay shuffle_prod -docker stack deploy --compose-file=orborus.yml shuffle_orborus diff --git a/functions/extensions/swarm/shuffle-apps/tmp b/functions/extensions/swarm/shuffle-apps/tmp deleted file mode 100644 index e69de29b..00000000 diff --git a/functions/extensions/swarm/shuffle-database/tmp b/functions/extensions/swarm/shuffle-database/tmp deleted file mode 100644 index e69de29b..00000000 diff --git a/functions/extensions/swarm/shuffle-files/tmp b/functions/extensions/swarm/shuffle-files/tmp deleted file mode 100644 index e69de29b..00000000 diff --git a/functions/extensions/swarm/stop.sh b/functions/extensions/swarm/stop.sh deleted file mode 100644 index c516f348..00000000 --- a/functions/extensions/swarm/stop.sh +++ /dev/null @@ -1,3 +0,0 @@ -docker stack rm shuffle_swarm - -# diff --git a/functions/extensions/wazuh/custom-shuffle b/functions/extensions/wazuh/custom-shuffle deleted file mode 100755 index bd540414..00000000 --- a/functions/extensions/wazuh/custom-shuffle +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh -# Created by Shuffle, AS. . - -WPYTHON_BIN="framework/python/bin/python3" - -SCRIPT_PATH_NAME="$0" - -DIR_NAME="$(cd $(dirname ${SCRIPT_PATH_NAME}); pwd -P)" -SCRIPT_NAME="$(basename ${SCRIPT_PATH_NAME})" - -case ${DIR_NAME} in - */active-response/bin | */wodles*) - if [ -z "${WAZUH_PATH}" ]; then - WAZUH_PATH="$(cd ${DIR_NAME}/../..; pwd)" - fi - - PYTHON_SCRIPT="${DIR_NAME}/${SCRIPT_NAME}.py" - ;; - */bin) - if [ -z "${WAZUH_PATH}" ]; then - WAZUH_PATH="$(cd ${DIR_NAME}/..; pwd)" - fi - - PYTHON_SCRIPT="${WAZUH_PATH}/framework/scripts/${SCRIPT_NAME}.py" - ;; - */integrations) - if [ -z "${WAZUH_PATH}" ]; then - WAZUH_PATH="$(cd ${DIR_NAME}/..; pwd)" - fi - - PYTHON_SCRIPT="${DIR_NAME}/${SCRIPT_NAME}.py" - ;; -esac - - -${WAZUH_PATH}/${WPYTHON_BIN} ${PYTHON_SCRIPT} "$@" diff --git a/functions/extensions/wazuh/custom-shuffle.py b/functions/extensions/wazuh/custom-shuffle.py deleted file mode 100755 index 8e555ea9..00000000 --- a/functions/extensions/wazuh/custom-shuffle.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 -# Created by Shuffle, AS. . -# Based on the Slack integration using Webhooks - -import json -import sys -import time -import os - -try: - import requests - from requests.auth import HTTPBasicAuth -except Exception as e: - print("No module 'requests' found. Install: pip install requests") - sys.exit(1) - -# ADD THIS TO ossec.conf configuration: -# -# custom-shuffle -# http://:3001/api/v1/hooks/ -# 3 -# json -# - -# Global vars -debug_enabled = False -pwd = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -json_alert = {} -now = time.strftime("%a %b %d %H:%M:%S %Z %Y") - -# Set paths -log_file = '{0}/logs/integrations.log'.format(pwd) - -try: - with open("/tmp/shuffle_start.txt", "w+") as tmp: - tmp.write("Script started") -except: - pass - - -def main(args): - debug("# Starting") - - # Read args - alert_file_location = args[1] - webhook = args[3] - - debug("# Webhook") - debug(webhook) - - debug("# File location") - debug(alert_file_location) - - # Load alert. Parse JSON object. - try: - with open(alert_file_location) as alert_file: - json_alert = json.load(alert_file) - except: - debug("# Alert file %s doesn't exist" % alert_file_location) - - debug("# Processing alert") - try: - debug(json_alert) - except Exception as e: - debug("Failed getting json_alert %s" % e) - sys.exit(1) - - debug("# Generating message") - msg = generate_msg(json_alert) - if isinstance(msg, str): - if len(msg) == 0: - return - debug(msg) - - debug("# Sending message") - - try: - with open("/tmp/shuffle_end.txt", "w+") as tmp: - tmp.write("Script done pre-msg sending") - except: - pass - - - send_msg(msg, webhook) - - -def debug(msg): - if debug_enabled: - msg = "{0}: {1}\n".format(now, msg) - print(msg) - f = open(log_file, "a") - f.write(msg) - f.close() - -# Skips container kills to stop self-recursion -def filter_msg(alert): - # These are things that recursively happen because Shuffle starts Docker containers - skip = ["87924", "87900", "87901", "87902", "87903", "87904", "86001", "86002", "86003", "87932", "80710", "87929", "87928", "5710"] - if alert["rule"]["id"] in skip: - return False - - #try: - # if "docker" in alert["rule"]["description"].lower() and " - #msg['text'] = alert.get('full_log') - #except: - # pass - #msg['title'] = alert['rule']['description'] if 'description' in alert['rule'] else "N/A" - - return True - -def generate_msg(alert): - if not filter_msg(alert): - print("Skipping rule %s" % alert["rule"]["id"]) - return "" - - level = alert['rule']['level'] - - if (level <= 4): - severity = 1 - elif (level >= 5 and level <= 7): - severity = 2 - else: - severity = 3 - - msg = {} - msg['severity'] = severity - msg['pretext'] = "WAZUH Alert" - msg['title'] = alert['rule']['description'] if 'description' in alert['rule'] else "N/A" - msg['text'] = alert.get('full_log') - msg['rule_id'] = alert["rule"]["id"] - msg['timestamp'] = alert["timestamp"] - msg['id'] = alert['id'] - msg["all_fields"] = alert - - #msg['fields'] = [] - # msg['fields'].append({ - # "title": "Agent", - # "value": "({0}) - {1}".format( - # alert['agent']['id'], - # alert['agent']['name'] - # ), - # }) - #if 'agentless' in alert: - # msg['fields'].append({ - # "title": "Agentless Host", - # "value": alert['agentless']['host'], - # }) - - #msg['fields'].append({"title": "Location", "value": alert['location']}) - #msg['fields'].append({ - # "title": "Rule ID", - # "value": "{0} _(Level {1})_".format(alert['rule']['id'], level), - #}) - - #attach = {'attachments': [msg]} - - return json.dumps(msg) - - -def send_msg(msg, url): - debug("# In send msg") - headers = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'} - res = requests.post(url, data=msg, headers=headers, verify=False) - debug("# After send msg: %s" % res) - - -if __name__ == "__main__": - try: - # Read arguments - bad_arguments = False - if len(sys.argv) >= 4: - msg = '{0} {1} {2} {3} {4}'.format( - now, - sys.argv[1], - sys.argv[2], - sys.argv[3], - sys.argv[4] if len(sys.argv) > 4 else '', - ) - #debug_enabled = (len(sys.argv) > 4 and sys.argv[4] == 'debug') - debug_enabled = True - else: - msg = '{0} Wrong arguments'.format(now) - bad_arguments = True - - # Logging the call - try: - f = open(log_file, 'a') - except: - f = open(log_file, 'w+') - f.write("") - f.close() - - f = open(log_file, 'a') - f.write(msg + '\n') - f.close() - - if bad_arguments: - debug("# Exiting: Bad arguments. Inputted: %s" % sys.argv) - sys.exit(1) - - # Main function - main(sys.argv) - - except Exception as e: - debug(str(e)) - raise diff --git a/functions/extensions/wazuh/ossec.conf b/functions/extensions/wazuh/ossec.conf deleted file mode 100755 index e87bc53d..00000000 --- a/functions/extensions/wazuh/ossec.conf +++ /dev/null @@ -1,6 +0,0 @@ - - custom-shuffle - 9 - http://:/api/v1/hooks/webhook_hookid - json - diff --git a/functions/kubernetes/charts/shuffle/README.md b/functions/kubernetes/charts/shuffle/README.md index 6c3a4ed2..c9810210 100644 --- a/functions/kubernetes/charts/shuffle/README.md +++ b/functions/kubernetes/charts/shuffle/README.md @@ -478,81 +478,74 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia ### worker Parameters -| Name | Description | Value | -| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | -| `worker.image.registry` | worker image registry | `ghcr.io` | -| `worker.image.repository` | worker image repository | `shuffle/shuffle-worker` | -| `worker.image.tag` | worker image tag (immutable tags are recommended, defaults to appVersion) | `""` | -| `worker.image.digest` | worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) | `""` | -| `worker.podSecurityContext.enabled` | Enable worker pods' Security Context | `true` | -| `worker.podSecurityContext.fsGroupChangePolicy` | Set filesystem group change policy for worker pods | `Always` | -| `worker.podSecurityContext.sysctls` | Set kernel settings using the sysctl interface for worker pods | `[]` | -| `worker.podSecurityContext.supplementalGroups` | Set filesystem extra groups for worker pods | `[]` | -| `worker.podSecurityContext.fsGroup` | Set fsGroup in worker pods' Security Context | `1001` | -| `worker.containerSecurityContext.enabled` | Enabled worker container' Security Context | `true` | -| `worker.containerSecurityContext.seLinuxOptions` | Set SELinux options in worker container | `{}` | -| `worker.containerSecurityContext.runAsUser` | Set runAsUser in worker container' Security Context | `1001` | -| `worker.containerSecurityContext.runAsGroup` | Set runAsGroup in worker container' Security Context | `1001` | -| `worker.containerSecurityContext.runAsNonRoot` | Set runAsNonRoot in worker container' Security Context | `true` | -| `worker.containerSecurityContext.readOnlyRootFilesystem` | Set readOnlyRootFilesystem in worker container' Security Context | `true` | -| `worker.containerSecurityContext.privileged` | Set privileged in worker container' Security Context | `false` | -| `worker.containerSecurityContext.allowPrivilegeEscalation` | Set allowPrivilegeEscalation in worker container' Security Context | `false` | -| `worker.containerSecurityContext.capabilities.drop` | List of capabilities to be dropped in worker container | `["ALL"]` | -| `worker.containerSecurityContext.seccompProfile.type` | Set seccomp profile in worker container | `RuntimeDefault` | -| `worker.serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | -| `worker.serviceAccount.name` | The name of the ServiceAccount to use. | `""` | -| `worker.serviceAccount.annotations` | Additional Service Account annotations (evaluated as a template) | `{}` | -| `worker.serviceAccount.automountServiceAccountToken` | Automount service account token for the worker service account | `true` | -| `worker.serviceAccount.imagePullSecrets` | Add image pull secrets to the worker service account | `[]` | -| `worker.rbac.create` | Specifies whether RBAC resources should be created | `true` | -| `worker.networkPolicy.enabled` | Specifies whether a NetworkPolicy should be created | `true` | -| `worker.networkPolicy.allowExternal` | Don't require server label for connections | `true` | -| `worker.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | -| `worker.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | -| `worker.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | +| Name | Description | Value | +| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | +| `worker.image.registry` | worker image registry | `ghcr.io` | +| `worker.image.repository` | worker image repository | `shuffle/shuffle-worker` | +| `worker.image.tag` | worker image tag (immutable tags are recommended, defaults to appVersion) | `""` | +| `worker.image.digest` | worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) | `""` | +| `worker.resourcesPreset` | Set worker container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if worker.resources is set (worker.resources is recommended for production). | `nano` | +| `worker.resources` | Set worker container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` | +| `worker.podSecurityContext.enabled` | Enable worker pods' Security Context | `true` | +| `worker.podSecurityContext.fsGroupChangePolicy` | Set filesystem group change policy for worker pods | `Always` | +| `worker.podSecurityContext.sysctls` | Set kernel settings using the sysctl interface for worker pods | `[]` | +| `worker.podSecurityContext.supplementalGroups` | Set filesystem extra groups for worker pods | `[]` | +| `worker.podSecurityContext.fsGroup` | Set fsGroup in worker pods' Security Context | `1001` | +| `worker.containerSecurityContext.enabled` | Enabled worker container' Security Context | `true` | +| `worker.containerSecurityContext.seLinuxOptions` | Set SELinux options in worker container | `{}` | +| `worker.containerSecurityContext.runAsUser` | Set runAsUser in worker container' Security Context | `1001` | +| `worker.containerSecurityContext.runAsGroup` | Set runAsGroup in worker container' Security Context | `1001` | +| `worker.containerSecurityContext.runAsNonRoot` | Set runAsNonRoot in worker container' Security Context | `true` | +| `worker.containerSecurityContext.readOnlyRootFilesystem` | Set readOnlyRootFilesystem in worker container' Security Context | `true` | +| `worker.containerSecurityContext.privileged` | Set privileged in worker container' Security Context | `false` | +| `worker.containerSecurityContext.allowPrivilegeEscalation` | Set allowPrivilegeEscalation in worker container' Security Context | `false` | +| `worker.containerSecurityContext.capabilities.drop` | List of capabilities to be dropped in worker container | `["ALL"]` | +| `worker.containerSecurityContext.seccompProfile.type` | Set seccomp profile in worker container | `RuntimeDefault` | +| `worker.serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | +| `worker.serviceAccount.name` | The name of the ServiceAccount to use. | `""` | +| `worker.serviceAccount.annotations` | Additional Service Account annotations (evaluated as a template) | `{}` | +| `worker.serviceAccount.automountServiceAccountToken` | Automount service account token for the worker service account | `true` | +| `worker.serviceAccount.imagePullSecrets` | Add image pull secrets to the worker service account | `[]` | +| `worker.rbac.create` | Specifies whether RBAC resources should be created | `true` | +| `worker.networkPolicy.enabled` | Specifies whether a NetworkPolicy should be created | `true` | +| `worker.networkPolicy.allowExternal` | Don't require server label for connections | `true` | +| `worker.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | +| `worker.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | +| `worker.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | ### app Parameters -| Name | Description | Value | -| ------------------------------------------------- | ---------------------------------------------------------------------------------- | ------ | -| `app.serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | -| `app.serviceAccount.name` | The name of the ServiceAccount to use. | `""` | -| `app.serviceAccount.annotations` | Additional Service Account annotations (evaluated as a template) | `{}` | -| `app.serviceAccount.automountServiceAccountToken` | Automount service account token for the app service account | `true` | -| `app.serviceAccount.imagePullSecrets` | Add image pull secrets to the app service account | `[]` | -| `app.rbac.create` | Specifies whether RBAC resources should be created | `true` | -| `app.networkPolicy.enabled` | Specifies whether a NetworkPolicy should be created | `true` | -| `app.networkPolicy.allowExternal` | Don't require server label for connections | `true` | -| `app.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | -| `app.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | -| `app.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | -| `app.exposedContainerPort` | The port that shuffle app containers will listen on for new requests. | `80` | -| `app.podSecurityContext.enabled` | Enable app pods' Security Context | `true` | -| `app.podSecurityContext.fsGroupChangePolicy` | Set filesystem group change policy for app pods | `Always` | -| `app.podSecurityContext.sysctls` | Set kernel settings using the sysctl interface for app pods | `[]` | -| `app.podSecurityContext.supplementalGroups` | Set filesystem extra groups for app pods | `[]` | -| `app.podSecurityContext.fsGroup` | Set fsGroup in app pods' Security Context | `1001` | -| `app.containerSecurityContext.enabled` | Enabled app container' Security Context | `true` | -| `app.containerSecurityContext.seLinuxOptions` | Set SELinux options in app container | `{}` | -| `app.containerSecurityContext.runAsUser` | Set runAsUser in app container' Security Context | `1001` | -| `app.containerSecurityContext.runAsGroup` | Set runAsGroup in app container' Security Context | `1001` | -| `app.containerSecurityContext.runAsNonRoot` | Set runAsNonRoot in app container' Security Context | `true` | -| `app.containerSecurityContext.readOnlyRootFilesystem` | Set readOnlyRootFilesystem in app container' Security Context | `true` | -| `app.containerSecurityContext.privileged` | Set privileged in app container' Security Context | `false` | -| `app.containerSecurityContext.allowPrivilegeEscalation` | Set allowPrivilegeEscalation in app container' Security Context | `false` | -| `app.containerSecurityContext.capabilities.drop` | List of capabilities to be dropped in app container | `["ALL"]` | -| `app.containerSecurityContext.seccompProfile.type` | Set seccomp profile in app container | `RuntimeDefault` | -| `app.serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | -| `app.serviceAccount.name` | The name of the ServiceAccount to use. | `""` | -| `app.serviceAccount.annotations` | Additional Service Account annotations (evaluated as a template) | `{}` | -| `app.serviceAccount.automountServiceAccountToken` | Automount service account token for the app service account | `true` | -| `app.serviceAccount.imagePullSecrets` | Add image pull secrets to the app service account | `[]` | -| `app.rbac.create` | Specifies whether RBAC resources should be created | `true` | -| `app.networkPolicy.enabled` | Specifies whether a NetworkPolicy should be created | `true` | -| `app.networkPolicy.allowExternal` | Don't require server label for connections | `true` | -| `app.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | -| `app.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | -| `app.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | +| Name | Description | Value | +| ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | +| `app.resourcesPreset` | Set app container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if app.resources is set (app.resources is recommended for production). | `nano` | +| `app.resources` | Set app container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` | +| `app.podSecurityContext.enabled` | Enable app pods' Security Context | `true` | +| `app.podSecurityContext.fsGroupChangePolicy` | Set filesystem group change policy for app pods | `Always` | +| `app.podSecurityContext.sysctls` | Set kernel settings using the sysctl interface for app pods | `[]` | +| `app.podSecurityContext.supplementalGroups` | Set filesystem extra groups for app pods | `[]` | +| `app.podSecurityContext.fsGroup` | Set fsGroup in app pods' Security Context | `1001` | +| `app.containerSecurityContext.enabled` | Enabled app container' Security Context | `true` | +| `app.containerSecurityContext.seLinuxOptions` | Set SELinux options in app container | `{}` | +| `app.containerSecurityContext.runAsUser` | Set runAsUser in app container' Security Context | `1001` | +| `app.containerSecurityContext.runAsGroup` | Set runAsGroup in app container' Security Context | `1001` | +| `app.containerSecurityContext.runAsNonRoot` | Set runAsNonRoot in app container' Security Context | `true` | +| `app.containerSecurityContext.readOnlyRootFilesystem` | Set readOnlyRootFilesystem in app container' Security Context | `true` | +| `app.containerSecurityContext.privileged` | Set privileged in app container' Security Context | `false` | +| `app.containerSecurityContext.allowPrivilegeEscalation` | Set allowPrivilegeEscalation in app container' Security Context | `false` | +| `app.containerSecurityContext.capabilities.drop` | List of capabilities to be dropped in app container | `["ALL"]` | +| `app.containerSecurityContext.seccompProfile.type` | Set seccomp profile in app container | `RuntimeDefault` | +| `app.serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | +| `app.serviceAccount.name` | The name of the ServiceAccount to use. | `""` | +| `app.serviceAccount.annotations` | Additional Service Account annotations (evaluated as a template) | `{}` | +| `app.serviceAccount.automountServiceAccountToken` | Automount service account token for the app service account | `true` | +| `app.serviceAccount.imagePullSecrets` | Add image pull secrets to the app service account | `[]` | +| `app.rbac.create` | Specifies whether RBAC resources should be created | `true` | +| `app.networkPolicy.enabled` | Specifies whether a NetworkPolicy should be created | `true` | +| `app.networkPolicy.allowExternal` | Don't require server label for connections | `true` | +| `app.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | +| `app.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | +| `app.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | +| `app.exposedContainerPort` | The port that shuffle app containers will listen on for new requests. | `80` | ### Traffic Exposure Parameters @@ -651,3 +644,4 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia ### Other Parameters + diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml index 504b674d..87486de5 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml +++ b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml @@ -13,7 +13,66 @@ data: TZ: "{{ .Values.shuffle.timezone }}" BASE_URL: "http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }}" KUBERNETES_NAMESPACE: "{{ .Release.Namespace }}" - KUBERNETES_SERVICE_ACCOUNT: {{ include "shuffle.orborus.serviceAccount.name" . }} - SHUFFLE_WORKER_IMAGE: "{{ include "shuffle.worker.image" . }}" REGISTRY_URL: "{{ .Values.shuffle.appRegistry }}" - SHUFFLE_SWARM_CONFIG: run + + # Shuffle worker configuration + SHUFFLE_WORKER_IMAGE: {{ include "shuffle.worker.image" . | quote }} + SHUFFLE_WORKER_SERVICE_ACCOUNT_NAME: {{ include "shuffle.worker.serviceAccount.name" . | quote }} + {{- if .Values.worker.podSecurityContext.enabled }} + SHUFFLE_WORKER_POD_SECURITY_CONTEXT: {{ omit .Values.worker.podSecurityContext "enabled" | mustToJson | quote }} + {{- end }} + {{- if .Values.worker.containerSecurityContext.enabled }} + SHUFFLE_WORKER_CONTAINER_SECURITY_CONTEXT: {{ include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.worker.containerSecurityContext "context" $) | fromYaml | mustToJson | quote }} + {{- end }} + + # Shuffle worker resources + {{- $workerResources := (.Values.worker.resources | default (include "common.resources.preset" (dict "type" .Values.worker.resourcesPreset)) | fromYaml) -}} + {{- if $workerResources.requests.cpu }} + SHUFFLE_WORKER_CPU_REQUEST: {{ $workerResources.requests.cpu | quote }} + {{- end }} + {{- if $workerResources.requests.memory}} + SHUFFLE_WORKER_MEMORY_REQUEST: {{ $workerResources.requests.memory | quote }} + {{- end }} + {{- if (index $workerResources.requests "ephemeral-storage") }} + SHUFFLE_WORKER_EPHEMERAL_STORAGE_REQUEST: {{ (index $workerResources.requests "ephemeral-storage") | quote }} + {{- end }} + {{- if $workerResources.limits.cpu }} + SHUFFLE_WORKER_CPU_LIMIT: {{ $workerResources.limits.cpu | quote }} + {{- end }} + {{- if $workerResources.limits.memory}} + SHUFFLE_WORKER_MEMORY_LIMIT: {{ $workerResources.limits.memory | quote }} + {{- end }} + {{- if (index $workerResources.limits "ephemeral-storage") }} + SHUFFLE_WORKER_EPHEMERAL_STORAGE_LIMIT: {{ (index $workerResources.limits "ephemeral-storage") | quote }} + {{- end }} + + # Shuffle app configuration + SHUFFLE_APP_EXPOSED_PORT: {{ .Values.app.exposedContainerPort | quote }} + SHUFFLE_APP_SERVICE_ACCOUNT_NAME: {{ include "shuffle.app.serviceAccount.name" . | quote }} + {{- if .Values.app.podSecurityContext.enabled }} + SHUFFLE_APP_POD_SECURITY_CONTEXT: {{ omit .Values.app.podSecurityContext "enabled" | mustToJson | quote }} + {{- end }} + {{- if .Values.app.containerSecurityContext.enabled }} + SHUFFLE_APP_CONTAINER_SECURITY_CONTEXT: {{ include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.app.containerSecurityContext "context" $) | fromYaml | mustToJson | quote }} + {{- end }} + + # Shuffle app resources + {{- $appResources := (.Values.app.resources | default (include "common.resources.preset" (dict "type" .Values.app.resourcesPreset)) | fromYaml) -}} + {{- if $appResources.requests.cpu }} + SHUFFLE_APP_CPU_REQUEST: {{ $appResources.requests.cpu | quote }} + {{- end }} + {{- if $appResources.requests.memory}} + SHUFFLE_APP_MEMORY_REQUEST: {{ $appResources.requests.memory | quote }} + {{- end }} + {{- if (index $appResources.requests "ephemeral-storage") }} + SHUFFLE_APP_EPHEMERAL_STORAGE_REQUEST: {{ (index $appResources.requests "ephemeral-storage") | quote }} + {{- end }} + {{- if $appResources.limits.cpu }} + SHUFFLE_APP_CPU_LIMIT: {{ $appResources.limits.cpu | quote }} + {{- end }} + {{- if $appResources.limits.memory}} + SHUFFLE_APP_MEMORY_LIMIT: {{ $appResources.limits.memory | quote }} + {{- end }} + {{- if (index $appResources.limits "ephemeral-storage") }} + SHUFFLE_APP_EPHEMERAL_STORAGE_LIMIT: {{ (index $appResources.limits "ephemeral-storage") | quote }} + {{- end }} diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml index fd73dd15..05ca1ac3 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml +++ b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml @@ -86,28 +86,8 @@ spec: value: kubernetes - name: IS_KUBERNETES value: "true" - - name: SHUFFLE_WORKER_SERVICE_ACCOUNT_NAME - value: {{ include "shuffle.worker.serviceAccount.name" . }} - - name: SHUFFLE_APP_EXPOSED_PORT - value: {{ .Values.app.exposedContainerPort | quote }} - {{- if .Values.worker.podSecurityContext.enabled }} - - name: SHUFFLE_WORKER_POD_SECURITY_CONTEXT - value: {{ omit .Values.worker.podSecurityContext "enabled" | mustToJson | quote }} - {{- end }} - {{- if .Values.worker.containerSecurityContext.enabled }} - - name: SHUFFLE_WORKER_CONTAINER_SECURITY_CONTEXT - value: {{ include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.worker.containerSecurityContext "context" $) | fromYaml | mustToJson | quote }} - {{- end }} - - name: SHUFFLE_APP_SERVICE_ACCOUNT_NAME - value: {{ include "shuffle.app.serviceAccount.name" . }} - {{- if .Values.app.podSecurityContext.enabled }} - - name: SHUFFLE_APP_POD_SECURITY_CONTEXT - value: {{ omit .Values.app.podSecurityContext "enabled" | mustToJson | quote }} - {{- end }} - {{- if .Values.app.containerSecurityContext.enabled }} - - name: SHUFFLE_APP_CONTAINER_SECURITY_CONTEXT - value: {{ include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.app.containerSecurityContext "context" $) | fromYaml | mustToJson | quote }} - {{- end }} + - name: SHUFFLE_SWARM_CONFIG + value: run {{- if .Values.orborus.extraEnvVars }} {{- include "common.tplvalues.render" (dict "value" .Values.orborus.extraEnvVars "context" $) | nindent 12 }} {{- end }} diff --git a/functions/kubernetes/charts/shuffle/values.schema.json b/functions/kubernetes/charts/shuffle/values.schema.json index 661eefc6..c2f9821c 100644 --- a/functions/kubernetes/charts/shuffle/values.schema.json +++ b/functions/kubernetes/charts/shuffle/values.schema.json @@ -2084,6 +2084,16 @@ } } }, + "resourcesPreset": { + "type": "string", + "description": "Set worker container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if worker.resources is set (worker.resources is recommended for production).", + "default": "nano" + }, + "resources": { + "type": "object", + "description": "Set worker container requests and limits for different resources like CPU or memory (essential for production workloads)", + "default": {} + }, "podSecurityContext": { "type": "object", "properties": { @@ -2259,6 +2269,16 @@ "app": { "type": "object", "properties": { + "resourcesPreset": { + "type": "string", + "description": "Set app container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if app.resources is set (app.resources is recommended for production).", + "default": "nano" + }, + "resources": { + "type": "object", + "description": "Set app container requests and limits for different resources like CPU or memory (essential for production workloads)", + "default": {} + }, "podSecurityContext": { "type": "object", "properties": { diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 08763cb6..57bccb31 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -1340,6 +1340,24 @@ worker: tag: "" digest: "" + ## worker resource requests and limits + ## ref: http://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + ## @param worker.resourcesPreset Set worker container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if worker.resources is set (worker.resources is recommended for production). + ## More information: https://github.com/bitnami/charts/blob/main/bitnami/common/templates/_resources.tpl#L15 + ## + resourcesPreset: "nano" + ## @param worker.resources Set worker container requests and limits for different resources like CPU or memory (essential for production workloads) + ## Example: + ## resources: + ## requests: + ## cpu: 2 + ## memory: 512Mi + ## limits: + ## cpu: 3 + ## memory: 1024Mi + ## + resources: {} + ## Configure Pods Security Context ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod ## @param worker.podSecurityContext.enabled Enable worker pods' Security Context @@ -1444,6 +1462,24 @@ worker: ## @section app Parameters ## app: + ## app resource requests and limits + ## ref: http://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + ## @param app.resourcesPreset Set app container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if app.resources is set (app.resources is recommended for production). + ## More information: https://github.com/bitnami/charts/blob/main/bitnami/common/templates/_resources.tpl#L15 + ## + resourcesPreset: "nano" + ## @param app.resources Set app container requests and limits for different resources like CPU or memory (essential for production workloads) + ## Example: + ## resources: + ## requests: + ## cpu: 2 + ## memory: 512Mi + ## limits: + ## cpu: 3 + ## memory: 1024Mi + ## + resources: {} + ## Configure Pods Security Context ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod ## @param app.podSecurityContext.enabled Enable app pods' Security Context diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 5d3ee295..1a7f57b1 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -7,10 +7,10 @@ toolchain go1.24.4 //replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared require ( - github.com/docker/docker v28.2.2+incompatible + github.com/docker/docker v28.3.3+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.84 + github.com/shuffle/shuffle-shared v0.9.14 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 ) @@ -58,7 +58,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/kin-openapi v0.42.0 // indirect - github.com/frikky/schemaless v0.0.16 // indirect + github.com/frikky/schemaless v0.0.20 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect @@ -95,11 +95,13 @@ require ( github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opensearch-project/opensearch-go v1.1.0 // indirect github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect + github.com/osteele/liquid v1.7.0 // indirect + github.com/osteele/tuesday v1.0.3 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect - github.com/sashabaranov/go-openai v1.40.1 // indirect + github.com/sashabaranov/go-openai v1.40.5 // indirect github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/sendgrid/sendgrid-go v3.16.1+incompatible // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect @@ -121,13 +123,13 @@ require ( go.opentelemetry.io/otel/trace v1.36.0 // indirect go.opentelemetry.io/proto/otlp v1.5.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.38.0 // indirect - golang.org/x/net v0.40.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/net v0.41.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.14.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/term v0.32.0 // indirect - golang.org/x/text v0.25.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/term v0.33.0 // indirect + golang.org/x/text v0.27.0 // indirect golang.org/x/time v0.11.0 // indirect google.golang.org/api v0.236.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 13ae08b5..2496c612 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -116,8 +116,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw= -github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -142,8 +142,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.16 h1:4d2ZktB9xGsAusbbKliOI8TuriSrdIMzD/6ToY3wkz8= -github.com/frikky/schemaless v0.0.16/go.mod h1:jT48kTcmr1q3o8i+8qe7g+eCsbwaz2Q9CjOJevQQzQs= +github.com/frikky/schemaless v0.0.20 h1:S/A2pQcRN9qa2RnufvxwCeM06trjG0JLTF3urt1tFQI= +github.com/frikky/schemaless v0.0.20/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -299,6 +299,10 @@ github.com/opensearch-project/opensearch-go v1.1.0 h1:eG5sh3843bbU1itPRjA9QXbxcg github.com/opensearch-project/opensearch-go v1.1.0/go.mod h1:+6/XHCuTH+fwsMJikZEWsucZ4eZMma3zNSeLrTtVGbo= github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ= github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8= +github.com/osteele/liquid v1.7.0 h1:VsbPSchE5D5S5scylAIvERET4dnCxsO6IDri2oSJ5Dk= +github.com/osteele/liquid v1.7.0/go.mod h1:xU0Z2dn2hOQIEFEWNmeltOmCtfhtoW/2fCyiNQeNG+U= +github.com/osteele/tuesday v1.0.3 h1:SrCmo6sWwSgnvs1bivmXLvD7Ko9+aJvvkmDjB5G4FTU= +github.com/osteele/tuesday v1.0.3/go.mod h1:pREKpE+L03UFuR+hiznj3q7j3qB1rUZ4XfKejwWFF2M= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= @@ -314,8 +318,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= -github.com/sashabaranov/go-openai v1.40.1 h1:bJ08Iwct5mHBVkuvG6FEcb9MDTfsXdTYPGjYLRdeTEU= -github.com/sashabaranov/go-openai v1.40.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/sashabaranov/go-openai v1.40.5 h1:SwIlNdWflzR1Rxd1gv3pUg6pwPc6cQ2uMoHs8ai+/NY= +github.com/sashabaranov/go-openai v1.40.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0= @@ -324,8 +328,8 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.84 h1:ElIMQYjKBVOiadbiGkSzt/lPU5xqaQwRxQvk9wx/xYM= -github.com/shuffle/shuffle-shared v0.8.84/go.mod h1:RdfNxqCPI+zU4jQKy3E/p4Io2injm7LpSKQUCDHNtLk= +github.com/shuffle/shuffle-shared v0.9.14 h1:POkTHO+bByuv8HiKuCMSGgtpDKk86ISr6ooLG8vQfuE= +github.com/shuffle/shuffle-shared v0.9.14/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -402,8 +406,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= -golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -452,8 +456,8 @@ golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= -golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -469,8 +473,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= -golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -495,14 +499,14 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= -golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= +golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -513,8 +517,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= @@ -546,8 +550,8 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= +golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 26f16017..507d5301 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -85,12 +85,11 @@ var appContainerSecurityContext = os.Getenv("SHUFFLE_APP_CONTAINER_SECURITY_CONT // var baseimagename = "docker.pkg.github.com/shuffle/shuffle" // var baseimagename = "ghcr.io/frikky" // var baseimagename = "shuffle/shuffle" -var baseimagename = os.Getenv("SHUFFLE_BASE_IMAGE_NAME") var baseimageregistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY") - +var baseimagename = os.Getenv("SHUFFLE_BASE_IMAGE_NAME") //var baseimagetagsuffix = os.Getenv("SHUFFLE_BASE_IMAGE_TAG_SUFFIX") -// Used for cloud with auth +// Used for cloud with auth. Onprem in certain cases too. var auth = os.Getenv("AUTH") var org = os.Getenv("ORG") @@ -497,6 +496,7 @@ func deployServiceWorkers(image string) { //} } + // Running 2 by default instead of 1. Higher scale mechanisms - es replicas := uint64(1) scaleReplicas := os.Getenv("SHUFFLE_SCALE_REPLICAS") if len(scaleReplicas) > 0 { @@ -522,7 +522,7 @@ func deployServiceWorkers(image string) { } appReplicas := os.Getenv("SHUFFLE_APP_REPLICAS") - appReplicaCnt := 1 + appReplicaCnt := 2 if len(appReplicas) > 0 { newCnt, err := strconv.Atoi(appReplicas) if err != nil { @@ -734,7 +734,7 @@ func deployServiceWorkers(image string) { var updatedNetworks []swarm.NetworkAttachmentConfig for _, net := range serviceSpec.Networks { if net.Target != "shuffle_shuffle" { - updatedNetworks = append(updatedNetworks, net) + updatedNetworks = append(updatedNetworks, net) } } serviceSpec.Networks = updatedNetworks @@ -813,20 +813,20 @@ func handleBackendImageDownload(ctx context.Context, images string) error { newImages = append(newImages, curimage) // Force remove the current image to avoid cached layers - // if swarmConfig == "run" || swarmConfig == "swarm" { - // _, err := dockercli.ImageRemove(ctx, curimage, image.RemoveOptions{ - // Force: true, - // PruneChildren: true, - // }) + // if swarmConfig == "run" || swarmConfig == "swarm" { + // _, err := dockercli.ImageRemove(ctx, curimage, image.RemoveOptions{ + // Force: true, + // PruneChildren: true, + // }) - // if err != nil { - // log.Printf("[ERROR] Failed removing image for re-download: %s", err) - // } else { - // log.Printf("[DEBUG] Removed image: %s", curimage) - // } - // } else { - // //log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig) - // } + // if err != nil { + // log.Printf("[ERROR] Failed removing image for re-download: %s", err) + // } else { + // log.Printf("[DEBUG] Removed image: %s", curimage) + // } + // } else { + // //log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig) + // } err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, curimage) if err != nil { @@ -887,7 +887,7 @@ func handleBackendImageDownload(ctx context.Context, images string) error { log.Printf("[ERROR] Failed updating service %s with the new image %s: %s. Resp: %#v", service.Spec.Annotations.Name, image, err, resp) } else { log.Printf("[DEBUG] Updated service %s with the new image %s. Resp: %#v", service.Spec.Annotations.Name, image, resp) - + found = true if !strings.Contains(fmt.Sprintf("%s", resp), "error") { @@ -1041,6 +1041,10 @@ func deployK8sWorker(image string, identifier string, env []string) error { env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_PORT=%s", os.Getenv("KUBERNETES_SERVICE_PORT"))) } + if len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0 { + env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_REGISTRY=%s", os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY"))) + } + if len(os.Getenv("REGISTRY_URL")) > 0 { env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL"))) } @@ -1065,6 +1069,10 @@ func deployK8sWorker(image string, identifier string, env []string) error { env = append(env, fmt.Sprintf("SHUFFLE_APP_CONTAINER_SECURITY_CONTEXT=%s", appContainerSecurityContext)) } + if len(os.Getenv("SHUFFLE_LOGS_DISABLED")) > 0 { + env = append(env, fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED"))) + } + clientset, _, err := shuffle.GetKubernetesClient() if err != nil { log.Printf("[ERROR] Error getting kubernetes client:", err) @@ -1096,8 +1104,13 @@ func deployK8sWorker(image string, identifier string, env []string) error { } } + // Required format: + // url/org/repo/appname:tag + // url/org/repo/appname:tag + + //env = append(env, fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", swarmConfig)) env = append(env, fmt.Sprintf("BASE_URL=%s", baseUrl)) - env = append(env, fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", swarmConfig)) + env = append(env, fmt.Sprintf("SHUFFLE_SWARM_CONFIG=run")) env = append(env, fmt.Sprintf("WORKER_HOSTNAME=%s", "shuffle-workers")) if len(kubernetesNamespace) == 0 { @@ -2044,6 +2057,11 @@ func main() { log.Printf("[DEBUG] Verbose mode. NOT cleaning up. Cleanup env: %s", cleanupEnv) } + // Default to 120 instead of default 30 + if len(os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")) == 0 { + os.Setenv("SHUFFLE_APP_SDK_TIMEOUT", "120") + } + workerTimeout := 600 if workerTimeoutEnv != "" { tmpInt, err := strconv.Atoi(workerTimeoutEnv) @@ -2133,16 +2151,17 @@ func main() { if isKubernetes != "true" { deployServiceWorkers(workerImage) + + err := setBackendToSwarmNetwork(ctx) + if err != nil { + log.Printf("[WARNING] Failed setting backend to swarm network: %s", err) + } + } else { deployK8sWorker(workerImage, "shuffle-workers", []string{}) runString = "Run: \"kubectl get pods\" for more info" } - err := setBackendToSwarmNetwork(ctx) - if err != nil { - log.Printf("[WARNING] Failed setting backend to swarm network: %s", err) - } - log.Printf("[DEBUG] Waiting 45 seconds to ensure workers are deployed. %s", runString) time.Sleep(time.Duration(45) * time.Second) @@ -2214,7 +2233,6 @@ func main() { log.Printf("[INFO] Waiting for executions at %s with Environment %#v", fullUrl, environment) - hasStarted := false for { if req.Method == "POST" { @@ -2298,7 +2316,7 @@ func main() { } if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" && os.Getenv("SHUFFLE_SCALE_REPLICAS") == "" { - // go AutoScale(ctx) + //go AutoScale(ctx) } hasStarted = true } @@ -3857,7 +3875,7 @@ func zombiecheck(ctx context.Context, workerTimeout int) error { var options container.StopOptions for _, containername := range stopContainers { log.Printf("[INFO] Stopping and removing container %s", containerNames[containername]) - dockercli.ContainerStop(ctx, containername, options) + go dockercli.ContainerStop(ctx, containername, options) removeContainers = append(removeContainers, containername) } @@ -3938,7 +3956,13 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, } } - client := &http.Client{} + client := &http.Client{ + //Transport: &http.Transport{ + // TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + //}, + Timeout: time.Duration(120 * time.Second), + } + req, err := http.NewRequest( "POST", streamUrl, @@ -4013,7 +4037,7 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, debugCommand := fmt.Sprintf("docker service logs shuffle-workers 2>&1 -f | grep %s", workflowExecution.ExecutionId) if isKubernetes == "true" { - debugCommand = fmt.Sprintf("kubectl logs -n %s container=shuffle-worker | grep %s", kubernetesNamespace, workflowExecution.ExecutionId) + debugCommand = fmt.Sprintf("kubectl logs -n %s deployment/shuffle-workers | grep %s", kubernetesNamespace, workflowExecution.ExecutionId) } log.Printf("[DEBUG] Ran worker from request with execution ID: %s. Worker URL: %s. DEBUGGING:\n%s", workflowExecution.ExecutionId, streamUrl, debugCommand) diff --git a/functions/onprem/worker/README.md b/functions/onprem/worker/README.md new file mode 100644 index 00000000..ba6c2a9d --- /dev/null +++ b/functions/onprem/worker/README.md @@ -0,0 +1,17 @@ +# Worker +A worker implementation in Golang. This runs ALL Shuffle workflows onprem. In general receives jobs from Orborus. + +## Development +The ideal way to test the Worker is with a single workflow execution, standalone. Here are some environment variables you can use: + +``` +# Control what to run +export ENVIRONMENT_NAME="" +export SHUFFLE_CLOUDRUN_URL="https://shuffler.io" +export AUTHORIZATION="" +export EXECUTIONID="" + +# Control debugging and shutdown mechanisms +export DEBUG="true" +export SHUFFLE_WORKER_SHUTDOWN_DISABLED="true" +``` diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 1df6cb59..ded2c196 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -5,12 +5,14 @@ go 1.24.0 toolchain go1.24.4 //replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared +//replace github.com/shuffle/singul => ../../../../singul require ( - github.com/docker/docker v28.2.2+incompatible + github.com/docker/docker v28.3.3+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.84 + github.com/shuffle/shuffle-shared v0.9.14 + github.com/shuffle/singul v0.0.16 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 k8s.io/client-go v0.33.1 @@ -32,7 +34,6 @@ require ( cloud.google.com/go/scheduler v1.11.7 // indirect cloud.google.com/go/storage v1.55.0 // indirect dario.cat/mergo v1.0.0 // indirect - github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect @@ -43,7 +44,6 @@ require ( github.com/algolia/algoliasearch-client-go/v3 v3.31.4 // indirect github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf // indirect github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect - github.com/cenkalti/backoff/v5 v5.0.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.6.1 // indirect github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect @@ -60,7 +60,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/kin-openapi v0.42.0 // indirect - github.com/frikky/schemaless v0.0.16 // indirect + github.com/frikky/schemaless v0.0.20 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect @@ -83,13 +83,13 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect github.com/googleapis/gax-go/v2 v2.14.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/sys/sequential v0.6.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -97,17 +97,19 @@ require ( github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opensearch-project/opensearch-go v1.1.0 // indirect github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect + github.com/osteele/liquid v1.7.0 // indirect + github.com/osteele/tuesday v1.0.3 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect - github.com/sashabaranov/go-openai v1.40.1 // indirect + github.com/sashabaranov/go-openai v1.40.5 // indirect github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/sendgrid/sendgrid-go v3.16.1+incompatible // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect @@ -116,20 +118,18 @@ require ( go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect go.opentelemetry.io/otel v1.36.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect go.opentelemetry.io/otel/metric v1.36.0 // indirect go.opentelemetry.io/otel/sdk v1.36.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect go.opentelemetry.io/otel/trace v1.36.0 // indirect - go.opentelemetry.io/proto/otlp v1.5.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.38.0 // indirect - golang.org/x/net v0.40.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/net v0.41.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.14.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/term v0.32.0 // indirect - golang.org/x/text v0.25.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/term v0.33.0 // indirect + golang.org/x/text v0.27.0 // indirect golang.org/x/time v0.11.0 // indirect google.golang.org/api v0.236.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index 0ef2d3a6..2834c50b 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -116,8 +116,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw= -github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -142,8 +142,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.16 h1:4d2ZktB9xGsAusbbKliOI8TuriSrdIMzD/6ToY3wkz8= -github.com/frikky/schemaless v0.0.16/go.mod h1:jT48kTcmr1q3o8i+8qe7g+eCsbwaz2Q9CjOJevQQzQs= +github.com/frikky/schemaless v0.0.20 h1:S/A2pQcRN9qa2RnufvxwCeM06trjG0JLTF3urt1tFQI= +github.com/frikky/schemaless v0.0.20/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -241,8 +241,8 @@ github.com/googleapis/gax-go/v2 v2.14.2 h1:eBLnkZ9635krYIPD+ag1USrOAI0Nr0QYF3+/3 github.com/googleapis/gax-go/v2 v2.14.2/go.mod h1:ON64QhlJkhVtSqp4v1uaK92VyZ2gmvDQsweuyLV+8+w= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 h1:VNqngBF40hVlDloBruUehVYC3ArSgIyScOAyMRqBxRg= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1/go.mod h1:RBRO7fro65R6tjKzYgLAFo0t1QEXY1Dp+i/bvpRiqiQ= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -301,6 +301,10 @@ github.com/opensearch-project/opensearch-go v1.1.0 h1:eG5sh3843bbU1itPRjA9QXbxcg github.com/opensearch-project/opensearch-go v1.1.0/go.mod h1:+6/XHCuTH+fwsMJikZEWsucZ4eZMma3zNSeLrTtVGbo= github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ= github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8= +github.com/osteele/liquid v1.7.0 h1:VsbPSchE5D5S5scylAIvERET4dnCxsO6IDri2oSJ5Dk= +github.com/osteele/liquid v1.7.0/go.mod h1:xU0Z2dn2hOQIEFEWNmeltOmCtfhtoW/2fCyiNQeNG+U= +github.com/osteele/tuesday v1.0.3 h1:SrCmo6sWwSgnvs1bivmXLvD7Ko9+aJvvkmDjB5G4FTU= +github.com/osteele/tuesday v1.0.3/go.mod h1:pREKpE+L03UFuR+hiznj3q7j3qB1rUZ4XfKejwWFF2M= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= @@ -316,8 +320,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= -github.com/sashabaranov/go-openai v1.40.1 h1:bJ08Iwct5mHBVkuvG6FEcb9MDTfsXdTYPGjYLRdeTEU= -github.com/sashabaranov/go-openai v1.40.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/sashabaranov/go-openai v1.40.5 h1:SwIlNdWflzR1Rxd1gv3pUg6pwPc6cQ2uMoHs8ai+/NY= +github.com/sashabaranov/go-openai v1.40.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0= @@ -326,8 +330,10 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.84 h1:ElIMQYjKBVOiadbiGkSzt/lPU5xqaQwRxQvk9wx/xYM= -github.com/shuffle/shuffle-shared v0.8.84/go.mod h1:RdfNxqCPI+zU4jQKy3E/p4Io2injm7LpSKQUCDHNtLk= +github.com/shuffle/shuffle-shared v0.9.14 h1:POkTHO+bByuv8HiKuCMSGgtpDKk86ISr6ooLG8vQfuE= +github.com/shuffle/shuffle-shared v0.9.14/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw= +github.com/shuffle/singul v0.0.16 h1:dW+0Mln9R1aUJ0fjikpWcxbjoQWqJHxe4kSxh2tQN5E= +github.com/shuffle/singul v0.0.16/go.mod h1:LYkp320A6gsoPlYbXUM+WvEPUVAuutlSsqnVKyRy4gs= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -335,8 +341,8 @@ github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnB github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -393,8 +399,8 @@ go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFw go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= -go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= -go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= +go.opentelemetry.io/proto/otlp v1.6.0 h1:jQjP+AQyTf+Fe7OKj/MfkDrmK4MNVtw2NpXsf9fefDI= +go.opentelemetry.io/proto/otlp v1.6.0/go.mod h1:cicgGehlFuNdgZkcALOCh3VE6K/u2tAjzlRhDwmVpZc= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -404,8 +410,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= -golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -454,8 +460,8 @@ golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= -golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -471,8 +477,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= -golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -491,20 +497,19 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= -golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= +golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -515,8 +520,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= @@ -548,8 +553,8 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= +golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 2fbf51e3..60efa0c7 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -2,6 +2,7 @@ package main import ( "github.com/shuffle/shuffle-shared" + "github.com/shuffle/singul/pkg" "bytes" "context" @@ -23,7 +24,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" - "github.com/docker/docker/api/types/image" + dockerimage "github.com/docker/docker/api/types/image" "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/network" dockerclient "github.com/docker/docker/client" @@ -57,6 +58,7 @@ var logsDisabled = os.Getenv("SHUFFLE_LOGS_DISABLED") var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP")) var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME") var dockerApiVersion = strings.ToLower(os.Getenv("DOCKER_API_VERSION")) +var shutdownDisabled = strings.ToLower(os.Getenv("SHUFFLE_WORKER_SHUTDOWN_DISABLED")) // Kubernetes settings var appServiceAccountName = os.Getenv("SHUFFLE_APP_SERVICE_ACCOUNT_NAME") @@ -289,7 +291,13 @@ func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.Workflo // removes every container except itself (worker) func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason string, handleResultSend bool) { log.Printf("[DEBUG][%s] Shutdown (%s) started with reason %#v. Result amount: %d. ResultsSent: %d, Send result: %#v, Parent: %#v", workflowExecution.ExecutionId, workflowExecution.Status, reason, len(workflowExecution.Results), requestsSent, handleResultSend, workflowExecution.ExecutionParent) - //reason := "Error in execution" + + // This is an escape hatch for development only + // Typically meant to be used when you aren't sure how to make a workflow run in bad scenarios, and want to rapidly debug it. + if shutdownDisabled == "true" { + log.Printf("[ERROR] Shutdown disabled: NOT shutting down. This should ONLY be used for development & debugging.") + os.Exit(3) + } sleepDuration := 1 if handleResultSend && requestsSent < 2 { @@ -404,6 +412,8 @@ func deployk8sApp(image string, identifier string, env []string) error { kubernetesNamespace = "default" } + log.Printf("[DEBUG] Deploying k8s app with identifier %s to namespace %s", identifier, kubernetesNamespace) + deployport, err := strconv.Atoi(os.Getenv("SHUFFLE_APP_EXPOSED_PORT")) if err != nil { deployport = 80 @@ -422,6 +432,10 @@ func deployk8sApp(image string, identifier string, env []string) error { envMap["SHUFFLE_SWARM_CONFIG"] = os.Getenv("SHUFFLE_SWARM_CONFIG") envMap["BASE_URL"] = "http://shuffle-workers:33333" + if len(os.Getenv("SHUFFLE_LOGS_DISABLED")) > 0 { + envMap["SHUFFLE_LOGS_DISABLED"] = os.Getenv("SHUFFLE_LOGS_DISABLED") + } + clientset, _, err := shuffle.GetKubernetesClient() if err != nil { log.Printf("[ERROR] Failed getting kubernetes: %s", err) @@ -598,8 +612,8 @@ func deployk8sApp(image string, identifier string, env []string) error { // use deployment instead of pod // then expose a service similarly. // number of replicas can be set to os.Getenv("SHUFFLE_SCALE_REPLICAS") - replicaNumberStr := os.Getenv("SHUFFLE_SCALE_REPLICAS") replicaNumber := 1 + replicaNumberStr := os.Getenv("SHUFFLE_SCALE_REPLICAS") if len(replicaNumberStr) > 0 { tmpInt, err := strconv.Atoi(replicaNumberStr) if err != nil { @@ -679,6 +693,10 @@ func deployk8sApp(image string, identifier string, env []string) error { return err } + // Giving the service time to start before we contineu anything + log.Printf("[DEBUG] Waiting 20 seconds before moving on to let app '%s' start properly. Service: %s (k8s)", name, image) + time.Sleep(20 * time.Second) + return nil } @@ -840,14 +858,17 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] // Running as coroutine for eventual completeness // FIXME: With goroutines it got too much trouble of deploying with an older version // Allowing slow startups, as long as it's eventually fast, and uses the same registry as on host. - shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image) + err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image) + if err == nil { + downloadedImages = append(downloadedImages, image) + } } var exposedPort int var err error if isKubernetes != "true" { - exposedPort, err = findAppInfo(image, appName) + exposedPort, err = findAppInfo(image, appName, false) if err != nil { log.Printf("[ERROR] Failed finding and creating port for %s: %s", appName, err) return err @@ -855,6 +876,11 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] } else { // ** STARTREMOVE ***/ exposedPort = 80 + //deployport, err := strconv.Atoi(os.Getenv("SHUFFLE_APP_EXPOSED_PORT")) + //if err == nil { + // exposedPort = deployport + //} + err = findAppInfoKubernetes(image, appName, env) if err != nil { log.Printf("[ERROR] Failed finding and creating port for %s: %s", appName, err) @@ -882,7 +908,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] waitTime := time.Duration(action.ExecutionDelay) * time.Second time.AfterFunc(waitTime, func() { - err = sendAppRequest(ctx, baseUrl, appName, exposedPort, &action, &workflowExecution) + err = sendAppRequest(ctx, baseUrl, appName, exposedPort, &action, &workflowExecution, image, 0) if err != nil { log.Printf("[ERROR] Failed sending SCHEDULED request to app %s on port %d: %s", appName, exposedPort, err) } @@ -897,7 +923,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() // Cancel the context to release resources even if not used - go sendAppRequest(ctx, baseUrl, appName, exposedPort, &action, &workflowExecution) + go sendAppRequest(ctx, baseUrl, appName, exposedPort, &action, &workflowExecution, image, 0) }) } @@ -1415,12 +1441,37 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { appname = strings.Replace(appname, ".", "-", -1) appversion = strings.Replace(appversion, ".", "-", -1) + action, _ = singul.HandleSingulStartnode(workflowExecution, action, []string{}) + parsedAppname := strings.Replace(strings.ToLower(action.AppName), " ", "-", -1) + //if strings.ToLower(parsedAppname) == "singul" { + // parsedAppname = "shuffle-ai" + // appversion = "1.0.0" + // appname = "shuffle-ai" + //} + imageName := fmt.Sprintf("%s:%s_%s", baseimagename, parsedAppname, action.AppVersion) if strings.Contains(imageName, " ") { imageName = strings.ReplaceAll(imageName, " ", "-") } + // Kubernetes specific. + // Should it be though? + if isKubernetes == "true" { + // Map it to: + // /baseimagename/: + localRegistry := os.Getenv("REGISTRY_URL") + if len(localRegistry) > 0 && len(baseimagename) > 0 { + + newImageName := fmt.Sprintf("%s/%s/%s:%s", localRegistry, baseimagename, parsedAppname, action.AppVersion) + + log.Printf("[INFO] Remapping image name %s to %s due to registry+image name existing on k8s", imageName, newImageName) + + imageName = newImageName + + } + } + askOtherWorkersToDownloadImage(imageName) // Added UUID to identifier just in case @@ -1584,7 +1635,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { // This is the weirdest shit ever looking back at // Needs optimization lol - pullOptions := image.PullOptions{} + pullOptions := dockerimage.PullOptions{} if strings.ToLower(cleanupEnv) == "true" { err = deployApp(dockercli, images[0], identifier, env, workflowExecution, action) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { @@ -1598,6 +1649,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { executed := false if err == nil { log.Printf("[DEBUG] Downloaded image %s from backend (CLEANUP)", imageName) + downloadedImages = append(downloadedImages, imageName) //err = deployApp(dockercli, image, identifier, env, workflow, action) err = deployApp(dockercli, imageName, identifier, env, workflowExecution, action) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { @@ -1712,6 +1764,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { executed := false if err == nil { log.Printf("[DEBUG] Downloaded image %s from backend (CLEANUP)", imageName) + downloadedImages = append(downloadedImages, imageName) //err = deployApp(dockercli, image, identifier, env, workflow, action) err = deployApp(dockercli, imageName, identifier, env, workflowExecution, action) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { @@ -1972,7 +2025,7 @@ func executionInit(workflowExecution shuffle.WorkflowExecution) error { return errors.New(fmt.Sprintf("No apps to handle onprem (%s)", environment)) } - pullOptions := image.PullOptions{} + pullOptions := dockerimage.PullOptions{} _ = pullOptions for _, image := range onpremApps { //log.Printf("[INFO] Image: %s", image) @@ -3126,10 +3179,35 @@ func findActiveSwarmNodes(dockercli *dockerclient.Client) (int64, error) { } /*** STARTREMOVE ***/ -func deploySwarmService(dockercli *dockerclient.Client, name, image string, deployport int) error { +func deploySwarmService(dockercli *dockerclient.Client, name, image string, deployport int, retry bool) error { log.Printf("[DEBUG] Deploying service for %s to swarm on port %d", name, deployport) //containerName := fmt.Sprintf("shuffle-worker-%s", parsedUuid) + + + // Check if the image exists or not - just in case + _, _, err := dockercli.ImageInspectWithRaw(context.Background(), image) + if err != nil { + log.Printf("[INFO] Image %s not found locally. Pulling from registry...", image) + + localRegistry := os.Getenv("REGISTRY_URL") + if !strings.HasPrefix(image, localRegistry) && len(localRegistry) > 0 { + image = fmt.Sprintf("%s/%s", localRegistry, image) + log.Printf("[DEBUG] Changed image to %s", image) + } + + _, err := dockercli.ImagePull( + context.Background(), + image, + dockerimage.PullOptions{}, + ) + if err != nil { + log.Printf("[ERROR] Failed pulling image %s: %s", image, err) + return err + } + } + + if len(baseimagename) == 0 || baseimagename == "/" { baseimagename = "frikky/shuffle" //var baseimagename = "frikky/shuffle" @@ -3142,7 +3220,11 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl networkName = swarmNetworkName } + // Apps used a lot should have 2 replicas (default) replicas := uint64(1) + //if (strings.Contains(strings.ToLower(name), "shuffle") && strings.Contains(strings.ToLower(name), "tools")) || strings.Contains(strings.ToLower(name), "http") { + // replicas = 2 + //} // Sent from Orborus // Should be equal to @@ -3287,6 +3369,21 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl _ = service if err != nil { + + if strings.Contains(fmt.Sprintf("%s", err), "network") && strings.Contains(fmt.Sprintf("%s", err), "not found") { + log.Printf("[DEBUG] Network %s not found. Trying to initialize it.", networkName) + networkErr := initSwarmNetwork() + if networkErr != nil { + log.Printf("[ERROR] Failed initializing swarm network: %s", err) + //return err + } + + // Retry deploying the service (once) + if !retry { + return deploySwarmService(dockercli, name, image, deployport, true) + } + } + log.Printf("[DEBUG] Failed deploying %s with image %s: %s", name, image, err) return err } @@ -3298,6 +3395,163 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl /*** ENDREMOVE ***/ +func findAppInfo(image, name string, redeploy bool) (int, error) { + + highest := baseport + exposedPort := -1 + + // Exists as a "cache" layer + if portMappings != nil { + for key, value := range portMappings { + if value > highest { + highest = value + } + + if key == name { + exposedPort = value + break + } + } + } else { + portMappings = make(map[string]int) + } + + //Filters: + if exposedPort == -1 || redeploy { + dockercli, err := dockerclient.NewEnvClient() + if err != nil { + log.Printf("[ERROR] Unable to create docker client (2): %s", err) + return -1, err + } + + serviceListOptions := types.ServiceListOptions{} + services, err := dockercli.ServiceList( + context.Background(), + serviceListOptions, + ) + + // Basic self-correction + if err != nil { + log.Printf("[ERROR] Unable to list services: %s (may continue anyway?)", err) + if strings.Contains(fmt.Sprintf("%s", err), "is too new") { + // Static for some reason + defaultVersion := "1.40" + dockerApiVersion = defaultVersion + os.Setenv("DOCKER_API_VERSION", defaultVersion) + log.Printf("[DEBUG] Setting Docker API to %s default and retrying listing requests", defaultVersion) + } else { + return -1, err + } + + services, err = dockercli.ServiceList( + context.Background(), + serviceListOptions, + ) + + if err != nil { + log.Printf("[ERROR] Unable to list services (2): %s", err) + return -1, err + } + } + + for _, service := range services { + //log.Printf("[INFO] Service: %#v. Ports: %#v", service.Spec.Annotations.Name, service.Spec.EndpointSpec) + + for _, endpoint := range service.Spec.EndpointSpec.Ports { + if !strings.Contains(endpoint.Name, "port") { + continue + } + + portMappings[service.Spec.Annotations.Name] = int(endpoint.PublishedPort) + if int(endpoint.PublishedPort) > highest { + highest = int(endpoint.PublishedPort) + } + + if service.Spec.Annotations.Name == name || service.Spec.Annotations.Name == strings.Replace(name, ".", "-", -1) { + exposedPort = int(endpoint.PublishedPort) + //break + } + } + + if service.Spec.Annotations.Name != name && service.Spec.Annotations.Name != strings.Replace(name, ".", "-", -1) { + continue + } + + if redeploy { + // Remove the service and redeploy it. + // There are cases where the service doesn't update properly + // Check when the last update happened. If it was within the last 5 minutes, skip + if int(time.Since(service.UpdatedAt).Seconds()) > 600 { + + log.Printf("[INFO] Attempting redeploy of app %s with image %s since it is more than 10 minutes since last attempt with failure.", name, image) + + err = dockercli.ServiceRemove( + context.Background(), + service.ID, + ) + if err != nil { + log.Printf("[ERROR] Failed auto-removing service %s: %s", name, err) + } else { + log.Printf("[INFO] Auto-removed service %s successfully (rebuild due to redeploy).", name) + time.Sleep(10 * time.Second) + err = deploySwarmService( + dockercli, + name, + image, + exposedPort, + false, + ) + if err != nil { + log.Printf("[ERROR] Failed re-deploying service %s: %s", name, err) + } else { + time.Sleep(10 * time.Second) + } + } + } else { + //log.Printf("[INFO] NOT redeploying service %s since it was updated less than 10 minutes ago.", name) + } + } + + // Break if it's the correct port, as it's the right service + if exposedPort >= 0 { + break + } + } + } + + //log.Printf("[DEBUG] Portmappings: %#v", portMappings) + + if exposedPort >= 0 { + //log.Printf("[INFO] Found service %s on port %d - no need to deploy another", name, exposedPort) + } else { + dockercli, err := dockerclient.NewEnvClient() + if err != nil { + log.Printf("[ERROR] Unable to create docker client (2): %s", err) + return -1, err + } + + // Increment by 1 for highest port + if highest <= baseport { + highest = baseport + } + + highest += 1 + err = deploySwarmService(dockercli, name, image, highest, false) + if err != nil { + log.Printf("[WARNING] NOT Found service: %s. error: %s", name, err) + return highest, err + } else { + log.Printf("[DEBUG] Waiting 20 seconds before moving on to let app '%s' start properly. Service: %s (swarm)", name, image) + time.Sleep(time.Duration(20) * time.Second) + } + + exposedPort = highest + //return exposedPort, errors.New("Deployed app %s") + } + + return exposedPort, nil +} + // Runs data discovery /*** STARTREMOVE ***/ @@ -3342,126 +3596,112 @@ func findAppInfoKubernetes(image, name string, env []string) error { return err } -func findAppInfo(image, name string) (int, error) { +// Backups in case networks are removed +func initSwarmNetwork() error { + ctx := context.Background() dockercli, err := dockerclient.NewEnvClient() if err != nil { log.Printf("[ERROR] Unable to create docker client (2): %s", err) - return -1, err + return err } - highest := baseport - exposedPort := -1 + // Create the network options with the specified MTU + options := make(map[string]string) + mtu := 1500 + options["com.docker.network.driver.mtu"] = fmt.Sprintf("%d", mtu) - // Exists as a "cache" layer - if portMappings != nil { - for key, value := range portMappings { - if value > highest { - highest = value - } - if key == name { - exposedPort = value - break - } - } - } else { - portMappings = make(map[string]int) + ingressOptions := network.CreateOptions{ + Driver: "overlay", + Attachable: false, + Ingress: true, + IPAM: &network.IPAM{ + Driver: "default", + Config: []network.IPAMConfig{ + network.IPAMConfig{ + Subnet: "10.225.225.0/24", + Gateway: "10.225.225.1", + }, + }, + }, } - //Filters: - if exposedPort == -1 { - serviceListOptions := types.ServiceListOptions{} - services, err := dockercli.ServiceList( - context.Background(), - serviceListOptions, - ) + _, err = dockercli.NetworkCreate( + ctx, + "ingress", + ingressOptions, + ) - // Basic self-correction - if err != nil { - log.Printf("[ERROR] Unable to list services: %s (may continue anyway?)", err) - if strings.Contains(fmt.Sprintf("%s", err), "is too new") { - // Static for some reason - defaultVersion := "1.40" - dockerApiVersion = defaultVersion - os.Setenv("DOCKER_API_VERSION", defaultVersion) - log.Printf("[DEBUG] Setting Docker API to %s default and retrying listing requests", defaultVersion) - } else { - return -1, err - } - - services, err = dockercli.ServiceList( - context.Background(), - serviceListOptions, - ) - - if err != nil { - log.Printf("[ERROR] Unable to list services (2): %s", err) - return -1, err - } - } - - for _, service := range services { - //log.Printf("[INFO] Service: %#v", service.Spec.Annotations.Name) - - for _, endpoint := range service.Spec.EndpointSpec.Ports { - if strings.Contains(endpoint.Name, "port") { - portMappings[service.Spec.Annotations.Name] = int(endpoint.PublishedPort) - if int(endpoint.PublishedPort) > highest { - highest = int(endpoint.PublishedPort) - } - - if service.Spec.Annotations.Name == name || service.Spec.Annotations.Name == strings.Replace(name, ".", "-", -1) { - exposedPort = int(endpoint.PublishedPort) - //break - } - } - } - - //log.Printf("%s - %s", service.Spec.Annotations.Name, strings.Replace(name, ".", "-", -1)) - if service.Spec.Annotations.Name != name && service.Spec.Annotations.Name != strings.Replace(name, ".", "-", -1) { - continue - } - - // Break if it's the correct port, as it's the right service - if exposedPort >= 0 { - break - } - } + if err != nil { + log.Printf("[WARNING] Ingress network may already exist: %s", err) } - //log.Printf("[DEBUG] Portmappings: %#v", portMappings) - - if exposedPort >= 0 { - //log.Printf("[INFO] Found service %s on port %d - no need to deploy another", name, exposedPort) - } else { - // Increment by 1 for highest port - if highest <= baseport { - highest = baseport - } - - highest += 1 - err = deploySwarmService(dockercli, name, image, highest) - if err != nil { - log.Printf("[WARNING] NOT Found service: %s. error: %s", name, err) - return highest, err - } else { - log.Printf("[DEBUG] Deployed app with name %s", name) - } - - exposedPort = highest - - if appsInitialized { - log.Printf("[DEBUG] Waiting 30 seconds before moving on to let app start") - time.Sleep(time.Duration(30) * time.Second) - } + //docker network create --driver=overlay workers + // Specific subnet? + networkName := "shuffle_swarm_executions" + if len(swarmNetworkName) > 0 { + networkName = swarmNetworkName } - return exposedPort, nil + networkCreateOptions := network.CreateOptions{ + Driver: "overlay", + Options: options, + Attachable: true, + Ingress: false, + IPAM: &network.IPAM{ + Driver: "default", + Config: []network.IPAMConfig{ + network.IPAMConfig{ + Subnet: "10.224.224.0/24", + Gateway: "10.224.224.1", + }, + }, + }, + } + _, err = dockercli.NetworkCreate( + ctx, + networkName, + networkCreateOptions, + ) + + if err != nil { + log.Printf("[WARNING] Swarm Executions network may already exist: %s", err) + } + + networkName = "shuffle-executions" + networkCreateOptions = network.CreateOptions{ + Driver: "overlay", + Options: options, + Attachable: true, + Ingress: false, + IPAM: &network.IPAM{ + Driver: "default", + Config: []network.IPAMConfig{ + network.IPAMConfig{ + Subnet: "10.223.223.0/24", + Gateway: "10.223.223.1", + }, + }, + }, + } + _, err = dockercli.NetworkCreate( + ctx, + networkName, + networkCreateOptions, + ) + + if err != nil { + log.Printf("[WARNING] Swarm Executions network may already exist: %s", err) + } + + return nil } + + /*** ENDREMOVE ***/ -func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, action *shuffle.Action, workflowExecution *shuffle.WorkflowExecution) error { +func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, action *shuffle.Action, workflowExecution *shuffle.WorkflowExecution, image string, attempts int64) error { parsedRequest := shuffle.OrborusExecutionRequest{ Cleanup: cleanupEnv, ExecutionId: workflowExecution.ExecutionId, @@ -3608,12 +3848,28 @@ func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, //var portMappings map[string]int } + // Try redeployment + attempts += 1 + if attempts < 2 { + // Check the service and fix it. + if isKubernetes == "true" { + log.Printf("[WARNING] App Redeployment in K8s isn't fully supported yet, but should be done for app %s with image %s.", appName, image) + } else { + _, err = findAppInfo(image, appName, true) + if err != nil { + log.Printf("[ERROR][%s] Error re-deploying app %s: %s", workflowExecution.ExecutionId, appName, err) + } + + return sendAppRequest(ctx, incomingUrl, appName, port, action, workflowExecution, image, attempts) + } + } + log.Printf("[ERROR][%s] Error running app run request: %s", workflowExecution.ExecutionId, err) actionResult := shuffle.ActionResult{ Action: *action, ExecutionId: workflowExecution.ExecutionId, Authorization: workflowExecution.Authorization, - Result: fmt.Sprintf(`{"success": false, "reason": "Failed to connect to app %s in swarm. Try the action again, restart Orborus if this is recurring, or contact support@shuffler.io.", "details": "%s"}`, streamUrl, newerr), + Result: fmt.Sprintf(`{"success": false, "attempts": %d, "reason": "Failed to connect to app %s in swarm. Try the action again, restart Orborus if this is recurring, or contact support@shuffler.io.", "details": "%s"}`, attempts, streamUrl, newerr), StartedAt: int64(time.Now().Unix()), CompletedAt: int64(time.Now().Unix()), Status: "FAILURE", @@ -3643,7 +3899,7 @@ func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, func baseDeploy() { var cli *dockerclient.Client - var err error + //var err error if isKubernetes != "true" { cli, err := dockerclient.NewEnvClient() @@ -3705,8 +3961,7 @@ func baseDeploy() { //deployApp(cli, value, identifier, env, workflowExecution, action) log.Printf("[DEBUG] Deploying app with identifier %s to ensure basic apps are available from the get-go", identifier) - err = deployApp(cli, value, identifier, env, workflowExecution, action) - _ = err + go deployApp(cli, value, identifier, env, workflowExecution, action) //err := deployApp(cli, value, identifier, env, workflowExecution, action) //if err != nil { // log.Printf("[DEBUG] Failed deploying app %s: %s", value, err) @@ -3735,7 +3990,7 @@ func getStreamResultsWrapper(client *http.Client, req *http.Request, workflowExe } if newresp.StatusCode != 200 { - log.Printf("[ERROR] %sStatusCode (1): %d", string(body), newresp.StatusCode) + log.Printf("[ERROR] StatusCode (1): %d - %s", newresp.StatusCode, string(body)) time.Sleep(time.Duration(sleepTime) * time.Second) return environments, errors.New(fmt.Sprintf("Bad status code: %d", newresp.StatusCode)) } @@ -3864,6 +4119,10 @@ func checkStandaloneRun() { // Check if the required argc/argv is set //log.Printf("ARGS: %#v", os.Args) if len(os.Args) < 4 { + if debug { + log.Printf("[DEBUG] You can run the worker in standalone mode with: go run worker.go standalone ") + } + return } @@ -4022,16 +4281,24 @@ func checkStandaloneRun() { // Initial loop etc func main() { + // Testing swarm auto-replacements. + //findAppInfo("frikky/shuffle:shuffle-ai_1.0.0", "shuffle-ai_1-0-0", true) + //findAppInfo("frikky/shuffle:shuffle-ai_1.0.0", "singul_1-0-0", true) + checkStandaloneRun() if os.Getenv("DEBUG") == "true" { debug = true + + log.Printf("[INFO] Disabled cleanup due to debug mode (DEBUG=true)") + cleanupEnv = "false" } /*** STARTREMOVE ***/ if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || os.Getenv("SHUFFLE_SWARM_CONFIG") == "swarm" { logsDisabled = "true" } + /*** ENDREMOVE ***/ // Elasticsearch necessary to ensure we'ren ot running with Datastore configurations for minimal/maximal data sizes // Recursive import kind of :) @@ -4384,7 +4651,7 @@ func handleDownloadImage(resp http.ResponseWriter, request *http.Request) { // check if images are already downloaded // Retrieve a list of Docker images - listOptions := image.ListOptions{} + listOptions := dockerimage.ListOptions{} images, err := client.ImageList(context.Background(), listOptions) if err != nil { log.Printf("[ERROR] listing images: %s", err) @@ -4415,7 +4682,10 @@ func handleDownloadImage(resp http.ResponseWriter, request *http.Request) { } log.Printf("[INFO] Downloading image %s", imageBody.Image) - shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, imageBody.Image) + err = shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, imageBody.Image) + if err == nil { + downloadedImages = append(downloadedImages, imageBody.Image) + } // return success resp.WriteHeader(200) diff --git a/functions/usecases/README.md b/functions/usecases/README.md deleted file mode 100644 index 553034c2..00000000 --- a/functions/usecases/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Mindmap exporter -Shuffle has a mindmap for Workflow use-cases. These can be changed and exported, with the most important piece being that they're explorable and editable. This has and will come in handy for us as we build it into the product. - -https://www.mindmeister.com/map/2172644474 - -## Editing the Mindmap -There are a few categories. To edit them, click the small plus next to the branch you want to change. - -## Exporting the Mindmap -Click "Export as RTF" in the top left corner of the URL. Download it there. - -## Generating the Shuffle-comaptible mindmap -1. Move the rtf file here -2. Rename it categories.rtf -3. Run the read_categories.py file (python3 read_categories.py) -4. You now have a file called categories.json locally with all the categories in JSON format, ready to be used in graphs. diff --git a/functions/usecases/categories.json b/functions/usecases/categories.json deleted file mode 100644 index e438a380..00000000 --- a/functions/usecases/categories.json +++ /dev/null @@ -1,260 +0,0 @@ -[ - { - "name": "1. Collect & Distribute", - "color": "#c51152", - "list": [ - { - "name": "2-way Ticket synchronization", - "items": {} - }, - { - "name": "Email management", - "items": { - "name": "Release a quarantined message", - "items": {} - } - }, - { - "name": "EDR to ticket", - "items": { - "name": "Get host information", - "items": {} - } - }, - { - "name": "SIEM to ticket", - "items": {} - }, - { - "name": "ChatOps", - "items": {} - }, - { - "name": "Threat Intel received", - "items": {} - }, - { - "name": "Domain investigation with LetsEncrypt", - "items": {} - }, - { - "name": "Botnet tracker", - "items": {} - }, - { - "name": "Get running containers", - "items": {} - }, - { - "name": "Assign tickets", - "items": {} - }, - { - "name": "Firewall alerts", - "items": { - "name": "URL filtering", - "items": {} - } - }, - { - "name": "IDS/IPS alerts", - "items": { - "name": "Manage policies", - "items": {} - } - }, - { - "name": "Deduplicate information", - "items": {} - }, - { - "name": "Correlate information", - "items": {} - } - ] - }, - { - "name": "2. Enrich", - "color": "#f4c20d", - "list": [ - { - "name": "Internal Enrichment", - "items": { - "name": "...", - "items": {} - } - }, - { - "name": "External historical Enrichment", - "items": { - "name": "...", - "items": {} - } - }, - { - "name": "Realtime", - "items": { - "name": "Analyze screenshots", - "items": {} - } - }, - { - "name": "Ticketing webhook verification", - "items": {} - } - ] - }, - { - "name": "3. Detect", - "color": "#3cba54", - "list": [ - { - "name": "Search SIEM (Sigma)", - "items": { - "name": "Endpoint", - "items": {} - } - }, - { - "name": "Search EDR (OSQuery)", - "items": {} - }, - { - "name": "Search emails (Phish)", - "items": { - "name": "Check headers and IOCs", - "items": {} - } - }, - { - "name": "Search IOCs (ioc-finder)", - "items": {} - }, - { - "name": "Search files (Yara)", - "items": {} - }, - { - "name": "Correlate tickets", - "items": {} - }, - { - "name": "Honeypot access", - "items": { - "name": "...", - "items": {} - } - } - ] - }, - { - "name": "4. Respond", - "color": "#4a148c", - "list": [ - { - "name": "Eradicate malware", - "items": {} - }, - { - "name": "Quarantine host(s)", - "items": {} - }, - { - "name": "Trigger scans", - "items": {} - }, - { - "name": "Update indicators (FW, EDR, SIEM...)", - "items": {} - }, - { - "name": "Autoblock activity when threat intel is received", - "items": {} - }, - { - "name": "Lock/Delete/Reset account", - "items": {} - }, - { - "name": "Lock vault", - "items": {} - }, - { - "name": "Increase authentication", - "items": {} - }, - { - "name": "Get policies from assets", - "items": {} - } - ] - }, - { - "name": "5. Verify", - "color": "#4885ed", - "list": [ - { - "name": "Discover vulnerabilities", - "items": {} - }, - { - "name": "Discover assets", - "items": {} - }, - { - "name": "Ensure policies are followed", - "items": {} - }, - { - "name": "Find Inactive users", - "items": {} - }, - { - "name": "Ensure access rights match HR systems", - "items": {} - }, - { - "name": "Ensure onboarding is followed", - "items": {} - }, - { - "name": "Third party apps in SaaS", - "items": {} - }, - { - "name": "Devices used for your cloud account", - "items": {} - }, - { - "name": "Too much access in GCP/Azure/AWS/ other clouds", - "items": {} - }, - { - "name": "Certificate validation", - "items": {} - }, - { - "name": "Monitor new DNS entries for domain with passive DNS", - "items": {} - }, - { - "name": "Monitor and track password dumps", - "items": {} - }, - { - "name": "Monitor for mentions of domain on darknet sites", - "items": {} - }, - { - "name": "Reporting", - "items": { - "name": "Monthly reports", - "items": { - "name": "...", - "items": {} - } - } - } - ] - } -] \ No newline at end of file diff --git a/functions/usecases/categories.rtf b/functions/usecases/categories.rtf deleted file mode 100644 index e19cf26a..00000000 --- a/functions/usecases/categories.rtf +++ /dev/null @@ -1,555 +0,0 @@ -{\rtf1\ansi\deff0\deflang2057\plain\fs24\fet1 -{\fonttbl -{\f0\froman Arial;} -} -{\info -{\createim\yr2022\mo2\dy20\hr1\min15} -} - -\paperw11907\paperh16840\margl1800\margr1800\margt1440\margb1440 -\slmult0\ltrpar\li0 -{\b\fs28 -Shuffle categories -} -\par\pard\plain -\slmult0\ltrpar\li200 -{\fs24 -1. Collect & Distribute -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -2-way Ticket synchronization -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Email management -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Attachments -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Manage senders -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Manage URLs -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Encode & Decode URLs -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Release a quarantined message -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -EDR to ticket -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Fetch incidents & events -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Quarantine files -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Quarantine host (respond) -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Get host information -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -SIEM to ticket -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -ChatOps -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Threat Intel received -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Domain investigation with LetsEncrypt -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Botnet tracker -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Get running containers -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Assign tickets -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Firewall alerts -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Block/accept policies -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Add addresses and ports to groups -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Support custom URL categories -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Fetch logs for specific address -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -URL filtering -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -IDS/IPS alerts -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Get/Fetch alerts -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Receive alerts real-time -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Get PCAP files -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Get network logs -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Manage policies -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Deduplicate information -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Correlate information -} -\par\pard\plain -\slmult0\ltrpar\li200 -{\fs24 -3. Detect -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Search SIEM (Sigma) -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Network -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Endpoint -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Search EDR (OSQuery) -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Search emails (Phish) -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Check malware -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Check targeted -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Check headers and IOCs -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Search IOCs (ioc-finder) -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Search files (Yara) -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Correlate tickets -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Honeypot access -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -S3 Honeypot -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -SSH Honeypot -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -FTP honeypot -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Network honeypot -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -... -} -\par\pard\plain -\slmult0\ltrpar\li200 -{\fs24 -rich -} -\par\pard\plain -\slmult0\ltrpar\li200 -{\fs24 -5. Verify -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Discover vulnerabilities -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Discover assets -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Ensure policies are followed -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Find Inactive users -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Ensure access rights match HR systems -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Ensure onboarding is followed -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Third party apps in SaaS -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Devices used for your cloud account -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Too much access in GCP/Azure/AWS/ other clouds -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Certificate validation -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Monitor new DNS entries for domain with passive DNS -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Monitor and track password dumps -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Monitor for mentions of domain on darknet sites -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Reporting -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Automation time saved -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Automation money saved -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Incident response report -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Department cost -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Monthly reports -} -\par\pard\plain -\slmult0\ltrpar\li800 -{\fs24 -EDR alerts -} -\par\pard\plain -\slmult0\ltrpar\li800 -{\fs24 -SIEM alerts -} -\par\pard\plain -\slmult0\ltrpar\li800 -{\fs24 -Emails quarantined -} -\par\pard\plain -\slmult0\ltrpar\li800 -{\fs24 -... -} -\par\pard\plain -\slmult0\ltrpar\li200 -{\fs24 -4. Respond -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Eradicate malware -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Quarantine host(s) -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Trigger scans -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Update indicators (FW, EDR, SIEM...) -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Autoblock activity when threat intel is received -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Lock/Delete/Reset account -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Lock vault -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Increase authentication -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Get policies from assets -} -\par\pard\plain -\slmult0\ltrpar\li200 -{\fs24 -2. Enrich -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Internal Enrichment -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Users -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Hostnames -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -IPs -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Departments -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Role -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Software -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -... -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -External historical Enrichment -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -IPs -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -URLs -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Hashes -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Files -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -... -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Realtime -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -File detonation -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -URL detonation -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -PCAP analysis -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Analyze screenshots -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Ticketing webhook verification -} -\par\pard\plain -} \ No newline at end of file diff --git a/functions/usecases/read_categories.py b/functions/usecases/read_categories.py deleted file mode 100644 index 65df43c9..00000000 --- a/functions/usecases/read_categories.py +++ /dev/null @@ -1,66 +0,0 @@ -data = "" -with open("categories.rtf", "r") as tmp: - data = tmp.read() - -fixed_json = [] -linearity = 0 -heading = "" -subheading = "" -subsubheading = "" - -cnt = -1 -subcnt = -1 - -colors = ["#c51152", "#3cba54", "#4885ed", "#4a148c", "#f4c20d"] -for line in data.split("\n"): - if line == "rich": - continue - - if "li" in line: - lisplit = line.split("\\") - try: - linearity = int(lisplit[-1][2]) - except: - pass - - #print("Linearity: %s" % linearity) - - if line.startswith("{") or line.startswith("}"): - continue - - if line.startswith("\\"): - continue - - if linearity == 0: - continue - - if linearity == 2: - #if cnt >= 0: - # for key, value in fixed_json[cnt].items(): - # print(key, value) - - - cnt += 1 - subcnt = -1 - fixed_json.append({"name": line, "color": colors[cnt], "list": []}) - heading = line - elif linearity == 4: - subheading = line - fixed_json[cnt]["list"].append({"name": line, "items": {}}) - subcnt += 1 - elif linearity == 6: - fixed_json[cnt]["list"][subcnt]["items"] = {"name": line, "items": {}} - elif linearity == 8: - fixed_json[cnt]["list"][subcnt]["items"]["items"] = {"name": line, "items": {}} - else: - print("No handler for %s" % line) - -#print(line) -#print(data) -import json -filename = "categories.json" -fixed_json.sort(key=lambda x: x["name"]) -with open(filename, "w+") as tmp: - tmp.write(json.dumps(fixed_json, indent=4)) - -print("Wrote to file %s" % filename)