diff --git a/.env b/.env
index 53204db2..0b1e8cee 100755
--- a/.env
+++ b/.env
@@ -1,5 +1,4 @@
# Default execution environment for workers
-ORG_ID=Shuffle
ENVIRONMENT_NAME=Shuffle
# Sanitize liquid.py input
diff --git a/.github/install-guide.md b/.github/install-guide.md
index 5233ff47..cf2186e6 100755
--- a/.github/install-guide.md
+++ b/.github/install-guide.md
@@ -18,7 +18,7 @@ The Docker setup is the default setup, and is ran with docker compose. This is [
**PS: if you're setting up Shuffle on Windows, go to the next step (Windows Docker setup)**
-1. Make sure you have [Docker](https://docs.docker.com/get-docker/) installed, and that you have a minimum of **2Gb of RAM** available.
+1. Make sure you have [Docker](https://docs.docker.com/get-docker/) installed, and that you have a minimum of **4Gb of RAM** available.
2. Download Shuffle
```bash
git clone https://github.com/Shuffle/Shuffle
diff --git a/.github/workflows/helm-test.yml b/.github/workflows/helm-test.yml
new file mode 100644
index 00000000..fb0e34aa
--- /dev/null
+++ b/.github/workflows/helm-test.yml
@@ -0,0 +1,31 @@
+name: Helm Test
+
+on:
+ push:
+ paths:
+ - "functions/kubernetes/charts/**"
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+jobs:
+ main:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+
+ - name: Install apt dependencies
+ run: |
+ curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null
+ sudo apt-get install apt-transport-https -y --no-install-recommends
+ echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list
+ sudo apt-get update
+ sudo apt-get install helm -y --no-install-recommends
+
+ - name: Update helm dependencies
+ run: helm dependency update ./functions/kubernetes/charts/shuffle
+
+ - name: Template helm chart using default values
+ run: helm template shuffle ./functions/kubernetes/charts/shuffle --debug --values ./functions/kubernetes/charts/shuffle/values.yaml
diff --git a/.github/workflows/tagged-nightly-release.yaml b/.github/workflows/tagged-nightly-release.yaml
new file mode 100644
index 00000000..c24b7e1d
--- /dev/null
+++ b/.github/workflows/tagged-nightly-release.yaml
@@ -0,0 +1,92 @@
+name: Tagged Nightly Release
+on:
+ release:
+ types: [published]
+ branches:
+ - nightly
+
+# there is a very clear point to this existing.
+# we want to also release versions that look like this:
+# v2.1.0-nightly-date, v2.1.0-nightly-date-1, v2.1.0-nightly-date-2
+# we NEVER want to send customers a "nightly" tag. We always want to send them
+# a tagged nightly tag. So that when something breaks, They can always
+# point to it.
+
+jobs:
+ main:
+ runs-on: ubuntu-latest
+ continue-on-error: ${{ matrix.experimental }}
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - app: frontend
+ path: frontend
+ experimental: true
+ - app: backend
+ path: backend
+ experimental: true
+ - app: app_sdk
+ path: backend/app_sdk
+ experimental: true
+ - app: orborus
+ path: functions/onprem/orborus
+ experimental: true
+ - app: worker
+ path: functions/onprem/worker
+ experimental: true
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+
+ - name: Set version
+ id: set_version
+ run: |
+ if [[ ${{ github.event_name }} == 'release' ]]; then
+ echo "VERSION=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
+ else
+ echo "VERSION=nightly-untagged-latest" >> $GITHUB_OUTPUT
+ fi
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v3
+ with:
+ platforms: "amd64,arm64,arm"
+
+ - name: Login to DockerHub
+ uses: docker/login-action@v3
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ - name: Login to Ghcr
+ uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Ghcr Build and push
+ id: docker_build
+ uses: docker/build-push-action@v4
+ env:
+ BUILDX_NO_DEFAULT_LOAD: true
+ with:
+ logout: false
+ context: ${{ matrix.path }}/
+ file: ${{ matrix.path }}/Dockerfile
+ platforms: linux/amd64,linux/arm64
+ push: true
+ cache-from: type=local,src=/tmp/.buildx-cache
+ cache-to: type=local,dest=/tmp/.buildx-cache
+ tags: |
+ ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }}
+ ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }}
+ frikky/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }}
+ frikky/shuffle:${{ matrix.app }}
+
+ - name: Image digest
+ run: echo ${{ steps.docker_build.outputs.digest }}
\ No newline at end of file
diff --git a/backend/Dockerfile b/backend/Dockerfile
index 9e27ffbe..16ffeff0 100755
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -27,7 +27,7 @@ FROM alpine:latest as certs
RUN apk add --update ca-certificates
# Sets up the final image
-FROM alpine:3.21.2
+FROM alpine:3.22.1
# FIXME: Install cgo because CGO_ENABLED=1 during build
RUN apk add --no-cache libc6-compat
diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod
index 61a1a032..d978245c 100644
--- a/backend/go-app/go.mod
+++ b/backend/go-app/go.mod
@@ -4,8 +4,8 @@ go 1.24.0
toolchain go1.24.3
-//replace github.com/frikky/schemaless => ../../../schemaless
//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared
+//replace github.com/frikky/schemaless => ../../../schemaless
//replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi
@@ -14,17 +14,19 @@ require (
cloud.google.com/go/storage v1.55.0
github.com/basgys/goxml2json v1.1.0
github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82
- github.com/docker/docker v28.2.2+incompatible
+ github.com/docker/docker v28.3.3+incompatible
github.com/frikky/kin-openapi v0.42.0
github.com/fsouza/go-dockerclient v1.12.1
github.com/ghodss/yaml v1.0.0
+ github.com/go-co-op/gocron v1.37.0
github.com/go-git/go-billy/v5 v5.6.2
github.com/go-git/go-git/v5 v5.16.1
github.com/gorilla/mux v1.8.1
github.com/h2non/filetype v1.1.3
github.com/satori/go.uuid v1.2.0
- github.com/shuffle/shuffle-shared v0.8.84
- golang.org/x/crypto v0.39.0
+ github.com/shuffle/shuffle-shared v0.9.14
+ github.com/shuffle/singul v0.0.16
+ golang.org/x/crypto v0.40.0
google.golang.org/api v0.236.0
google.golang.org/grpc v1.72.2
gopkg.in/yaml.v3 v3.0.1
@@ -55,7 +57,6 @@ require (
github.com/bitly/go-simplejson v0.5.1 // indirect
github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf // indirect
github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect
- github.com/cenkalti/backoff/v5 v5.0.2 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cloudflare/circl v1.6.1 // indirect
github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect
@@ -72,7 +73,7 @@ require (
github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
- github.com/frikky/schemaless v0.0.16 // indirect
+ github.com/frikky/schemaless v0.0.20 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-jose/go-jose/v4 v4.0.5 // indirect
@@ -114,18 +115,21 @@ require (
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/opensearch-project/opensearch-go v1.1.0 // indirect
github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect
+ github.com/osteele/liquid v1.7.0 // indirect
+ github.com/osteele/tuesday v1.0.3 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
github.com/pjbgf/sha1cd v0.3.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
- github.com/sashabaranov/go-openai v1.40.1 // indirect
+ github.com/robfig/cron/v3 v3.0.1 // indirect
+ github.com/sashabaranov/go-openai v1.40.5 // indirect
github.com/sendgrid/rest v2.6.9+incompatible // indirect
github.com/sendgrid/sendgrid-go v3.16.1+incompatible // indirect
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/skeema/knownhosts v1.3.1 // indirect
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
- github.com/spf13/pflag v1.0.5 // indirect
+ github.com/spf13/pflag v1.0.6 // indirect
github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
@@ -135,19 +139,18 @@ require (
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
go.opentelemetry.io/otel v1.36.0 // indirect
- go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect
go.opentelemetry.io/otel/metric v1.36.0 // indirect
go.opentelemetry.io/otel/sdk v1.36.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect
go.opentelemetry.io/otel/trace v1.36.0 // indirect
- go.opentelemetry.io/proto/otlp v1.5.0 // indirect
+ go.uber.org/atomic v1.9.0 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
- golang.org/x/net v0.40.0 // indirect
+ golang.org/x/net v0.41.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
- golang.org/x/sync v0.15.0 // indirect
- golang.org/x/sys v0.33.0 // indirect
- golang.org/x/term v0.32.0 // indirect
- golang.org/x/text v0.26.0 // indirect
+ golang.org/x/sync v0.16.0 // indirect
+ golang.org/x/sys v0.34.0 // indirect
+ golang.org/x/term v0.33.0 // indirect
+ golang.org/x/text v0.27.0 // indirect
golang.org/x/time v0.11.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect
diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum
index 34b04706..71508c8f 100644
--- a/backend/go-app/go.sum
+++ b/backend/go-app/go.sum
@@ -126,8 +126,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
-github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw=
-github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
+github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
@@ -152,8 +152,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0=
github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4=
-github.com/frikky/schemaless v0.0.16 h1:4d2ZktB9xGsAusbbKliOI8TuriSrdIMzD/6ToY3wkz8=
-github.com/frikky/schemaless v0.0.16/go.mod h1:jT48kTcmr1q3o8i+8qe7g+eCsbwaz2Q9CjOJevQQzQs=
+github.com/frikky/schemaless v0.0.20 h1:S/A2pQcRN9qa2RnufvxwCeM06trjG0JLTF3urt1tFQI=
+github.com/frikky/schemaless v0.0.20/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
github.com/fsouza/go-dockerclient v1.12.1 h1:FMoLq+Zhv9Oz/rFmu6JWkImfr6CBgZOPcL+bHW4gS0o=
github.com/fsouza/go-dockerclient v1.12.1/go.mod h1:OqsgJJcpCwqyM3JED7TdfM9QVWS5O7jSYwXxYKmOooY=
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
@@ -162,6 +162,8 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
+github.com/go-co-op/gocron v1.37.0 h1:ZYDJGtQ4OMhTLKOKMIch+/CY70Brbb1dGdooLEhh7b0=
+github.com/go-co-op/gocron v1.37.0/go.mod h1:3L/n6BkO7ABj+TrfSVXLRzsP26zmikL4ISkLQ0O8iNY=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM=
@@ -243,6 +245,7 @@ github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAx
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
+github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4=
@@ -278,6 +281,7 @@ github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zt
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -325,10 +329,15 @@ github.com/opensearch-project/opensearch-go v1.1.0 h1:eG5sh3843bbU1itPRjA9QXbxcg
github.com/opensearch-project/opensearch-go v1.1.0/go.mod h1:+6/XHCuTH+fwsMJikZEWsucZ4eZMma3zNSeLrTtVGbo=
github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ=
github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8=
+github.com/osteele/liquid v1.7.0 h1:VsbPSchE5D5S5scylAIvERET4dnCxsO6IDri2oSJ5Dk=
+github.com/osteele/liquid v1.7.0/go.mod h1:xU0Z2dn2hOQIEFEWNmeltOmCtfhtoW/2fCyiNQeNG+U=
+github.com/osteele/tuesday v1.0.3 h1:SrCmo6sWwSgnvs1bivmXLvD7Ko9+aJvvkmDjB5G4FTU=
+github.com/osteele/tuesday v1.0.3/go.mod h1:pREKpE+L03UFuR+hiznj3q7j3qB1rUZ4XfKejwWFF2M=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4=
github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
@@ -336,12 +345,16 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
-github.com/sashabaranov/go-openai v1.40.1 h1:bJ08Iwct5mHBVkuvG6FEcb9MDTfsXdTYPGjYLRdeTEU=
-github.com/sashabaranov/go-openai v1.40.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
+github.com/sashabaranov/go-openai v1.40.5 h1:SwIlNdWflzR1Rxd1gv3pUg6pwPc6cQ2uMoHs8ai+/NY=
+github.com/sashabaranov/go-openai v1.40.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0=
@@ -350,8 +363,10 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G
github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8=
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
-github.com/shuffle/shuffle-shared v0.8.84 h1:ElIMQYjKBVOiadbiGkSzt/lPU5xqaQwRxQvk9wx/xYM=
-github.com/shuffle/shuffle-shared v0.8.84/go.mod h1:RdfNxqCPI+zU4jQKy3E/p4Io2injm7LpSKQUCDHNtLk=
+github.com/shuffle/shuffle-shared v0.9.14 h1:POkTHO+bByuv8HiKuCMSGgtpDKk86ISr6ooLG8vQfuE=
+github.com/shuffle/shuffle-shared v0.9.14/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw=
+github.com/shuffle/singul v0.0.16 h1:dW+0Mln9R1aUJ0fjikpWcxbjoQWqJHxe4kSxh2tQN5E=
+github.com/shuffle/singul v0.0.16/go.mod h1:LYkp320A6gsoPlYbXUM+WvEPUVAuutlSsqnVKyRy4gs=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
@@ -359,8 +374,8 @@ github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnB
github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY=
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
+github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE=
github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -417,8 +432,10 @@ go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFw
go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4=
go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w=
go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA=
-go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4=
-go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4=
+go.opentelemetry.io/proto/otlp v1.6.0 h1:jQjP+AQyTf+Fe7OKj/MfkDrmK4MNVtw2NpXsf9fefDI=
+go.opentelemetry.io/proto/otlp v1.6.0/go.mod h1:cicgGehlFuNdgZkcALOCh3VE6K/u2tAjzlRhDwmVpZc=
+go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc=
go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -428,8 +445,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
-golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
+golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM=
+golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -478,8 +495,8 @@ golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
-golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
+golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
+golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -495,8 +512,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
-golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -521,14 +538,14 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
-golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
+golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
-golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
+golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg=
+golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -539,8 +556,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
-golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
+golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
+golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
@@ -572,8 +589,8 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
-golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
+golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo=
+golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/backend/go-app/main.go b/backend/go-app/main.go
index cfbc768a..720ca901 100755
--- a/backend/go-app/main.go
+++ b/backend/go-app/main.go
@@ -3,6 +3,7 @@ package main
import (
uuid "github.com/satori/go.uuid"
"github.com/shuffle/shuffle-shared"
+ "github.com/shuffle/singul/pkg"
"archive/zip"
"bufio"
@@ -62,6 +63,7 @@ var registryName = "registry.hub.docker.com"
var runningEnvironment = "onprem"
var syncUrl = "https://shuffler.io"
+var debug = false
//var syncUrl = "http://localhost:5002"
type retStruct struct {
@@ -319,7 +321,7 @@ func checkError(cmdName string, cmdArgs []string) error {
scanner := bufio.NewScanner(cmdReader)
go func() {
for scanner.Scan() {
- fmt.Printf("Out: %s\n", scanner.Text())
+ log.Printf("Out: %s\n", scanner.Text())
}
}()
@@ -638,7 +640,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) {
} else {
log.Printf("[DEBUG] Successfully created the default org!")
- defaultEnv := os.Getenv("ORG_ID")
+ defaultEnv := os.Getenv("ENVIRONMENT_NAME")
if len(defaultEnv) == 0 {
defaultEnv = "Shuffle"
log.Printf("[DEBUG] Setting default environment for org to %s", defaultEnv)
@@ -1068,6 +1070,74 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) {
activatedAppIds = append(activatedAppIds, app.ID)
}
+ parsedStatus := []string{}
+ if len(org.ManagerOrgs) > 0 || userInfo.ActiveOrg.CreatorOrg != "" {
+ parsedStatus = append(parsedStatus, "sub_org")
+
+ parentOrgId := userInfo.ActiveOrg.CreatorOrg
+ if len(userInfo.ActiveOrg.CreatorOrg) == 0 {
+ parentOrgId = org.ManagerOrgs[0].Id
+ }
+
+ // Check for licensing/branding of parent and override
+ parentOrg, err := shuffle.GetOrg(ctx, parentOrgId)
+ if err == nil {
+ if parentOrg.LeadInfo.IntegrationPartner {
+ parsedStatus = append(parsedStatus, "integration_partner")
+
+ // except theme take from parent org
+ org.Branding.EnableChat = parentOrg.Branding.EnableChat
+ org.Branding.HomeUrl = parentOrg.Branding.HomeUrl
+ org.Branding.DocumentationLink = parentOrg.Defaults.DocumentationReference
+ org.Branding.SupportEmail = parentOrg.Branding.SupportEmail
+ org.Branding.LogoutUrl = parentOrg.Branding.LogoutUrl
+ org.Branding.BrandColor = parentOrg.Branding.BrandColor
+ org.Branding.BrandName = parentOrg.Branding.BrandName
+ org.Branding.GlobalUser = parentOrg.Branding.GlobalUser
+
+ if len(org.Branding.Theme) == 0 {
+ org.Branding.Theme = parentOrg.Branding.Theme
+ }
+
+ userInfo.ActiveOrg.Branding = parentOrg.Branding
+ userInfo.ActiveOrg.Image = parentOrg.Image
+ userInfo.ActiveOrg.Branding.DocumentationLink = parentOrg.Defaults.DocumentationReference
+ userInfo.ActiveOrg.Branding.BrandColor = parentOrg.Branding.BrandColor
+ userInfo.ActiveOrg.Branding.SupportEmail = org.Branding.SupportEmail
+ userInfo.ActiveOrg.Branding.LogoutUrl = org.Branding.LogoutUrl
+ userInfo.ActiveOrg.Branding.BrandName = parentOrg.Branding.BrandName
+
+ if len(org.Branding.Theme) == 0 {
+ userInfo.ActiveOrg.Branding.Theme = parentOrg.Branding.Theme
+ } else {
+ userInfo.ActiveOrg.Branding.Theme = org.Branding.Theme
+ }
+
+ // check whether current is global user or not? means is user part of parent org or not
+ for _, user := range parentOrg.Users {
+ if user.Id == userInfo.Id && user.Role == "admin" {
+ userInfo.ActiveOrg.Branding.GlobalUser = true
+ break
+ }
+ }
+ }
+ }
+ } else {
+ // for parent org branding
+ if org.LeadInfo.IntegrationPartner {
+ userInfo.ActiveOrg.Branding.Theme = org.Branding.Theme
+ userInfo.ActiveOrg.Branding.DocumentationLink = org.Defaults.DocumentationReference
+ userInfo.ActiveOrg.Branding.SupportEmail = org.Branding.SupportEmail
+ userInfo.ActiveOrg.Branding.LogoutUrl = org.Branding.LogoutUrl
+ userInfo.ActiveOrg.Branding.BrandColor = org.Branding.BrandColor
+ userInfo.ActiveOrg.Branding.BrandName = org.Branding.BrandName
+
+ parsedStatus = append(parsedStatus, "integration_partner")
+ }
+ }
+
+ aiEnabled := os.Getenv("OPENAI_API_URL") != "" && os.Getenv("AI_MODEL") != ""
+
returnValue := shuffle.HandleInfo{
Success: true,
Username: userInfo.Username,
@@ -1091,6 +1161,8 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) {
Licensed: licensed,
ActiveApps: activatedAppIds,
Theme: userInfo.Theme,
+ OrgStatus: parsedStatus,
+ AIEnabled: aiEnabled,
}
returnData, err := json.Marshal(returnValue)
@@ -1330,7 +1402,7 @@ func parseWorkflowParameters(resp http.ResponseWriter, request *http.Request) (m
return t, err
}
- //fmt.Println(curjson.String())
+ //log.Println(curjson.String())
//log.Printf("Parsing json a second time: %s", string(curjson.String()))
err = json.Unmarshal(curjson.Bytes(), &t)
@@ -2484,7 +2556,7 @@ func execSubprocess(cmdName string, cmdArgs []string) error {
scanner := bufio.NewScanner(cmdReader)
go func() {
for scanner.Scan() {
- fmt.Printf("Out: %s\n", scanner.Text())
+ log.Printf("Out: %s\n", scanner.Text())
}
}()
@@ -3456,7 +3528,7 @@ func handleAppHotload(ctx context.Context, location string, forceUpdate bool) er
return err
}
- _, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate)
+ _, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate, false)
if err != nil {
log.Printf("[WARNING] Githubfolders error: %s", err)
return err
@@ -3909,7 +3981,7 @@ func runInitCloudSetup() {
if err != nil {
log.Printf("[INFO] Failed initial setup: %s", err)
} else {
- log.Printf("[INFO] Ran initial setup!")
+ log.Printf("[INFO] Finished initial cloudsync setup!")
}
}
@@ -3925,7 +3997,7 @@ func runInitEs(ctx context.Context) {
log.Printf("[INFO] Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
}
- defaultEnv := os.Getenv("ORG_ID")
+ defaultEnv := os.Getenv("ENVIRONMENT_NAME")
if len(defaultEnv) == 0 {
defaultEnv = "Shuffle"
log.Printf("[DEBUG] Setting default environment for org to %s", defaultEnv)
@@ -4031,8 +4103,9 @@ func runInitEs(ctx context.Context) {
time.Sleep(30 * time.Second)
}
- // FIXME: This should ONLY run on one backend instance
+ shuffle.InitOpensearchIndexes()
+ // FIXME: This should ONLY run on one backend instance. This may cause interference.
schedules, err := shuffle.GetAllSchedules(ctx, "ALL")
if err != nil {
log.Printf("[WARNING] Failed getting schedules during service init: %s", err)
@@ -4078,14 +4151,23 @@ func runInitEs(ctx context.Context) {
//log.Printf("Schedule: %#v", schedule)
//log.Printf("Schedule time: every %d seconds", schedule.Seconds)
- jobret, err := newscheduler.Every(schedule.Seconds).Seconds().NotImmediately().Run(job(schedule))
- if err != nil {
- log.Printf("[ERROR] Failed to start schedule for workflow %s: %s", schedule.WorkflowId, err)
+ if schedule.Seconds == 0 && len(schedule.Frequency) > 0 {
+ cronJob, err := CronScheduler.Cron(schedule.Frequency).Do(job(schedule))
+ if err != nil {
+ log.Printf("[ERROR] Failed to start schedule for workflow %s: %s", schedule.WorkflowId, err)
+ } else {
+ log.Printf("[DEBUG] Successfully started schedule for workflow %s", schedule.WorkflowId)
+ }
+ cronJobs[schedule.Id] = cronJob
} else {
- log.Printf("[DEBUG] Successfully started schedule for workflow %s", schedule.WorkflowId)
+ jobret, err := newscheduler.Every(schedule.Seconds).Seconds().NotImmediately().Run(job(schedule))
+ if err != nil {
+ log.Printf("[ERROR] Failed to start schedule for workflow %s: %s", schedule.WorkflowId, err)
+ } else {
+ log.Printf("[DEBUG] Successfully started schedule for workflow %s", schedule.WorkflowId)
+ }
+ scheduledJobs[schedule.Id] = jobret
}
-
- scheduledJobs[schedule.Id] = jobret
}
}
@@ -4337,20 +4419,8 @@ func runInitEs(ctx context.Context) {
}
// Getting apps to see if we should initialize a test
- // FIXME: Isn't this a little backwards?
workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000, 0)
log.Printf("[INFO] Getting and validating workflowapps. Got %d with err %#v", len(workflowapps), err)
-
- // accept any certificate (might be useful for testing)
- //customGitClient := &http.Client{
- // Transport: &http.Transport{
- // TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
- // },
- // Timeout: 15 * time.Second,
- //}
- //client.InstallProtocol("http", githttp.NewClient(customGitClient))
- //client.InstallProtocol("https", githttp.NewClient(customGitClient))
-
if err != nil && len(workflowapps) == 0 {
log.Printf("[WARNING] Failed getting apps (runInit): %s", err)
} else if err == nil && len(workflowapps) < 10 {
@@ -4360,8 +4430,9 @@ func runInitEs(ctx context.Context) {
url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION")
if len(url) == 0 {
- log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/shuffle/shuffle-apps")
- url = "https://github.com/shuffle/shuffle-apps"
+ log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/shuffle/python-apps")
+
+ url = "https://github.com/shuffle/python-apps"
//url = ""
//return
}
@@ -4401,7 +4472,7 @@ func runInitEs(ctx context.Context) {
_ = r
//iterateAppGithubFolders(fs, dir, "", "testing")
- _, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate)
+ _, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate, true)
if err != nil {
log.Printf("[WARNING] Error from app load in init: %s", err)
}
@@ -4409,6 +4480,10 @@ func runInitEs(ctx context.Context) {
// Hotloads locally
location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER")
+ if len(location) == 0 {
+ location = "./shuffle-apps"
+ }
+
if len(location) != 0 {
handleAppHotload(ctx, location, false)
}
@@ -5090,6 +5165,7 @@ func handleAppZipUpload(resp http.ResponseWriter, request *http.Request) {
func initHandlers() {
var err error
ctx := context.Background()
+ CronScheduler.StartAsync()
log.Printf("[DEBUG] Starting Shuffle backend - initializing database connection")
//requestCache = cache.New(5*time.Minute, 10*time.Minute)
@@ -5176,11 +5252,13 @@ func initHandlers() {
r.HandleFunc("/api/v1/workflows/queue", handleGetWorkflowqueue).Methods("GET", "POST")
r.HandleFunc("/api/v1/workflows/queue/confirm", handleGetWorkflowqueueConfirm).Methods("POST")
- // App specific
- // From here down isnt checked for org specific
+ // App specific. Partially Singul.
+ r.HandleFunc("/api/v1/apps/categories", shuffle.GetActiveCategories).Methods("GET", "OPTIONS")
+ r.HandleFunc("/api/v1/apps/categories/run", singul.RunCategoryAction).Methods("POST", "OPTIONS")
+
r.HandleFunc("/api/v1/apps/{key}/execute", executeSingleAction).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/{key}/run", executeSingleAction).Methods("POST", "OPTIONS")
- r.HandleFunc("/api/v1/apps/categories", shuffle.GetActiveCategories).Methods("GET", "OPTIONS")
+
//r.HandleFunc("/api/v1/apps/categories/run", shuffle.RunCategoryAction).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/upload", handleAppZipUpload).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appId}/activate", activateWorkflowAppDocker).Methods("GET", "OPTIONS")
@@ -5248,6 +5326,8 @@ func initHandlers() {
// First v2 API
r.HandleFunc("/api/v2/workflows/{key}/executions", shuffle.GetWorkflowExecutionsV2).Methods("GET", "OPTIONS")
+ r.HandleFunc("/api/v2/workflows/generate/llm", shuffle.HandleWorkflowGenerationResponse).Methods("POST", "OPTIONS")
+ r.HandleFunc("/api/v2/workflows/edit/llm", shuffle.HandleEditWorkflowWithLLM).Methods("POST", "OPTIONS")
// New for recommendations in Shuffle
r.HandleFunc("/api/v1/recommendations/get_actions", shuffle.HandleActionRecommendation).Methods("POST", "OPTIONS")
@@ -5308,8 +5388,8 @@ func initHandlers() {
// EVERYTHING below here is NEW for 0.8.0 (written 25.05.2021)
r.HandleFunc("/api/v1/workflows/{key}/publish", makeWorkflowPublic).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/cloud/setup", handleCloudSetup).Methods("POST", "OPTIONS")
- //r.HandleFunc("/api/v1/orgs", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS")
- //r.HandleFunc("/api/v1/orgs/", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS")
+ r.HandleFunc("/api/v1/orgs", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS")
+ r.HandleFunc("/api/v1/orgs/", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleGetOrg).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleEditOrg).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgid}/forms", shuffle.HandleGetOrgForms).Methods("GET", "OPTIONS")
@@ -5404,6 +5484,10 @@ func initHandlers() {
// Had to move away from mux, which means Method is fucked up right now.
func main() {
+ if os.Getenv("DEBUG") == "true" {
+ debug = true
+ }
+
initHandlers()
hostname, err := os.Hostname()
if err != nil {
diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go
index d33cee2c..3cf355ab 100755
--- a/backend/go-app/walkoff.go
+++ b/backend/go-app/walkoff.go
@@ -11,7 +11,7 @@ import (
"io"
"io/ioutil"
"log"
- "math/rand"
+ //"math/rand"
"net/http"
"net/url"
"os"
@@ -26,6 +26,7 @@ import (
uuid "github.com/satori/go.uuid"
newscheduler "github.com/carlescere/scheduler"
+ "github.com/go-co-op/gocron"
"github.com/frikky/kin-openapi/openapi3"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/memfs"
@@ -42,17 +43,22 @@ var baseEnvironment = "onprem"
var cloudname = "cloud"
var scheduledJobs = map[string]*newscheduler.Job{}
+var cronJobs = map[string]*gocron.Job{}
var scheduledOrgs = map[string]*newscheduler.Job{}
+var CronScheduler = gocron.NewScheduler(time.UTC)
+
// Frequency = cronjob OR minutes between execution
func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode, frequency, orgId string, body []byte) error {
var err error
testSplit := strings.Split(frequency, "*")
cronJob := ""
+ isCron := false
newfrequency := 0
if len(testSplit) > 5 {
cronJob = frequency
+ isCron = true
} else {
newfrequency, err = strconv.Atoi(frequency)
if err != nil {
@@ -65,12 +71,7 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode
//} else if int(newfrequency) <
}
- // Reverse. Can't handle CRON, only numbers
- if len(cronJob) > 0 {
- return errors.New("cronJob isn't formatted correctly")
- }
-
- if newfrequency < 1 {
+ if newfrequency < 1 && !isCron {
return errors.New("Frequency has to be more than 0")
}
@@ -96,17 +97,27 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode
}
}
- log.Printf("[INFO] Starting frequency for execution: %d", newfrequency)
+ log.Printf("[INFO] Starting frequency for execution: %s", frequency)
- //jobret, err := newscheduler.Every(newfrequency).Seconds().NotImmediately().Run(job)
- jobret, err := newscheduler.Every(newfrequency).Seconds().Run(job)
- if err != nil {
- log.Printf("Failed to schedule workflow: %s", err)
- return err
+ if isCron {
+ cronJob, err := CronScheduler.Cron(cronJob).Do(job)
+ if err != nil {
+ log.Printf("[ERROR] Failed to start schedule with cron(%s): %s", cronJob, err)
+ }
+
+ cronJobs[scheduleId] = cronJob
+ } else {
+ //jobret, err := newscheduler.Every(newfrequency).Seconds().NotImmediately().Run(job)
+ jobret, err := newscheduler.Every(newfrequency).Seconds().Run(job)
+ if err != nil {
+ log.Printf("Failed to schedule workflow: %s", err)
+ return err
+ }
+
+ scheduledJobs[scheduleId] = jobret
}
//scheduledJobs = append(scheduledJobs, jobret)
- scheduledJobs[scheduleId] = jobret
// Doesn't need running/not running. If stopped, we just delete it.
timeNow := int64(time.Now().Unix())
@@ -117,6 +128,7 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode
Argument: string(body),
WrappedArgument: bodyWrapper,
Seconds: newfrequency,
+ Frequency: frequency,
CreationTime: timeNow,
LastModificationtime: timeNow,
LastRuntime: timeNow,
@@ -245,7 +257,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
return
}
- // This is really the environment's name - NOT org-id
+ // This is really the environment's name - NOT OrgId
environment := request.Header.Get("Org-Id")
if len(environment) == 0 {
log.Printf("[AUDIT] No org-id header set")
@@ -254,7 +266,8 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
return
}
- orgId := request.Header.Get("org")
+ // Org => Org ID here
+ orgId := request.Header.Get("Org")
if len(orgId) == 0 {
//log.Printf("[AUDIT] No 'org' header set (get workflow queue). ")
/*
@@ -263,9 +276,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
return
*/
}
-
- orborusLabel := request.Header.Get("x-orborus-label")
-
+
// This section is cloud custom for now
auth := request.Header.Get("Authorization")
if len(auth) == 0 {
@@ -277,192 +288,46 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
*/
}
- //log.Printf("[AUDIT] Get workflow queue for org %s, env %s, orborus label %s", orgId, environment, orborusLabel)
-
ctx := shuffle.GetContext(request)
- // Get all env and check the name?
envs, err := shuffle.GetEnvironments(ctx, orgId)
-
if err != nil || len(envs) == 0 {
- log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", environment, err)
+ //log.Printf("[WARNING] No env found for orgId %s during queue loading", orgId)
}
var env *shuffle.Environment
- for _, e := range envs {
- if e.Name == environment {
- env = &e
+ found := false
+ for i := range envs {
+ if envs[i].Name == environment {
+ env = &envs[i]
+ found = true
break
}
}
- timeNow := time.Now().Unix()
- err = shuffle.HandleOrborusFailover(ctx, request, resp, env)
- if err != nil {
- if !strings.Contains(err.Error(), "mismatch") {
- log.Printf("[WARNING] Failed handling Orborus failover: %s", err)
+ // Only works onprem - shared queues across tenants
+ // without tenancy
+ if !found {
+ env, err = shuffle.GetEnvironment(ctx, environment, "")
+ if err != nil {
+ log.Printf("[WARNING] Failed to find the environment(%s) in org(%s). Could cause with Failover test", environment, orgId)
}
-
- return
}
- //log.Printf("Found env: %#v", env)
+ // Handles failover control between Orborus'
+ // Further tracks checkin time to ensure this works properly
+ // across instances
+ err = shuffle.HandleOrborusFailover(ctx, request, resp, env)
+ if err != nil {
+ log.Printf("[WARNING] Failed handling Orborus failover: %s", err)
+ }
+
if len(env.OrgId) > 0 {
orgId = env.OrgId
}
- // FIXME: Workflow stats disabled for now
- // as it caused too many problems
- // goal: track docker stuff once a minute and graph it
- // For now: Disable this as it caused too many problems
- if request.Method == "POST" && true == false {
- //log.Printf("[DEBUG] POST to workflowqueue")
- if rand.Intn(10) == 0 {
- // Parse out body
- body, err := ioutil.ReadAll(request.Body)
- if err == nil {
-
- // Parse out CPU, memory and disk.
-
- var envData shuffle.OrborusStats
- err = json.Unmarshal(body, &envData)
- if err == nil && !envData.Swarm && !envData.Kubernetes && (envData.CPU > 0 || envData.Memory > 0 || envData.Disk > 0) {
-
- // Set the input in memory
- envData.OrgId = orgId
- envData.Environment = environment
- envData.OrborusLabel = orborusLabel
- envData.Timestamp = time.Now().Unix()
-
- if envData.CPU > 0 && envData.MaxCPU > 0 {
- envData.CPUPercent = float64(envData.CPU) / float64(envData.MaxCPU)
- }
-
- if envData.Memory > 0 && envData.MaxMemory > 0 {
- envData.MemoryPercent = float64(envData.Memory) / float64(envData.MaxMemory)
- }
-
- // Check if CPU percent constantly has stayed above X% for the last Y requests
- percentageCheck := 90
- concurrentChecks := 2
-
- //if int(envData.CPUPercent) > percentageCheck {
- // Get cached data
- percentages := []float64{}
- cacheKey := fmt.Sprintf("%s_%s_percent", environment , strings.ToLower(orgId))
-
- // Marshal float list into []byte
- cacheData := []byte{}
- cache, err := shuffle.GetCache(ctx, cacheKey)
- if err == nil {
- // Unmarshal into percentages
- cacheData := []byte(cache.([]uint8))
- err = json.Unmarshal(cacheData, &percentages)
- if err != nil {
- log.Printf("[INFO] error in cache unmarshal for percentages: %s", err)
- }
-
- if len(percentages) > concurrentChecks {
- percentages = percentages[:concurrentChecks]
- }
-
- percentages = append(percentages, envData.CPUPercent)
- if len(percentages) > concurrentChecks {
- //log.Printf("[INFO] Checking percentages: %v", percentages)
-
- // percentageCheck := 1
- sendAlert := true
- for _, p := range percentages {
- if int(p) < percentageCheck {
- //log.Printf("[AUDIT] CPU percent is below %d: %d", percentageCheck, int(p))
- sendAlert = false
- break
- }
- }
-
- if sendAlert {
- log.Printf("[INFO] CPU percent has been above %d percent for the last 5 requests. Sending alert. Env: %s, org: %s", percentageCheck, environment, orgId)
-
- // Set notification + alert for organization
- err = shuffle.CreateOrgNotification(
- ctx,
- fmt.Sprintf("CPU percent has been above %d percent", percentageCheck),
- fmt.Sprintf("A environment %s has been using more than %d\\% CPU for the last 5 requests.", environment, percentageCheck),
- fmt.Sprintf("/admin?tab=environments"),
- environment,
- true,
- )
-
- if err != nil {
- log.Printf("[ERROR] error creating notification: %s", err)
- }
-
- org, err := shuffle.GetOrg(ctx, environment)
- if err == nil {
- foundRecommendation := false
- for _, recommendation := range org.Priorities {
- if strings.Contains(recommendation.Name, "CPU") {
- foundRecommendation = true
- break
- }
- }
-
- if !foundRecommendation {
- // Add to start of org.Priorities
- org, _ = shuffle.AddPriority(*org, shuffle.Priority{
- Name: fmt.Sprintf("High CPU in environment %s", orgId),
- Description: fmt.Sprintf("The environment %s has been using more than %d percent CPU. This indicates you may need to look at scaling.", orgId, percentageCheck),
- Type: "scale",
- Active: true,
- URL: fmt.Sprintf("/admin?tab=environments"),
- Severity: 1,
- }, false)
-
- //Make last item the first item
- org.Priorities = append([]shuffle.Priority{org.Priorities[len(org.Priorities)-1]}, org.Priorities[:len(org.Priorities)-1]...)
- err = shuffle.SetOrg(ctx, *org, org.Id)
- if err != nil {
- log.Printf("[ERROR] Problem setting org: %s", err)
- }
- }
- }
- }
-
- if len(percentages) > 1 {
- percentages = percentages[1:]
- }
- }
-
- // Marshal float list into []byte
- } else {
- //log.Printf("[ERROR] Failed getting cache: %s", err)
- percentages = append(percentages, envData.CPUPercent)
- }
-
- if len(percentages) > 0 {
- //log.Printf("[DEBUG] Setting cache for %s: %#v", cacheKey, percentages)
- cacheData, err = json.Marshal(percentages)
- if err != nil {
- log.Printf("[INFO] error in cache marshal: %s", err)
- }
-
- // Add the new data
- go shuffle.SetCache(ctx, cacheKey, cacheData, 5)
- }
- }
-
- //log.Printf("CPU percent: %f", envData.CPUPercent)
- //log.Printf("Memory percent: %f", envData.MemoryPercent*100)
-
- go shuffle.SetenvStats(ctx, envData)
- }
- }
- }
-
executionRequests, err := shuffle.GetWorkflowQueue(ctx, environment, 100)
if err != nil {
- // Skipping as this comes up over and over
- //log.Printf("(2) Failed reading body for workflowqueue: %s", err)
- resp.WriteHeader(401)
+ resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
@@ -471,10 +336,11 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
if len(executionRequests.Data) == 0 {
executionRequests.Data = []shuffle.ExecutionRequest{}
} else {
- //log.Printf("In workflowqueue with %d", len(executionRequests.Data))
- // Try again :)
+ // Try again? I don't think this is necessary, and shouldn't really ever occur.
+ /*
if len(env.Id) == 0 && len(env.Name) == 0 {
+ timeNow := int64(time.Now().Unix())
foundId := ""
for _, requestData := range executionRequests.Data {
execution, err := shuffle.GetWorkflowExecution(ctx, requestData.ExecutionId)
@@ -487,9 +353,10 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
}
if len(environment) > 0 {
- env, err := shuffle.GetEnvironment(ctx, foundId, environment)
+
+ env, err := shuffle.GetEnvironment(ctx, environment, foundId)
if err != nil {
- log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", orgId, err)
+ log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", environment, err)
//resp.WriteHeader(401)
//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No env found matching %s"}`, id)))
//return
@@ -505,6 +372,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
}
}
}
+ */
if len(executionRequests.Data) > 50 {
executionRequests.Data = executionRequests.Data[0:49]
@@ -1685,7 +1553,15 @@ func deleteSchedule(ctx context.Context, id string) error {
value.Lock()
} else {
// FIXME - allow it to kind of stop anyway?
- return errors.New("Can't find the schedule.")
+ if j, ok := cronJobs[id]; ok {
+ err := CronScheduler.RemoveByID(j)
+ if err != nil {
+ log.Printf("[ERROR] Failed to remove the scheduler %s", err)
+ return err
+ }
+ } else {
+ return errors.New("Can't find the schedule.")
+ }
}
}
@@ -2141,14 +2017,14 @@ func validateAppInput(resp http.ResponseWriter, request *http.Request) {
//fmt.Printf("File type: %s. MIME: %s\n", kind.Extension, kind.MIME.Value)
if kind == filetype.Unknown {
- fmt.Println("Unknown file type")
+ log.Println("Unknown file type")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if kind.MIME.Value != "application/zip" {
- fmt.Println("Not zip, can't unzip")
+ log.Println("Not zip, can't unzip")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
@@ -2307,17 +2183,24 @@ func handleSingleAppHotloadRequest(resp http.ResponseWriter, request *http.Reque
resp.Write([]byte(`{"success": false}`))
return
}
+
if user.Role != "admin" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Must be admin to hotload apps"}`))
return
}
+
location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER")
+ if len(location) == 0 {
+ location = "./shuffle-apps"
+ }
+
if len(location) == 0 {
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "SHUFFLE_APP_HOTLOAD_FOLDER not specified in .env"}`)))
return
}
+
requestUrlFields := strings.Split(request.URL.String(), "/")
var appName string
if requestUrlFields[1] == "api" {
@@ -2381,6 +2264,10 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) {
}
location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER")
+ if len(location) == 0 {
+ location = "./shuffle-apps"
+ }
+
if len(location) == 0 {
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "SHUFFLE_APP_HOTLOAD_FOLDER not specified in .env"}`)))
@@ -2982,24 +2869,24 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) {
}
location := strings.Split(request.URL.String(), "/")
- var fileId string
+ var appId string
if location[1] == "api" {
if len(location) <= 4 {
- resp.WriteHeader(401)
+ resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
- fileId = location[4]
+ appId = location[4]
}
//log.Printf("[AUDIT] User Authentication failed in execute SINGLE action - CONTINUING ANYWAY: %s. Found OrgID: %#v", err, user.ActiveOrg.Id)
- log.Printf("[AUDIT] User %s (%s) in org %s (%s) is running SINGLE App run for App ID '%s'", user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id, fileId)
+ log.Printf("[AUDIT] User %s (%s) in org %s (%s) is running SINGLE App run for App ID '%s'", user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id, appId)
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[INFO] Failed single execution POST body read: %s", err)
- resp.WriteHeader(401)
+ resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
@@ -3024,7 +2911,15 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) {
decisionId = decision[0]
}
- workflowExecution, err := shuffle.PrepareSingleAction(ctx, user, fileId, body, runValidationAction, decisionId)
+ log.Printf("\n\nACTION TO RUN: %s. Body: %s. Source URL: %s\n\n", appId, string(body), request.URL.String())
+
+ workflowExecution, err := shuffle.PrepareSingleAction(ctx, user, appId, body, runValidationAction, decisionId)
+ if appId == "agent_starter" {
+ log.Printf("[INFO] Returning early for agent_starter single action execution: %s", workflowExecution.ExecutionId)
+ resp.WriteHeader(200)
+ resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization)))
+ return
+ }
debugUrl := fmt.Sprintf("/workflows/%s?execution_id=%s", workflowExecution.Workflow.ID, workflowExecution.ExecutionId)
resp.Header().Add("X-Debug-Url", debugUrl)
@@ -3065,6 +2960,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) {
go shuffle.IncrementCache(ctx, workflowExecution.OrgId, "workflow_executions")
executionRequest := shuffle.ExecutionRequest{
+ Priority: 15,
ExecutionId: workflowExecution.ExecutionId,
WorkflowId: workflowExecution.Workflow.ID,
Authorization: workflowExecution.Authorization,
@@ -3110,7 +3006,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) {
}
// Onlyname is used to
-func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) ([]shuffle.BuildLaterStruct, []shuffle.BuildLaterStruct, error) {
+func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate, duringStartup bool) ([]shuffle.BuildLaterStruct, []shuffle.BuildLaterStruct, error) {
var err error
allapps := []shuffle.WorkflowApp{}
@@ -3122,7 +3018,14 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
"YARA",
"ATTACK-PREDICTOR",
}
- //if strings.ToUpper(workflowapp.Name) == strings.ToUpper(appname) {
+
+ startupNames := []string{
+ "shuffle-tools",
+ "http",
+ "email",
+ "shuffle-ai",
+ "shuffle-subflow",
+ }
// It's here to prevent getting them in every iteration
buildLaterFirst := []shuffle.BuildLaterStruct{}
@@ -3132,6 +3035,19 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
continue
}
+ //duringStartup
+ if duringStartup {
+ // Look for names: shuffle tools, http, email, shuffle ai
+ if shuffle.ArrayContains(startupNames, strings.ToLower(file.Name())) {
+ // Allowed to build during startup
+
+ //log.Printf("\n\n\nFOUND MATCHING APP: %s\n\n\n", file.Name())
+ } else {
+ //log.Printf("\n\n\nWRONG APP (2): %s\n\n\n", file.Name())
+ continue
+ }
+ }
+
// Folder?
switch mode := file.Mode(); {
case mode.IsDir():
@@ -3148,7 +3064,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
}
// Go routine? Hmm, this can be super quick I guess
- buildFirst, buildLast, err := IterateAppGithubFolders(ctx, fs, dir, tmpExtra, "", forceUpdate)
+ buildFirst, buildLast, err := IterateAppGithubFolders(ctx, fs, dir, tmpExtra, "", forceUpdate, false)
for _, item := range buildFirst {
buildLaterFirst = append(buildLaterFirst, item)
@@ -3160,7 +3076,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
if err != nil {
log.Printf("[WARNING] Error reading folder: %s", err)
- //buildFirst, buildLast, err := IterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate)
+ //buildFirst, buildLast, err := IterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate, false)
if !forceUpdate {
continue
@@ -3460,6 +3376,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
"http",
"email",
}
+
for _, buildLater := range buildLaterFirst {
found := false
for _, appname := range initApps {
@@ -3478,12 +3395,19 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
}
// Prepend newSortedList to buildLaterFirst
+ handledImages := []string{}
buildLaterFirst = append(newSortedList, buildLaterFirst...)
-
if len(extra) == 0 {
log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst))
for _, item := range buildLaterFirst {
+
+ if len(item.Tags) > 0 && shuffle.ArrayContains(handledImages, item.Tags[0]) {
+ continue
+ }
+
+ handledImages = append(handledImages, item.Tags[0])
err = buildImageMemory(fs, item.Tags, item.Extra, true)
+
if err != nil {
orgId := ""
@@ -3513,6 +3437,12 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
if len(buildLaterList) > 0 {
log.Printf("[INFO] Starting build of %d skipped docker images", len(buildLaterList))
for _, item := range buildLaterList {
+ if len(item.Tags) > 0 && shuffle.ArrayContains(handledImages, item.Tags[0]) {
+ continue
+ }
+
+ handledImages = append(handledImages, item.Tags[0])
+
err = buildImageMemory(fs, item.Tags, item.Extra, true)
if err != nil {
log.Printf("[INFO] Failed image build memory: %s", err)
@@ -3646,7 +3576,7 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) {
}
}
- IterateAppGithubFolders(ctx, fs, dir, "", "", tmpBody.ForceUpdate)
+ IterateAppGithubFolders(ctx, fs, dir, "", "", tmpBody.ForceUpdate, false)
} else if strings.Contains(tmpBody.URL, "s3") {
//https://docs.aws.amazon.com/sdk-for-go/api/service/s3/
diff --git a/docker-compose.yml b/docker-compose.yml
index f6683a2a..980e0764 100755
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -61,7 +61,7 @@ services:
security_opt:
- seccomp:unconfined
opensearch:
- image: opensearchproject/opensearch:3.0.0
+ image: opensearchproject/opensearch:3.2.0
hostname: shuffle-opensearch
container_name: shuffle-opensearch
environment:
diff --git a/frontend/public/aiGenerateWorkflowSteps.svg b/frontend/public/aiGenerateWorkflowSteps.svg
new file mode 100644
index 00000000..3f294fcb
--- /dev/null
+++ b/frontend/public/aiGenerateWorkflowSteps.svg
@@ -0,0 +1,85 @@
+
diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx
index 13289352..9425a1c6 100755
--- a/frontend/src/App.jsx
+++ b/frontend/src/App.jsx
@@ -92,7 +92,7 @@ const App = (message, props) => {
const [dataset, setDataset] = useState(false)
const [isLoaded, setIsLoaded] = useState(false)
const [curpath, setCurpath] = useState(typeof window === "undefined" || window.location === undefined ? "" : window.location.pathname)
- const { themeMode, handleThemeChange, setBrandColor, brandColor,setThemeMode} = useContext(Context);
+ const { themeMode, handleThemeChange, setBrandColor, brandColor,setThemeMode, setSupportEmail, setLogoutUrl, setBrandName} = useContext(Context);
const currentTheme = getTheme(themeMode, brandColor);
const mainColor = currentTheme?.palette?.backgroundColor
const [isPreviousThemeLight, setIsPreviousThemeLight] = useState(false)
@@ -192,11 +192,44 @@ const App = (message, props) => {
{ path: "/" }
);
}
- if (responseJson?.theme?.length > 0) {
- handleThemeChange(responseJson.theme)
- }else{
- handleThemeChange("dark")
- }
+ if (responseJson?.org_status?.includes("integration_partner")) {
+ if (responseJson?.active_org?.branding?.enable_chat !== true) {
+
+ // Find the drift chatbox and remove it
+ }
+
+ if (responseJson?.active_org?.branding?.theme?.length > 0 ) {
+ handleThemeChange(responseJson?.active_org?.branding?.theme)
+ }
+
+ if (responseJson?.active_org?.branding?.brand_color?.length > 0 ) {
+ setBrandColor(responseJson?.active_org?.branding?.brand_color)
+ localStorage.setItem("brandColor", responseJson?.active_org?.branding?.brand_color)
+ }
+
+ if (responseJson?.active_org?.branding?.brand_name?.length > 0 ) {
+ setBrandName(responseJson?.active_org?.branding?.brand_name)
+ localStorage.setItem("brandName", responseJson?.active_org?.branding?.brand_name)
+ }
+
+ if (responseJson?.active_org?.branding?.support_email?.length > 0 ) {
+ setSupportEmail(responseJson?.active_org?.branding?.support_email)
+ }
+
+ if (responseJson?.active_org?.branding?.logout_url?.length > 0 ) {
+ setLogoutUrl(responseJson?.active_org?.branding?.logout_url)
+ }
+ } else {
+ localStorage.removeItem("brandColor")
+ setBrandColor("#ff8544")
+ setBrandName("Shuffle")
+ localStorage.removeItem("brandName")
+ if(responseJson?.theme?.length > 0){
+ handleThemeChange(responseJson?.theme)
+ }else{
+ handleThemeChange("dark")
+ }
+ }
}else {
handleThemeChange("dark")
setThemeMode("dark")
diff --git a/frontend/src/components/AdminNavBar.jsx b/frontend/src/components/AdminNavBar.jsx
index 2a480c95..fbbc391d 100644
--- a/frontend/src/components/AdminNavBar.jsx
+++ b/frontend/src/components/AdminNavBar.jsx
@@ -92,9 +92,14 @@ const AdminNavBar = (props) => {
const HandleVisibleTabs = () => {
if (userdata?.id?.length > 0) {
if (userdata?.active_org?.role === "admin" || userdata?.support) {
- setVisibleItems(items);
+ if (isChildOrg) {
+ const filteredItems = items.filter(item => item.text !== "Partner");
+ setVisibleItems(filteredItems);
+ }else {
+ setVisibleItems(items);
+ }
}else {
- const filteredItems = items.filter(item => item.text !== "Users" && item.text !== "Files" && item.text !== "Datastore" && item.text !== "Triggers" && item.text !== "Locations");
+ const filteredItems = items.filter(item => item.text !== "Users" && item.text !== "Files" && item.text !== "Datastore" && item.text !== "Triggers" && item.text !== "Locations" && item.text !== "Partner");
setVisibleItems(filteredItems);
}
}
@@ -105,12 +110,12 @@ const AdminNavBar = (props) => {
// Filter out Users and Tenants tabs
if (userdata?.active_org?.role === "admin" || userdata?.support) {
const filteredItems = items.filter(item =>
- item.text !== "Users" && item.text !== "Tenants"
+ item.text !== "Users" && item.text !== "Tenants" && item.text !== "Partner"
);
setVisibleItems(filteredItems);
}else {
const filteredItems = items.filter(item =>
- item.text !== "Users" && item.text !== "Tenants" && item.text !== "Files" && item.text !== "Datastore" && item.text !== "Triggers" && item.text !== "Locations"
+ item.text !== "Users" && item.text !== "Partner" && item.text !== "Tenants" && item.text !== "Files" && item.text !== "Datastore" && item.text !== "Triggers" && item.text !== "Locations"
);
setVisibleItems(filteredItems);
}
@@ -191,7 +196,7 @@ const AdminNavBar = (props) => {
const params = new URLSearchParams(location.search);
const tab = params?.get('tab')?.toLowerCase();
- if (tab === "users" || tab === "tenants") {
+ if (tab === "users" || tab === "tenants" || tab === "partner") {
toast.info("You are not allowed to access this tab. Please contact your admin for more information. Redirecting to Organization Configuration tab.");
setTimeout(() => {
setSelectedItem("Organization");
@@ -200,6 +205,7 @@ const AdminNavBar = (props) => {
}
, 3000);
}
+
} else if (userdata && isOrgLoaded && isUserDataLoaded && userdata?.active_org?.role !== "admin" && !userdata?.support) {
const queryParams = new URLSearchParams(location.search);
const tabName = queryParams?.get('admin_tab')?.toLowerCase();
diff --git a/frontend/src/components/AnalyticsTab.jsx b/frontend/src/components/AnalyticsTab.jsx
index c97560ed..c64dee56 100644
--- a/frontend/src/components/AnalyticsTab.jsx
+++ b/frontend/src/components/AnalyticsTab.jsx
@@ -2,7 +2,6 @@
// import Switch from '@mui/material/Switch';
// import { Typography, Button } from '@mui/material';
// import { useNavigate, Link, useParams } from "react-router-dom";
-// import { Bar } from 'react-chartjs-2';
// import Grid from '@mui/material/Grid';
// import SearchIcon from '@mui/icons-material/Search';
// import NewReleasesIcon from '@mui/icons-material/NewReleases';
diff --git a/frontend/src/components/Billing.jsx b/frontend/src/components/Billing.jsx
index 0b9daa6f..8b14eaf5 100644
--- a/frontend/src/components/Billing.jsx
+++ b/frontend/src/components/Billing.jsx
@@ -91,18 +91,21 @@ const Billing = memo((props) => {
const [currentTab, setCurrentTab] = useState(0)
const [allChildOrgs, setAllChildOrgs] = useState([])
const [allChildOrgsStats, setAllChildOrgsStats] = useState([])
+ const [statistics, setStatistics] = useState([])
+ const [monthlyAppRunsParent, setMonthlyAppRunsParent] = useState(0)
+ const [monthlyAllSuborgExecutions, setMonthlyAllSuborgExecutions] = useState(0)
useEffect(() => {
- if (userdata.app_execution_limit !== undefined && userdata.app_execution_usage !== undefined) {
- const percentage = (userdata.app_execution_usage / userdata.app_execution_limit) * 100;
+ if (monthlyAppRunsParent > 0 || monthlyAllSuborgExecutions > 0) {
+ const percentage = ((monthlyAppRunsParent + monthlyAllSuborgExecutions) / userdata.app_execution_limit) * 100;
setCurrentAppRunsInPercentage(Math.round(percentage));
- setCurrentAppRunsInNumber(userdata.app_execution_limit - userdata.app_execution_usage);
+ setCurrentAppRunsInNumber(userdata.app_execution_limit - userdata.app_execution_usage - userdata.app_executions_suborgs);
}
if (userdata?.id?.length > 0 && isLoggedIn === false){
setIsLoggedIn(true)
}
- }, [userdata]);
+ }, [monthlyAppRunsParent, monthlyAllSuborgExecutions, userdata]);
const [BillingEmail, setBillingEmail] = useState(selectedOrganization?.Billing?.Email);
@@ -199,6 +202,48 @@ const Billing = memo((props) => {
}
}, [])
+
+ const getStats = (orgid) => {
+
+ if (orgid === undefined || orgid === null) {
+ return
+ }
+
+ fetch(`${globalUrl}/api/v1/orgs/${orgid}/stats`, {
+ method: "GET",
+ headers: {
+ "Content-Type": "application/json",
+ Accept: "application/json",
+ },
+ credentials: "include",
+ })
+ .then((response) => {
+ if (response.status !== 200) {
+ console.log("Status not 200 for workflows :O!: ", response.status);
+ return;
+ }
+
+ return response.json();
+ })
+ .then((responseJson) => {
+ if (responseJson["success"] === false) {
+ return
+ }
+
+ setStatistics(responseJson);
+ })
+ .catch((error) => {
+ console.log("error: ", error)
+ });
+ }
+
+
+ useEffect(() => {
+ if (selectedOrganization && selectedOrganization?.id?.length > 0) {
+ getStats(selectedOrganization.id);
+ }
+ }, [selectedOrganization]);
+
const paperStyle = {
padding: 20,
// maxWidth: 400,
@@ -1860,19 +1905,6 @@ const Billing = memo((props) => {
const updateAlertThreshold = (index, field, value) => {
- if (field === 'percentage') {
- if (value > 100 || value < 0) {
- value = 0
- toast("The percentage value should be between 0 and 100")
- }
- } else if (field === 'count') {
- if (value < 0 || value >= userdata.app_execution_limit) {
- value = 0
- toast("The count value should be greater than 0 and less than the total app execution limit")
- }
- }
-
-
const totalValue = userdata.app_execution_limit;
const newAlertThresholds = alertThresholds.map((threshold, i) => {
if (i === index) {
@@ -1986,7 +2018,13 @@ const Billing = memo((props) => {
};
const isChildOrg = userdata?.active_org?.creator_org !== "" && userdata?.active_org?.creator_org !== undefined && userdata?.active_org?.creator_org !== null
-
+
+ useEffect(() => {
+ if (isChildOrg && currentTab === 0) {
+ setCurrentTab(1);
+ }
+ }, [isChildOrg, currentTab]);
+
return (
@@ -2386,8 +2424,8 @@ const Billing = memo((props) => {
We offer priority support through consultations and training to help you make the most of our product. If you have any questions, please reach out to us at {supportEmail}.
We offer priority support through consultations and training to help you make the most of our product. If you have any questions, please reach out to us at
-
@@ -2423,9 +2461,21 @@ const Billing = memo((props) => {
}}
/>
- You have used {currentAppRunsInPercentage}% of total app execution limit or {userdata.app_execution_usage} app runs out of {userdata.app_execution_limit} app runs.
+ You have used {currentAppRunsInPercentage}% of total app execution limit or {Number(monthlyAppRunsParent ?? 0) + Number(monthlyAllSuborgExecutions ?? 0)} app runs out of {userdata.app_execution_limit} app runs this month.
-
+
+ {userdata?.active_org?.creator_org?.length > 0 ? null :
+ (
+ <>
+
+ Parent Organization App Executions: {monthlyAppRunsParent}
+
+
+ Sub-Organization App Executions: {monthlyAllSuborgExecutions || "N/A"}
+
+ >
+ )}
+
Set email alert thresholds for app runs
@@ -2442,8 +2492,8 @@ const Billing = memo((props) => {
: " " + 0 + " "}
app runs.
-
- Please note: Once your app runs reach the set alert threshold, all admins in the organization will receive an email notification.
+
+ Please note: Once your app runs reach the set alert threshold, all admins in the organization will receive an email notification. For Parent organizations, the alert will be sent base on the total app runs from both parent and sub-organizations. For Sub-organizations, the alert will be sent based on the app runs of the sub-organization only.
- )
-}
-
const AppStats = (defaultprops) => {
const {
@@ -92,6 +45,12 @@ const AppStats = (defaultprops) => {
inputWorkflows,
clickedFromOrgTab,
syncStats,
+ statistics,
+ monthlyAppRunsParent,
+ setMonthlyAppRunsParent,
+ monthlyAllSuborgExecutions,
+ setMonthlyAllSuborgExecutions,
+ currentTab
} = defaultprops;
const [keys, setKeys] = useState([])
@@ -104,7 +63,6 @@ const AppStats = (defaultprops) => {
const [endTime, setEndTime] = useState("")
const [startTime, setStartTime] = useState("")
- const [statistics, setStatistics] = useState(undefined);
const [filteredStatistics, setFilteredStatistics] = useState(undefined);
const [apprunCost, setApprunCost] = useState(0)
@@ -125,6 +83,11 @@ const AppStats = (defaultprops) => {
}
}, [])
+ useEffect(() => {
+ if (statistics && statistics?.org_id?.length > 0) {
+ handleDataSetting(statistics, "day")
+ }
+}, [statistics])
const getWorkflowStats = async (workflow, startTime, endTime) => {
@@ -266,6 +229,7 @@ const AppStats = (defaultprops) => {
const statKey = syncStats === true ? "onprem_stats" : "daily_statistics"
if (statistics[statKey] === undefined || statistics[statKey] === null) {
setFilteredStatistics(statistics)
+ setMonthlyAppRunsParent(statistics["monthly_app_executions"] ?? 0)
return
}
@@ -312,20 +276,27 @@ const AppStats = (defaultprops) => {
}
}
- // Make a date at the 1st of the current month
+ // Make a date at the 1st of the current month - only when no start time is selected
var foundstarttime = (new Date())
- foundstarttime.setDate(1)
if (startTime !== "" && startTime !== undefined && startTime !== null) {
- foundstarttime = startTime
+ foundstarttime = new Date(startTime)
+ // Set to start of day to include the entire start date
+ foundstarttime.setHours(0, 0, 0, 0)
+ } else {
+ // Default to 1st of current month when no start time is selected
+ foundstarttime.setDate(1)
+ foundstarttime.setHours(0, 0, 0, 0)
}
- // Set to tomorrow by default
+ // Set end time properly
var foundendtime = (new Date())
- foundendtime.setDate(foundendtime.getDate() + 1)
-
- // Check if endtime is after the daily statistics["date"] string
if (endTime !== "" && endTime !== undefined && endTime !== null) {
- foundendtime = endTime
+ foundendtime = new Date(endTime)
+ // Set to end of day to include the entire end date
+ foundendtime.setHours(23, 59, 59, 999)
+ } else {
+ // Default to current date when no end time is selected
+ foundendtime.setHours(23, 59, 59, 999)
}
// Check if start time is before the daily statistics["date"] string
@@ -337,8 +308,20 @@ const AppStats = (defaultprops) => {
}
const date = new Date(item["date"])
- if (date >= foundstarttime) {
- if (date <= foundendtime) {
+ // Normalize the date to start of day for comparison
+ const normalizedDate = new Date(date)
+ normalizedDate.setHours(0, 0, 0, 0)
+
+ // Normalize foundstarttime for comparison
+ const normalizedStartTime = new Date(foundstarttime)
+ normalizedStartTime.setHours(0, 0, 0, 0)
+
+ // Normalize foundendtime for comparison
+ const normalizedEndTime = new Date(foundendtime)
+ normalizedEndTime.setHours(0, 0, 0, 0)
+
+ if (normalizedDate >= normalizedStartTime) {
+ if (normalizedDate <= normalizedEndTime) {
newlist.push(item)
}
}
@@ -373,6 +356,10 @@ const AppStats = (defaultprops) => {
workflowexecutions += item["workflow_executions"]
appexecutions += item["app_executions"]
+ if (currentTab === 0) {
+ appexecutions += (item["child_app_executions"] ?? 0)
+ }
+
estimatedcost += (item["app_executions"] * invocationCost)
}
@@ -390,7 +377,16 @@ const AppStats = (defaultprops) => {
}
setFilteredStatistics(tmpstats)
- handleDataSetting(tmpstats, "day")
+ handleDataSetting(tmpstats, "day")
+ // if we have done monthly reset than only show monthly app runs as current month app run
+ const currentMonth = new Date().getMonth() + 1
+ if (!monthlyAppRunsParent && statistics["monthly_app_executions"] > 0 && currentMonth === statistics["last_monthly_reset_month"]) {
+ setMonthlyAppRunsParent(statistics["monthly_app_executions"])
+ }
+
+ if (!monthlyAllSuborgExecutions && statistics["monthly_child_app_executions"]> 0 && currentMonth === statistics["last_monthly_reset_month"]) {
+ setMonthlyAllSuborgExecutions(statistics["monthly_child_app_executions"])
+ }
if (workflows !== undefined && workflows !== null && workflows.length > 0) {
@@ -476,7 +472,7 @@ const AppStats = (defaultprops) => {
if (item["child_app_executions"] !== undefined && item["child_app_executions"] !== null) {
childorgappRuns["data"].push({
key: new Date(item["date"]),
- data: inputdata["child_app_executions"]
+ data: item["child_app_executions"]
})
}
@@ -496,40 +492,45 @@ const AppStats = (defaultprops) => {
}
}
- // Adds data for today
- if (inputdata["daily_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) {
- appRuns["data"].push({
- key: new Date(),
- data: inputdata["daily_app_executions"]
- })
+ // Only add today's data if endTime is not set or if today falls within the selected date range
+ const today = new Date()
+ const shouldAddTodayData = endTime === "" || endTime === undefined || endTime === null ||
+ (new Date(endTime) >= today.setHours(0, 0, 0, 0))
- appcostRuns["data"].push({
- key: new Date(),
- data: (inputdata["daily_app_executions"] * invocationCost).toFixed(2)
- })
- }
+ if (shouldAddTodayData) {
+ // Adds data for today
+ if (inputdata["daily_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) {
+ appRuns["data"].push({
+ key: new Date(),
+ data: inputdata["daily_app_executions"]
+ })
- if (inputdata["daily_child_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) {
- childorgappRuns["data"].push({
- key: new Date(),
- data: inputdata["daily_child_app_executions"]
- })
+ appcostRuns["data"].push({
+ key: new Date(),
+ data: (inputdata["daily_app_executions"] * invocationCost).toFixed(2)
+ })
+ }
- //setApprunCosts(appcostRuns)
- }
+ if (inputdata["daily_child_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) {
+ childorgappRuns["data"].push({
+ key: new Date(),
+ data: inputdata["daily_child_app_executions"]
+ })
+ }
- if (inputdata["daily_workflow_executions"] !== undefined && inputdata["daily_workflow_executions"] !== null) {
- workflowRuns["data"].push({
- key: new Date(),
- data: inputdata["daily_workflow_executions"]
- })
- }
+ if (inputdata["daily_workflow_executions"] !== undefined && inputdata["daily_workflow_executions"] !== null) {
+ workflowRuns["data"].push({
+ key: new Date(),
+ data: inputdata["daily_workflow_executions"]
+ })
+ }
- if (inputdata["daily_subflow_executions"] !== undefined && inputdata["daily_subflow_executions"] !== null) {
- subflowRuns["data"].push({
- key: new Date(),
- data: inputdata["daily_subflow_executions"]
- })
+ if (inputdata["daily_subflow_executions"] !== undefined && inputdata["daily_subflow_executions"] !== null) {
+ subflowRuns["data"].push({
+ key: new Date(),
+ data: inputdata["daily_subflow_executions"]
+ })
+ }
}
// Only for parent orgs
@@ -541,49 +542,8 @@ const AppStats = (defaultprops) => {
setWorkflowRuns(workflowRuns)
setAppruns(appRuns)
setApprunCosts(appcostRuns)
- }
-
- const getStats = (orgid) => {
-
- if (orgid === undefined || orgid === null) {
- return
- }
-
- fetch(`${globalUrl}/api/v1/orgs/${orgid}/stats`, {
- method: "GET",
- headers: {
- "Content-Type": "application/json",
- Accept: "application/json",
- },
- credentials: "include",
- })
- .then((response) => {
- if (response.status !== 200) {
- console.log("Status not 200 for workflows :O!: ", response.status);
- return;
- }
-
- return response.json();
- })
- .then((responseJson) => {
- if (responseJson["success"] === false) {
- return
- }
-
- setStatistics(responseJson)
- handleDataSetting(responseJson, "day")
- })
- .catch((error) => {
- console.log("error: ", error)
- });
}
- useEffect(() => {
- if(selectedOrganization?.id?.length > 0) {
- getStats(selectedOrganization.id)
- }
- }, [selectedOrganization])
-
const paperStyle = {
textAlign: "center",
padding: "40px",
@@ -703,15 +663,19 @@ const AppStats = (defaultprops) => {
]
const data = (
-
+
All shown statistics are gathered from Your Organisation Statistics.
- It exists to give you more insight into your workflows, and to understand your utilization of the Shuffle platform. The billing tracker is in Beta, and is always calculated manually before being invoiced.
+ {currentTab === 0 ?
+
+ All Organization app runs are calculated base on addition of parent org app runs + all child org app runs.
+ : It exists to give you more insight into your workflows, and to
+ understand your utilization of the Shuffle platform.{" "}}
{syncStats !== true ? null :
"PS: You are currently looking at data from your onprem synced org"}
@@ -722,7 +686,7 @@ const AppStats = (defaultprops) => {
{filteredStatistics !== undefined ?
- {syncStats == true ? null :
+ {/* {syncStats == true ? null :
The cost of app runs in the selected period based on {filteredStatistics.monthly_app_executions} App Runs. These numbers do not exclude your included 10.000/month or {includedExecutions} App Runs per month. App Run cost: ${invocationCost}.
@@ -742,7 +706,7 @@ const AppStats = (defaultprops) => {
- }
+ } */}
{syncStats === true ? null :
{
}
- {syncStats === true ? null :
+ {syncStats === true || currentTab === 0 ? null :
Workflow runs in the selected period
@@ -778,7 +742,7 @@ const AppStats = (defaultprops) => {
}
- {syncStats === true ? null :
+ {/* {syncStats === true ? null :
Estimated cost to be billed at the end of the current month. Subtracted contractually included app runs. Actual cost month to date: ${monthToDateCost}. App Run cost: ${invocationCost}.
@@ -793,15 +757,9 @@ const AppStats = (defaultprops) => {
- }
-
+ Protected keys are encrypted, only available to admins, and will be masked when used in workflows. This is a basic protection, and is NOT bulletproof.
+
{
- Beta Feature: When a workflow run is done, the data from the selected actions will be removed by replacing it with a default value. This is useful for cleaning up sensitive data, or data that is no longer needed. This is done after a workflow run is finished or aborted, and is not reversible. Data will remain in the workflow run result (last node value) even if the action result itself is cleaned up.
+ When a workflow run is done, the data from the selected actions will be removed by replacing it with a default value. This is useful for cleaning up sensitive data, or data that is no longer needed. This is done after a workflow run is finished or aborted, and is not reversible. Data will remain in the workflow run result (last node value) even if the action result itself is cleaned up.
@@ -1285,7 +1743,7 @@ const EditWorkflow = (props) => {
- {newWorkflow === true ?
+ {/*newWorkflow === true ?
Relevant Workflows
@@ -1311,9 +1769,8 @@ const EditWorkflow = (props) => {
}
- : null}
-
- {/*newWorkflow === true && name.length > 2 ?
+ : null*/}
+ {/*newWorkflow === true && name.length > 2 ?
{
//toast("No Orborus necessary for environment cloud. Create and use a different environment to run executions on-premises.",)
return
}
-
+
if (
props.userdata.active_org === undefined ||
props.userdata.active_org === null
) {
return;
}
-
+
const elementName = "copy_element_shuffle";
var auth =
environment.auth === ""
? "cb5st3d3Z!3X3zaJ*Pc"
: environment.auth
-
- // Escape exclamation marks for copying
- auth = auth.replace("\\!", "!").replace(/!/g, "\\!")
-
+ // Escape exclamation marks for copying
+ auth = auth.replace("\\!", "!").replace(/!/g, "\\!")
const newUrl =
globalUrl === "https://shuffler.io"
? "https://shuffle-backend-stbuwivzoq-nw.a.run.app"
: globalUrl;
-
+
var skipPipeline = false
if (commandController.pipelines === true) {
skipPipeline = true
}
-
+
var addProxy = false
if (commandController.proxies === true) {
addProxy = true
}
-
+
if (installationTab === 1) {
return (`docker run -d \\
--restart=always \\
@@ -428,14 +426,32 @@ const EnvironmentTab = memo((props) => {
-e SHUFFLE_SWARM_CONFIG=run \\
-e SHUFFLE_LOGS_DISABLED=true \\
-e BASE_URL="${newUrl}" \\${addProxy ? `
- -e HTTPS_PROXY=IP:PORT \\` : ""}${skipPipeline ? `
- -e SHUFFLE_SKIP_PIPELINES=true \\` : ""}
+ -e HTTPS_PROXY=IP:PORT \\` : ""}${skipPipeline ? `
+ -e SHUFFLE_SKIP_PIPELINES=true \\` : ""}
ghcr.io/shuffle/shuffle-orborus:latest
`)
} else if (installationTab === 2) {
- return `https://shuffler.io/docs/configuration#kubernetes`
+ return `helm install shuffle-orborus oci://ghcr.io/shuffle/charts/shuffle \\
+ --namespace shuffle --create-namespace \\
+ --set opensearch.enabled=false \\
+ --set backend.replicaCount=0 \\
+ --set frontend.replicaCount=0 \\
+ --set 'orborus.extraEnvVars[0].name=AUTH' \\
+ --set 'orborus.extraEnvVars[0].value=${auth}' \\
+ --set 'orborus.extraEnvVars[1].name=BASE_URL' \\
+ --set 'orborus.extraEnvVars[1].value=${newUrl}' \\
+ --set 'orborus.extraEnvVars[2].name=ENVIRONMENT_NAME' \\
+ --set 'orborus.extraEnvVars[2].value=${environment.Name}' \\
+ --set 'orborus.extraEnvVars[3].name=ORG' \\
+ --set 'orborus.extraEnvVars[3].value=${environment.org_id}' \\
+ --set shuffle.org="${environment.org_id}"${addProxy ? ` \\
+ --set 'orborus.extraEnvVars[4].name=HTTPS_PROXY' \\
+ --set 'orborus.extraEnvVars[4].value=IP:PORT'` : ""}${skipPipeline ? ` \\
+ --set 'orborus.extraEnvVars[${addProxy ? 5 : 4}].name=SHUFFLE_SKIP_PIPELINES' \\
+ --set 'orborus.extraEnvVars[${addProxy ? 5 : 4}].value=true'` : ""} \\
+ --set persistence.storageClass=standard`
}
-
+
const commandData = `docker rm shuffle-orborus --force; \\\ndocker run -d \\
--restart=always \\
--name="shuffle-orborus" \\
@@ -445,12 +461,13 @@ const EnvironmentTab = memo((props) => {
-e ENVIRONMENT_NAME="${environment.Name}" \\
-e ORG="${props.userdata.active_org.id}" \\
-e BASE_URL="${newUrl}" \\${addProxy ? `
- -e HTTPS_PROXY=IP:PORT \\` : ""}${skipPipeline ? `
- -e SHUFFLE_SKIP_PIPELINES=true \\` : ""}
+ -e HTTPS_PROXY=IP:PORT \\` : ""}${skipPipeline ? `
+ -e SHUFFLE_SKIP_PIPELINES=true \\` : ""}
ghcr.io/shuffle/shuffle-orborus:latest`
-
+
return commandData
- };
+ };
+
const submitEnvironment = (data) => {
// FIXME - add some check here ROFL
@@ -1475,7 +1492,7 @@ const EnvironmentTab = memo((props) => {
{installationTab === 2 ?
- Check our Kubernetes documentation for more information on how to run Shuffle on Kubernetes. The status of the node will change when connected.
+ Simply connect to your Kubernetes cluster and run the following command:
:
@@ -1489,7 +1506,6 @@ const EnvironmentTab = memo((props) => {
"2. Run this command on the server you want to run workflows or store Pipeline data on"}
- {installationTab === 2 ? null :
You can see these buttons because you may have the correct access rights as a creator to help modify this workflow.
@@ -21115,20 +21628,6 @@ const AngularWorkflow = (defaultprops) => {
+ : null}
+
{data.action.app_name === "shuffle-subflow" &&
validate.result.success !== undefined &&
validate.result.success === true ? (
@@ -22944,7 +23496,7 @@ const AngularWorkflow = (defaultprops) => {
>
Action Logs
-
+
More log details for this action are not available without an onprem environment with the SHUFFLE_LOGS_DISABLED environment variable set to false: SHUFFLE_LOGS_DISABLED=false. Logs are enabled by default, except in scale mode.
@@ -854,7 +932,7 @@ const LoginPage = props => {
helperText={
handleValidateForm(username, password)
? ""
- : "Password must be at least 9 characters long"
+ : "Password must be at least 10 characters long"
}
/>
diff --git a/frontend/src/views/LoginPageOld.jsx b/frontend/src/views/LoginPageOld.jsx
index aaee73d6..c0996436 100755
--- a/frontend/src/views/LoginPageOld.jsx
+++ b/frontend/src/views/LoginPageOld.jsx
@@ -344,7 +344,7 @@ const LoginDialog = (props) => {
variant="body2"
style={{ marginBottom: 20, color: "white" }}
>
- 1. Make sure shuffle-database folder has correct access, and that you have a minimum of 2Gb of RAM available:{" "}
+ 1. Make sure shuffle-database folder has correct access, and that you have a minimum of 4Gb of RAM available:{" "}