diff --git a/.github/install-guide.md b/.github/install-guide.md
index cf2186e6..113ac1fc 100755
--- a/.github/install-guide.md
+++ b/.github/install-guide.md
@@ -18,16 +18,15 @@ The Docker setup is the default setup, and is ran with docker compose. This is [
**PS: if you're setting up Shuffle on Windows, go to the next step (Windows Docker setup)**
-1. Make sure you have [Docker](https://docs.docker.com/get-docker/) installed, and that you have a minimum of **4Gb of RAM** available.
+1. Make sure you have [Docker](https://docs.docker.com/get-docker/) and [git](https://git-scm.com/downloads)(for downloading) installed, and that you have a minimum of **4Gb of RAM** available. More RAM = better.
2. Download Shuffle
```bash
git clone https://github.com/Shuffle/Shuffle
cd Shuffle
```
-3. Fix prerequisites for the Opensearch database (Elasticsearch):
+3. Fix prerequisites for the Opensearch database (Elasticsearch also works). This requires the `shuffle-database` folder to exist.
```bash
-mkdir shuffle-database # Create a database folder
sudo chown -R 1000:1000 shuffle-database # IF you get an error using 'chown', add the user first with 'sudo useradd opensearch'
sudo swapoff -a # Disable swap
@@ -35,7 +34,7 @@ sudo swapoff -a # Disable swap
4. Run docker-compose.
```bash
-docker-compose up -d
+docker compose up -d
```
5. Recommended for Opensearch to work well
diff --git a/.github/workflows/dockerbuild.yaml b/.github/workflows/dockerbuild.yaml
index 3aa07ae6..28675b83 100644
--- a/.github/workflows/dockerbuild.yaml
+++ b/.github/workflows/dockerbuild.yaml
@@ -20,19 +20,19 @@ jobs:
include:
- app: frontend
path: frontend
- version: 2.1.0
+ version: 2.1.1
experimental: true
- app: backend
path: backend
- version: 2.1.0
+ version: 2.1.1
experimental: true
- app: orborus
path: functions/onprem/orborus
- version: 2.1.0
+ version: 2.1.1
experimental: true
- app: worker
path: functions/onprem/worker
- version: 2.1.0
+ version: 2.1.1
experimental: true
steps:
- name: Checkout
diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod
index 5bf206d9..5c39faf8 100644
--- a/backend/go-app/go.mod
+++ b/backend/go-app/go.mod
@@ -8,6 +8,7 @@ toolchain go1.24.3
//replace github.com/frikky/schemaless => ../../../schemaless
//replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi
+//replace github.com/shuffle/opensearch-go => ../../../opensearch-go
require (
cloud.google.com/go/datastore v1.20.0
@@ -24,8 +25,8 @@ require (
github.com/gorilla/mux v1.8.1
github.com/h2non/filetype v1.1.3
github.com/satori/go.uuid v1.2.0
- github.com/shuffle/shuffle-shared v0.9.28
- github.com/shuffle/singul v0.0.16
+ github.com/shuffle/shuffle-shared v0.9.37
+ github.com/shuffle/singul v0.0.18
golang.org/x/crypto v0.40.0
google.golang.org/api v0.236.0
google.golang.org/grpc v1.72.2
@@ -73,7 +74,7 @@ require (
github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
- github.com/frikky/schemaless v0.0.20 // indirect
+ github.com/frikky/schemaless v0.0.23 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-jose/go-jose/v4 v4.0.5 // indirect
@@ -113,8 +114,6 @@ require (
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
- github.com/opensearch-project/opensearch-go v1.1.0 // indirect
- github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect
github.com/osteele/liquid v1.7.0 // indirect
github.com/osteele/tuesday v1.0.3 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
@@ -126,6 +125,7 @@ require (
github.com/sendgrid/rest v2.6.9+incompatible // indirect
github.com/sendgrid/sendgrid-go v3.16.1+incompatible // indirect
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
+ github.com/shuffle/opensearch-go/v4 v4.0.0 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/skeema/knownhosts v1.3.1 // indirect
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum
index dc8366b0..39338b02 100644
--- a/backend/go-app/go.sum
+++ b/backend/go-app/go.sum
@@ -72,20 +72,6 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
-github.com/aws/aws-sdk-go v1.42.27/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc=
-github.com/aws/aws-sdk-go v1.44.263/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
-github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4=
-github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw=
-github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk=
-github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8=
-github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/basgys/goxml2json v1.1.0 h1:4ln5i4rseYfXNd86lGEB+Vi652IsIXIvggKM/BhUKVw=
github.com/basgys/goxml2json v1.1.0/go.mod h1:wH7a5Np/Q4QoECFIU8zTQlZwZkrilY0itPfecMw41Dw=
github.com/bitly/go-simplejson v0.5.1 h1:xgwPbetQScXt1gh9BmoJ6j9JMr3TElvuIyjR8pgdoow=
@@ -152,8 +138,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0=
github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4=
-github.com/frikky/schemaless v0.0.20 h1:S/A2pQcRN9qa2RnufvxwCeM06trjG0JLTF3urt1tFQI=
-github.com/frikky/schemaless v0.0.20/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
+github.com/frikky/schemaless v0.0.23 h1:fa1JR8HJat2obbAKB5+Rnwmr09LNUXlyjTMgKG/G7BA=
+github.com/frikky/schemaless v0.0.23/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
github.com/fsouza/go-dockerclient v1.12.1 h1:FMoLq+Zhv9Oz/rFmu6JWkImfr6CBgZOPcL+bHW4gS0o=
github.com/fsouza/go-dockerclient v1.12.1/go.mod h1:OqsgJJcpCwqyM3JED7TdfM9QVWS5O7jSYwXxYKmOooY=
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
@@ -223,7 +209,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
@@ -265,8 +250,6 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
-github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@@ -325,10 +308,6 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
-github.com/opensearch-project/opensearch-go v1.1.0 h1:eG5sh3843bbU1itPRjA9QXbxcg8LaZ+DjEzQH9aLN3M=
-github.com/opensearch-project/opensearch-go v1.1.0/go.mod h1:+6/XHCuTH+fwsMJikZEWsucZ4eZMma3zNSeLrTtVGbo=
-github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ=
-github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8=
github.com/osteele/liquid v1.7.0 h1:VsbPSchE5D5S5scylAIvERET4dnCxsO6IDri2oSJ5Dk=
github.com/osteele/liquid v1.7.0/go.mod h1:xU0Z2dn2hOQIEFEWNmeltOmCtfhtoW/2fCyiNQeNG+U=
github.com/osteele/tuesday v1.0.3 h1:SrCmo6sWwSgnvs1bivmXLvD7Ko9+aJvvkmDjB5G4FTU=
@@ -363,10 +342,12 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G
github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8=
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
-github.com/shuffle/shuffle-shared v0.9.21 h1:ZxA/g0Lk93miHJpVYAhtTQmtWLJWcCtm6kFNF772Ynk=
-github.com/shuffle/shuffle-shared v0.9.21/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw=
-github.com/shuffle/singul v0.0.16 h1:dW+0Mln9R1aUJ0fjikpWcxbjoQWqJHxe4kSxh2tQN5E=
-github.com/shuffle/singul v0.0.16/go.mod h1:LYkp320A6gsoPlYbXUM+WvEPUVAuutlSsqnVKyRy4gs=
+github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU=
+github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0=
+github.com/shuffle/shuffle-shared v0.9.37 h1:rk/TKhcqXFxRdlMiUjPfm2dCRjoGI360i12znF/Rq+A=
+github.com/shuffle/shuffle-shared v0.9.37/go.mod h1:sYRBTvGlYRsh9bFJeSbA1Rq9DA274+rGzvIH+Q1j3GU=
+github.com/shuffle/singul v0.0.18 h1:SMM18+MHC41UexDToNdVkBTkUCxI9kRf8y30TPzvTnA=
+github.com/shuffle/singul v0.0.18/go.mod h1:8zZevcJhlQ9hW8UfDS/X4e6UYS8h03gy5qkFtno+lJM=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
@@ -393,8 +374,18 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
+github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
+github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
+github.com/wI2L/jsondiff v0.7.0 h1:1lH1G37GhBPqCfp/lrs91rf/2j3DktX6qYAKZkLuCQQ=
+github.com/wI2L/jsondiff v0.7.0/go.mod h1:KAEIojdQq66oJiHhDyQez2x+sRit0vIzC9KeK0yizxM=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
@@ -491,9 +482,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
@@ -536,13 +525,11 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg=
golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0=
@@ -554,7 +541,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=
diff --git a/backend/go-app/main.go b/backend/go-app/main.go
index 6845056e..78dde288 100755
--- a/backend/go-app/main.go
+++ b/backend/go-app/main.go
@@ -14,18 +14,18 @@ import (
"crypto/md5"
"strconv"
- "os"
- "io"
- "log"
- "fmt"
- "errors"
- "net/url"
- "os/exec"
- "net/http"
- "io/ioutil"
- "math/rand"
"encoding/hex"
"encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "log"
+ "math/rand"
+ "net/http"
+ "net/url"
+ "os"
+ "os/exec"
"net/http/httptest"
"strings"
@@ -67,6 +67,7 @@ var runningEnvironment = "onprem"
var syncUrl = "https://shuffler.io"
var debug = false
+
//var syncUrl = "http://localhost:5002"
type retStruct struct {
@@ -76,6 +77,7 @@ type retStruct struct {
IntervalSeconds int64 `json:"interval_seconds"`
Reason string `json:"reason"`
Subscriptions []shuffle.PaymentSubscription `json:"subscriptions"`
+ Licensed bool `json:"licensed"`
}
type Contact struct {
@@ -951,42 +953,62 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) {
return childOrgs[i].Created < childOrgs[j].Created
})
+ parentOrg := &shuffle.Org{}
+ if len(org.CreatorOrg) > 0 {
+ parentOrg, err = shuffle.GetOrg(ctx, org.CreatorOrg)
+ if err != nil {
+ log.Printf("[ERROR] Failed to get parent org during getinfo: %s", err)
+ parentOrg = &shuffle.Org{}
+ }
+ licenseOrg := shuffle.HandleCheckLicense(ctx, *parentOrg)
+ parentOrg = &licenseOrg
+ }
+
+ limit := 3
+
+ if parentOrg.Licensed && parentOrg.SyncFeatures.MultiTenant.Active {
+ limit = int(parentOrg.SyncFeatures.MultiTenant.Limit)
+ }
+
for index, org := range childOrgs {
- if index < 3 {
+ if index < limit {
continue
}
failToLoadOrgs = append(failToLoadOrgs, org.Id)
}
- if len(org.CreatorOrg) > 0 && len(childOrgs) > 3 && shuffle.ArrayContains(failToLoadOrgs, userInfo.ActiveOrg.Id) {
- parentOrg, err := shuffle.GetOrg(ctx, org.CreatorOrg)
- if err != nil {
- log.Printf("[ERROR] Failed to get parent org during getinfo: %s", err)
- } else {
- parent := shuffle.HandleCheckLicense(ctx, *parentOrg)
- parentOrg := &parent
- if !parentOrg.SyncFeatures.MultiTenant.Active {
- userInfo.ActiveOrg = shuffle.OrgMini{
- Id: parentOrg.Id,
- Name: parentOrg.Name,
- Role: userInfo.Role,
- Branding: parentOrg.Branding,
- Image: parentOrg.Image,
- }
- log.Printf("[INFO] Parent org %s has more than 3 child orgs and is not licensed. Moving user %s to parent org %s", parentOrg.Name, userInfo.Username, parentOrg.Name)
+ if len(org.CreatorOrg) > 0 && len(childOrgs) > limit && shuffle.ArrayContains(failToLoadOrgs, userInfo.ActiveOrg.Id) {
- err = shuffle.SetUser(ctx, &userInfo, false)
- if err != nil {
- log.Printf("[WARNING] Failed setting user to parent org: %s", err)
- }
-
- resp.WriteHeader(200)
- resp.Write([]byte(`{"success": true, "reason": "Parent org has more than 3 child orgs and is not licensed. Moving to parent org. Contact support@shuffler.io for more information", "switch_parent": true}`))
- return
- }
+ userInfo.ActiveOrg = shuffle.OrgMini{
+ Id: parentOrg.Id,
+ Name: parentOrg.Name,
+ Role: userInfo.Role,
+ Branding: parentOrg.Branding,
+ Image: parentOrg.Image,
}
+
+ if parentOrg.Licensed {
+ log.Printf("[INFO] Parent org %s is licensed. But Multi-tenant feature is not active. Moving user %s to parent org %s", parentOrg.Name, userInfo.Username)
+ } else {
+ log.Printf("[INFO] Parent org %s has more than 3 child orgs and is not licensed. Moving user %s to parent org %s", parentOrg.Name, userInfo.Username)
+ }
+
+ err = shuffle.SetUser(ctx, &userInfo, false)
+ if err != nil {
+ log.Printf("[WARNING] Failed setting user to parent org: %s", err)
+ }
+
+ reason := "Parent org has more than 3 child orgs and is not licensed. Moving user to parent org. Contact support@shuffler.io for more information"
+
+ if parentOrg.Licensed {
+ reason = fmt.Sprintf("Parent organization is licensed, but the maximum number of sub-organizations (%d) has been reached. You have been moved to the parent organization. Please contact support@shuffler.io for further assistance.", parentOrg.SyncFeatures.MultiTenant.Limit)
+ }
+
+ resp.WriteHeader(200)
+ resp.Write([]byte(`{"success": true, "reason": "` + reason + `", "switch_parent": true}`))
+ return
}
//if err == nil {
@@ -1138,7 +1160,8 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) {
// Check for licensing/branding of parent and override
parentOrg, err := shuffle.GetOrg(ctx, parentOrgId)
if err == nil {
- if parentOrg.LeadInfo.IntegrationPartner {
+ parent := shuffle.HandleCheckLicense(ctx, *parentOrg)
+ if parent.SyncFeatures.Branding.Active {
parsedStatus = append(parsedStatus, "integration_partner")
// except theme take from parent org
@@ -1176,11 +1199,15 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) {
break
}
}
+ } else {
+ userInfo.ActiveOrg.Branding = shuffle.OrgBranding{}
}
}
} else {
// for parent org branding
- if org.LeadInfo.IntegrationPartner {
+ licenseOrg := shuffle.HandleCheckLicense(ctx, *org)
+ org = &licenseOrg
+ if org.SyncFeatures.Branding.Active {
userInfo.ActiveOrg.Branding.Theme = org.Branding.Theme
userInfo.ActiveOrg.Branding.DocumentationLink = org.Defaults.DocumentationReference
userInfo.ActiveOrg.Branding.SupportEmail = org.Branding.SupportEmail
@@ -1189,6 +1216,8 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) {
userInfo.ActiveOrg.Branding.BrandName = org.Branding.BrandName
parsedStatus = append(parsedStatus, "integration_partner")
+ } else {
+ userInfo.ActiveOrg.Branding = shuffle.OrgBranding{}
}
}
@@ -1254,7 +1283,6 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) {
}
count := len(users)
-
if count == 0 {
log.Printf("[WARNING] No users - redirecting for management user")
resp.WriteHeader(200)
@@ -3389,7 +3417,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User, s
// Read and copy the baseline Dockerfile
dockerfileContent, err := ioutil.ReadFile(dockerfileSource)
if err != nil {
- foundDockerfile := shuffle.GetBaseDockerfile()
+ foundDockerfile := shuffle.GetBaseDockerfile()
if len(foundDockerfile) > 0 {
dockerfileContent = foundDockerfile
} else {
@@ -3872,11 +3900,12 @@ func handleCloudJob(job shuffle.CloudSyncJob) error {
// Handles jobs from remote (cloud)
func remoteOrgJobController(org shuffle.Org, body []byte) error {
type retStruct struct {
- Success bool `json:"success"`
- Reason string `json:"reason"`
- Jobs []shuffle.CloudSyncJob `json:"jobs"`
- SyncFeatures shuffle.SyncFeatures `json:"sync_features"`
+ Success bool `json:"success"`
+ Reason string `json:"reason"`
+ Jobs []shuffle.CloudSyncJob `json:"jobs"`
+ SyncFeatures shuffle.SyncFeatures `json:"sync_features"`
Subscriptions []shuffle.PaymentSubscription `json:"subscriptions"`
+ Licensed bool `json:"licensed"`
}
responseData := retStruct{}
@@ -3959,6 +3988,15 @@ func remoteOrgJobController(org shuffle.Org, body []byte) error {
shuffle.SetCache(ctx, subscriptionCacheKey, subscriptionsBytes, 1800)
}
+ licenseCacheKey := fmt.Sprintf("org_licensed_%s", org.Id)
+ licensedBytes, err := json.Marshal(responseData.Licensed)
+ if err != nil {
+ log.Printf("[ERROR] Failed to marshal Licensed for cache: %s", err)
+ } else {
+
+ shuffle.SetCache(ctx, licenseCacheKey, licensedBytes, 1800)
+ }
+
for _, job := range responseData.Jobs {
err = handleCloudJob(job)
if err != nil {
@@ -3996,7 +4034,7 @@ func remoteOrgJobHandler(org shuffle.Org, interval int) error {
}
// Just to prevent it from spamming large outbound requests
- if shouldBackupData {
+ if shouldBackupData {
if org.SyncConfig.WorkflowBackup {
workflows, err := shuffle.GetAllWorkflowsByQuery(ctx, foundUser, 250, "")
if err != nil {
@@ -4030,7 +4068,7 @@ func remoteOrgJobHandler(org shuffle.Org, interval int) error {
}
// Send stats once every 10 times or so..?
- // For now, just send every time
+ // For now, just send every time
info, err := shuffle.GetOrgStatistics(ctx, org.Id)
if err != nil {
log.Printf("[ERROR] Failed getting org statistics backup for org %s: %s", org.Id, err)
@@ -4206,10 +4244,8 @@ func runInitEs(ctx context.Context) {
}
}
- if strings.Contains(os.Getenv("SHUFFLE_OPENSEARCH_URL"), "https") {
- log.Printf("[INFO] Waiting 30 seconds during init to make sure the opensearch instance is up and running with security features enabled")
- time.Sleep(30 * time.Second)
- }
+ log.Printf("[INFO] Waiting 30 seconds during init to make sure the opensearch instance is up and running with security features enabled")
+ time.Sleep(30 * time.Second)
shuffle.InitOpensearchIndexes()
@@ -4366,7 +4402,7 @@ func runInitEs(ctx context.Context) {
}
//interval := int(org.SyncConfig.Interval)
- interval := 30
+ interval := 30
if interval == 0 {
log.Printf("[WARNING] Skipping org %s because sync isn't set (0).", org.Id)
continue
@@ -4468,7 +4504,6 @@ func runInitEs(ctx context.Context) {
continue
}
-
respBody, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("[ERROR] Failed setting respbody %s for execution stop. Status: %d", err, newresp.StatusCode)
@@ -4539,7 +4574,7 @@ func runInitEs(ctx context.Context) {
url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION")
if len(url) == 0 {
log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/shuffle/python-apps")
-
+
url = "https://github.com/shuffle/python-apps"
//url = ""
//return
@@ -4624,6 +4659,45 @@ func runInitEs(ctx context.Context) {
}
}
+ // Self-cleaning
+ go func() {
+ cursor := ""
+ cnt := 0
+ newCtx := context.Background()
+ for _, org := range activeOrgs {
+ if len(org.Id) == 0 {
+ log.Printf("[DEBUG] No ID found for org with name '%s'. Why was it made?", org.Name)
+ continue
+ }
+
+ log.Printf("[INFO] Starting self-cleanup of cache keys for org %s", org.Id)
+
+ for {
+ keys, newCursor, err := shuffle.GetAllCacheKeys(newCtx, org.Id, "", 1000, cursor)
+ if err != nil {
+ //log.Printf("[ERROR] Failed getting all cache keys for cleanup: %s", err)
+ break
+ }
+
+ if newCursor == cursor || len(newCursor) == 0 {
+ break
+ }
+
+ if len(keys) == 0 {
+ break
+ }
+
+ cursor = newCursor
+ cnt += 1
+ if cnt > 10 {
+ break
+ }
+ }
+
+ log.Printf("[INFO] Finished self-cleanup of cache keys for org %s", org.Id)
+ }
+ }()
+
log.Printf("[INFO] Finished INIT (ES)")
}
@@ -4978,7 +5052,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) {
Interval: responseData.IntervalSeconds,
WorkflowBackup: true,
- AppBackup: true,
+ AppBackup: true,
}
interval := int(responseData.IntervalSeconds)
@@ -5443,6 +5517,11 @@ func initHandlers() {
r.HandleFunc("/api/v2/workflows/{key}/executions", shuffle.GetWorkflowExecutionsV2).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v2/workflows/generate/llm", shuffle.HandleWorkflowGenerationResponse).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v2/workflows/edit/llm", shuffle.HandleEditWorkflowWithLLM).Methods("POST", "OPTIONS")
+ r.HandleFunc("/api/v2/workflows/generate", shuffle.GenerateSingulWorkflows).Methods("POST", "OPTIONS")
+ r.HandleFunc("/api/v2/datastore", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS")
+ r.HandleFunc("/api/v2/datastore", shuffle.HandleSetDatastoreKey).Methods("POST", "OPTIONS")
+ r.HandleFunc("/api/v2/datastore/category/{category_key}", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS")
+ r.HandleFunc("/api/v2/datastore/automate", shuffle.HandleDatastoreCategoryConfig).Methods("POST", "OPTIONS")
// New for recommendations in Shuffle
r.HandleFunc("/api/v1/recommendations/get_actions", shuffle.HandleActionRecommendation).Methods("POST", "OPTIONS")
@@ -5537,6 +5616,10 @@ func initHandlers() {
r.HandleFunc("/api/v1/orgs/{orgId}/stats/{key}", shuffle.GetSpecificStats).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/statistics", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS")
+ r.HandleFunc("/api/v1/stats", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS")
+ r.HandleFunc("/api/v1/stats", shuffle.HandleAppendStatistics).Methods("POST", "OPTIONS")
+ r.HandleFunc("/api/v1/stats/{key}", shuffle.GetSpecificStats).Methods("GET", "OPTIONS")
+
r.HandleFunc("/api/v1/orgs/{orgId}/cache", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/cache", shuffle.HandleSetCacheKey).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/cache/{cache_key}", shuffle.HandleDeleteCacheKey).Methods("DELETE", "OPTIONS")
@@ -5592,7 +5675,7 @@ func initHandlers() {
r.HandleFunc("/api/v1/dashboards/{key}/widgets", shuffle.HandleNewWidget).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/dashboards/{key}/widgets/{widget_id}", shuffle.HandleGetWidget).Methods("GET", "OPTIONS")
- if (strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" || strings.ToLower(os.Getenv("DEBUG_MEMORY")) == "true") {
+ if strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" || strings.ToLower(os.Getenv("DEBUG_MEMORY")) == "true" {
log.Printf("[DEBUG] Memory debugging is enabled on /debug/pprof")
r.HandleFunc("/debug/pprof/", pprof.Index)
r.HandleFunc("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP)
@@ -5609,7 +5692,6 @@ func initHandlers() {
// Had to move away from mux, which means Method is fucked up right now.
func main() {
-
if os.Getenv("DEBUG") == "true" {
debug = true
}
diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go
index 090d524e..032af73a 100755
--- a/backend/go-app/walkoff.go
+++ b/backend/go-app/walkoff.go
@@ -20,21 +20,21 @@ import (
"strings"
"time"
- dockerclient "github.com/docker/docker/client"
"github.com/docker/docker/api/types/image"
+ dockerclient "github.com/docker/docker/client"
"github.com/h2non/filetype"
uuid "github.com/satori/go.uuid"
newscheduler "github.com/carlescere/scheduler"
- "github.com/go-co-op/gocron"
"github.com/frikky/kin-openapi/openapi3"
+ gyaml "github.com/ghodss/yaml"
+ "github.com/go-co-op/gocron"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
- "github.com/go-git/go-git/v5/storage/memory"
"github.com/go-git/go-git/v5/plumbing"
http2 "github.com/go-git/go-git/v5/plumbing/transport/http"
- gyaml "github.com/ghodss/yaml"
+ "github.com/go-git/go-git/v5/storage/memory"
)
var localBase = "http://localhost:5001"
@@ -249,7 +249,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque
resp.Write([]byte(`{"success": true}`))
}
-// FIXME: Authenticate this one? Can org ID be auth enough?
+// FIXME: Authenticate this one. Can org ID be auth enough?
// (especially since we have a default: shuffle)
func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
@@ -257,7 +257,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
return
}
- // This is really the environment's name - NOT OrgId
+ // This is really the environment's name - NOT OrgId
environment := request.Header.Get("Org-Id")
if len(environment) == 0 {
log.Printf("[AUDIT] No org-id header set")
@@ -276,7 +276,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
return
*/
}
-
+
// This section is cloud custom for now
auth := request.Header.Get("Authorization")
if len(auth) == 0 {
@@ -305,7 +305,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
}
// Only works onprem - shared queues across tenants
- // without tenancy
+ // without tenancy
if !found {
env, err = shuffle.GetEnvironment(ctx, environment, "")
if err != nil {
@@ -325,7 +325,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
orgId = env.OrgId
}
- executionRequests, err := shuffle.GetWorkflowQueue(ctx, environment, 100)
+ executionRequests, err := shuffle.GetWorkflowQueue(ctx, environment, 100, *env)
if err != nil {
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
@@ -336,47 +336,11 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
if len(executionRequests.Data) == 0 {
executionRequests.Data = []shuffle.ExecutionRequest{}
} else {
-
- // Try again? I don't think this is necessary, and shouldn't really ever occur.
- /*
- if len(env.Id) == 0 && len(env.Name) == 0 {
- timeNow := int64(time.Now().Unix())
- foundId := ""
- for _, requestData := range executionRequests.Data {
- execution, err := shuffle.GetWorkflowExecution(ctx, requestData.ExecutionId)
- if err == nil {
- if len(execution.ExecutionOrg) > 0 {
- foundId = execution.ExecutionOrg
- break
- }
- }
- }
-
- if len(environment) > 0 {
-
- env, err := shuffle.GetEnvironment(ctx, environment, foundId)
- if err != nil {
- log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", environment, err)
- //resp.WriteHeader(401)
- //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No env found matching %s"}`, id)))
- //return
- } else {
- if timeNow > env.Edited+60 {
- env.RunningIp = request.RemoteAddr
- env.Checkin = timeNow
- err = shuffle.SetEnvironment(ctx, env)
- if err != nil {
- log.Printf("[WARNING] Failed updating environment: %s", err)
- }
- }
- }
- }
- }
- */
-
+ // Try again? I don't think this is necessary, and shouldn't really ever occur.
if len(executionRequests.Data) > 50 {
executionRequests.Data = executionRequests.Data[0:49]
}
+
}
newjson, err := json.Marshal(executionRequests)
@@ -493,16 +457,16 @@ func handleGetWorkflowExecutionResult(resp http.ResponseWriter, request *http.Re
if workflowExecution.Workflow.Sharing == "form" {
newWorkflow := shuffle.Workflow{
- Name: workflowExecution.Workflow.Name,
- ID: workflowExecution.Workflow.ID,
- Owner: workflowExecution.Workflow.Owner,
- OrgId: workflowExecution.Workflow.OrgId,
+ Name: workflowExecution.Workflow.Name,
+ ID: workflowExecution.Workflow.ID,
+ Owner: workflowExecution.Workflow.Owner,
+ OrgId: workflowExecution.Workflow.OrgId,
- Sharing: workflowExecution.Workflow.Sharing,
+ Sharing: workflowExecution.Workflow.Sharing,
Description: workflowExecution.Workflow.Description,
InputQuestions: workflowExecution.Workflow.InputQuestions,
- FormControl: workflowExecution.Workflow.FormControl,
+ FormControl: workflowExecution.Workflow.FormControl,
}
workflowExecution.Results = []shuffle.ActionResult{}
@@ -801,7 +765,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) {
}
if user.Id != workflow.Owner || len(user.Id) == 0 {
- if workflow.OrgId == user.ActiveOrg.Id {
+ if workflow.OrgId == user.ActiveOrg.Id {
log.Printf("[INFO] User %s is deleting workflow %s as admin. Owner: %s", user.Username, workflow.ID, workflow.Owner)
} else {
log.Printf("[WARNING] Wrong user (%s) for workflow %s (delete workflow)", user.Username, workflow.ID)
@@ -818,7 +782,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) {
childWorkflows, err := shuffle.ListChildWorkflows(ctx, workflow.ID)
if err != nil {
log.Printf("[ERROR] Failed to list child workflows: %s", err)
- } else {
+ } else {
//log.Printf("\n\n[DEBUG] Found %d child workflows for workflow %s\n\n", len(childWorkflows), workflow.ID)
// Find cookies and append them to request.Header to replicate current request as closely as possible
@@ -934,10 +898,10 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request
}
/*
- if !workflow.IsValid {
- log.Printf("[ERROR] Stopped execution as workflow %s is not valid.", workflow.ID)
- return shuffle.WorkflowExecution{}, fmt.Sprintf(`workflow %s is invalid`, workflow.ID), errors.New("Failed getting workflow")
- }
+ if !workflow.IsValid {
+ log.Printf("[ERROR] Stopped execution as workflow %s is not valid.", workflow.ID)
+ return shuffle.WorkflowExecution{}, fmt.Sprintf(`workflow %s is invalid`, workflow.ID), errors.New("Failed getting workflow")
+ }
*/
maxExecutionDepth := 10
@@ -950,7 +914,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request
workflowExecution, execInfo, _, workflowExecErr := shuffle.PrepareWorkflowExecution(ctx, workflow, request, int64(maxExecutionDepth))
if workflowExecErr != nil {
- if len(workflowExecution.Workflow.Actions) > 0 && len(workflowExecution.Results) > 0 && len(workflowExecution.ExecutionId) > 0 {
+ if len(workflowExecution.Workflow.Actions) > 0 && len(workflowExecution.Results) > 0 && len(workflowExecution.ExecutionId) > 0 {
err := shuffle.SetWorkflowExecution(ctx, workflowExecution, true)
if err != nil {
log.Printf("[ERROR] Failed setting workflow execution during init (2): %s", err)
@@ -964,7 +928,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request
return shuffle.WorkflowExecution{}, "", nil
} else {
log.Printf("[ERROR] Failed in prepareExecution: '%s'", workflowExecErr)
- return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed running: %s", workflowExecErr), workflowExecErr
+ return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed running: %s", workflowExecErr), workflowExecErr
}
}
@@ -1373,7 +1337,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) {
}
if user.Id != workflow.Owner || len(user.Id) == 0 {
- if workflow.OrgId == user.ActiveOrg.Id {
+ if workflow.OrgId == user.ActiveOrg.Id {
log.Printf("[AUDIT] User %s is accessing workflow %s as admin (stop schedule)", user.Username, workflow.ID)
} else {
log.Printf("[WARNING] Wrong user (%s) for workflow %s (stop schedule)", user.Username, workflow.ID)
@@ -1618,7 +1582,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) {
}
if user.Id != workflow.Owner || len(user.Id) == 0 {
- if workflow.OrgId == user.ActiveOrg.Id {
+ if workflow.OrgId == user.ActiveOrg.Id {
log.Printf("[INFO] User %s is deleting workflow %s as admin. Owner: %s", user.Username, workflow.ID, workflow.Owner)
} else {
log.Printf("[WARNING] Wrong user (%s) for workflow %s (schedule start). Owner: %s", user.Username, workflow.ID, workflow.Owner)
@@ -2058,7 +2022,7 @@ func loadGithubWorkflows(url, username, password, userId, branch, orgId string)
cloneOptions.ReferenceName = plumbing.ReferenceName(branch)
}
- cloneOptions = checkGitProxy(cloneOptions)
+ cloneOptions = checkGitProxy(cloneOptions)
storer := memory.NewStorage()
r, err := git.Clone(storer, fs, cloneOptions)
@@ -2913,7 +2877,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) {
log.Printf("\n\nACTION TO RUN: %s. Body: %s. Source URL: %s\n\n", appId, string(body), request.URL.String())
workflowExecution, err := shuffle.PrepareSingleAction(ctx, user, appId, body, runValidationAction, decisionId)
- if appId == "agent_starter" {
+ if appId == "agent_starter" {
log.Printf("[INFO] Returning early for agent_starter single action execution: %s", workflowExecution.ExecutionId)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization)))
@@ -2941,7 +2905,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) {
return
}
- workflowExecution.ProjectId = ""
+ workflowExecution.ProjectId = ""
workflowExecution.Locations = []string{""}
foundEnv := ""
@@ -2959,7 +2923,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) {
go shuffle.IncrementCache(ctx, workflowExecution.OrgId, "workflow_executions")
executionRequest := shuffle.ExecutionRequest{
- Priority: 15,
+ Priority: 15,
ExecutionId: workflowExecution.ExecutionId,
WorkflowId: workflowExecution.Workflow.ID,
Authorization: workflowExecution.Authorization,
@@ -3034,7 +2998,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
continue
}
- //duringStartup
+ //duringStartup
if duringStartup {
// Look for names: shuffle tools, http, email, shuffle ai
if shuffle.ArrayContains(startupNames, strings.ToLower(file.Name())) {
@@ -3402,7 +3366,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
if len(item.Tags) > 0 && shuffle.ArrayContains(handledImages, item.Tags[0]) {
continue
- }
+ }
handledImages = append(handledImages, item.Tags[0])
err = buildImageMemory(fs, item.Tags, item.Extra, true)
@@ -3438,7 +3402,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
for _, item := range buildLaterList {
if len(item.Tags) > 0 && shuffle.ArrayContains(handledImages, item.Tags[0]) {
continue
- }
+ }
handledImages = append(handledImages, item.Tags[0])
@@ -3529,7 +3493,7 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) {
}
}
- cloneOptions = checkGitProxy(cloneOptions)
+ cloneOptions = checkGitProxy(cloneOptions)
storer := memory.NewStorage()
r, err := git.Clone(storer, fs, cloneOptions)
diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx
index 75363d20..18083aea 100755
--- a/frontend/src/App.jsx
+++ b/frontend/src/App.jsx
@@ -25,6 +25,7 @@ import AgentUI from "./views/AgentUI.jsx";
import Welcome from "./views/Welcome.jsx";
import Dashboard from "./views/Dashboard.jsx";
import DashboardView from "./views/DashboardViews.jsx";
+import NewDashboard from "./views/NewDashboard.jsx";
import AdminSetup from "./views/AdminSetup.jsx";
import Admin from "./views/Admin.jsx";
import Docs from "./views/Docs.jsx";
@@ -167,6 +168,47 @@ const App = (message, props) => {
});
};
+ useEffect(() => {
+ if (userdata &&
+ userdata?.org_status?.includes("integration_partner") &&
+ userdata?.active_org?.image) {
+
+ const existingLinks = document.querySelectorAll("link[rel*='icon']");
+ existingLinks.forEach(link => link.parentNode.removeChild(link));
+ const newLink = document.createElement('link');
+ newLink.rel = 'icon';
+ newLink.type = 'image/x-icon';
+ try {
+ if (typeof userdata.active_org.image === 'string') {
+ if (userdata.active_org.image.startsWith('data:')) {
+ newLink.href = userdata.active_org.image;
+ } else if (userdata.active_org.image.startsWith('/')) {
+ newLink.href = userdata.active_org.image;
+ } else {
+ let mimeType = 'image/png';
+ newLink.href = `data:${mimeType};base64,${userdata.active_org.image}`;
+ }
+ } else {
+ console.error("Image data is not in expected format:", userdata.active_org.image);
+ return;
+ }
+
+ // Add the new favicon link to head
+ document.head.appendChild(newLink);
+ const iframe = document.createElement('iframe');
+ iframe.style.display = 'none';
+ document.body.appendChild(iframe);
+ iframe.contentDocument.write('');
+ setTimeout(() => {
+ document.body.removeChild(iframe);
+ }, 100);
+
+ } catch (error) {
+ console.error("Error updating favicon:", error);
+ }
+ }
+ }, [userdata]);
+
const checkLogin = () => {
var baseurl = globalUrl;
fetch(`${globalUrl}/api/v1/getinfo`, {
@@ -831,6 +873,21 @@ const App = (message, props) => {
/>
}
/>
+
+
+ }
+ />
+
{
App Authentication
- Control the authentication options for individual apps.
+ Control the authentication options for individual apps. These keys are write-only, and cannot be viewed after creation. If you want editable secrets (e.g. for use in code), use Protected Keys.
- Learn more about App Authentication
+ Learn more
@@ -1787,7 +1787,7 @@ const Hits = ({
if (selectedAppData.authentication === undefined || selectedAppData.authentication === null) {
setAuthenticationType({
- type: "",
+ type: "",
})
selectedAppData.authentication = {
@@ -1955,6 +1955,7 @@ const Hits = ({
if (data === undefined || data === null) {
return;
}
+
const filteredData = data.filter((appAuth) => appAuth?.app?.id === appid);
if (filteredData.length === 0) {
setAppAuthentication([]);
@@ -1965,7 +1966,7 @@ const Hits = ({
}
};
- const HandleAppAuthentication = ()=>{
+ const HandleAppAuthentication = () => {
const url = `${globalUrl}/api/v1/apps/authentication`;
diff --git a/frontend/src/components/AppCreationModal.jsx b/frontend/src/components/AppCreationModal.jsx
index 28dea461..6a66f0e0 100644
--- a/frontend/src/components/AppCreationModal.jsx
+++ b/frontend/src/components/AppCreationModal.jsx
@@ -21,8 +21,9 @@ import AutoFixHighIcon from '@mui/icons-material/AutoFixHigh'
import CreateIcon from '@mui/icons-material/Create'
import { toast } from 'react-toastify'
import YAML from "yaml";
+import Dropzone from "./Dropzone.jsx";
-const AppCreationModal = ({ open, onClose, theme, globalUrl, isCloud }) => {
+const AppCreationModal = ({ open, onClose, theme, globalUrl, isCloud, startOpenApi = false, prefillOpenApiData = "" }) => {
const [openApiModal, setOpenApiModal] = useState(false)
const [generateAppModal, setGenerateAppModal] = useState(false)
const [openApi, setOpenApi] = useState("")
@@ -35,6 +36,16 @@ const AppCreationModal = ({ open, onClose, theme, globalUrl, isCloud }) => {
const navigate = useNavigate()
const upload = useRef()
+ useEffect(() => {
+ if (open && (startOpenApi || (prefillOpenApiData && prefillOpenApiData.length > 0))) {
+ if (prefillOpenApiData && prefillOpenApiData.length > 0) {
+ setOpenApiData(prefillOpenApiData)
+ setIsDropzone(true)
+ }
+ setOpenApiModal(true)
+ }
+ }, [open, startOpenApi, prefillOpenApiData])
+
// Style for the create options
const AppCreateButton = ({ text, func, icon }) => {
const [hover, setHover] = React.useState(false)
@@ -467,6 +478,7 @@ const AppCreationModal = ({ open, onClose, theme, globalUrl, isCloud }) => {
}
}}
>
+ {
Paste in the URI for the OpenAPI or find out
-
How to find URI for openAPI?
@@ -568,31 +583,54 @@ const AppCreationModal = ({ open, onClose, theme, globalUrl, isCloud }) => {
Must point to a version 2 or 3 OpenAPI specification.
-
+
Or upload a YAML or JSON specification
-
-
+
+
+ Drag & drop your OpenAPI (YAML/JSON) anywhere
+
+
+ or click to browse files
+
+