diff --git a/backend/Dockerfile b/backend/Dockerfile old mode 100755 new mode 100644 index 647f74cf..5ccc3de8 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -27,7 +27,7 @@ FROM alpine:latest as certs RUN apk add --update ca-certificates # Sets up the final image -FROM alpine:3.22.1 +FROM alpine:3.22.2 # FIXME: Install cgo because CGO_ENABLED=1 during build RUN apk add --no-cache libc6-compat diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 1711153b..e3a1b921 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -4,12 +4,12 @@ go 1.24.0 toolchain go1.24.3 -//replace github.com/shuffle/opensearch-go => ../../../opensearch-go //replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared //replace github.com/frikky/schemaless => ../../../schemaless //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi +//replace github.com/shuffle/opensearch-go => ../../../opensearch-go require ( cloud.google.com/go/datastore v1.20.0 @@ -26,8 +26,8 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.34 - github.com/shuffle/singul v0.0.17 + github.com/shuffle/shuffle-shared v0.9.53 + github.com/shuffle/singul v0.0.20 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 google.golang.org/grpc v1.72.2 @@ -75,7 +75,7 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frikky/schemaless v0.0.22 // indirect + github.com/frikky/schemaless v0.0.24 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect @@ -115,8 +115,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/opensearch-project/opensearch-go v1.1.0 // indirect - github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect github.com/osteele/liquid v1.7.0 // indirect github.com/osteele/tuesday v1.0.3 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect @@ -128,6 +126,7 @@ require ( github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/sendgrid/sendgrid-go v3.16.1+incompatible // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/shuffle/opensearch-go/v4 v4.0.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index d7ba4a0e..ea6056ba 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -72,20 +72,6 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/aws/aws-sdk-go v1.42.27/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= -github.com/aws/aws-sdk-go v1.44.263/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4= -github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/basgys/goxml2json v1.1.0 h1:4ln5i4rseYfXNd86lGEB+Vi652IsIXIvggKM/BhUKVw= github.com/basgys/goxml2json v1.1.0/go.mod h1:wH7a5Np/Q4QoECFIU8zTQlZwZkrilY0itPfecMw41Dw= github.com/bitly/go-simplejson v0.5.1 h1:xgwPbetQScXt1gh9BmoJ6j9JMr3TElvuIyjR8pgdoow= @@ -152,8 +138,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.22 h1:aMc7cc/lr1zpogjGWbY0j6J2f6QqyfPbbW6Y9JgTAqE= -github.com/frikky/schemaless v0.0.22/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= +github.com/frikky/schemaless v0.0.24 h1:1bNg96gm82yLbRlDS0haQH/96X3GlCAEuTCSasXIJZE= +github.com/frikky/schemaless v0.0.24/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= github.com/fsouza/go-dockerclient v1.12.1 h1:FMoLq+Zhv9Oz/rFmu6JWkImfr6CBgZOPcL+bHW4gS0o= github.com/fsouza/go-dockerclient v1.12.1/go.mod h1:OqsgJJcpCwqyM3JED7TdfM9QVWS5O7jSYwXxYKmOooY= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= @@ -223,7 +209,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -265,8 +250,6 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -325,10 +308,6 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/opensearch-project/opensearch-go v1.1.0 h1:eG5sh3843bbU1itPRjA9QXbxcg8LaZ+DjEzQH9aLN3M= -github.com/opensearch-project/opensearch-go v1.1.0/go.mod h1:+6/XHCuTH+fwsMJikZEWsucZ4eZMma3zNSeLrTtVGbo= -github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ= -github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8= github.com/osteele/liquid v1.7.0 h1:VsbPSchE5D5S5scylAIvERET4dnCxsO6IDri2oSJ5Dk= github.com/osteele/liquid v1.7.0/go.mod h1:xU0Z2dn2hOQIEFEWNmeltOmCtfhtoW/2fCyiNQeNG+U= github.com/osteele/tuesday v1.0.3 h1:SrCmo6sWwSgnvs1bivmXLvD7Ko9+aJvvkmDjB5G4FTU= @@ -363,10 +342,12 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.9.34 h1:4RtGRMoYtBQia2CLbuH7aPFM+EMFXbH0ccIQsBXs8n8= -github.com/shuffle/shuffle-shared v0.9.34/go.mod h1:WbLPLyBrh86ieRBgDyjC43M2qBF8a7E8jO8Pv7t8SOk= -github.com/shuffle/singul v0.0.17 h1:mxaPtj6z85Nf6tl7L2gwDliTfEZtRQqApuu9iKcP75o= -github.com/shuffle/singul v0.0.17/go.mod h1:8c42n1NahhCIPxzLxwp9eYbWkvY4+ct0jfbhkRsRRsY= +github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU= +github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0= +github.com/shuffle/shuffle-shared v0.9.53 h1:ykziiRA82FZX0YiuErSmFLQMxvAFi6yUajgRjd3J5xE= +github.com/shuffle/shuffle-shared v0.9.53/go.mod h1:g03P/ZM8pUwBbdxW0l9A1siMMuemEQshgDpO9hF23eE= +github.com/shuffle/singul v0.0.20 h1:Lz+K4l2GJQ5W6o8ePr/2qlIcGuZuMC7W8Wj/JFw1XDI= +github.com/shuffle/singul v0.0.20/go.mod h1:qY3ZmwaNwmRApLWMTqZaNEMXO6X8bjr/zY44xDU62hQ= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -393,8 +374,18 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/wI2L/jsondiff v0.7.0 h1:1lH1G37GhBPqCfp/lrs91rf/2j3DktX6qYAKZkLuCQQ= +github.com/wI2L/jsondiff v0.7.0/go.mod h1:KAEIojdQq66oJiHhDyQez2x+sRit0vIzC9KeK0yizxM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= @@ -491,9 +482,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= @@ -536,13 +525,11 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= @@ -554,7 +541,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 5b26b726..5f54465a 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -14,18 +14,18 @@ import ( "crypto/md5" "strconv" - "os" - "io" - "log" - "fmt" - "errors" - "net/url" - "os/exec" - "net/http" - "io/ioutil" - "math/rand" "encoding/hex" "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "math/rand" + "net/http" + "net/url" + "os" + "os/exec" "net/http/httptest" "strings" @@ -67,6 +67,7 @@ var runningEnvironment = "onprem" var syncUrl = "https://shuffler.io" var debug = false + //var syncUrl = "http://localhost:5002" type retStruct struct { @@ -3416,7 +3417,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User, s // Read and copy the baseline Dockerfile dockerfileContent, err := ioutil.ReadFile(dockerfileSource) if err != nil { - foundDockerfile := shuffle.GetBaseDockerfile() + foundDockerfile := shuffle.GetBaseDockerfile() if len(foundDockerfile) > 0 { dockerfileContent = foundDockerfile } else { @@ -4033,7 +4034,7 @@ func remoteOrgJobHandler(org shuffle.Org, interval int) error { } // Just to prevent it from spamming large outbound requests - if shouldBackupData { + if shouldBackupData { if org.SyncConfig.WorkflowBackup { workflows, err := shuffle.GetAllWorkflowsByQuery(ctx, foundUser, 250, "") if err != nil { @@ -4067,7 +4068,7 @@ func remoteOrgJobHandler(org shuffle.Org, interval int) error { } // Send stats once every 10 times or so..? - // For now, just send every time + // For now, just send every time info, err := shuffle.GetOrgStatistics(ctx, org.Id) if err != nil { log.Printf("[ERROR] Failed getting org statistics backup for org %s: %s", org.Id, err) @@ -4243,10 +4244,8 @@ func runInitEs(ctx context.Context) { } } - if strings.Contains(os.Getenv("SHUFFLE_OPENSEARCH_URL"), "https") { - log.Printf("[INFO] Waiting 30 seconds during init to make sure the opensearch instance is up and running with security features enabled") - time.Sleep(30 * time.Second) - } + log.Printf("[INFO] Waiting 30 seconds during init to make sure the opensearch instance is up and running with security features enabled") + time.Sleep(30 * time.Second) shuffle.InitOpensearchIndexes() @@ -4403,7 +4402,7 @@ func runInitEs(ctx context.Context) { } //interval := int(org.SyncConfig.Interval) - interval := 30 + interval := 30 if interval == 0 { log.Printf("[WARNING] Skipping org %s because sync isn't set (0).", org.Id) continue @@ -4505,7 +4504,6 @@ func runInitEs(ctx context.Context) { continue } - respBody, err := ioutil.ReadAll(newresp.Body) if err != nil { log.Printf("[ERROR] Failed setting respbody %s for execution stop. Status: %d", err, newresp.StatusCode) @@ -4576,7 +4574,7 @@ func runInitEs(ctx context.Context) { url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION") if len(url) == 0 { log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/shuffle/python-apps") - + url = "https://github.com/shuffle/python-apps" //url = "" //return @@ -4662,9 +4660,9 @@ func runInitEs(ctx context.Context) { } // Self-cleaning - go func() { + go func() { cursor := "" - cnt := 0 + cnt := 0 newCtx := context.Background() for _, org := range activeOrgs { if len(org.Id) == 0 { @@ -4674,7 +4672,7 @@ func runInitEs(ctx context.Context) { log.Printf("[INFO] Starting self-cleanup of cache keys for org %s", org.Id) - for { + for { keys, newCursor, err := shuffle.GetAllCacheKeys(newCtx, org.Id, "", 1000, cursor) if err != nil { //log.Printf("[ERROR] Failed getting all cache keys for cleanup: %s", err) @@ -4958,6 +4956,17 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { resp.Write(b) } else { + + // Delete cache for the sync features + cacheKey := fmt.Sprintf("org_sync_features_%s", org.Id) + shuffle.DeleteCache(ctx, cacheKey) + + subscriptionCacheKey := fmt.Sprintf("org_subscriptions_%s", org.Id) + shuffle.DeleteCache(ctx, subscriptionCacheKey) + + licenseCacheKey := fmt.Sprintf("org_licensed_%s", org.Id) + shuffle.DeleteCache(ctx, licenseCacheKey) + resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Successfully disabled cloud sync for org."}`))) } @@ -5049,12 +5058,29 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { shuffle.SetCache(ctx, cacheKey, featuresBytes, 1800) } + subscriptionCacheKey := fmt.Sprintf("org_subscriptions_%s", org.Id) + subscriptionsBytes, err := json.Marshal(responseData.Subscriptions) + if err != nil { + log.Printf("[ERROR] Failed to marshal Subscriptions for cache: %s", err) + } else { + shuffle.SetCache(ctx, subscriptionCacheKey, subscriptionsBytes, 1800) + } + + licenseCacheKey := fmt.Sprintf("org_licensed_%s", org.Id) + licensedBytes, err := json.Marshal(responseData.Licensed) + if err != nil { + log.Printf("[ERROR] Failed to marshal Licensed for cache: %s", err) + } else { + + shuffle.SetCache(ctx, licenseCacheKey, licensedBytes, 1800) + } + org.SyncConfig = shuffle.SyncConfig{ Apikey: responseData.SessionKey, Interval: responseData.IntervalSeconds, WorkflowBackup: true, - AppBackup: true, + AppBackup: true, } interval := int(responseData.IntervalSeconds) @@ -5677,7 +5703,7 @@ func initHandlers() { r.HandleFunc("/api/v1/dashboards/{key}/widgets", shuffle.HandleNewWidget).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/dashboards/{key}/widgets/{widget_id}", shuffle.HandleGetWidget).Methods("GET", "OPTIONS") - if (strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" || strings.ToLower(os.Getenv("DEBUG_MEMORY")) == "true") { + if strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" || strings.ToLower(os.Getenv("DEBUG_MEMORY")) == "true" { log.Printf("[DEBUG] Memory debugging is enabled on /debug/pprof") r.HandleFunc("/debug/pprof/", pprof.Index) r.HandleFunc("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 54635430..0c015470 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -20,21 +20,21 @@ import ( "strings" "time" - dockerclient "github.com/docker/docker/client" "github.com/docker/docker/api/types/image" + dockerclient "github.com/docker/docker/client" "github.com/h2non/filetype" uuid "github.com/satori/go.uuid" newscheduler "github.com/carlescere/scheduler" - "github.com/go-co-op/gocron" "github.com/frikky/kin-openapi/openapi3" + gyaml "github.com/ghodss/yaml" + "github.com/go-co-op/gocron" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/storage/memory" "github.com/go-git/go-git/v5/plumbing" http2 "github.com/go-git/go-git/v5/plumbing/transport/http" - gyaml "github.com/ghodss/yaml" + "github.com/go-git/go-git/v5/storage/memory" ) var localBase = "http://localhost:5001" @@ -257,7 +257,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { return } - // This is really the environment's name - NOT OrgId + // This is really the environment's name - NOT OrgId environment := request.Header.Get("Org-Id") if len(environment) == 0 { log.Printf("[AUDIT] No org-id header set") @@ -276,7 +276,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { return */ } - + // This section is cloud custom for now auth := request.Header.Get("Authorization") if len(auth) == 0 { @@ -305,7 +305,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } // Only works onprem - shared queues across tenants - // without tenancy + // without tenancy if !found { env, err = shuffle.GetEnvironment(ctx, environment, "") if err != nil { @@ -336,7 +336,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { if len(executionRequests.Data) == 0 { executionRequests.Data = []shuffle.ExecutionRequest{} } else { - // Try again? I don't think this is necessary, and shouldn't really ever occur. + // Try again? I don't think this is necessary, and shouldn't really ever occur. if len(executionRequests.Data) > 50 { executionRequests.Data = executionRequests.Data[0:49] } @@ -457,16 +457,16 @@ func handleGetWorkflowExecutionResult(resp http.ResponseWriter, request *http.Re if workflowExecution.Workflow.Sharing == "form" { newWorkflow := shuffle.Workflow{ - Name: workflowExecution.Workflow.Name, - ID: workflowExecution.Workflow.ID, - Owner: workflowExecution.Workflow.Owner, - OrgId: workflowExecution.Workflow.OrgId, + Name: workflowExecution.Workflow.Name, + ID: workflowExecution.Workflow.ID, + Owner: workflowExecution.Workflow.Owner, + OrgId: workflowExecution.Workflow.OrgId, - Sharing: workflowExecution.Workflow.Sharing, + Sharing: workflowExecution.Workflow.Sharing, Description: workflowExecution.Workflow.Description, InputQuestions: workflowExecution.Workflow.InputQuestions, - FormControl: workflowExecution.Workflow.FormControl, + FormControl: workflowExecution.Workflow.FormControl, } workflowExecution.Results = []shuffle.ActionResult{} @@ -765,7 +765,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { } if user.Id != workflow.Owner || len(user.Id) == 0 { - if workflow.OrgId == user.ActiveOrg.Id { + if workflow.OrgId == user.ActiveOrg.Id { log.Printf("[INFO] User %s is deleting workflow %s as admin. Owner: %s", user.Username, workflow.ID, workflow.Owner) } else { log.Printf("[WARNING] Wrong user (%s) for workflow %s (delete workflow)", user.Username, workflow.ID) @@ -782,7 +782,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { childWorkflows, err := shuffle.ListChildWorkflows(ctx, workflow.ID) if err != nil { log.Printf("[ERROR] Failed to list child workflows: %s", err) - } else { + } else { //log.Printf("\n\n[DEBUG] Found %d child workflows for workflow %s\n\n", len(childWorkflows), workflow.ID) // Find cookies and append them to request.Header to replicate current request as closely as possible @@ -898,10 +898,10 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request } /* - if !workflow.IsValid { - log.Printf("[ERROR] Stopped execution as workflow %s is not valid.", workflow.ID) - return shuffle.WorkflowExecution{}, fmt.Sprintf(`workflow %s is invalid`, workflow.ID), errors.New("Failed getting workflow") - } + if !workflow.IsValid { + log.Printf("[ERROR] Stopped execution as workflow %s is not valid.", workflow.ID) + return shuffle.WorkflowExecution{}, fmt.Sprintf(`workflow %s is invalid`, workflow.ID), errors.New("Failed getting workflow") + } */ maxExecutionDepth := 10 @@ -914,7 +914,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request workflowExecution, execInfo, _, workflowExecErr := shuffle.PrepareWorkflowExecution(ctx, workflow, request, int64(maxExecutionDepth)) if workflowExecErr != nil { - if len(workflowExecution.Workflow.Actions) > 0 && len(workflowExecution.Results) > 0 && len(workflowExecution.ExecutionId) > 0 { + if len(workflowExecution.Workflow.Actions) > 0 && len(workflowExecution.Results) > 0 && len(workflowExecution.ExecutionId) > 0 { err := shuffle.SetWorkflowExecution(ctx, workflowExecution, true) if err != nil { log.Printf("[ERROR] Failed setting workflow execution during init (2): %s", err) @@ -928,7 +928,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request return shuffle.WorkflowExecution{}, "", nil } else { log.Printf("[ERROR] Failed in prepareExecution: '%s'", workflowExecErr) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed running: %s", workflowExecErr), workflowExecErr + return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed running: %s", workflowExecErr), workflowExecErr } } @@ -1337,7 +1337,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { } if user.Id != workflow.Owner || len(user.Id) == 0 { - if workflow.OrgId == user.ActiveOrg.Id { + if workflow.OrgId == user.ActiveOrg.Id { log.Printf("[AUDIT] User %s is accessing workflow %s as admin (stop schedule)", user.Username, workflow.ID) } else { log.Printf("[WARNING] Wrong user (%s) for workflow %s (stop schedule)", user.Username, workflow.ID) @@ -1505,30 +1505,39 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { return } -func deleteSchedule(ctx context.Context, id string) error { - log.Printf("[DEBUG] Should stop schedule %s!", id) +func deleteKeySchedule(ctx context.Context, id string) error { err := shuffle.DeleteKey(ctx, "schedules", id) if err != nil { - log.Printf("[ERROR] Failed to delete schedule: %s", err) return err + } + return nil +} + +func deleteSchedule(ctx context.Context, id string) error { + log.Printf("[DEBUG] Should stop schedule %s!", id) + if value, exists := scheduledJobs[id]; exists { + // Stops the schedule properly + value.Lock() } else { - if value, exists := scheduledJobs[id]; exists { - // Stops the schedule properly - value.Lock() - } else { - // FIXME - allow it to kind of stop anyway? - if j, ok := cronJobs[id]; ok { - err := CronScheduler.RemoveByID(j) - if err != nil { - log.Printf("[ERROR] Failed to remove the scheduler %s", err) - return err - } - } else { - return errors.New("Can't find the schedule.") + // FIXME - allow it to kind of stop anyway? + if j, ok := cronJobs[id]; ok { + err := CronScheduler.RemoveByID(j) + if err != nil { + log.Printf("[ERROR] Failed to remove the scheduler %s", err) + return err } + } else { + // Just stop and delete anyway if not in memory + deleteKeySchedule(ctx, id) + return errors.New("Can't find the schedule.") } } + err := deleteKeySchedule(ctx, id) + if err != nil { + log.Printf("[ERROR] Failed to stop schedule in db %s: %s", id, err) + return err + } return nil } @@ -1582,7 +1591,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { } if user.Id != workflow.Owner || len(user.Id) == 0 { - if workflow.OrgId == user.ActiveOrg.Id { + if workflow.OrgId == user.ActiveOrg.Id { log.Printf("[INFO] User %s is deleting workflow %s as admin. Owner: %s", user.Username, workflow.ID, workflow.Owner) } else { log.Printf("[WARNING] Wrong user (%s) for workflow %s (schedule start). Owner: %s", user.Username, workflow.ID, workflow.Owner) @@ -2022,7 +2031,7 @@ func loadGithubWorkflows(url, username, password, userId, branch, orgId string) cloneOptions.ReferenceName = plumbing.ReferenceName(branch) } - cloneOptions = checkGitProxy(cloneOptions) + cloneOptions = checkGitProxy(cloneOptions) storer := memory.NewStorage() r, err := git.Clone(storer, fs, cloneOptions) @@ -2877,7 +2886,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { log.Printf("\n\nACTION TO RUN: %s. Body: %s. Source URL: %s\n\n", appId, string(body), request.URL.String()) workflowExecution, err := shuffle.PrepareSingleAction(ctx, user, appId, body, runValidationAction, decisionId) - if appId == "agent_starter" { + if appId == "agent_starter" { log.Printf("[INFO] Returning early for agent_starter single action execution: %s", workflowExecution.ExecutionId) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization))) @@ -2905,7 +2914,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { return } - workflowExecution.ProjectId = "" + workflowExecution.ProjectId = "" workflowExecution.Locations = []string{""} foundEnv := "" @@ -2923,7 +2932,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { go shuffle.IncrementCache(ctx, workflowExecution.OrgId, "workflow_executions") executionRequest := shuffle.ExecutionRequest{ - Priority: 15, + Priority: 15, ExecutionId: workflowExecution.ExecutionId, WorkflowId: workflowExecution.Workflow.ID, Authorization: workflowExecution.Authorization, @@ -2998,7 +3007,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. continue } - //duringStartup + //duringStartup if duringStartup { // Look for names: shuffle tools, http, email, shuffle ai if shuffle.ArrayContains(startupNames, strings.ToLower(file.Name())) { @@ -3366,7 +3375,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. if len(item.Tags) > 0 && shuffle.ArrayContains(handledImages, item.Tags[0]) { continue - } + } handledImages = append(handledImages, item.Tags[0]) err = buildImageMemory(fs, item.Tags, item.Extra, true) @@ -3402,7 +3411,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. for _, item := range buildLaterList { if len(item.Tags) > 0 && shuffle.ArrayContains(handledImages, item.Tags[0]) { continue - } + } handledImages = append(handledImages, item.Tags[0]) @@ -3493,7 +3502,7 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { } } - cloneOptions = checkGitProxy(cloneOptions) + cloneOptions = checkGitProxy(cloneOptions) storer := memory.NewStorage() r, err := git.Clone(storer, fs, cloneOptions) diff --git a/docker-compose.yml b/docker-compose.yml index 8a8d2cd2..0c39f664 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -122,11 +122,10 @@ services: # - MEMCACHED_MAX_CONNECTIONS=2500 # ports: # - 11211:11211 - # deploy: - # resources: - # limits: - # memory: 2048M - + # networks: + # - shuffle + # - swarm_executions + #docker-socket-proxy: # image: tecnativa/docker-socket-proxy # container_name: docker-socket-proxy diff --git a/frontend/Dockerfile b/frontend/Dockerfile old mode 100755 new mode 100644 index 20051475..5df910ee --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -27,7 +27,7 @@ COPY ./*.json /usr/src/app/ RUN npm run build --loglevel verbose 2>&1 # Production environment -FROM nginx:1.29.0 +FROM nginx:1.29.3 RUN mkdir -p /usr/share/nginx/html/build RUN mkdir -p /usr/share/nginx/html/css diff --git a/frontend/src/components/AdminNavBar.jsx b/frontend/src/components/AdminNavBar.jsx index 9ebea75b..f32a284f 100644 --- a/frontend/src/components/AdminNavBar.jsx +++ b/frontend/src/components/AdminNavBar.jsx @@ -23,7 +23,7 @@ import { } from '@mui/icons-material'; import theme, { getTheme } from '../theme.jsx'; -import { Button, Skeleton, Tooltip } from '@mui/material'; +import { Box, Button, Skeleton, Tooltip, Typography } from '@mui/material'; import { Index } from 'react-instantsearch-dom'; import { Context } from '../context/ContextApi.jsx'; import { toast } from 'react-toastify'; @@ -265,7 +265,7 @@ const AdminNavBar = (props) => { : selectedOrganization?.image; return ( - !isOrgLoaded && !isUserDataLoaded ? : + !isOrgLoaded || !isUserDataLoaded ? :