diff --git a/functions/onprem/orborus/Dockerfile b/functions/onprem/orborus/Dockerfile deleted file mode 100644 index d59aea8d..00000000 --- a/functions/onprem/orborus/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -FROM golang:1.24 as builder - -WORKDIR /app - -COPY orborus.go /app/orborus.go -COPY go.mod /app/go.mod - -#RUN go get -RUN go mod download -RUN go mod tidy -#RUN go build -v - -# Enabled CGO January 2025 (?) -#RUN CGO_ENABLED=1 GOOS=linux go build -a -installsuffix cgo -o orborus. -RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o /app/orborus . - -FROM alpine:3.22.1 - -RUN apk add --no-cache bash tzdata -#COPY --from=builder /app/orborus orborus -COPY --from=builder /app/ / -ENV ENVIRONMENT_NAME=Shuffle \ - BASE_URL=http://shuffle-backend:5001 \ - DOCKER_API_VERSION=1.40 \ - SHUFFLE_OPENSEARCH_URL=https://opensearch:9200 - -CMD ["./orborus"] diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh deleted file mode 100755 index bbec5067..00000000 --- a/functions/onprem/orborus/build.sh +++ /dev/null @@ -1,16 +0,0 @@ -NAME=shuffle-orborus -VERSION=1.3.0 - -echo "Running docker build with $NAME:$VERSION" -#docker rmi frikky/shuffle:$NAME --force -docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly -t ghcr.io/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:nightly - -#docker push frikky/$NAME:$VERSION -# docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -docker push frikky/shuffle:$NAME -docker push ghcr.io/frikky/$NAME:$VERSION -docker push ghcr.io/frikky/$NAME:nightly - -docker push shuffle/shuffle:$NAME -docker push ghcr.io/shuffle/$NAME:$VERSION -docker push ghcr.io/shuffle/$NAME:nightly diff --git a/functions/onprem/orborus/docker-compose.yml b/functions/onprem/orborus/docker-compose.yml deleted file mode 100644 index ef6132e2..00000000 --- a/functions/onprem/orborus/docker-compose.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: '3' -services: - orborus: - image: ghcr.io/frikky/shuffle-orborus:nightly - container_name: shuffle-orborus - hostname: shuffle-orborus - volumes: - - /var/run/docker.sock:/var/run/docker.sock - environment: - - SHUFFLE_APP_SDK_VERSION=nightly - - SHUFFLE_WORKER_VERSION=nightly - - ORG_ID=Shuffle - - ENVIRONMENT_NAME=Shuffle - - BASE_URL=http://192.168.86.39:5001 - - DOCKER_API_VERSION=1.40 - - SHUFFLE_SCALE_REPLICAS=5 - - SHUFFLE_SWARM_CONFIG=run - restart: unless-stopped - networks: - - shuffle-executions -networks: - shuffle-executions: - driver: overlay - external: true - diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod deleted file mode 100644 index 08977c36..00000000 --- a/functions/onprem/orborus/go.mod +++ /dev/null @@ -1,160 +0,0 @@ -module orborus - -go 1.24.0 - -toolchain go1.24.4 - -replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared - -require ( - github.com/docker/docker v28.3.3+incompatible - github.com/docker/go-connections v0.5.0 - github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v1.2.24 - k8s.io/api v0.34.2 - k8s.io/apimachinery v0.34.2 -) - -require ( - cloud.google.com/go/auth v0.16.1 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect -) - -require ( - cel.dev/expr v0.20.0 // indirect - cloud.google.com/go v0.121.1 // indirect - cloud.google.com/go/compute/metadata v0.7.0 // indirect - cloud.google.com/go/datastore v1.20.0 // indirect - cloud.google.com/go/iam v1.5.2 // indirect - cloud.google.com/go/monitoring v1.24.2 // indirect - cloud.google.com/go/scheduler v1.11.7 // indirect - cloud.google.com/go/storage v1.55.0 // indirect - dario.cat/mergo v1.0.0 // indirect - github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect - github.com/Masterminds/semver v1.5.0 // indirect - github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/ProtonMail/go-crypto v1.1.6 // indirect - github.com/adrg/strutil v0.3.1 // indirect - github.com/algolia/algoliasearch-client-go/v3 v3.31.4 // indirect - github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf // indirect - github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect - github.com/cenkalti/backoff/v5 v5.0.2 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cloudflare/circl v1.6.1 // indirect - github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect - github.com/containerd/errdefs v1.0.0 // indirect - github.com/containerd/errdefs/pkg v0.3.0 // indirect - github.com/cyphar/filepath-securejoin v0.4.1 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/distribution/reference v0.6.0 // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/emicklei/go-restful/v3 v3.12.2 // indirect - github.com/emirpasic/gods v1.18.1 // indirect - github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect - github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frikky/kin-openapi v0.42.0 // indirect - github.com/frikky/schemaless v0.0.33 // indirect - github.com/fxamacker/cbor/v2 v2.9.0 // indirect - github.com/ghodss/yaml v1.0.0 // indirect - github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.6.2 // indirect - github.com/go-git/go-git/v5 v5.16.5 // indirect - github.com/go-jose/go-jose/v4 v4.1.3 // indirect - github.com/go-logr/logr v1.4.2 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.23.0 // indirect - github.com/goccy/go-json v0.10.5 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.7.0 // indirect - github.com/google/go-github/v28 v28.1.1 // indirect - github.com/google/go-querystring v1.1.0 // indirect - github.com/google/s2a-go v0.1.9 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect - github.com/googleapis/gax-go/v2 v2.14.2 // indirect - github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/sys/sequential v0.6.0 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/openai/openai-go/v3 v3.8.1 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/osteele/liquid v1.7.0 // indirect - github.com/osteele/tuesday v1.0.3 // indirect - github.com/patrickmn/go-cache v2.1.0+incompatible // indirect - github.com/pjbgf/sha1cd v0.3.2 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect - github.com/sashabaranov/go-openai v1.40.5 // indirect - github.com/sendgrid/rest v2.6.9+incompatible // indirect - github.com/sendgrid/sendgrid-go v3.16.1+incompatible // indirect - github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect - github.com/shuffle/opensearch-go/v4 v4.0.0 // indirect - github.com/skeema/knownhosts v1.3.1 // indirect - github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect - github.com/spf13/pflag v1.0.6 // indirect - github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect - github.com/tidwall/gjson v1.18.0 // indirect - github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.1 // indirect - github.com/tidwall/sjson v1.2.5 // indirect - github.com/x448/float16 v0.8.4 // indirect - github.com/xanzy/ssh-agent v0.3.3 // indirect - github.com/zeebo/errs v1.4.0 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect - go.opentelemetry.io/otel v1.36.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect - go.opentelemetry.io/otel/metric v1.36.0 // indirect - go.opentelemetry.io/otel/sdk v1.36.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect - go.opentelemetry.io/otel/trace v1.36.0 // indirect - go.opentelemetry.io/proto/otlp v1.5.0 // indirect - go.yaml.in/yaml/v2 v2.4.2 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect - go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.18.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/term v0.37.0 // indirect - golang.org/x/text v0.31.0 // indirect - golang.org/x/time v0.11.0 // indirect - google.golang.org/api v0.236.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250519155744-55703ea1f237 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect - google.golang.org/grpc v1.72.2 // indirect - google.golang.org/protobuf v1.36.6 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/client-go v0.34.2 // indirect - k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect - k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect - sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect - sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect -) diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum deleted file mode 100644 index 38fe9d50..00000000 --- a/functions/onprem/orborus/go.sum +++ /dev/null @@ -1,651 +0,0 @@ -cel.dev/expr v0.20.0 h1:OunBvVCfvpWlt4dN7zg3FM6TDkzOePe1+foGJ9AXeeI= -cel.dev/expr v0.20.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.121.1 h1:S3kTQSydxmu1JfLRLpKtxRPA7rSrYPRPEUmL/PavVUw= -cloud.google.com/go v0.121.1/go.mod h1:nRFlrHq39MNVWu+zESP2PosMWA0ryJw8KUBZ2iZpxbw= -cloud.google.com/go/auth v0.16.1 h1:XrXauHMd30LhQYVRHLGvJiYeczweKQXZxsTbV9TiguU= -cloud.google.com/go/auth v0.16.1/go.mod h1:1howDHJ5IETh/LwYs3ZxvlkXF48aSqqJUM+5o02dNOI= -cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= -cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= -cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.20.0 h1:NNpXoyEqIJmZFc0ACcwBEaXnmscUpcG4NkKnbCePmiM= -cloud.google.com/go/datastore v1.20.0/go.mod h1:uFo3e+aEpRfHgtp5pp0+6M0o147KoPaYNaPAKpfh8Ew= -cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= -cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= -cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= -cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= -cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= -cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= -cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= -cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/scheduler v1.11.7 h1:zkMEJ0UbEJ3O7NwEUlKLIp6eXYv1L7wHjbxyxznajKM= -cloud.google.com/go/scheduler v1.11.7/go.mod h1:gqYs8ndLx2M5D0oMJh48aGS630YYvC432tHCnVWN13s= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.55.0 h1:NESjdAToN9u1tmhVqhXCaCwYBuvEhZLLv0gBr+2znf0= -cloud.google.com/go/storage v1.55.0/go.mod h1:ztSmTTwzsdXe5syLVS0YsbFxXuvEmEyZj7v7zChEmuY= -cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= -cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= -github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0/go.mod h1:BnBReJLvVYx2CS/UHOgVz2BXKXD9wsQPxZug20nZhd0= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0 h1:OqVGm6Ei3x5+yZmSJG1Mh2NwHvpVmZ08CB5qJhT9Nuk= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0/go.mod h1:SZiPHWGOOk3bl8tkevxkoiwPgsIl6CwrWcbwjfHZpdM= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 h1:6/0iUd0xrnX7qt+mLNRwg5c0PGv8wpE8K90ryANQwMI= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0= -github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= -github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= -github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= -github.com/adrg/strutil v0.3.1 h1:OLvSS7CSJO8lBii4YmBt8jiK9QOtB9CzCzwl4Ic/Fz4= -github.com/adrg/strutil v0.3.1/go.mod h1:8h90y18QLrs11IBffcGX3NW/GFBXCMcNg4M7H6MspPA= -github.com/algolia/algoliasearch-client-go/v3 v3.31.4 h1:UJhx6AhZCYf0qZygDz2c1x1+1q2q2sfzsRaQM6yswWk= -github.com/algolia/algoliasearch-client-go/v3 v3.31.4/go.mod h1:i7tLoP7TYDmHX3Q7vkIOL4syVse/k5VJ+k0i8WqFiJk= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf h1:TqhNAT4zKbTdLa62d2HDBFdvgSbIGB3eJE8HqhgiL9I= -github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf/go.mod h1:r5xuitiExdLAJ09PR7vBVENGvp4ZuTBeWTGtxuX3K+c= -github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 h1:/P9/RL0xgWE+ehnCUUN5h3RpG3dmoMCOONO1CCvq23Y= -github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013/go.mod h1:pccXHIvs3TV/TUqSNyEvF99sxjX2r4FFRIyw6TZY9+w= -github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8= -github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= -github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= -github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 h1:Om6kYQYDUk5wWbT0t0q6pvyM49i9XZAv9dDrkDA7gjk= -github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= -github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= -github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= -github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= -github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= -github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= -github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= -github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= -github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= -github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= -github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= -github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= -github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= -github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= -github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= -github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= -github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= -github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= -github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= -github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= -github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.33 h1:5Soj6VQc+ozqLh4R6MatWOl/atAeNpdon+nV5EKwjOI= -github.com/frikky/schemaless v0.0.33/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= -github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= -github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= -github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM= -github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.16.5 h1:mdkuqblwr57kVfXri5TTH+nMFLNUxIj9Z7F5ykFbw5s= -github.com/go-git/go-git/v5 v5.16.5/go.mod h1:QOMLpNf1qxuSY4StA/ArOdfFR2TrKEjJiye2kel2m+M= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= -github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= -github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= -github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= -github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= -github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-github/v28 v28.1.1 h1:kORf5ekX5qwXO2mGzXXOjMe/g6ap8ahVe0sBEulhSxo= -github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= -github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= -github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= -github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= -github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.14.2 h1:eBLnkZ9635krYIPD+ag1USrOAI0Nr0QYF3+/3GqO0k0= -github.com/googleapis/gax-go/v2 v2.14.2/go.mod h1:ON64QhlJkhVtSqp4v1uaK92VyZ2gmvDQsweuyLV+8+w= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 h1:VNqngBF40hVlDloBruUehVYC3ArSgIyScOAyMRqBxRg= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1/go.mod h1:RBRO7fro65R6tjKzYgLAFo0t1QEXY1Dp+i/bvpRiqiQ= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= -github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= -github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= -github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= -github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= -github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= -github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= -github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= -github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= -github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= -github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= -github.com/openai/openai-go/v3 v3.8.1 h1:b+YWsmwqXnbpSHWQEntZAkKciBZ5CJXwL68j+l59UDg= -github.com/openai/openai-go/v3 v3.8.1/go.mod h1:UOpNxkqC9OdNXNUfpNByKOtB4jAL0EssQXq5p8gO0Xs= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= -github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/osteele/liquid v1.7.0 h1:VsbPSchE5D5S5scylAIvERET4dnCxsO6IDri2oSJ5Dk= -github.com/osteele/liquid v1.7.0/go.mod h1:xU0Z2dn2hOQIEFEWNmeltOmCtfhtoW/2fCyiNQeNG+U= -github.com/osteele/tuesday v1.0.3 h1:SrCmo6sWwSgnvs1bivmXLvD7Ko9+aJvvkmDjB5G4FTU= -github.com/osteele/tuesday v1.0.3/go.mod h1:pREKpE+L03UFuR+hiznj3q7j3qB1rUZ4XfKejwWFF2M= -github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= -github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= -github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= -github.com/sashabaranov/go-openai v1.40.5 h1:SwIlNdWflzR1Rxd1gv3pUg6pwPc6cQ2uMoHs8ai+/NY= -github.com/sashabaranov/go-openai v1.40.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0= -github.com/sendgrid/rest v2.6.9+incompatible/go.mod h1:kXX7q3jZtJXK5c5qK83bSGMdV6tsOE70KbHoqJls4lE= -github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1GjJohAA0p6hVEaDtHWWs= -github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU= -github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0= -github.com/shuffle/shuffle-shared v1.2.24 h1:5jH7/QE4Lf+Yt55oPuszE93Ce34hn1lBVuyEdkX9nic= -github.com/shuffle/shuffle-shared v1.2.24/go.mod h1:RSKyexqkGDB+WbboGfWMj1Sfl4e49MI2CvW3pFmirg4= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= -github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= -github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= -github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= -github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= -github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= -github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= -github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= -github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -github.com/wI2L/jsondiff v0.7.0 h1:1lH1G37GhBPqCfp/lrs91rf/2j3DktX6qYAKZkLuCQQ= -github.com/wI2L/jsondiff v0.7.0/go.mod h1:KAEIojdQq66oJiHhDyQez2x+sRit0vIzC9KeK0yizxM= -github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= -github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= -github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= -github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw= -go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= -go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= -go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 h1:1fTNlAIJZGWLP5FVu0fikVry1IsiUnXjf7QFvoNN3Xw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0/go.mod h1:zjPK58DtkqQFn+YUMbx0M2XV3QgKU0gS9LeGohREyK4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0 h1:nRVXXvf78e00EwY6Wp0YII8ww2JVWshZ20HfTlE11AM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0/go.mod h1:r49hO7CgrxY9Voaj3Xe8pANWtr0Oq916d0XAmOoCZAQ= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= -go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= -go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= -go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= -go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= -go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= -go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= -go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= -go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= -go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= -go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= -go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= -go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= -go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= -go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= -golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= -golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.236.0 h1:CAiEiDVtO4D/Qja2IA9VzlFrgPnK3XVMmRoJZlSWbc0= -google.golang.org/api v0.236.0/go.mod h1:X1WF9CU2oTc+Jml1tiIxGmWFK/UZezdqEu09gcxZAj4= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 h1:1tXaIXCracvtsRxSBsYDiSBN0cuJvM7QYW+MrpIRY78= -google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:49MsLSx0oWMOZqcpB3uL8ZOkAh1+TndpJ8ONoCBWiZk= -google.golang.org/genproto/googleapis/api v0.0.0-20250519155744-55703ea1f237 h1:Kog3KlB4xevJlAcbbbzPfRG0+X9fdoGM+UBRKVz6Wr0= -google.golang.org/genproto/googleapis/api v0.0.0-20250519155744-55703ea1f237/go.mod h1:ezi0AVyMKDWy5xAncvjLWH7UcLBB5n7y2fQ8MzjJcto= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= -google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= -gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -k8s.io/api v0.34.2 h1:fsSUNZhV+bnL6Aqrp6O7lMTy6o5x2C4XLjnh//8SLYY= -k8s.io/api v0.34.2/go.mod h1:MMBPaWlED2a8w4RSeanD76f7opUoypY8TFYkSM+3XHw= -k8s.io/apimachinery v0.34.2 h1:zQ12Uk3eMHPxrsbUJgNF8bTauTVR2WgqJsTmwTE/NW4= -k8s.io/apimachinery v0.34.2/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= -k8s.io/client-go v0.34.2 h1:Co6XiknN+uUZqiddlfAjT68184/37PS4QAzYvQvDR8M= -k8s.io/client-go v0.34.2/go.mod h1:2VYDl1XXJsdcAxw7BenFslRQX28Dxz91U9MWKjX97fE= -k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= -k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= -sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= -sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= -sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= -sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go deleted file mode 100755 index a22768a5..00000000 --- a/functions/onprem/orborus/orborus.go +++ /dev/null @@ -1,5055 +0,0 @@ -package main - -/* - Orborus exists to listen for new jobs from Shuffle. This is to run workflows, pipelines, and other tasks. -*/ -import ( - "archive/zip" - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "log" - "math" - "net" - "net/http" - "os" - "os/exec" - "os/signal" - "path/filepath" - "runtime" - "strconv" - "strings" - "sync" - "syscall" - "time" - - "github.com/shuffle/shuffle-shared" - - "math/rand" - //"os/signal" - //"syscall" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/filters" - "github.com/docker/docker/api/types/image" - "github.com/docker/docker/api/types/mount" - "github.com/docker/docker/api/types/network" - "github.com/docker/docker/api/types/swarm" - "github.com/docker/go-connections/nat" - - //"github.com/docker/docker/api/types/filters" - dockerclient "github.com/docker/docker/client" - uuid "github.com/satori/go.uuid" - - //"github.com/mackerelio/go-osstat/disk" - //"github.com/mackerelio/go-osstat/memory" - //"github.com/shirou/gopsutil/cpu" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -// Starts jobs in bulk, so this could be increased or decreased based on who the user is -var sleepTime = 2 - -// Making it work on low-end machines even during busy times :) -// May cause some things to run slowly -var maxConcurrency = 25 - -// Timeout if something rashes -var workerTimeoutEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_TIMEOUT") -var concurrencyEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY") -var appSdkVersion = os.Getenv("SHUFFLE_APP_SDK_VERSION") -var workerVersion = os.Getenv("SHUFFLE_WORKER_VERSION") -var newWorkerImage = os.Getenv("SHUFFLE_WORKER_IMAGE") -var dockerSwarmBridgeMTU = os.Getenv("SHUFFLE_SWARM_BRIDGE_DEFAULT_MTU") -var dockerSwarmBridgeInterface = os.Getenv("SHUFFLE_SWARM_BRIDGE_DEFAULT_INTERFACE") -var maxCPUPercent = 90 - -// Kubernetes settings -var isKubernetes = os.Getenv("IS_KUBERNETES") -var kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") -var workerServiceAccountName = os.Getenv("SHUFFLE_WORKER_SERVICE_ACCOUNT_NAME") -var workerPodSecurityContext = os.Getenv("SHUFFLE_WORKER_POD_SECURITY_CONTEXT") -var workerContainerSecurityContext = os.Getenv("SHUFFLE_WORKER_CONTAINER_SECURITY_CONTEXT") -var appServiceAccountName = os.Getenv("SHUFFLE_APP_SERVICE_ACCOUNT_NAME") -var appPodSecurityContext = os.Getenv("SHUFFLE_APP_POD_SECURITY_CONTEXT") -var appContainerSecurityContext = os.Getenv("SHUFFLE_APP_CONTAINER_SECURITY_CONTEXT") -var debug = os.Getenv("DEBUG") == "true" - -// var baseimagename = "docker.pkg.github.com/shuffle/shuffle" -// var baseimagename = "ghcr.io/frikky" -// var baseimagename = "shuffle/shuffle" -var baseimageregistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY") -var baseimagename = os.Getenv("SHUFFLE_BASE_IMAGE_NAME") - -//var baseimagetagsuffix = os.Getenv("SHUFFLE_BASE_IMAGE_TAG_SUFFIX") - -// Used for cloud with auth. Onprem in certain cases too. -var auth = os.Getenv("AUTH") -var org = os.Getenv("ORG") - -// var orgId = os.Getenv("ORG_ID") -var baseUrl = os.Getenv("BASE_URL") -var workerServerUrl = os.Getenv("SHUFFLE_WORKER_SERVER_URL") -var environment = os.Getenv("ENVIRONMENT_NAME") -var dockerApiVersion = os.Getenv("DOCKER_API_VERSION") -var runningMode = strings.ToLower(os.Getenv("RUNNING_MODE")) -var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP")) -var timezone = os.Getenv("TZ") -var containerName = os.Getenv("ORBORUS_CONTAINER_NAME") -var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG") -var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME") -var orborusLabel = os.Getenv("SHUFFLE_ORBORUS_LABEL") -var memcached = os.Getenv("SHUFFLE_MEMCACHED") -var queuePerMinute = os.Getenv("SHUFFLE_EXECUTION_PER_MINIUTE") -var queuePerMinuteInt int - -// Used to download file categories. Not required since 2.1.1 -var pipelineApikey = "" -var pipelineUrl = os.Getenv("SHUFFLE_PIPELINE_URL") - -var executionIds = []string{} -var pipelines = []shuffle.PipelineInfo{} -var namespacemade = false // For K8s -var skipPipelineMount = false -var tenzirDisabled = false - -var dockercli *dockerclient.Client -var containerId string -var executionCount = 0 -var orborusUuid = os.Getenv("SHUFFLE_ORBORUS_UUID") - -var imagedownloadTimeout = time.Second * 300 -var window = shuffle.NewTimeWindow(1 * time.Minute) - -func init() { - var err error - // Look for argc/argv and map environment variables - sensorMode := false - for _, arg := range os.Args { - if !strings.HasPrefix(arg, "--") { - continue - } - - // Split away = - value := "" - if strings.Contains(arg, "=") { - newArg := strings.Split(arg, "=")[0] - value = strings.Split(arg, "=")[1] - - arg = newArg - } else { - continue - } - - if len(value) == 0 { - continue - } - - parsedArg := strings.TrimPrefix(arg, "--") - parsedArg = strings.ReplaceAll(strings.ToUpper(parsedArg), " ", "_") - if !strings.HasPrefix(parsedArg, "SHUFFLE_") { - parsedArg = "SHUFFLE_" + parsedArg - } - - if parsedArg == "SHUFFLE_SENSOR_MODE" { - parsedArg = "SHUFFLE_AGENT_SENSOR_MODE" - } else if parsedArg == "SHUFFLE_AGENT_MODE" { - parsedArg = "SHUFFLE_AGENT_SENSOR_MODE" - } - - if parsedArg == "SHUFFLE_AGENT_SENSOR_MODE" && strings.ToLower(value) == "true" { - sensorMode = true - } - - os.Setenv(parsedArg, value) - } - - if sensorMode { - log.Printf("[INFO] Enabling sensormode (init check)") - for _, arg := range os.Args { - if !strings.HasPrefix(arg, "--") { - continue - } - - // Split away = - value := "" - if strings.Contains(arg, "=") { - newArg := strings.Split(arg, "=")[0] - value = strings.Split(arg, "=")[1] - - arg = newArg - } else { - continue - } - - if len(value) == 0 { - continue - } - - arg = strings.TrimPrefix(arg, "--") - - if arg == "queue" { - os.Setenv("ENVIRONMENT_NAME", value) - environment = value - } else if arg == "auth" { - os.Setenv("AUTH", value) - auth = value - } else if arg == "org_id" { - os.Setenv("ORG", value) - org = value - } else if arg == "base_url" { - os.Setenv("BASE_URL", value) - baseUrl = value - } - } - } else { - // dockercli, err = dockerclient.NewEnvClient() - dockercli, dockerApiVersion, err = shuffle.GetDockerClient() - if err != nil { - log.Printf("Unable to create docker client: %s", err) - } - - if os.Getenv("SHUFFLE_EC2_INSTANCE") == "true" { - log.Printf("[INFO] Detected AWS EC2 instance. Setting up Docker Swarm with AWS optimizations.") - containers, err := dockercli.ContainerList(context.Background(), container.ListOptions{}) - if err == nil { - for _, container := range containers { - if strings.Contains(container.Image, "shuffle-orborus") { - if len(container.Names) != 0 { - if strings.Contains(container.Names[0], "shuffle-orborus") { - containerName = container.Names[0] - containerName = strings.TrimPrefix(containerName, "/") - os.Setenv("ORBORUS_CONTAINER_NAME", containerName) - log.Printf("[DEBUG] Found orborus container name: %s", containerName) - break - } - } - } - } - } else { - log.Printf("[ERROR] Failed to find orborus container: %s", err) - } - } - - getThisContainerId() - - if len(pipelineApikey) == 0 { - if len(os.Getenv("SHUFFLE_AUTHORIZATION")) > 0 { - log.Printf("[DEBUG] No pipeline API key found. Overriding with api key from SHUFFLE_AUTHORIZATION") - - pipelineApikey = os.Getenv("SHUFFLE_AUTHORIZATION") - os.Setenv("SHUFFLE_PIPELINE_AUTH", pipelineApikey) - } - } - } -} - -// form id of current running container -func getThisContainerId() { - fCol := "" - - // some adjusting based on current running mode - switch runningMode { - case "kubernetes": - // cgroup will be like: - // 11:net_cls,net_prio:/kubepods/besteffort/podf132b44d-cfcf-43f7-9906-79f58e268333/851466f8b5ed5aa0f265b1c95c6d2bafbc51a38dd5c5a1621b6e586572150009 - fCol = "5" - log.Printf("[INFO] Running containerized in Kubernetes!") - - case "docker": - // cgroup will be like: - // 12:perf_event:/docker/0f06810364f52a2cd6e80bfba27419cb8a29758a204cd676388f4913bb366f2b - fCol = "3" - log.Printf("[INFO] Running containerized in Docker!") - - default: - fCol = "3" // for backward-compatibility with production - log.Printf("[WARNING] RUNNING_MODE not set - defaulting to Docker (NOT Kubernetes).") - } - - if fCol != "" { - cmd := fmt.Sprintf("cat /proc/self/cgroup | grep memory | tail -1 | cut -d/ -f%s | grep -o -E '[0-9A-z]{64}'", fCol) - out, err := exec.Command("bash", "-c", cmd).Output() - if err == nil { - containerId = strings.TrimSpace(string(out)) - log.Printf("[DEBUG] Set containerId network to %s", containerId) - - // cgroup error. Use fallback strategy below. - // https://github.com/moby/moby/issues/7015 - //log.Printf("Checking if %s is in %s", ".scope", string(out)) - if strings.Contains(string(out), ".scope") { - log.Printf("[DEBUG] ContainerId contains scope. setting to empty.") - containerId = "" - //docker-76c537e9a4b7c7233011f5d70e6b7f2d600b6413ac58a96519b8dca7a3f7117a.scope - } - } else { - log.Printf("[WARNING] Failed getting container ID: %s", err) - } - } - - if containerId == "" { - if containerName != "" { - containerId = containerName - log.Printf("[INFO] Falling back to ORBORUS_CONTAINER_NAME as container ID") - } else { - containerId = "shuffle-orborus" - log.Printf(`[WARNING] ORBORUS_CONTAINER_NAME env is not set. Falling back to default name "%s" as container ID. This may cause issues on the same server`, containerId) - } - } - - log.Printf(`[INFO] Started with containerId "%s"`, containerId) -} - -func skipCheckInCleanup(name string) bool { - return strings.HasPrefix(name, "backend") || - strings.HasPrefix(name, "shuffle-backend") || - strings.HasPrefix(name, "frontend") || - strings.HasPrefix(name, "shuffle-frontend") || - strings.HasPrefix(name, "orborus") || - strings.HasPrefix(name, "shuffle-orborus") || - strings.HasPrefix(name, "opensearch") || - strings.HasPrefix(name, "shuffle-opensearch") || - strings.HasPrefix(name, "memcached") || - strings.HasPrefix(name, "shuffle-memcached") -} - -func cleanupExistingNodes(ctx context.Context) error { - if cleanupEnv != "true" { - log.Printf("[INFO] Skipping cleanup of existing workers as CLEANUP is NOT set to true. Swarm actions are being auto-discovered during executions then instead.") - return nil - } - - if isKubernetes == "true" { - // Cleanup all workers created by orborus and all apps created by workers. - - if kubernetesNamespace == "" { - kubernetesNamespace = "default" - } - - clientset, _, err := shuffle.GetKubernetesClient() - if err != nil { - log.Printf("[ERROR] Error getting kubernetes client:", err) - return err - } - - // Delete all services - services, err := clientset.CoreV1().Services(kubernetesNamespace).List(context.Background(), metav1.ListOptions{ - LabelSelector: "app.kubernetes.io/name in (shuffle-worker, shuffle-app),app.kubernetes.io/managed-by in (shuffle-orborus, shuffle-worker)", - }) - if err != nil { - log.Printf("[ERROR] Failed listing services: %s", err) - return err - } - - for _, service := range services.Items { - err := clientset.CoreV1().Services(kubernetesNamespace).Delete(context.Background(), service.Name, metav1.DeleteOptions{}) - if err != nil { - log.Printf("[ERROR] Failed deleting service %s: %s", service.Name, err) - } - } - - deployments, err := clientset.AppsV1().Deployments(kubernetesNamespace).List(context.Background(), metav1.ListOptions{ - LabelSelector: "app.kubernetes.io/name in (shuffle-worker, shuffle-app),app.kubernetes.io/managed-by in (shuffle-orborus, shuffle-worker)", - }) - if err != nil { - log.Printf("[ERROR] Failed listing deployments: %s", err) - return err - } - - for _, deployment := range deployments.Items { - err := clientset.AppsV1().Deployments(kubernetesNamespace).Delete(context.Background(), deployment.Name, metav1.DeleteOptions{}) - if err != nil { - log.Printf("[ERROR] Failed deleting deployment %s: %s", deployment.Name, err) - } - } - - log.Printf("[INFO] Cleaned up all services and deployments in namespace %s. Waiting 10 seconds for cleanup to reflect", kubernetesNamespace) - - time.Sleep(10 * time.Second) - - return nil - } - - serviceListOptions := types.ServiceListOptions{} - services, err := dockercli.ServiceList( - context.Background(), - serviceListOptions, - ) - - if err != nil { - log.Printf("[DEBUG] Failed finding containers: %s", err) - return err - } - - //log.Printf("\n\nFound %d contaienrs", len(services)) - - for _, service := range services { - - //portFound := false - //for _, endpoint := range service.Spec.EndpointSpec.Ports { - // if strings.Contains(endpoint.Name, "port") { - // //portFound = true - // } - //} - - if strings.Contains(service.Spec.Annotations.Name, "opensearch") { - continue - } - - if strings.Contains(service.Spec.TaskTemplate.ContainerSpec.Image, "shuffle") { - - if !strings.Contains(service.Spec.TaskTemplate.ContainerSpec.Image, "shuffle-frontend") && - !strings.Contains(service.Spec.TaskTemplate.ContainerSpec.Image, "shuffle-backend") && - !strings.Contains(service.Spec.TaskTemplate.ContainerSpec.Image, "shuffle-orborus") { - - err = dockercli.ServiceRemove(ctx, service.ID) - if err != nil { - log.Printf("[DEBUG] Failed to remove service %s", service.Spec.Annotations.Name) - } else { - log.Printf("[DEBUG] Removed service %#v", service.Spec.TaskTemplate.ContainerSpec.Image) - } - } - } - } - - return nil -} - -func deployServiceWorkers(image string) { - log.Printf("[DEBUG] Validating deployment of workers as services IF swarmConfig = run (value: %#v)", swarmConfig) - if swarmConfig != "run" && swarmConfig != "swarm" { - log.Printf("[DEBUG] Skipping deployment of workers as services as swarmConfig is not set to run or swarm. Value: %#v", swarmConfig) - return - } - ctx := context.Background() - - // Looks for and cleans up all existing items in swarm we can't re-use (Shuffle only) - - // frikky@debian:~/git/shuffle/functions/onprem/worker$ docker service create --replicas 5 --name shuffle-workers --env SHUFFLE_SWARM_CONFIG=run --publish published=33333,target=33333 ghcr.io/shuffle/shuffle-worker:nightly - - // Get a list of network interfaces - interfaces, err := net.Interfaces() - if err != nil { - log.Printf("[ERROR] Failed to get network interfaces: %s", err) - } - - mtu := 1500 - if len(dockerSwarmBridgeMTU) == 0 { - mtu, err = strconv.Atoi(dockerSwarmBridgeMTU) // by default - if err != nil { - if debug { - log.Printf("[DEBUG] Failed to convert the default MTU to int: %s. Using 1500 instead. Input: %s", err, dockerSwarmBridgeMTU) - } - - mtu = 1500 - } - } - - bridgeName := dockerSwarmBridgeInterface - if bridgeName == "" { - bridgeName = "eth0" - } - - // Check if there is at least one interface - if len(interfaces) < 2 { - // this assumes that the machine should have at least 2 network - // interfaces. If not, we will use the default MTU. - // interface 1 is the loopback interface - // interface 2 is eth0, The eth0 interface inside a - // Docker container corresponds to the virtual Ethernet - // interface that connects the container to the docker0 - log.Printf("[ERROR] Failed to get enough network interfaces") - } else { - // Get the preferred interface - for _, iface := range interfaces { - if strings.Contains(iface.Name, bridgeName) { - targetInterface := iface - mtu = targetInterface.MTU - log.Printf("[INFO] Using MTU %d from interface %s", mtu, targetInterface.Name) - break - } - } - } - - // Create the network options with the specified MTU - options := make(map[string]string) - options["com.docker.network.driver.mtu"] = fmt.Sprintf("%d", mtu) - - ingressOptions := network.CreateOptions{ - Driver: "overlay", - Attachable: false, - Ingress: true, - IPAM: &network.IPAM{ - Driver: "default", - Config: []network.IPAMConfig{ - network.IPAMConfig{ - Subnet: "10.225.225.0/24", - Gateway: "10.225.225.1", - }, - }, - }, - } - - _, err = dockercli.NetworkCreate( - ctx, - "ingress", - ingressOptions, - ) - - if err != nil { - log.Printf("[WARNING] Ingress network may already exist: %s", err) - } - - //docker network create --driver=overlay workers - // Specific subnet? - networkName := "shuffle_swarm_executions" - if len(swarmNetworkName) > 0 { - networkName = swarmNetworkName - } - - networkCreateOptions := network.CreateOptions{ - Driver: "overlay", - Options: options, - Attachable: true, - Ingress: false, - IPAM: &network.IPAM{ - Driver: "default", - Config: []network.IPAMConfig{ - network.IPAMConfig{ - Subnet: "10.224.224.0/24", - Gateway: "10.224.224.1", - }, - }, - }, - } - _, err = dockercli.NetworkCreate( - ctx, - networkName, - networkCreateOptions, - ) - - if err != nil { - if strings.Contains(fmt.Sprintf("%s", err), "already exists") { - // Try patching for attachable - if debug { - log.Printf("[DEBUG] Network %s already exists", networkName) - } - } else { - log.Printf("[DEBUG] Failed to create network %s for workers: %s. This is not critical, and containers will still be added", networkName, err) - } - } - - networkID := "" - - // find network ID - networks, err := dockercli.NetworkList(ctx, network.ListOptions{}) - if err == nil { - for _, net := range networks { - if net.Name == networkName { - if net.Scope == "swarm" { - log.Printf("[DEBUG] Found swarm-scoped network: %s (%s)", networkName, net.ID) - networkID = net.ID - } else { - log.Printf("[WARNING] Network %s exists but is not swarm scoped (scope=%s)", networkName, net.Scope) - } - break - } - } - } - - /* - isMemcachedRunning, err := checkMemcached(ctx, dockercli) - if err != nil { - log.Printf("[ERROR] Failed checking memcached: %s", err) - } - if isMemcachedRunning == false { - log.Printf("[ERROR] Memcached is not running. Will try to deploy it.") - deployMemcached(dockercli) - } - - ip := "shuffle-cache" - if len(os.Getenv("SHUFFLE_MEMCACHED")) == 0 { - os.Setenv("SHUFFLE_MEMCACHED", fmt.Sprintf("%s:11211", ip)) - } - */ - - if networkID == "" { - log.Printf("[ERROR] Network %s does not exist", networkName) - networkID = networkName - } - - defaultNetworkAttach := false - if containerId != "" { - log.Printf("[DEBUG] Should connect orborus container to worker network as it's running in Docker with name %#v!", containerId) - // https://pkg.go.dev/github.com/docker/docker@v20.10.12+incompatible/api/types/network#EndpointSettings - networkConfig := &network.EndpointSettings{} - err := dockercli.NetworkConnect(ctx, networkID, containerId, networkConfig) - if err != nil { - log.Printf("[ERROR] Failed connecting Orborus to docker network %s: %s", networkName, err) - } - - if len(containerId) == 64 && baseUrl == "http://shuffle-backend:5001" { - log.Printf("[WARNING] Network MAY not work due to backend being %s and container length 64. Will try to attach shuffle_shuffle network", baseUrl) - defaultNetworkAttach = true - } - } - - if len(os.Getenv("DOCKER_HOST")) > 0 { - log.Printf("[DEBUG] Deploying docker socket proxy to the network %s as the DOCKER_HOST variable is set", networkName) - - listOptions := container.ListOptions{ - All: true, - } - containers, err := dockercli.ContainerList(ctx, listOptions) - - if err == nil { - for _, container := range containers { - if strings.Contains(strings.ToLower(container.Image), "docker-socket-proxy") { - networkConfig := &network.EndpointSettings{} - err := dockercli.NetworkConnect(ctx, networkID, container.ID, networkConfig) - if err != nil { - log.Printf("[ERROR] Failed connecting Docker socket proxy to docker network %s: %s", networkName, err) - } else { - log.Printf("[INFO] Attached the docker socket proxy to the execution network") - } - - break - } - } - } else { - log.Printf("[ERROR] Failed listing containers when deploying socket proxy on swarm: %s", err) - } - //} else { - // log.Printf("[ERROR] Failed listing and finding the right image for docker socket proxy: %s", err) - //} - } - - // Running 2 by default instead of 1. Higher scale mechanisms - es - replicas := uint64(1) - scaleReplicas := os.Getenv("SHUFFLE_SCALE_REPLICAS") - if len(scaleReplicas) > 0 { - tmpInt, err := strconv.Atoi(scaleReplicas) - if err != nil { - log.Printf("[ERROR] %s is not a valid number for replication", scaleReplicas) - } else { - replicas = uint64(tmpInt) - } - - log.Printf("[DEBUG] SHUFFLE_SCALE_REPLICAS set to value %#v. Trying to overwrite default (%d/node)", scaleReplicas, replicas) - } - - innerContainerName := fmt.Sprintf("shuffle-workers") - cnt, err := findActiveSwarmNodes() - if err != nil { - log.Printf("[ERROR] Failed to find active swarm nodes: %s. Defaulting to 1", err) - } - - nodeCount := uint64(1) - if cnt > 0 { - nodeCount = uint64(cnt) - } - - appReplicas := os.Getenv("SHUFFLE_APP_REPLICAS") - appReplicaCnt := 2 - if len(appReplicas) > 0 { - newCnt, err := strconv.Atoi(appReplicas) - if err != nil { - log.Printf("[ERROR] %s is not a valid number for SHUFFLE_APP_REPLICAS", appReplicas) - } else { - appReplicaCnt = newCnt - } - } - - log.Printf("[DEBUG] Found %d node(s) to replicate over. Defaulting to 1 IF we can't auto-discover them.", cnt) - - // FIXME: From September 2025 - This is set back to 1, as this doesn't really reflect how scale works at all. It is just confusing, and makes number larger/smaller "arbitrarily" instead of using default docker scale - nodeCount = 1 - replicatedJobs := uint64(replicas * nodeCount) - - log.Printf("[DEBUG] Deploying %d container(s) for worker with swarm to each node. Service name: %s. Image: %s", replicas, innerContainerName, image) - - if timezone == "" { - timezone = "Europe/Amsterdam" - } - - // FIXME: May not need ingress ports. Could use internal services and DNS of swarm itself - // https://github.com/moby/moby/blob/e2f740de442bac52b280bc485a3ca5b31567d938/api/types/swarm/service.go#L46 - serviceSpec := swarm.ServiceSpec{ - Annotations: swarm.Annotations{ - Name: innerContainerName, - Labels: map[string]string{}, - }, - Mode: swarm.ServiceMode{ - Replicated: &swarm.ReplicatedService{ - Replicas: &replicatedJobs, - }, - }, - Networks: []swarm.NetworkAttachmentConfig{ - swarm.NetworkAttachmentConfig{ - Target: networkID, - }, - swarm.NetworkAttachmentConfig{ - Target: "ingress", - }, - }, - EndpointSpec: &swarm.EndpointSpec{ - Mode: "vip", - Ports: []swarm.PortConfig{ - swarm.PortConfig{ - Protocol: swarm.PortConfigProtocolTCP, - PublishMode: swarm.PortConfigPublishModeIngress, - Name: "worker-port", - PublishedPort: 33333, - TargetPort: 33333, - }, - }, - }, - TaskTemplate: swarm.TaskSpec{ - Resources: &swarm.ResourceRequirements{ - Reservations: &swarm.Resources{}, - }, - LogDriver: &swarm.Driver{ - Name: "json-file", - Options: map[string]string{ - "max-size": "10m", - }, - }, - ContainerSpec: &swarm.ContainerSpec{ - Image: image, - Env: []string{ - fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", os.Getenv("SHUFFLE_SWARM_CONFIG")), - fmt.Sprintf("SHUFFLE_SWARM_NETWORK_NAME=%s", networkName), - fmt.Sprintf("SHUFFLE_APP_REPLICAS=%d", appReplicaCnt), - fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED")), - fmt.Sprintf("DEBUG_MEMORY=%s", os.Getenv("DEBUG_MEMORY")), - fmt.Sprintf("SHUFFLE_APP_SDK_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")), - fmt.Sprintf("SHUFFLE_MAX_SWARM_NODES=%s", os.Getenv("SHUFFLE_MAX_SWARM_NODES")), - fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", os.Getenv("SHUFFLE_BASE_IMAGE_NAME")), - fmt.Sprintf("SHUFFLE_APP_REQUEST_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_REQUEST_TIMEOUT")), - }, - //Hosts: []string{ - // innerContainerName, - //}, - }, - RestartPolicy: &swarm.RestartPolicy{ - Condition: swarm.RestartPolicyConditionOnFailure, - }, - Placement: &swarm.Placement{ - Constraints: []string{}, - }, - }, - } - - if defaultNetworkAttach == true || strings.ToLower(os.Getenv("SHUFFLE_DEFAULT_NETWORK_ATTACH")) == "true" { - targetName := "shuffle_shuffle" - isAttachable := false - networks, err := dockercli.NetworkList(ctx, network.ListOptions{}) - if err == nil { - for _, net := range networks { - if net.Name == targetName { - if net.Scope == "swarm" { - log.Printf("[DEBUG] Found swarm-scoped network: %s", targetName) - isAttachable = true - } else { - log.Printf("[WARNING] Network %s exist but is not swarm scoped (scope=%s)", targetName, net.Scope) - } - break - } - } - } - - if isAttachable { - log.Printf("[DEBUG] Adding network attach for network %s to worker in swarm", targetName) - serviceSpec.Networks = append(serviceSpec.Networks, swarm.NetworkAttachmentConfig{ - Target: targetName, - }) - - // FIXM: Remove this if deployment fails? - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_SWARM_OTHER_NETWORK=%s", targetName)) - } - } - - if dockerApiVersion != "" { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("DOCKER_API_VERSION=%s", dockerApiVersion)) - } - - if len(os.Getenv("SHUFFLE_SCALE_REPLICAS")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_SCALE_REPLICAS=%s", os.Getenv("SHUFFLE_SCALE_REPLICAS"))) - } - - if len(os.Getenv("SHUFFLE_MEMCACHED")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_MEMCACHED=%s", os.Getenv("SHUFFLE_MEMCACHED"))) - } - - if strings.ToLower(os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) == "true" { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))) - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("no_proxy=%s", os.Getenv("no_proxy"))) - } - - if len(workerServerUrl) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_WORKER_SERVER_URL=%s", os.Getenv("SHUFFLE_WORKER_SERVER_URL"))) - } - - // Handles backend - if len(os.Getenv("BASE_URL")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("BASE_URL=%s", os.Getenv("BASE_URL"))) - } - - if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_CLOUDRUN_URL=%s", os.Getenv("SHUFFLE_CLOUDRUN_URL"))) - } - - if len(os.Getenv("SHUFFLE_AUTO_IMAGE_DOWNLOAD")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_AUTO_IMAGE_DOWNLOAD=%s", os.Getenv("SHUFFLE_AUTO_IMAGE_DOWNLOAD"))) - } - - if len(os.Getenv("DOCKER_HOST")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("DOCKER_HOST=%s", os.Getenv("DOCKER_HOST"))) - } else { - if runtime.GOOS == "windows" { - serviceSpec.TaskTemplate.ContainerSpec.Mounts = []mount.Mount{ - mount.Mount{ - Source: `\\.\pipe\docker_engine`, - Target: `\\.\pipe\docker_engine`, - Type: mount.TypeBind, - }, - } - } else { - serviceSpec.TaskTemplate.ContainerSpec.Mounts = []mount.Mount{ - mount.Mount{ - Source: "/var/run/docker.sock", - Target: "/var/run/docker.sock", - Type: mount.TypeBind, - }, - } - - } - } - - // Look for SHUFFLE_VOLUME_BINDS - if len(os.Getenv("SHUFFLE_VOLUME_BINDS")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_VOLUME_BINDS=%s", os.Getenv("SHUFFLE_VOLUME_BINDS"))) - } - - overrideHttpProxy := os.Getenv("SHUFFLE_INTERNAL_HTTP_PROXY") - overrideHttpsProxy := os.Getenv("SHUFFLE_INTERNAL_HTTPS_PROXY") - if len(overrideHttpProxy) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_INTERNAL_HTTP_PROXY=%s", overrideHttpProxy)) - } - - if len(overrideHttpsProxy) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_INTERNAL_HTTPS_PROXY=%s", overrideHttpsProxy)) - } - - serviceOptions := types.ServiceCreateOptions{} - _, err = dockercli.ServiceCreate( - ctx, - serviceSpec, - serviceOptions, - ) - - // Force deploy if it's not disabled - deployTenzirNode() - - if err == nil { - log.Printf("[DEBUG] Successfully deployed workers with %d replica(s) on %d node(s)", replicas, cnt) - // wait for service to be ready - time.Sleep(time.Duration(rand.Intn(4)+1) * time.Second) - - //log.Printf("[DEBUG] Servicecreate request: %#v %#v", service, err) - // patch service network - // this is an edgecase that we noticed on docker version 29 - // and API version 1.44 - services, serr := dockercli.ServiceList(ctx, types.ServiceListOptions{}) - if serr == nil { - for _, svc := range services { - if svc.Spec.Annotations.Name == innerContainerName { - log.Printf("[DEBUG] Found service %s (%s) — patching network attach", innerContainerName, svc.ID) - - spec := svc.Spec - spec.TaskTemplate.Networks = append(spec.TaskTemplate.Networks, swarm.NetworkAttachmentConfig{ - Target: networkID, - }) - - _, uerr := dockercli.ServiceUpdate(ctx, svc.ID, svc.Version, spec, types.ServiceUpdateOptions{}) - if uerr != nil { - log.Printf("[WARNING] Failed to patch service %s with network %s: %v", innerContainerName, networkID, uerr) - } else { - log.Printf("[INFO] Successfully attached network %s to service %s", networkID, innerContainerName) - } - break - } - } - } else { - log.Printf("[WARNING] Failed to list services for patching network attach: %v", serr) - } - } else { - if !strings.Contains(fmt.Sprintf("%s", err), "Already Exists") && !strings.Contains(fmt.Sprintf("%s", err), "is already in use by service") { - log.Printf("[ERROR] Failed making service: %s", err) - if strings.Contains(fmt.Sprintf("%s", err), "networks scoped to the swarm can be used") { - log.Printf("[WARNING] Swarm network attachment failed, retrying without shuffle_shuffle") - - var updatedNetworks []swarm.NetworkAttachmentConfig - for _, net := range serviceSpec.Networks { - if net.Target != "shuffle_shuffle" { - updatedNetworks = append(updatedNetworks, net) - } - } - serviceSpec.Networks = updatedNetworks - - var updatedEnv []string - for _, env := range serviceSpec.TaskTemplate.ContainerSpec.Env { - if !strings.HasPrefix(env, "SHUFFLE_SWARM_OTHER_NETWORK=") { - updatedEnv = append(updatedEnv, env) - } - } - serviceSpec.TaskTemplate.ContainerSpec.Env = updatedEnv - serviceOptions := types.ServiceCreateOptions{} - _, err = dockercli.ServiceCreate( - ctx, - serviceSpec, - serviceOptions, - ) - if err != nil { - log.Printf("[ERROR] Failed to deploy service even without shuffle_shuffle network: %s", err) - } - } - } else { - log.Printf("[WARNING] Failed deploying workers: %s", err) - if len(serviceSpec.Networks) > 1 { - serviceSpec.Networks = []swarm.NetworkAttachmentConfig{ - swarm.NetworkAttachmentConfig{ - Target: "shuffle_shuffle", - }, - } - - _, _ = dockercli.ServiceCreate( - ctx, - serviceSpec, - serviceOptions, - ) - } - } - } -} - -// Deploys the worker with the current available environments -// https://docs.docker.com/engine/api/sdk/examples/ -func buildEnvVars(envMap map[string]string) []corev1.EnvVar { - var envVars []corev1.EnvVar - for key, value := range envMap { - envVars = append(envVars, corev1.EnvVar{Name: key, Value: value}) - } - - return envVars -} - -func buildResourcesFromEnv() corev1.ResourceRequirements { - requests := corev1.ResourceList{} - limits := corev1.ResourceList{} - - type item struct { - env string - resourceName corev1.ResourceName - resourceList corev1.ResourceList - } - - items := []item{ - // kubernetes requests - {env: "SHUFFLE_WORKER_CPU_REQUEST", resourceName: corev1.ResourceCPU, resourceList: requests}, - {env: "SHUFFLE_WORKER_MEMORY_REQUEST", resourceName: corev1.ResourceMemory, resourceList: requests}, - {env: "SHUFFLE_WORKER_EPHEMERAL_STORAGE_REQUEST", resourceName: corev1.ResourceEphemeralStorage, resourceList: requests}, - // kubernetes limits - {env: "SHUFFLE_WORKER_CPU_LIMIT", resourceName: corev1.ResourceCPU, resourceList: limits}, - {env: "SHUFFLE_WORKER_MEMORY_LIMIT", resourceName: corev1.ResourceMemory, resourceList: limits}, - {env: "SHUFFLE_WORKER_EPHEMERAL_STORAGE_LIMIT", resourceName: corev1.ResourceEphemeralStorage, resourceList: limits}, - } - - for _, it := range items { - if value := strings.TrimSpace(os.Getenv(it.env)); value != "" { - if quantity, err := resource.ParseQuantity(value); err == nil { - it.resourceList[it.resourceName] = quantity - } else { - log.Printf("[WARNING] Cannot parse %s=%q as resource quantity: %v", it.env, value, err) - } - } - } - - rr := corev1.ResourceRequirements{} - if len(requests) > 0 { - rr.Requests = requests - } - if len(limits) > 0 { - rr.Limits = limits - } - - return rr -} - -func handleBackendImageDownload(ctx context.Context, images string) error { - - // Replicate images with lowercase, as the name may be wrong - // Most of the time lowercase is correct. Swapping to have that first - originalImages := images - images = strings.ToLower(images) + "," + originalImages - - // Remove the image - handled := []string{} - //log.Printf("[DEBUG] Removing existing image (s): %s", images) - newImages := []string{} - - successful := []string{} - for _, curimage := range strings.Split(images, ",") { - curimage = strings.TrimSpace(curimage) - if shuffle.ArrayContains(handled, curimage) { - continue - } - - handled = append(handled, curimage) - if !strings.Contains(curimage, "/") { - curimage = fmt.Sprintf("frikky/shuffle:%s", curimage) - } - - newImages = append(newImages, curimage) - - // Force remove the current image to avoid cached layers - // if swarmConfig == "run" || swarmConfig == "swarm" { - // _, err := dockercli.ImageRemove(ctx, curimage, image.RemoveOptions{ - // Force: true, - // PruneChildren: true, - // }) - - // if err != nil { - // log.Printf("[ERROR] Failed removing image for re-download: %s", err) - // } else { - // log.Printf("[DEBUG] Removed image: %s", curimage) - // } - // } else { - // //log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig) - // } - - err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, curimage) - if err != nil { - //log.Printf("[ERROR] Failed downloading image: %s", err) - } else { - //log.Printf("[DEBUG] Downloaded image: %s", curimage) - successful = append(successful, curimage) - } - } - - if len(successful) == 0 { - log.Printf("[ERROR] Failed downloading image copies: %s. This means the app may not have been updated.", strings.Join(handled, ", ")) - } else { - log.Printf("[DEBUG] Successfully downloaded image copies: %s", strings.Join(successful, ", ")) - } - - if swarmConfig == "run" || swarmConfig == "swarm" { - log.Printf("[DEBUG] Should update service with new image after updating(s): %s. \n\nBETA REPLACEMENT IMPLEMENTATION: Contact support@shuffler.io for support.", strings.Join(newImages, "\n")) - - // 1. Download the image - // 2. Find the existing service using the image - // 3. Update the service with the new image in a rolling restart - - // Find the existing service - serviceListOptions := types.ServiceListOptions{} - services, err := dockercli.ServiceList( - ctx, - serviceListOptions, - ) - - if err != nil { - log.Printf("[ERROR] Failed finding services: %s", err) - } else { - found := false - for _, service := range services { - //log.Printf("Service image: %s", service.Spec.TaskTemplate.ContainerSpec.Image) - - for _, image := range newImages { - if !strings.Contains(service.Spec.TaskTemplate.ContainerSpec.Image, image) { - continue - } - - log.Printf("[DEBUG] Found service for image: %#v", service.Spec.Annotations.Name) - - // Update the service to run with the new image - //docker service update --image username/imagename:latest servicename --force - serviceUpdateOptions := types.ServiceUpdateOptions{} - service.Spec.TaskTemplate.ForceUpdate++ - resp, err := dockercli.ServiceUpdate( - ctx, - service.ID, - service.Version, - service.Spec, - serviceUpdateOptions, - ) - - if err != nil { - log.Printf("[ERROR] Failed updating service %s with the new image %s: %s. Resp: %#v", service.Spec.Annotations.Name, image, err, resp) - } else { - log.Printf("[DEBUG] Updated service %s with the new image %s. Resp: %#v", service.Spec.Annotations.Name, image, resp) - - found = true - - if !strings.Contains(fmt.Sprintf("%s", resp), "error") { - break - } else { - log.Printf("[ERROR] Failed updating service %s with the new image %s: %s. Resp: %#v", service.Spec.Annotations.Name, image, err, resp) - } - } - } - - if found { - break - } - } - - if !found { - log.Printf("[DEBUG] Failed to find service to update for service %s", newImages) - } - - } - - } - - return nil -} - -func fixk8sRoles() { - clientset, _, err := shuffle.GetKubernetesClient() - if err != nil { - log.Printf("[ERROR] Error getting kubernetes client: %s", err) - os.Exit(1) - } - - kubernetesNamespace := "default" - - // Check if namespace exist as variable. If so, make it - if len(os.Getenv("KUBERNETES_NAMESPACE")) > 0 { - kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") - } - - // fix roles - // check if "service-creator" role is assigned to the service account "default" - // roleBindingNames := []string{"service-creator-binding", "pod-creator-binding", "deployment-creator-binding"} - serviceAccountName := "default" - roleBindingName := "creator-all" - - resourceTypes := []string{"services", "pods", "deployments"} - - // Check if the RoleBinding exists - roleBinding, err := clientset.RbacV1().RoleBindings(kubernetesNamespace).Get(context.TODO(), roleBindingName, metav1.GetOptions{}) - if err != nil { - log.Printf("[WARNING] Failed to get RoleBinding %s: %s", roleBindingName, err) - - // create role and rolebinding - role := &rbacv1.Role{ - ObjectMeta: metav1.ObjectMeta{ - Name: roleBindingName, - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"", "apps"}, - Resources: resourceTypes, - Verbs: []string{"create", "list"}, - }, - }, - } - - ctx := context.TODO() - - _, err := clientset.RbacV1().Roles(kubernetesNamespace).Create(ctx, role, metav1.CreateOptions{}) - if err != nil { - log.Printf("[ERROR] Failed to create Role %s: %s", roleBindingName, err) - if !strings.Contains(fmt.Sprintf("%s", err), "already exists") { - log.Printf("[INFO] role %s already exists", roleBindingName) - } - } - - roleBinding := &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: roleBindingName, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: serviceAccountName, - Namespace: kubernetesNamespace, - }, - }, - RoleRef: rbacv1.RoleRef{ - Kind: "Role", - Name: roleBindingName, - }, - } - - _, err = clientset.RbacV1().RoleBindings(kubernetesNamespace).Create(ctx, roleBinding, metav1.CreateOptions{}) - if err != nil { - log.Printf("[ERROR] Failed to create RoleBinding %s: %s", roleBindingName, err) - if strings.Contains(fmt.Sprintf("%s", err), "already exists") { - log.Printf("[INFO] rolebinding %s already exists", roleBindingName) - } - } - - log.Printf("[INFO] Created Role %s and RoleBinding %s", roleBindingName, roleBindingName) - } else { - log.Printf("[INFO] RoleBinding %s exists", roleBindingName) - } - - // Check if the RoleBinding is assigned to the service account - var found bool - for _, subject := range roleBinding.Subjects { - if subject.Kind == "ServiceAccount" && subject.Name == serviceAccountName { - found = true - break - } - } - - if !found { - log.Printf("[WARNING] Service account %s is not assigned to RoleBinding %s\n", serviceAccountName, roleBindingName) - // assign the service account to the rolebinding - roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{ - Kind: "ServiceAccount", - Name: serviceAccountName, - Namespace: kubernetesNamespace, - }) - - ctx := context.TODO() - - _, err := clientset.RbacV1().RoleBindings(kubernetesNamespace).Update(ctx, roleBinding, metav1.UpdateOptions{}) - if err != nil { - log.Printf("[ERROR](ns - %s) Failed to update RoleBinding %s: %s", kubernetesNamespace, roleBindingName, err) - if !strings.Contains(fmt.Sprintf("%s", err), "already exists") { - log.Printf("[INFO] rolebinding %s already exists", roleBindingName) - } - } - } -} - -// TODO: Check if deployment or service already exist by labels and only create if not already exists -func deployK8sWorker(image string, identifier string, env []string) error { - env = append(env, fmt.Sprintf("IS_KUBERNETES=true")) - env = append(env, fmt.Sprintf("KUBERNETES_NAMESPACE=%s", os.Getenv("KUBERNETES_NAMESPACE"))) - - // app resource env - for _, k := range []string{ - "SHUFFLE_APP_CPU_REQUEST", - "SHUFFLE_APP_MEMORY_REQUEST", - "SHUFFLE_APP_EPHEMERAL_STORAGE_REQUEST", - "SHUFFLE_APP_CPU_LIMIT", - "SHUFFLE_APP_MEMORY_LIMIT", - "SHUFFLE_APP_EPHEMERAL_STORAGE_LIMIT", - } { - if v := os.Getenv(k); v != "" { - env = append(env, fmt.Sprintf("%s=%s", k, v)) - } - } - - if len(os.Getenv("KUBERNETES_SERVICE_HOST")) > 0 { - env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_HOST=%s", os.Getenv("KUBERNETES_SERVICE_HOST"))) - } - - if len(os.Getenv("SHUFFLE_MEMCACHED")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_MEMCACHED=%s", os.Getenv("SHUFFLE_MEMCACHED"))) - } - - if len(os.Getenv("KUBERNETES_SERVICE_PORT")) > 0 { - env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_PORT=%s", os.Getenv("KUBERNETES_SERVICE_PORT"))) - } - - if len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_REGISTRY=%s", os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY"))) - } - - if len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", os.Getenv("SHUFFLE_BASE_IMAGE_NAME"))) - } else { - log.Printf("[INFO] SHUFFLE_BASE_IMAGE_NAME is not set. Defaulting to %s", baseimagename) - env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", baseimagename)) - } - - if len(os.Getenv("REGISTRY_URL")) > 0 { - env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL"))) - } - - if len(os.Getenv("SHUFFLE_USE_GHCR_OVERRIDE_FOR_AUTODEPLOY")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_USE_GHCR_OVERRIDE_FOR_AUTODEPLOY=%s", os.Getenv("SHUFFLE_USE_GHCR_OVERRIDE_FOR_AUTODEPLOY"))) - } - - if len(os.Getenv("SHUFFLE_APP_EXPOSED_PORT")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_APP_EXPOSED_PORT=%s", os.Getenv("SHUFFLE_APP_EXPOSED_PORT"))) - } - - if len(appServiceAccountName) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_APP_SERVICE_ACCOUNT_NAME=%s", appServiceAccountName)) - } - - if len(appPodSecurityContext) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_APP_POD_SECURITY_CONTEXT=%s", appPodSecurityContext)) - } - - if len(appContainerSecurityContext) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_APP_CONTAINER_SECURITY_CONTEXT=%s", appContainerSecurityContext)) - } - - if len(os.Getenv("SHUFFLE_APP_MOUNT_TMP_VOLUME")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_APP_MOUNT_TMP_VOLUME=%s", os.Getenv("SHUFFLE_APP_MOUNT_TMP_VOLUME"))) - } - - if len(os.Getenv("SHUFFLE_LOGS_DISABLED")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED"))) - } - - if len(os.Getenv("SHUFFLE_APP_REPLICAS")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_APP_REPLICAS=%s", os.Getenv("SHUFFLE_APP_REPLICAS"))) - } - - clientset, _, err := shuffle.GetKubernetesClient() - if err != nil { - log.Printf("[ERROR] Error getting kubernetes client:", err) - return err - } - - //env = append(env, fmt.Sprintf("KUBERNETES_CONFIG=%s", config.String())) - - // Check if namespace exist as variable. If so, make it - if len(os.Getenv("KUBERNETES_NAMESPACE")) > 0 && !namespacemade { - kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") - - // Make the namespace - namespace := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: os.Getenv("KUBERNETES_NAMESPACE"), - }, - } - - _, err := clientset.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{}) - if err != nil { - if !strings.Contains(strings.ToLower(fmt.Sprintf("%s", err)), "already exists") { - log.Printf("[ERROR] Failed creating Kubernetes namespace: %s", err) - } else { - namespacemade = true - } - } else { - namespacemade = true - } - } - - // Required format: - // url/org/repo/appname:tag - // url/org/repo/appname:tag - - //env = append(env, fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", swarmConfig)) - env = append(env, fmt.Sprintf("BASE_URL=%s", baseUrl)) - env = append(env, fmt.Sprintf("SHUFFLE_SWARM_CONFIG=run")) - env = append(env, fmt.Sprintf("WORKER_HOSTNAME=%s", "shuffle-workers")) - - if len(kubernetesNamespace) == 0 { - foundNamespace, err := shuffle.GetKubernetesNamespace() - if err != nil { - //log.Printf("[ERROR] Failed getting Kubernetes namespace: %s", err) - } - - if len(foundNamespace) > 0 { - kubernetesNamespace = foundNamespace - os.Setenv("KUBERNETES_NAMESPACE", kubernetesNamespace) - } - } - - if len(kubernetesNamespace) == 0 { - kubernetesNamespace = "default" - } - - kubernetesImage := os.Getenv("SHUFFLE_KUBERNETES_WORKER") - if len(kubernetesImage) == 0 { - kubernetesImage = image - } - log.Printf("[DEBUG] Using Kubernetes worker image '%s'", kubernetesImage) - // image = "shuffle-worker:v1" //hard coded image name to test locally - - envMap := make(map[string]string) - for _, envStr := range env { - parts := strings.SplitN(envStr, "=", 2) - if len(parts) == 2 { - envMap[parts[0]] = parts[1] - } - } - - labels := map[string]string{ - // Well-known Kubernetes labels - "app.kubernetes.io/name": "shuffle-worker", - "app.kubernetes.io/instance": identifier, - "app.kubernetes.io/part-of": "shuffle", - "app.kubernetes.io/managed-by": "shuffle-orborus", - // Keep legacy labels for backward compatibility - "container": "shuffle-worker", - } - - matchLabels := map[string]string{ - "app.kubernetes.io/name": "shuffle-worker", - "app.kubernetes.io/instance": identifier, - } - - // Parse security contexts from env - var podSecurityContext *corev1.PodSecurityContext - var containerSecurityContext *corev1.SecurityContext - - if len(workerPodSecurityContext) > 0 { - podSecurityContext = &corev1.PodSecurityContext{} - err = json.Unmarshal([]byte(workerPodSecurityContext), podSecurityContext) - if err != nil { - log.Printf("[ERROR] Failed to unmarshal worker pod security context: %v", err) - return fmt.Errorf("failed to unmarshal worker pod security context: %v", err) - } - } - - if len(workerContainerSecurityContext) > 0 { - containerSecurityContext = &corev1.SecurityContext{} - err = json.Unmarshal([]byte(workerContainerSecurityContext), containerSecurityContext) - if err != nil { - log.Printf("[ERROR] Failed to unmarshal worker container security context: %v", err) - return fmt.Errorf("failed to unmarshal worker container security context: %v", err) - } - } - - containerAttachment := corev1.Container{ - Name: identifier, - Image: kubernetesImage, - Env: buildEnvVars(envMap), - SecurityContext: containerSecurityContext, - Resources: buildResourcesFromEnv(), - - //ImagePullPolicy: "Never", - ImagePullPolicy: corev1.PullIfNotPresent, - } - - if len(os.Getenv("REGISTRY_URL")) > 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 { - log.Printf("[INFO] Setting image pull policy to Always as private registry is used.") - containerAttachment.ImagePullPolicy = corev1.PullAlways - } - - podname := shuffle.GetPodName() - ctx := context.Background() - if len(podname) > 0 { - _, err := shuffle.GetCurrentPodNetworkConfig(ctx, clientset, kubernetesNamespace, podname) - if err != nil { - log.Printf("[ERROR] Failed getting current pod network: %s", err) - } else { - log.Printf("[DEBUG] Current pod found!") - // currentPodStatus = k8s.io/api/core/v1.PodStatus - } - } - - // While testing: - // kubectl delete pods --all --all-namespaces; kubectl delete services --all --all-namespaces - // pod := &corev1.Pod{ - // ObjectMeta: metav1.ObjectMeta{ - // Name: identifier, - // Labels: containerLabels, - // }, - // Spec: corev1.PodSpec{ - // RestartPolicy: "Never", - // // DNSPolicy: "Default", - // DNSPolicy: corev1.DNSClusterFirst, - // // NodeSelector: map[string]string{ - // // "node": "master", - // // }, - // Containers: []corev1.Container{ - // containerAttachment, - // }, - // }, - // } - - // // Check if running on ARM or x86 to download the correct image - - // // Get current pod's network so we can make the pod in it - - // _, err = clientset.CoreV1().Pods(kubernetesNamespace).List(context.Background(), metav1.ListOptions{}) - // if err != nil { - // log.Printf("[ERROR] Failed listing pods: %s", err) - // } - - // createdPod, err := clientset.CoreV1().Pods(kubernetesNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) - // if err != nil { - // //log.Printf("[ERROR] Failed creating pod: %v", err) - // return err - // } - - // log.Printf("[INFO] Created pod %q in namespace %q\n", createdPod.Name, createdPod.Namespace) - - // // kubectl expose pod shuffle-workers --type=LoadBalancer --port=33333 - // service := &corev1.Service{ - // ObjectMeta: metav1.ObjectMeta{ - // Name: identifier, - // }, - // Spec: corev1.ServiceSpec{ - // Selector: map[string]string{ - // "container": "shuffle-workers", - // }, - // Ports: []corev1.ServicePort{ - // { - // Protocol: "TCP", - // Port: 33333, - // TargetPort: intstr.FromInt(33333), - // }, - // }, - // Type: corev1.ServiceTypeLoadBalancer, - // }, - // } - - // _, err = clientset.CoreV1().Services(kubernetesNamespace).Create(context.TODO(), service, metav1.CreateOptions{}) - // if err != nil { - // log.Printf("[ERROR] Failed creating service: %v", err) - // return err - // } - - replicaNumberStr := os.Getenv("SHUFFLE_SCALE_REPLICAS") - replicaNumber := 1 - if len(replicaNumberStr) > 0 { - tmpInt, err := strconv.Atoi(replicaNumberStr) - if err != nil { - log.Printf("[ERROR] %s is not a valid number for replication", replicaNumberStr) - } else { - replicaNumber = tmpInt - - } - } - - existing, err := clientset.AppsV1().Deployments(kubernetesNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: "app.kubernetes.io/name=shuffle-worker", - }) - if err != nil { - log.Printf("[ERROR] Failed listing existing deployments: %v", err) - } - - if len(existing.Items) > 0 { - log.Printf("[INFO] Found existing deployments, skipping creation") - return nil - } - - replicaNumberInt32 := int32(replicaNumber) - // worker makes authenticated requests to the k8s api to create app deployments. - // Therefore, it needs to have access to the service account token. - automountServiceAccountToken := true - - deployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: identifier, - Labels: labels, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: &replicaNumberInt32, - Selector: &metav1.LabelSelector{ - MatchLabels: matchLabels, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - containerAttachment, - }, - DNSPolicy: corev1.DNSClusterFirst, - ServiceAccountName: workerServiceAccountName, - AutomountServiceAccountToken: &automountServiceAccountToken, - SecurityContext: podSecurityContext, - }, - }, - }, - } - - _, err = clientset.AppsV1().Deployments(kubernetesNamespace).Create(context.Background(), deployment, metav1.CreateOptions{}) - if err != nil { - log.Printf("[ERROR] Failed creating deployment: %v", err) - return err - } - - svcAppProtocol := "http" - service := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: identifier, - Labels: labels, - }, - Spec: corev1.ServiceSpec{ - Selector: matchLabels, - Ports: []corev1.ServicePort{ - { - Protocol: "TCP", - AppProtocol: &svcAppProtocol, - Port: 33333, - TargetPort: intstr.FromInt(33333), - }, - }, - Type: corev1.ServiceTypeClusterIP, - }, - } - - _, err = clientset.CoreV1().Services(kubernetesNamespace).Create(context.Background(), service, metav1.CreateOptions{}) - if err != nil { - log.Printf("[ERROR] Failed creating service: %v", err) - return err - } - - return nil -} - -func deployWorker(image string, identifier string, env []string, executionRequest shuffle.ExecutionRequest) error { - - if len(os.Getenv("REGISTRY_URL")) > 0 && os.Getenv("REGISTRY_URL") != "" { - env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL"))) - } - - if swarmConfig == "run" || swarmConfig == "swarm" || isKubernetes == "true" { - // FIXME: Should we handle replies properly? - // In certain cases, a workflow may e.g. be aborted already. If it's aborted, that returns - // a 401 from the worker, which returns an error here - go sendWorkerRequest(executionRequest, image, env) - - return nil - } - - // Binds is the actual "-v" volume. - // Max 20% CPU every second - - //CPUQuota: 25000, - //CPUPeriod: 100000, - //CPUShares: 256, - hostConfig := &container.HostConfig{ - LogConfig: container.LogConfig{ - Type: "json-file", - Config: map[string]string{ - "max-size": "10m", - }, - }, - Resources: container.Resources{}, - } - - // This is just to test the mounting locally so - // I can control from what source I'm mounting - // the certs to. Default behaviour is: - // /certs:/certs. - certPath := "/certs" - if os.Getenv("SHUFFLE_CERT_PATH") != "" { - certPath = os.Getenv("SHUFFLE_CERT_PATH") - } - - _, err := os.ReadDir(certPath) - if certPath != "" && err == nil { - certVol := mount.Mount{ - Type: mount.TypeBind, - Source: certPath, - Target: "/certs", - } - - hostConfig.Mounts = append(hostConfig.Mounts, certVol) - } - - if len(os.Getenv("DOCKER_HOST")) == 0 { - if runtime.GOOS == "windows" { - hostConfig.Binds = []string{`\\.\pipe\docker_engine:\\.\pipe\docker_engine`} - } else { - hostConfig.Binds = []string{"/var/run/docker.sock:/var/run/docker.sock:rw"} - } - } - - //var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG") - parsedUuid := uuid.NewV4() - - config := &container.Config{ - Image: image, - Env: env, - } - - if isKubernetes != "true" { - hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) - - if strings.ToLower(cleanupEnv) == "true" { - hostConfig.AutoRemove = true - } - } - - //log.Printf("[INFO] Identifier: %s", identifier) - cont, err := dockercli.ContainerCreate( - context.Background(), - config, - hostConfig, - nil, - nil, - identifier, - ) - - if err != nil { - if strings.Contains(fmt.Sprintf("%s", err), "Conflict. The container name ") { - identifier = fmt.Sprintf("%s-%s", identifier, parsedUuid) - //log.Printf("[INFO] 2 - Identifier: %s", identifier) - cont, err = dockercli.ContainerCreate( - context.Background(), - config, - hostConfig, - nil, - nil, - identifier, - ) - - if err != nil { - log.Printf("[ERROR][%s] Container create error(2): %s", executionRequest.ExecutionId, err) - return err - } - } else { - log.Printf("[ERROR][%s] Container create error: %s", executionRequest.ExecutionId, err) - return err - } - } - - // FIXME: Verbosity for testing - //log.Printf("WORKER STARTING WITH ENV: %#v", env) - - ctx := context.Background() - containerStartOptions := container.StartOptions{} - err = dockercli.ContainerStart(ctx, cont.ID, containerStartOptions) - if err != nil { - // Trying to recreate and start WITHOUT network if it's possible. No extended checks. Old execution system (<0.9.30) - if strings.Contains(fmt.Sprintf("%s", err), "cannot join network") || strings.Contains(fmt.Sprintf("%s", err), "No such container") { - hostConfig.NetworkMode = "" - //container.NetworkMode(fmt.Sprintf("container:%s", containerId)) - cont, err = dockercli.ContainerCreate( - context.Background(), - config, - hostConfig, - nil, - nil, - identifier+"-2", - ) - if err != nil { - log.Printf("[ERROR][%s] Failed to CREATE container (2): %s", executionRequest.ExecutionId, err) - } - - err = dockercli.ContainerStart(context.Background(), cont.ID, containerStartOptions) - if err != nil { - log.Printf("[ERROR][%s] Failed to start container (2): %s", executionRequest.ExecutionId, err) - } - } else { - log.Printf("[ERROR][%s] Failed initial container start. Quitting as this is NOT a simple network issue. Err: %s", executionRequest.ExecutionId, err) - } - - if err != nil { - log.Printf("[ERROR][%s] Failed to start worker container in environment '%s': %s", executionRequest.ExecutionId, environment, err) - return err - } else { - log.Printf("[INFO][%s] Worker Container created (2). Runtime Location '%s': docker logs -f %s", executionRequest.ExecutionId, environment, cont.ID) - } - - stats, err := dockercli.ContainerInspect(ctx, cont.ID) - if err != nil { - log.Printf("[WARNING][%s] Failed checking worker '%s': %s", executionRequest.ExecutionId, cont.ID, err) - return nil - } - - containerStatus := stats.ContainerJSONBase.State.Status - if containerStatus != "running" { - log.Printf("[ERROR][%s] Status of %s is %s. Should be running. Contact support@shuffler.io if this persists.", executionRequest.ExecutionId, cont.ID, containerStatus) - } - /* - err = stopWorker(containerName) - if err != nil { - log.Printf("Failed stopping worker %s", execution.ExecutionId) - return nil - } - - err = deployWorker(dockercli, workerImage, containerName, env) - if err != nil { - log.Printf("Failed executing worker %s in state %s", execution.ExecutionId, containerStatus) - return nil - } - } - */ - } else { - log.Printf("[INFO][%s] New Worker created. Environment %s: docker logs %s", executionRequest.ExecutionId, environment, cont.ID) - } - - return nil -} - -func stopWorker(containername string) error { - ctx := context.Background() - - // containers, err := cli.ContainerList(ctx, types.ContainerListOptions{ - // All: true, - // }) - - //if err := dockercli.ContainerStop(ctx, containername, nil); err != nil { - var options container.StopOptions - if err := dockercli.ContainerStop(ctx, containername, options); err != nil { - log.Printf("[ERROR] Unable to stop container %s - running removal anyway, just in case: %s", containername, err) - } - - removeOptions := container.RemoveOptions{ - RemoveVolumes: true, - Force: true, - } - - if err := dockercli.ContainerRemove(ctx, containername, removeOptions); err != nil { - log.Printf("[ERROR] Unable to remove container: %s", err) - } - - return nil -} - -func initializeImages() { - ctx := context.Background() - - if appSdkVersion == "" { - appSdkVersion = "latest" - log.Printf("[INFO] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %#v", appSdkVersion) - } - - if workerVersion == "" { - workerVersion = "latest" - log.Printf("[INFO] SHUFFLE_WORKER_VERSION not defined. Defaulting to %#v", workerVersion) - } - - if baseimageregistry == "" { - //baseimageregistry = "ghcr.io" // Github - baseimageregistry = "docker.io" // Dockerhub - - if len(os.Getenv("REGISTRY_URL")) > 0 { - baseimageregistry = os.Getenv("REGISTRY_URL") - } else { - // os.Setenv("REGISTRY_URL", baseimageregistry) - } - - os.Setenv("SHUFFLE_BASE_IMAGE_REGISTRY", baseimageregistry) - - log.Printf("[INFO] Setting baseimageregistry to %#v", baseimageregistry) - } - - if baseimagename == "" { - // FIXME: This is probably the problem for image names tbh - //baseimagename = "shuffle" // Github (ghcr.io) - baseimagename = "frikky/shuffle" // Dockerhub - - os.Setenv("SHUFFLE_BASE_IMAGE_NAME", baseimagename) - log.Printf("[INFO] Setting baseimagename to %#v", baseimagename) - } - - // Old sane default overrides: - if baseimageregistry == "ghcr.io" && baseimagename == "shuffle" { - baseimageregistry = "docker.io" - baseimagename = "frikky/shuffle" - - os.Setenv("REGISTRY_URL", baseimageregistry) - os.Setenv("SHUFFLE_BASE_IMAGE_REGISTRY", baseimageregistry) - os.Setenv("SHUFFLE_BASE_IMAGE_NAME", baseimagename) - - log.Printf("[WARNING] Overriding bad defaults of ghcr.io/shuffle") - } - - log.Printf("[DEBUG] Setting swarm config to %#v. Default is empty.", swarmConfig) - - // This is now always static - newWorker := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) - if len(newWorkerImage) > 0 { - newWorker = newWorkerImage - } - - // Check whether they are the same first - if os.Getenv("SHUFFLE_AUTO_IMAGE_DOWNLOAD") == "false" { - log.Printf("[DEBUG] Skipping image download as SHUFFLE_AUTO_IMAGE_DOWNLOAD is set to false") - } else { - images := []string{ - fmt.Sprintf("frikky/shuffle:app_sdk"), - newWorker, - } - - pullOptions := image.PullOptions{} - for _, image := range images { - if isKubernetes == "true" { - log.Printf("[DEBUG] Skipping image pull of '%s' because Kubernetes does it in realtime instead", image) - } else { - log.Printf("[DEBUG] Pulling image %s", image) - reader, err := dockercli.ImagePull(ctx, image, pullOptions) - if err != nil { - log.Printf("[ERROR] Failed getting image %s: %s", image, err) - - continue - } - - io.Copy(os.Stdout, reader) - log.Printf("[DEBUG] Successfully downloaded and built %s", image) - } - } - } -} - -func findActiveSwarmNodes() (int64, error) { - ctx := context.Background() - nodes, err := dockercli.NodeList(ctx, types.NodeListOptions{}) - if err != nil { - return 1, err - } - - nodeCount := int64(0) - for _, node := range nodes { - //log.Printf("ID: %s - %#v", node.ID, node.Status.State) - if node.Status.State == "ready" { - nodeCount += 1 - } - } - - // Check for SHUFFLE_MAX_NODES - // Make it into a number and check if it's lower than nodeCount - maxNodesString := os.Getenv("SHUFFLE_MAX_SWARM_NODES") - if len(maxNodesString) > 0 { - maxNodes, err := strconv.ParseInt(maxNodesString, 10, 64) - if err != nil { - return nodeCount, err - } - - if nodeCount > maxNodes { - nodeCount = maxNodes - } - } - - return nodeCount, nil -} - -// Get IP -func getLocalIP() string { - addrs, err := net.InterfaceAddrs() - if err != nil { - return "" - } - - for _, address := range addrs { - // check the address type and if it is not a loopback the display it - if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { - if ipnet.IP.To4() != nil { - return ipnet.IP.String() - } - } - } - - return "" -} - -// Get all local IPs in the system -func getLocalIPs() ([]string, error) { - var ipv4s []string - var ipv6s []string - - ifaces, err := net.Interfaces() - if err != nil { - return nil, err - } - - for _, iface := range ifaces { - if iface.Flags&net.FlagUp == 0 { - continue - } - if iface.Flags&net.FlagLoopback != 0 { - continue - } - - addrs, err := iface.Addrs() - if err != nil { - continue - } - - for _, address := range addrs { - ipnet, ok := address.(*net.IPNet) - if !ok || ipnet.IP == nil { - continue - } - - ip := ipnet.IP - if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() { - continue - } - - if ip4 := ip.To4(); ip4 != nil { - ipv4s = append(ipv4s, ip4.String()) - continue - } - - if ip.To16() != nil { - ipv6s = append(ipv6s, ip.String()) - } - } - } - - return append(ipv4s, ipv6s...), nil -} - -func checkSwarmService(ctx context.Context) { - // https://docs.docker.com/engine/reference/commandline/swarm_init/ - ip := getLocalIP() - log.Printf("[DEBUG] Attempting swarm setup on %s", ip) - - info, err := dockercli.Info(ctx) - if err != nil { - log.Printf("[WARNING] Failed to get Docker Info: %s", err) - } - - if info.Swarm.ControlAvailable { - log.Printf("[INFO] Already part of swarm as a manager") - return - } - - listenAddr := "0.0.0.0" - req := swarm.InitRequest{ - ListenAddr: fmt.Sprintf("%s:2377", listenAddr), - AdvertiseAddr: fmt.Sprintf("%s:2377", ip), - } - - id, err := dockercli.SwarmInit(ctx, req) - if err != nil { - log.Printf("[ERROR] Swarm init issue: %s. Retrying with a failover IP address from interface.", err) - - // Dummy message used for testing - //err = errors.New("Error response from daemon: could not choose an IP address to advertise since this system has multiple addresses on different interfaces (10.52.208.221 on eno1 and 192.168.122.1 on virbr0) - specify one with --advertise-addr") - // Update 28 Jan 2026: The error message updated and not as clear - - candidates, err := getLocalIPs() - if len(candidates) > 0 && err == nil { - for cnt, candidate := range candidates { - if cnt > 5 { - break - } - - req.AdvertiseAddr = fmt.Sprintf("%s:2377", candidate) - id, err = dockercli.SwarmInit(context.Background(), req) - if err != nil { - continue - } - - log.Printf("[INFO] Swarm init ID: '%s'.", id) - return - } - } - - log.Printf("[ERROR] Swarm init failed after advertise-addr retries: %s, try running swarm init manually: docker swarm init", err) - return - } -} - -func getContainerResourceUsage(ctx context.Context, cli *dockerclient.Client, containerID string) (float64, float64, error) { - // Get container stats - stats, err := cli.ContainerStats(ctx, containerID, false) - if err != nil { - return 0, 0, err - } - - defer stats.Body.Close() - // Parse and return CPU and memory utilization - cpuUsage, memoryUsage, err := parseResourceUsage(stats.Body) - if err != nil { - return 0, 0, err - } - - return cpuUsage, memoryUsage, nil -} - -func parseResourceUsage(body io.Reader) (float64, float64, error) { - //var stats types.StatsJSON - var stats container.Stats - - // Decode the stream of stats as JSON - decoder := json.NewDecoder(body) - if err := decoder.Decode(&stats); err != nil { - return 0, 0, err - } - - //log.Printf("[DEBUG] CPU : %d", stats.CPUStats.CPUUsage.TotalUsage) - //log.Printf("[DEBUG] CPU2: %d", stats.PreCPUStats.CPUUsage.TotalUsage) - if stats.CPUStats.CPUUsage.TotalUsage == 0 || stats.PreCPUStats.CPUUsage.TotalUsage == 0 { - //log.Printf("[DEBUG] BODY: %#v", stats) - return 0, 0, nil - } - - // Calculate time difference between current and previous stats in nanoseconds - timeDelta := float64(stats.Read.Sub(stats.PreRead).Nanoseconds()) - - // Calculate CPU usage percentage - cpuDelta := float64(stats.CPUStats.CPUUsage.TotalUsage - stats.PreCPUStats.CPUUsage.TotalUsage) - cpuUsage := (cpuDelta / timeDelta) * 100.0 - - // Calculate memory usage percentage - memoryUsage := float64(stats.MemoryStats.Usage) / float64(stats.MemoryStats.Limit) * 100.0 - - return cpuUsage, memoryUsage, nil - -} - -func getHostname() (string, error) { - hostname, err := os.Hostname() - if err != nil { - return "", fmt.Errorf("failed to get hostname: %w", err) - } - - // Split away TLD - parts := strings.Split(hostname, ".") - if len(parts) > 0 { - hostname = parts[0] - } - - hostname = strings.ToUpper(hostname) - return hostname, nil -} - -func getOrborusStats(ctx context.Context, sensorMode shuffle.SensorMode) shuffle.OrborusStats { - newStats := shuffle.OrborusStats{ - OrgId: org, - Environment: environment, - OrborusLabel: orborusLabel, - Timestamp: time.Now().Unix(), - - Uuid: orborusUuid, - } - - if (swarmConfig == "run" || swarmConfig == "swarm") && strings.Contains(newWorkerImage, "scale") { - newStats.Swarm = true - } - - newStats.PollTime = sleepTime - newStats.MaxQueue = maxConcurrency - newStats.Queue = executionCount - - if isKubernetes == "true" || runningMode == "kubernetes" || runningMode == "k8s" { - newStats.Kubernetes = true - return newStats - } - - // Handles Orborus in sensor mode. Sends minimal data per request, but - // once in a while (30 minutes) sends a lot of details like software etc - if sensorMode.Enabled { - cacheKey := fmt.Sprintf("orborus_sensorDetails_cache") - cached, err := shuffle.GetCache(ctx, cacheKey) - if err == nil { - cacheData := []byte(cached.([]uint8)) - err := json.Unmarshal(cacheData, &newStats.SensorDetails) - if err == nil && len(newStats.SensorDetails.Hostname) > 0 { - newStats.SensorDetails.SensorMode = true - - // Not necessary to always send as it's big - // Backend optimises this anyway - if len(newStats.SensorDetails.Serial) > 100 { - newStats.SensorDetails.Serial = "" - } - - newStats.SensorDetails.InstalledSoftware = []shuffle.Software{} - return newStats - } - // If there's an error, we ignore the cache and continue to gather details - log.Printf("[WARNING] Failed to unmarshal cached sensor details: %s. Gathering new details.", err) - } - - newStats.SensorDetails.SensorMode = true - hostname, err := getHostname() - if err == nil { - newStats.SensorDetails.Hostname = hostname - } - - newStats.SensorDetails.OS = runtime.GOOS - newStats.SensorDetails.Arch = runtime.GOARCH - newStats.SensorDetails.ElevatedAccess = shuffle.IsElevated() - newStats.SensorDetails.Serial = shuffle.GetProfiler() - - if sensorMode.SoftwareListEnabled { - // Check cache first before running the command - newStats.SensorDetails.InstalledSoftware = shuffle.ListInstalledSoftware() - } - - if sensorMode.HdEncryptedCheck { - newStats.SensorDetails.HdEncrypted = fmt.Sprintf("%t", shuffle.IsDiskEncrypted()) - } - - if sensorMode.ScreenlockCheck { - newStats.SensorDetails.AutomaticScreenlockEnabled = fmt.Sprintf("%t", shuffle.IsAutomaticScreenlockEnabled()) - } - - if len(sensorMode.LogForwarding) > 0 { - newStats.SensorDetails.LogForwarding = fmt.Sprintf("not implemented: %s", sensorMode.LogForwarding) - } - - if len(sensorMode.ResponseActions) > 0 { - newStats.SensorDetails.ResponseActions = sensorMode.ResponseActions - } - - marshalledStats, err := json.Marshal(newStats.SensorDetails) - if err == nil { - shuffle.SetCache(ctx, cacheKey, marshalledStats, 30) // Cache for 10 minutes - } - - return newStats - } else { - return newStats - } - - // FIXME: Should we reach here anymore? Can it be useful? Primarily used for stats. - // Disable orborus stats - if os.Getenv("SHUFFLE_STATS_DISABLED") == "true" { - return newStats - } - - // Use the docker API to get the CPU usage of the docker engine machine - pers, err := dockercli.Info(ctx) - if err != nil { - log.Printf("[ERROR] Failed getting docker info: %s. This is normal IF there are many containers running.", err) - return newStats - } else { - newStats.TotalContainers = pers.Containers - newStats.StoppedContainers = pers.ContainersStopped - - // Calculate the amount of CPU utilization on the host - newStats.CPU = int(pers.NCPU) - newStats.MaxCPU = int(pers.NCPU) - newStats.Memory = int(pers.MemTotal) - newStats.MaxMemory = int(pers.MemTotal) - } - - // Get list of all running containers - containers, err := dockercli.ContainerList(ctx, container.ListOptions{}) - - if err != nil { - log.Printf("[ERROR] Failed getting container list: %s", err) - return newStats - } - - // Use a WaitGroup to wait for all goroutines to finish - var wg sync.WaitGroup - - // Channel to collect results - resultCh := make(chan struct { - containerID string - cpuUsage float64 - memoryUsage float64 - }) - - // Iterate through containers and start a goroutine for each container - for _, container := range containers { - // Check if container is running - if container.State != "running" { - continue - } - - wg.Add(1) - go func(container types.Container) { - defer wg.Done() - - // Get CPU and memory usage for the container - cpuUsage, memoryUsage, err := getContainerResourceUsage(ctx, dockercli, container.ID) - if err != nil { - //log.Printf("[DEBUG] Error getting resource usage for container %s: %v\n", container.ID, err) - } - - // Send the result to the channel - resultCh <- struct { - containerID string - cpuUsage float64 - memoryUsage float64 - }{container.ID, cpuUsage, memoryUsage} - }(container) - } - - // Close the result channel after all goroutines are done - go func() { - wg.Wait() - close(resultCh) - }() - - // Collect results from the channel - - // Iterate through containers and get CPU usage - totalCPU := float64(0.0) - memUsage := float64(0.0) - for result := range resultCh { - //log.Printf("[DEBUG] Container %s CPU utilization: %.2f%%, Memory utilization: %.2f%%\n", result.containerID, result.cpuUsage, result.memoryUsage) - - // check if it's NaN or Inf - if !math.IsNaN(result.cpuUsage) { - totalCPU += float64(result.cpuUsage) - } - - if !math.IsNaN(result.memoryUsage) { - memUsage += float64(result.memoryUsage) - } - } - - newStats.CPUPercent = totalCPU / float64(newStats.CPU) - newStats.MemoryPercent = memUsage - - //log.Printf("[DEBUG] CPU: %.2f, Memory: %.2f", newStats.CPUPercent, newStats.MemoryPercent) - - /* - cpuPercent, err := cpu.Percent(250*time.Millisecond, false) - if err == nil && len(cpuPercent) > 0 { - newStats.CPUPercent = cpuPercent[0] - } - //Percent(interval time.Duration, percpu bool) ([]float64, error) - - // Get memory usage - memory, err := memory.Get() - if err != nil { - log.Printf("[ERROR] Failed getting memory stats: %s", err) - } else { - newStats.Memory = int(memory.Used) - newStats.MaxMemory = int(memory.Total) - } - */ - - // Get disk usage - /* - disk, err := disk.Get() - if err != nil { - log.Printf("[ERROR] Failed getting disk stats: %s", err) - } else { - newStats.Disk = int(disk.Used) - newStats.MaxDisk = int(disk.Total) - } - */ - - /* - // General - Disk int `json:"disk"` - - // Docker - AppContainers int `json:"app_containers"` - WorkerContainers int `json:"worker_containers"` - TotalContainers int `json:"total_containers"` - } - */ - return newStats -} - -func sendRemoveRequest(client *http.Client, toBeRemoved shuffle.ExecutionRequestWrapper, baseUrl, environment, auth, org string, sleepTime int) error { - confirmUrl := fmt.Sprintf("%s/api/v1/workflows/queue/confirm", baseUrl) - data, err := json.Marshal(toBeRemoved) - if err != nil { - log.Printf("[WARNING] Failed removal marshalling: %s", err) - time.Sleep(time.Duration(sleepTime) * time.Second) - return err - } - - result, err := http.NewRequest( - "POST", - confirmUrl, - bytes.NewBuffer([]byte(data)), - ) - - if err != nil { - log.Printf("[ERROR] Failed building confirm request: %s", err) - time.Sleep(time.Duration(sleepTime) * time.Second) - return err - } - - result.Header.Add("Content-Type", "application/json") - result.Header.Add("Org-Id", environment) - - if len(auth) > 0 { - result.Header.Add("Authorization", auth) - } - - if len(org) > 0 { - result.Header.Add("Org", org) - } - - if len(orborusLabel) > 0 { - result.Header.Add("X-Orborus-Label", orborusLabel) - } - - resultResp, err := client.Do(result) - if err != nil { - if !strings.Contains(fmt.Sprintf("%s", err), "timeout") { - log.Printf("[ERROR] Failed making confirm request: %s", err) - } - - time.Sleep(time.Duration(sleepTime) * time.Second) - return err - } - - defer resultResp.Body.Close() - body, err := ioutil.ReadAll(resultResp.Body) - if err != nil { - log.Printf("[ERROR] Failed reading confirm body: %s", err) - time.Sleep(time.Duration(sleepTime) * time.Second) - return err - } - - _ = body - //log.Printf("[DEBUG] Confirm response: %s", string(body)) - - return nil -} - -func cleanup() { - log.Printf("[INFO] Cleaning up during shutdown") - ctx := context.Background() - cleanupExistingNodes(ctx) - zombiecheck(ctx, 600, shuffle.SensorMode{ - Enabled: os.Getenv("SHUFFLE_AGENT_MODE") == "true", - }) - os.Exit(0) -} - -func StartAgentSensor(sensorMode shuffle.SensorMode) error { - if sensorMode.Enabled == false { - return errors.New("Sensor mode is not enabled. Set SHUFFLE_AGENT_MODE to true to enable it.") - } - - log.Printf("[INFO] Starting Orborus - host monitoring mode (sensor/agent)") - - // Check if inside Docker/Kubernetes. Use Docker/Kubernetes libraries - if isKubernetes == "true" || shuffle.IsRunningInCluster() { - log.Printf("[INFO] Detected Kubernetes environment. Not valid for Sensor Mode. Exiting.") - return errors.New("Kubernetes environment detected. Sensor mode is not valid in Kubernetes. Exiting.") - } else if swarmConfig == "run" || swarmConfig == "swarm" { - log.Printf("[INFO] Detected Docker Swarm environment. Not valid for Sensor Mode. Exiting.") - return errors.New("Docker Swarm environment detected. Sensor mode is not valid in Docker Swarm. Exiting.") - } - - if len(sensorMode.LogForwarding) > 0 { - log.Printf("[INFO] Audit log monitoring is enabled (SHUFFLE_LOG_FORWARDING=)") - - // Initialize telemetry configuration - telemetryConfig := shuffle.TelemetryConfig{ - Enabled: true, - Modes: []string{"audit_log"}, - BufferSize: 1000, - FlushInterval: 10 * time.Second, - } - - if excludePatterns := os.Getenv("SHUFFLE_AUDIT_LOG_EXCLUDE"); excludePatterns != "" { - patterns := strings.Split(excludePatterns, ",") - telemetryConfig.Filters = append(telemetryConfig.Filters, shuffle.TelemetryFilter{ - Type: "message", - Exclude: patterns, - }) - } - - if includePatterns := os.Getenv("SHUFFLE_AUDIT_LOG_INCLUDE"); includePatterns != "" { - patterns := strings.Split(includePatterns, ",") - telemetryConfig.Filters = append(telemetryConfig.Filters, shuffle.TelemetryFilter{ - Type: "message", - Include: patterns, - }) - } - - collector, err := shuffle.NewAuditLogCollector(telemetryConfig) - if err != nil { - log.Printf("[ERROR] Failed to create audit log collector: %v", err) - } else { - ctx := context.Background() - if err := collector.LogCollectorStart(ctx); err != nil { - log.Printf("[ERROR] Failed to start audit log collector: %v", err) - } else { - log.Printf("[INFO] Audit log collector started successfully") - - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - - go func() { - <-sigChan - log.Printf("[INFO] Received shutdown signal, stopping audit log collector...") - collector.Stop() - - os.Exit(0) - }() - } - } - } else { - log.Printf("[INFO] Audit log monitoring is NOT enabled (SHUFFLE_LOG_FORWARDING=") - } - - return nil -} - -// Initial loop etc -func main() { - - // Checks for whether sensor mode is enabled for detection/response - sensorMode := shuffle.SensorMode{ - Enabled: os.Getenv("SHUFFLE_AGENT_SENSOR_MODE") == "true", - - SoftwareListEnabled: os.Getenv("SHUFFLE_SOFTWARE_LIST_ENABLED") == "true", - HdEncryptedCheck: os.Getenv("SHUFFLE_HD_ENCRYPTED_CHECK") == "true", - ScreenlockCheck: os.Getenv("SHUFFLE_SCREENLOCK_CHECK") == "true", - - LogForwarding: os.Getenv("SHUFFLE_LOG_FORWARDING"), - ResponseActions: os.Getenv("SHUFFLE_RESPONSE_ACTIONS"), - } - - ctx := context.Background() - workerTimeout := 600 - workerImage := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) - if len(newWorkerImage) > 0 { - workerImage = newWorkerImage - } - - if len(orborusUuid) == 0 { - orborusUuid = uuid.NewV4().String() - } - - client := shuffle.GetExternalClient(baseUrl) - fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl) - - // Increases default concurrency to 50 for swarm - if maxConcurrency < 50 && (swarmConfig == "run" || swarmConfig == "swarm") { - fullUrl += "?amount=50" - } - - if len(os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) > 0 { - log.Printf("[INFO] Trying to set Orborus sleep time between polls to %s", os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) - - tmpInt, err := strconv.Atoi(os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) - if err == nil { - sleepTime = tmpInt - } - } - - log.Println("[INFO] Setting up execution environment for env '%s'", environment) - if baseUrl == "" { - baseUrl = "https://uk.shuffler.io" - } - - //if orgId == "" { - // log.Printf("[ERROR] Org not defined. Set variable ORG_ID based on your org") - // os.Exit(3) - //} - if environment == "" { - log.Printf("[ERROR] Environment not defined. Set variable ENVIRONMENT_NAME to configure it.") - os.Exit(3) - } - - if timezone == "" { - timezone = "Europe/Amsterdam" - } - - log.Printf("[INFO] Using environment '%s' with timezone %s", environment, timezone) - - if sensorMode.Enabled { - - // Start high on purpose for now (no overloads) - if sleepTime < 15 { - - // For prod - if strings.Contains(baseUrl, "shuffler.io") || strings.Contains(baseUrl, ".run.app") { - log.Printf("[INFO] Running in hosted environment. Setting default sleep time to 30 seconds to avoid hitting rate limits. You can adjust this with SHUFFLE_ORBORUS_PULL_TIME.", sleepTime) - sleepTime = 15 - } - } - - if sensorMode.ResponseActions != "full" && sensorMode.ResponseActions != "controlled" { - log.Printf("[WARNING] Invalid response actions mode '%s'. Disabling. Valid options are 'full', 'controlled', or empty.", sensorMode.ResponseActions) - sensorMode.ResponseActions = "" - - } - - log.Printf("[INFO] Running in sensor/agent mode. Starting the agent.") - err := StartAgentSensor(sensorMode) - if err != nil { - log.Printf("[ERROR] Failed to start sensor/agent mode: %#v", err) - return - } - } else { - if os.Getenv("SHUFFLE_PIPELINE_STANDALONE") == "true" { - log.Printf("[INFO] Allowing use of standalone pipeline (tenzir). URL: %s", pipelineUrl) - - tenzirDisabled = false - os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - os.Setenv("SHUFFLE_PIPELINE_ENABLED", "true") - } - - // Block until a signal is received - if shuffle.IsRunningInCluster() { - log.Printf("[INFO] Running inside k8s cluster") - } - - if isKubernetes == "true" { - fixk8sRoles() - } - - startupDelay := os.Getenv("SHUFFLE_ORBORUS_STARTUP_DELAY") - if len(startupDelay) > 0 { - log.Printf("[DEBUG] Setting startup delay to %#v", startupDelay) - - tmpInt, err := strconv.Atoi(startupDelay) - if err == nil { - time.Sleep(time.Duration(tmpInt) * time.Second) - } else { - log.Printf("[WARNING] Env SHUFFLE_ORBORUS_STARTUP_DELAY must be a number, not '%s'. Using default.", startupDelay) - } - } - - // Auto enables pipelines IF they are not mentioned - if len(os.Getenv("SHUFFLE_SKIP_PIPELINES")) == 0 { - tenzirDisabled = false - os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - os.Setenv("SHUFFLE_PIPELINE_ENABLED", "true") - } - - if os.Getenv("SHUFFLE_SKIP_PIPELINES") != "true" && os.Getenv("SHUFFLE_PIPELINE_ENABLED") != "false" { - // Run in 15 seconds in a goroutine - go func() { - time.Sleep(15 * time.Second) - log.Printf("[INFO] Auto-downloading Sigma rules during startup") - ruleType := "sigma" - err := handleFileCategoryChange(ruleType) - if err != nil { - log.Printf("[WARNING] Failed downloading %s rules: %s", ruleType, err) - } - }() - } - - // Handle Cleanup - made it cleanup by default - if strings.ToLower(os.Getenv("SHUFFLE_CONTAINER_AUTO_CLEANUP")) != "false" && os.Getenv("CLEANUP") == "" { - cleanupEnv = "true" - } - - if len(cleanupEnv) > 0 { - log.Printf("[DEBUG] Verbose mode. NOT cleaning up. Cleanup env: %s", cleanupEnv) - } - - // Default to 120 instead of default 30 - if len(os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")) == 0 { - os.Setenv("SHUFFLE_APP_SDK_TIMEOUT", "120") - } - - if workerTimeoutEnv != "" { - tmpInt, err := strconv.Atoi(workerTimeoutEnv) - if err == nil { - workerTimeout = tmpInt - } else { - log.Printf("[WARNING] Env SHUFFLE_ORBORUS_EXECUTION_TIMEOUT must be a number, not %s", workerTimeoutEnv) - } - - log.Printf("[INFO] Cleanup process running every %d seconds", workerTimeout) - } - - if concurrencyEnv != "" { - //var concurrencyEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY") - tmpInt, err := strconv.Atoi(concurrencyEnv) - if err == nil { - maxConcurrency = tmpInt - log.Printf("[INFO] Max workflow execution concurrency set to %d", maxConcurrency) - } else { - log.Printf("[WARNING] Env SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY must be a number, not %s. Defaulted to %d", workerTimeoutEnv, maxConcurrency) - } - } - - if len(os.Getenv("DOCKER_HOST")) > 0 { - log.Printf("[DEBUG] Running docker with socket proxy %s instead of default", os.Getenv("DOCKER_HOST")) - - } else { - log.Printf(`[DEBUG] Running docker with default socket /var/run/docker.sock or `) - } - - // Run by default from now - //commenting for now as its stoppoing minikube - - log.Printf("[INFO] Running towards %s (BASE_URL) with environment name %s", baseUrl, environment) - - if environment == "" { - environment = "onprem" - log.Printf("[WARNING] Defaulting to environment name %s. Set environment variable ENVIRONMENT_NAME to change. This should be the same as in the frontend action.", environment) - } - - if pipelineUrl == "" { - pipelineUrl = "http://localhost:5160" - - // Find the IP in baseUrl. Base format is http://: - if baseUrl != "" && !strings.Contains(baseUrl, "shuffle") && !strings.Contains(baseUrl, "localhost") && !strings.Contains(baseUrl, "run.app") { - urlSplit := strings.Split(baseUrl, "://") - if len(urlSplit) > 1 { - // Find the IP - ipSplit := strings.Split(urlSplit[1], ":") - if len(ipSplit) > 0 { - pipelineUrl = fmt.Sprintf("http://%s:5160", ipSplit[0]) - } - } - } - - if len(containerId) > 0 { - pipelineUrl = "http://tenzir-node:5160" - } - - log.Printf("[WARNING] SHUFFLE_PIPELINE_URL not set, falling back to default URL: %s. If BASE_URL is set, we use the external IP for that", pipelineUrl) - os.Setenv("SHUFFLE_PIPELINE_URL", pipelineUrl) - } - - // FIXME - during init, BUILD and/or LOAD worker and app_sdk - // Build/load app_sdk so it can be loaded as 127.0.0.1:5000/walkoff_app_sdk - log.Printf("[INFO] Setting up Docker environment. Downloading worker and App SDK!") - - initializeImages() - - if swarmConfig == "run" || swarmConfig == "swarm" || isKubernetes == "true" { - - if isKubernetes != "true" { - checkSwarmService(ctx) - } - - log.Printf("[DEBUG] Cleaning up containers from previous run") - cleanupExistingNodes(ctx) - time.Sleep(time.Duration(5) * time.Second) - - log.Printf("[DEBUG] Deploying worker image %s to swarm", workerImage) - - runString := "Run: \"docker service ls\" for more info" - - if isKubernetes != "true" { - deployServiceWorkers(workerImage) - - err := setBackendToSwarmNetwork(ctx) - if err != nil { - log.Printf("[WARNING] Failed setting backend to swarm network: %s", err) - } - - } else { - deployK8sWorker(workerImage, "shuffle-workers", []string{}) - runString = "Run: \"kubectl get pods\" for more info" - } - - log.Printf("[DEBUG] Waiting 45 seconds to ensure workers are deployed. %s", runString) - time.Sleep(time.Duration(45) * time.Second) - - //deployServiceWorkers(workerImage) - } - - zombiecheck(ctx, workerTimeout, sensorMode) - - if isKubernetes == "true" { - log.Printf("[INFO] Finished configuring kubernetes environment. Connecting to %s", fullUrl) - } else { - log.Printf("[INFO] Finished configuring docker environment. Connecting to %s", fullUrl) - } - } - - forwardData := bytes.NewBuffer([]byte{}) - forwardMethod := "POST" - - req, err := http.NewRequest( - forwardMethod, - fullUrl, - forwardData, - ) - - if err != nil { - log.Printf("[ERROR] Failed making request builder during init: %s", err) - return - } - - zombiecounter := 0 - - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Org-Id", environment) - if len(auth) > 0 { - req.Header.Add("Authorization", auth) - } - - if len(org) > 0 { - req.Header.Add("Org", org) - } - - if len(orborusLabel) > 0 { - log.Printf("[DEBUG] Sending with Label '%s'", orborusLabel) - req.Header.Add("X-Orborus-Label", orborusLabel) - } - - if swarmConfig != "run" && swarmConfig != "swarm" { - req.Header.Add("X-Orborus-Runmode", "Default") - } else { - req.Header.Add("X-Orborus-Runmode", "Docker Swarm") - } - - if os.Getenv("SHUFFLE_MAX_CPU") != "" { - // parse - tmpInt, err := strconv.Atoi(os.Getenv("SHUFFLE_MAX_CPU")) - if err == nil { - maxCPUPercent = tmpInt - } - } - - swarmPollingTime := time.Now() - swarmRequestsMade := 0 - swarmControlMode := false - if os.Getenv("SHUFFLE_SWARM_CONTROL_MODE") == "true" { - swarmControlMode = true - } - - log.Printf("[INFO] Waiting for executions at %s with Environment %#v. Sensormode: %#v", fullUrl, environment, sensorMode.Enabled) - - hostname, err := getHostname() - hasStarted := false - for { - if req.Method == "POST" && !sensorMode.Enabled { - // Should find data to send (memory etc.) - - // Create timeout of max a few seconds just in case - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - // Marshal and set body - orborusStats := getOrborusStats(ctx, sensorMode) - - pipelinePayload, pipelineerr := sendPipelineHealthStatus(sensorMode) - - if pipelineerr != nil { - // Too verbose to be enabled. - //log.Printf("[ERROR] Failed sending pipeline health status: %s", pipelineerr) - } - - orborusStats.DataLake = pipelinePayload - jsonData, err := json.Marshal(orborusStats) - if err == nil { - req.Body = ioutil.NopCloser(bytes.NewBuffer(jsonData)) - } else { - log.Printf("[ERROR] Failed marshalling. Maybe max 4 second timeout? %s", err) - } - - if int(orborusStats.CPUPercent) > maxCPUPercent { - log.Printf("[DEBUG] CPU usage is at %f%%. This is more than the max limit the machine should be running at (%d). Waiting before continue.", orborusStats.CPUPercent, maxCPUPercent) - time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - } else if sensorMode.Enabled { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - orborusStats := getOrborusStats(ctx, sensorMode) - - jsonData, err := json.Marshal(orborusStats) - if err == nil { - req.Body = ioutil.NopCloser(bytes.NewBuffer(jsonData)) - } else { - log.Printf("[ERROR] Failed marshalling. Maybe max 4 second timeout? %s", err) - } - } - - newresp, err := client.Do(req) - if err != nil { - log.Printf("[WARNING] Failed making request to %s: %s", fullUrl, err) - - zombiecounter += 1 - if zombiecounter*sleepTime > workerTimeout { - go zombiecheck(ctx, workerTimeout, sensorMode) - zombiecounter = 0 - } - time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - - //defer newresp.Body.Close() - if newresp.StatusCode == 405 { - log.Printf("[WARNING] Received 405 from %s. This is likely due to a misconfigured base URL. Automatically swapping to GET request (backwards compatibility)", fullUrl) - - req.Method = "GET" - req.Body = nil - - //time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - - body, err := ioutil.ReadAll(newresp.Body) - if err != nil { - log.Printf("[ERROR] Failed reading body from Shuffle: %s", err) - zombiecounter += 1 - if zombiecounter*sleepTime > workerTimeout { - go zombiecheck(ctx, workerTimeout, sensorMode) - zombiecounter = 0 - } - time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - - // Controls Leader/Follower mode - if newresp.StatusCode == 409 { - log.Printf("[INFO] Another Orborus is already handling jobs. Polling every 30 seconds in case Leader stops. Resp: %s", string(body)) - time.Sleep(time.Duration(30) * time.Second) - continue - } else if newresp.StatusCode != 200 { - log.Printf("[ERROR] Backend connection failed for url '%s', or is missing (%d): %s", fullUrl, newresp.StatusCode, string(body)) - } else { - if !hasStarted { - log.Printf("[DEBUG] Starting iteration on environment %#v (default: Shuffle). Got statuscode %d from backend on first request", environment, newresp.StatusCode) - } - - if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" && os.Getenv("SHUFFLE_SCALE_REPLICAS") == "" { - //go AutoScale(ctx) - } - hasStarted = true - } - - var executionRequests shuffle.ExecutionRequestWrapper - err = json.Unmarshal(body, &executionRequests) - if err != nil { - log.Printf("[WARNING] Failed executionrequest in queue unmarshaling: %s", err) - if !sensorMode.Enabled { - sleepTime = 10 - } - - zombiecounter += 1 - if zombiecounter*sleepTime > workerTimeout { - go zombiecheck(ctx, workerTimeout, sensorMode) - zombiecounter = 0 - } - time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - - if hasStarted && len(executionRequests.Data) > 0 { - //log.Printf("[INFO] Body: %s", string(body)) - // Type string `json:"type"` - } - - // do things on behalf of backend - var toBeRemoved shuffle.ExecutionRequestWrapper - if len(executionRequests.Data) > 0 { - newrequests := []shuffle.ExecutionRequest{} - - // Deduplicating in case same job shows up multiple times - // This is specifically to handle data pipelines better - deduplicatedJobs := []shuffle.ExecutionRequest{} - for _, incRequest := range executionRequests.Data { - if !strings.Contains(incRequest.Type, "DOCKER") && !strings.Contains(incRequest.Type, "PIPELINE") && !strings.Contains(incRequest.Type, "SIGMA") && !strings.Contains(incRequest.Type, "TENZIR") { - deduplicatedJobs = append(deduplicatedJobs, incRequest) - continue - } - - found := false - for _, dedupRequest := range deduplicatedJobs { - if incRequest.ExecutionArgument == dedupRequest.ExecutionArgument && incRequest.Type == dedupRequest.Type { - found = true - break - } - } - - if found { - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - continue - } - - deduplicatedJobs = append(deduplicatedJobs, incRequest) - } - - executionRequests.Data = deduplicatedJobs - for _, incRequest := range executionRequests.Data { - - // Handles sensormode. ELSE handles Docker/K8s etc. - if sensorMode.Enabled { - - if len(incRequest.ExecutionSource) > 0 || len(incRequest.ExecutionArgument) == 0 { - parsedHostname := incRequest.ExecutionSource - if strings.Contains(parsedHostname, ".") { - parsedHostnameSplit := strings.Split(parsedHostname, ".") - parsedHostname = strings.ToUpper(parsedHostnameSplit[0]) - } - - if parsedHostname == hostname { - if debug { - log.Printf("[DEBUG] CORRECT HOSTNAME: %#v matches sensor hostname %#v. Removing from queue without processing.", parsedHostname, hostname) - } - - if sensorMode.ResponseActions != "" { - go shuffle.HandleSensorResponseAction(sensorMode, incRequest) - } - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - } else { - // Just ignore as other machines will handle it. - //log.Printf("[WARNING] Hostname '%s' from job does not match sensor hostname '%s'. Removing from queue without processing. Job: %#v", parsedHostname, hostname, incRequest) - } - } else { - // Invalid command - if debug { - log.Printf("[DEBUG] Removing invalid sensor command from queue: %#v", incRequest) - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - } - - } else { - // Looking for specific jobs - if incRequest.Type == "PIPELINE_CREATE" || incRequest.Type == "PIPELINE_START" || incRequest.Type == "PIPELINE_STOP" || incRequest.Type == "PIPELINE_DELETE" || incRequest.Type == "PIPELINE_UPDATE" { - log.Printf("[INFO] Handling pipeline request from backend: '%s' with argument '%s'", incRequest.Type, incRequest.ExecutionArgument) - - os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - os.Setenv("SHUFFLE_PIPELINE_ENABLED", "true") - tenzirDisabled = false - - // Running NEW or editing pipelines - err := handlePipeline(incRequest) - if err != nil { - log.Printf("[ERROR] Failed handling pipeline ('%s' '%s'): %s. Deleting job anyway.", incRequest.Type, incRequest.ExecutionSource, err) - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - } else if incRequest.Type == "DOCKER_IMAGE_DOWNLOAD" { - log.Printf("[INFO] Re-downloading new image(s) due to backend request: %#v", incRequest.ExecutionArgument) - - if len(incRequest.ExecutionArgument) > 0 { - go handleBackendImageDownload(ctx, incRequest.ExecutionArgument) - } else { - log.Printf("[ERROR] No image name provided for download. Removing job from queue.") - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - - } else if incRequest.Type == "CATEGORY_UPDATE" { - os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - - tenzirDisabled = false - err = handleFileCategoryChange("sigma") - if err != nil { - log.Printf("[ERROR] Failed to download the file category: %s", err) - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - - } else if incRequest.Type == "DISABLE_SIGMA_FOLDER" { - log.Printf("[INFO] Got job to disable sigma rules") - - err = removeFileCategory("sigma") - if err != nil { - log.Printf("[ERROR] Failed to disable the sigma rules: %s", err) - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - - } else if incRequest.Type == "DISABLE_SIGMA_FILE" { - fileName := incRequest.ExecutionArgument - log.Printf("[INFO] Got job to disable sigma file %s", fileName) - - err = disableRule(fileName) - if err != nil { - log.Printf("[ERROR] Failed to disable the sigma file %s, reason: %s", fileName, err) - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - - } else if incRequest.Type == "ENABLE_SIGMA_FILE" { - fileName := incRequest.ExecutionArgument - log.Printf("[INFO] Got job to enable sigma file %s", fileName) - - err = enableRule(fileName) - if err != nil { - log.Printf("[ERROR] Failed to disable the sigma file %s, reason: %s", fileName, err) - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - } else if incRequest.Type == "START_TENZIR" { - log.Printf("[INFO] Got job to start tenzir") - - // Manual command = overrides to allow starting of Tenzir from the frontend anyway. - //os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - tenzirDisabled = false - - // Removed either way - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - - err := deployTenzirNode() - if err != nil { - if strings.Contains(fmt.Sprintf("%s", err), "node available") { - // Disabling until UI is updated - //os.Setenv("SHUFFLE_SKIP_PIPELINES", "true") - //tenzirDisabled = true - - log.Printf("[ERROR] Failed to start tenzir, reason: %s", err) - err = shuffle.CreateOrgNotification( - ctx, - fmt.Sprintf("Failed to start Tenzir: %s", err), - fmt.Sprintf("Tenzir failed to start due to: %s", err), - fmt.Sprintf("/detections/Sigma"), - org, - true, - "LOW", - "TENZIR_START", - ) - - if err != nil { - log.Printf("[ERROR] Failed to send notification: %s", err) - return - } - } - } - - } else { - if debug { - log.Printf("[DEBUG] Passing execution ID request to normal queue: %#v", incRequest.ExecutionId) - } - - newrequests = append(newrequests, incRequest) - } - } - } - - if len(toBeRemoved.Data) > 0 { - err = sendRemoveRequest(client, toBeRemoved, baseUrl, environment, auth, org, sleepTime) - if err != nil { - log.Printf("[ERROR] Failed sending remove request: %s", err) - } else { - toBeRemoved.Data = []shuffle.ExecutionRequest{} - } - } - - // Remove the download image request - executionRequests.Data = newrequests - } - - // Skipping throttling with swarm - if sensorMode.Enabled { - // Pass - } else if swarmConfig != "run" && swarmConfig != "swarm" { - if len(executionRequests.Data) == 0 { - zombiecounter += 1 - if zombiecounter*sleepTime > workerTimeout { - go zombiecheck(ctx, workerTimeout, sensorMode) - zombiecounter = 0 - } - time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - - // Anything below here verifies concurrency - executionCount = getRunningWorkers(ctx, workerTimeout) - if executionCount >= maxConcurrency { - if zombiecounter*sleepTime > workerTimeout { - go zombiecheck(ctx, workerTimeout, sensorMode) - zombiecounter = 0 - } - time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - - allowed := maxConcurrency - executionCount - if len(executionRequests.Data) > allowed { - log.Printf("[WARNING] Throttle - Cutting down requests from %d to %d (MAX: %d, CUR: %d)", len(executionRequests.Data), allowed, maxConcurrency, executionCount) - executionRequests.Data = executionRequests.Data[0:allowed] - } - } else if swarmControlMode && (swarmConfig == "run" || swarmConfig == "swarm") { - // any reason it is not maxConcurrency instead of - // hardcoded 50? - if len(executionRequests.Data) > 50 { - executionRequests.Data = executionRequests.Data[0:50] - } - - if swarmRequestsMade > 100 && time.Since(swarmPollingTime).Seconds() > 5 { - log.Printf("[DEBUG] Swarm requests made: %d", swarmRequestsMade) - time.Sleep(time.Duration(sleepTime) * time.Second) - - swarmPollingTime = time.Now() - swarmRequestsMade = 0 - } - - swarmRequestsMade += len(executionRequests.Data) - } - - // New, abortable version. Should check executionid and remove everything else - for _, execution := range executionRequests.Data { - if len(execution.ExecutionArgument) > 0 { - log.Printf("[INFO] Argument: %s", execution.ExecutionArgument) - } - - if execution.Type == "schedule" { - log.Printf("[INFO] Schedule type! Weird deployment. Type: %s", execution.Type) - continue - } - - if len(execution.ExecutionId) == 0 { - log.Printf("[WARNING] Execution ID is empty: %#v", execution) - continue - } - - if execution.Status == "ABORT" || execution.Status == "FAILED" { - log.Printf("[INFO][%s] Executionstatus issue: ", execution.ExecutionId, execution.Status) - } - - if shuffle.ArrayContains(executionIds, execution.ExecutionId) { - log.Printf("[INFO][%s] Execution already handled (rerunning old execution)", execution.ExecutionId) - toBeRemoved.Data = append(toBeRemoved.Data, execution) - - // Should check when last this was ran, and if it's more than 10 minutes ago and it's not finished, we should run it again? - /* - if swarmConfig != "run" && swarmConfig != "swarm" { - continue - } - */ - } - - // Now, how do I execute this one? - containerName := fmt.Sprintf("worker-%s", execution.ExecutionId) - env := []string{ - fmt.Sprintf("AUTHORIZATION=%s", execution.Authorization), - fmt.Sprintf("EXECUTIONID=%s", execution.ExecutionId), - fmt.Sprintf("ENVIRONMENT_NAME=%s", environment), - fmt.Sprintf("BASE_URL=%s", baseUrl), - fmt.Sprintf("CLEANUP=%s", cleanupEnv), - fmt.Sprintf("TZ=%s", timezone), - fmt.Sprintf("SHUFFLE_PASS_APP_PROXY=%s", os.Getenv("SHUFFLE_PASS_APP_PROXY")), - fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", os.Getenv("SHUFFLE_SWARM_CONFIG")), - fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED")), - fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", os.Getenv("SHUFFLE_BASE_IMAGE_NAME")), - fmt.Sprintf("SHUFFLE_ALLOW_PACKAGE_INSTAL=%s", os.Getenv("SHUFFLE_ALLOW_PACKAGE_INSTALL")), - } - - //log.Printf("Running worker with proxy? %s", os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) - if strings.ToLower(os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) == "true" { - env = append(env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) - env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) - env = append(env, fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))) - } - - if dockerApiVersion != "" { - env = append(env, fmt.Sprintf("DOCKER_API_VERSION=%s", dockerApiVersion)) - } - - if len(os.Getenv("DOCKER_HOST")) > 0 { - env = append(env, fmt.Sprintf("DOCKER_HOST=%s", os.Getenv("DOCKER_HOST"))) - } - - if len(os.Getenv("SHUFFLE_MEMCACHED")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_MEMCACHED=%s", os.Getenv("SHUFFLE_MEMCACHED"))) - } - - if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_CLOUDRUN_URL=%s", os.Getenv("SHUFFLE_CLOUDRUN_URL"))) - } - - if len(os.Getenv("SHUFFLE_SKIPSSL_VERIFY")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_SKIPSSL_VERIFY=%s", os.Getenv("SHUFFLE_SKIPSSL_VERIFY"))) - } - - if len(os.Getenv("SHUFFLE_DEBUG_MEMORY")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_DEBUG_MEMORY=%s", os.Getenv("SHUFFLE_DEBUG_MEMORY"))) - } - - // Look for volume binds - if len(os.Getenv("SHUFFLE_VOLUME_BINDS")) > 0 { - //log.Printf("[DEBUG] Added volume binds: %s", os.Getenv("SHUFFLE_VOLUME_BINDS")) - env = append(env, fmt.Sprintf("SHUFFLE_VOLUME_BINDS=%s", os.Getenv("SHUFFLE_VOLUME_BINDS"))) - } - - if len(os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_APP_SDK_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_SDK_TIMEOUT"))) - } - - // Setting up internal proxy config for Shuffle -> shuffle comms - overrideHttpProxy := os.Getenv("SHUFFLE_INTERNAL_HTTP_PROXY") - overrideHttpsProxy := os.Getenv("SHUFFLE_INTERNAL_HTTPS_PROXY") - if len(overrideHttpProxy) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_INTERNAL_HTTP_PROXY=%s", overrideHttpProxy)) - } - - if len(overrideHttpsProxy) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_INTERNAL_HTTPS_PROXY=%s", overrideHttpsProxy)) - } - - if len(os.Getenv("SHUFFLE_MAX_SWARM_NODES")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_MAX_SWARM_NODES=%s", os.Getenv("SHUFFLE_MAX_SWARM_NODES"))) - } - - err = deployWorker(workerImage, containerName, env, execution) - zombiecounter += 1 - if err == nil { - //log.Printf("[DEBUG] ExecutionID %s was deployed and to be removed from queue.", execution.ExecutionId) - toBeRemoved.Data = append(toBeRemoved.Data, execution) - executionIds = append(executionIds, execution.ExecutionId) - } else { - log.Printf("[WARNING][%s] Failed to deploy: %s", execution.ExecutionId, err) - - if strings.Contains(err.Error(), "already exists") { - toBeRemoved.Data = append(toBeRemoved.Data, execution) - executionIds = append(executionIds, execution.ExecutionId) - } else if strings.Contains(err.Error(), "No such image") { - // Download the image - - if isKubernetes == "true" { - log.Printf("[DEBUG] Skipping image pull of '%s' because Kubernetes does it in realtime instead", workerImage) - } else { - log.Printf("[DEBUG] Re-pulling image %s as it doesn't exist, and is necessary for worker to run (autofix)", workerImage) - pullOptions := image.PullOptions{} - _, err = dockercli.ImagePull(ctx, workerImage, pullOptions) - if err != nil { - log.Printf("[ERROR] Failed to pull image %s: %s", workerImage, err) - } - } - } - } - } - - // Removes handled workflows (worker is made) - //log.Printf("\n\n[INFO] Removing %d executions from queue\n\n", len(toBeRemoved.Data)) - if len(toBeRemoved.Data) > 0 { - - err = sendRemoveRequest(client, toBeRemoved, baseUrl, environment, auth, org, sleepTime) - if err != nil { - log.Printf("[ERROR] Failed to remove executions from queue: %s", err) - } - - } - time.Sleep(time.Duration(sleepTime) * time.Second) - } -} - -// Tenzir command samples -// docker pull ghcr.io/dominiklohmann/tenzir-arm64:latest -// docker tag ghcr.io/dominiklohmann/tenzir-arm64:latest tenzir/tenzir:latest - -// Read from Cache and send it to a webhook -// docker run tenzir/tenzir:latest 'from http://192.168.86.44:5002/api/v1/orgs/7e9b9007-5df2-4b47-bca5-c4d267ef2943/cache/CIDR%20ranges?type=text&authorization=cec9d01f-09b2-4419-8a0a-76c6046e3fef read lines | to http://192.168.86.44:5002/api/v1/hooks/webhook_665ace5f-f27b-496a-a365-6e07eb61078c write lines' -func handlePipeline(incRequest shuffle.ExecutionRequest) error { - - log.Printf("[INFO] Pipeline: '%s' with source '%s'", incRequest.Type, incRequest.ExecutionSource) - - err := deployTenzirNode() - if err != nil { - log.Printf("[ERROR] Failed to deploy the pipeline, reason: %s", err) - return err - } - - // no need of execution arguments for STOP and DELETE - if (incRequest.Type != "PIPELINE_STOP" && incRequest.Type != "PIPELINE_DELETE") && len(incRequest.ExecutionArgument) == 0 { - log.Printf("[ERROR] No execution argument found for pipeline type %s. Skipping", incRequest.Type) - - return errors.New("no execution argument found for pipeline create. Skipping") - } - - identifier := strings.ToLower(strings.ReplaceAll(incRequest.ExecutionSource, " ", "-")) - if !strings.HasPrefix(strings.ToLower(incRequest.ExecutionSource), "shuffle") { - identifier = fmt.Sprintf("shuffle-%s", strings.ToLower(strings.ReplaceAll(incRequest.ExecutionSource, " ", "-"))) - } - - command := incRequest.ExecutionArgument - pipelines = []shuffle.PipelineInfo{} - if incRequest.Type == "PIPELINE_CREATE" { - log.Printf("[INFO] Should delete -> recreate new pipeline with id %#v", identifier) - //err := deployPipeline(image, identifier, command) - _, err := createPipeline(command, identifier) - if err != nil { - log.Printf("[ERROR] Failed to create pipeline: %s", err) - return err - } - } else if incRequest.Type == "PIPELINE_DELETE" || incRequest.Type == "PIPELINE_STOP" { - pipelineId := incRequest.ExecutionId - log.Printf("[INFO] Should delete pipeline %#v. PipelineID: %s", identifier, pipelineId) - //pipelineId, err := searchPipeline(identifier) - //if err != nil { - //} - - err = deletePipeline(pipelineId) - if err != nil { - log.Printf("[ERROR] Failed Deleting Pipeline %s", err) - return err - } - - /* - } else if incRequest.Type == "PIPELINE_STOP" { - log.Printf("[INFO] Should stop the pipeline %#v", identifier) - pipelineId, err := searchPipeline(identifier) - if err != nil { - log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - return err - } - _, err = updatePipelineState(command, pipelineId, "stop") - if err != nil { - log.Printf("[ERROR] Failed to stop Pipeline: %s reason:%s ", pipelineId, err) - return err - } else { - log.Printf("[INFO] Successfully stopped the Pipeline: %s", pipelineId) - } - */ - - } else if incRequest.Type == "PIPELINE_START" { - log.Printf("[INFO] Should start the pipeline %#v", identifier) - pipelineId, err := searchPipeline(identifier) - if err != nil { - if err.Error() == "no existing pipeline found with name" { - log.Printf("[INFO] Starting a new pipeline with command '%s' and identifier '%s'", command, identifier) - var createErr error - pipelineId, createErr = createPipeline(command, identifier) - if createErr != nil { - return createErr - } - } else { - log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) - return err - } - } - - log.Printf("[INFO] Starting existing pipeline with ID %s", pipelineId) - _, err = updatePipelineState(command, pipelineId, "start") - if err != nil { - log.Printf("[ERROR] Failed to start Pipeline: %s reason:%s ", pipelineId, err) - return err - } else { - log.Printf("[INFO] Successfully started pipeline: %s", pipelineId) - } - - } else { - log.Printf("[ERROR] Unknown type for pipeline: %s", incRequest.Type) - return errors.New("unknown type for pipeline") - } - - return nil -} - -func deployTenzirNode() error { - // Specifically for standalone tenzir - if os.Getenv("SHUFFLE_PIPELINE_STANDALONE") == "true" { - return nil - } - - // Disabled all pipeline features - if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "true" { - return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES (1)") - } - - if isKubernetes == "true" { - return errors.New("Tenzir not implemented for k8s") - } - - err := checkTenzirNode() - if err == nil { - return nil - } - - ctx := context.Background() - cacheKey := "tenzir-key" - _, err = shuffle.GetCache(ctx, cacheKey) - if err == nil { - return nil - } - - //imageName := "frikky/shuffle:tenzir" - imageName := "tenzir/tenzir:main" - if os.Getenv("TENZIR_IMAGE_NAME") != "" { - imageName = os.Getenv("TENZIR_IMAGE_NAME") - log.Printf("[INFO] Using custom Tenzir image name: %s", imageName) - } - - containerName := "tenzir-node" - containerStartOptions := container.StartOptions{} - - containerInfo, err := dockercli.ContainerInspect(ctx, containerName) - if err != nil { - if dockerclient.IsErrNotFound(err) { - // Create network if it doesn't exist - networkName := "tenzir-network" - networkSubnet := "192.168.102.0/24" - networkGateway := "192.168.102.1" - - err = createNetworkIfNotExists(ctx, networkName, networkSubnet, networkGateway) - if err != nil { - log.Printf("[ERROR] Failed to create network %s: %s", networkName, err) - //return err - } - - // Trying to connect orborus to the tenzir network as well - err = dockercli.NetworkConnect(ctx, networkName, containerId, nil) - if err != nil { - log.Printf("[ERROR] Error connecting tenzir container to network: %s", err) - } - - // Check if image exists - _, _, err := dockercli.ImageInspectWithRaw(ctx, imageName) - if dockerclient.IsErrNotFound(err) { - log.Printf("[DEBUG] Pulling image %s. This may take a while.", imageName) - pullOptions := image.PullOptions{} - out, err := dockercli.ImagePull(ctx, imageName, pullOptions) - if err != nil { - log.Printf("[ERROR] Failed to pull the Tenzir image: %s", err) - return err - } - defer out.Close() - - io.Copy(io.Discard, out) - } else if err != nil { - return err - } - - err = createAndStartTenzirNode(ctx, containerName, imageName, containerStartOptions) - if err != nil { - return err - } - } else { - return err - } - } else { - if !containerInfo.State.Running { - log.Printf("[DEBUG] Tenzir Node exists, but is not running. Restarting it.") - err := dockercli.ContainerStart(ctx, containerName, containerStartOptions) - if err != nil { - log.Printf("[ERROR] Failed to start Tenzir Node container: %v", err) - return err - } - - time.Sleep(10 * time.Second) - log.Printf("[INFO] Waiting for Tenzir to become available ...") - err = checkTenzirNode() - if err != nil { - return err - } - } - } - - tenzirStatus := struct { - ContainerStatus string `json:"container_status"` - }{ - ContainerStatus: "running", - } - - cacheData, err := json.Marshal(tenzirStatus) - if err != nil { - log.Printf("[WARNING] Failed marshalling execution: %s", err) - } - - err = shuffle.SetCache(ctx, cacheKey, cacheData, 1) - if err != nil { - log.Printf("[WARNING] Failed updating cache for tenzir: %s", err) - } - - return nil -} - -func createAndStartTenzirNode(ctx context.Context, containerName, imageName string, containerStartOptions container.StartOptions) error { - healthconfig := &container.HealthConfig{ - Test: []string{"tenzir --connection-timeout=30s --connection-retry-delay=1s 'api /ping'"}, - Interval: 30 * time.Second, - Retries: 1, - } - - // Ensure restart policy is there - config := &container.Config{ - Hostname: containerName, - Cmd: []string{"--commands=web server --mode=dev --bind=0.0.0.0"}, - Image: imageName, - Healthcheck: healthconfig, - ExposedPorts: nat.PortSet{ - "5160/tcp": struct{}{}, - "1514/udp": struct{}{}, - "1514/tcp": struct{}{}, - }, - Entrypoint: []string{containerName}, - Env: []string{}, - } - - tenzirApikey := os.Getenv("TENZIR_PLUGINS__PLATFORM__API_KEY") - tenzirControlEndpoint := os.Getenv("TENZIR_PLUGINS__PLATFORM__CONTROL_ENDPOINT") - tenzirPluginsPlatform := os.Getenv("TENZIR_PLUGINS__PLATFORM__TENANT_ID") - - anyFound := false - if len(tenzirApikey) > 0 { - config.Env = append(config.Env, fmt.Sprintf("TENZIR_PLUGINS__PLATFORM__API_KEY=%s", tenzirApikey)) - anyFound = true - } - - if len(tenzirControlEndpoint) > 0 { - config.Env = append(config.Env, fmt.Sprintf("TENZIR_PLUGINS__PLATFORM__CONTROL_ENDPOINT=%s", tenzirControlEndpoint)) - anyFound = true - } - - if len(tenzirPluginsPlatform) > 0 { - config.Env = append(config.Env, fmt.Sprintf("TENZIR_PLUGINS__PLATFORM__TENANT_ID=%s", tenzirPluginsPlatform)) - anyFound = true - } - - tenzirStorageFolder := os.Getenv("SHUFFLE_STORAGE_FOLDER") - if len(tenzirStorageFolder) > 0 { - tenzirStorageFolder = tenzirStorageFolder - - if !strings.HasSuffix(tenzirStorageFolder, "/") { - tenzirStorageFolder = tenzirStorageFolder + "/" - } - } else { - tenzirStorageFolder = "/tmp/" - log.Printf("[DEBUG] Using base folder %s for Tenzir storage. Change it using environment variable SHUFFLE_STORAGE_FOLDER=/filepath/", tenzirStorageFolder) - } - - if !anyFound { - //log.Printf("[DEBUG] No Tenzir Plugin environment variables found.") - } else { - //log.Printf("[DEBUG] Attempting Tenzir connection with app.tenzir.com tenant '%s'", tenzirPluginsPlatform) - } - - hostConfig := &container.HostConfig{ - PortBindings: nat.PortMap{ - "1514/tcp": []nat.PortBinding{{HostPort: "1514"}}, - "1514/udp": []nat.PortBinding{{HostPort: "1514"}}, - "5160/tcp": []nat.PortBinding{{HostPort: "5160"}}, - }, - Mounts: []mount.Mount{ - { - Type: "bind", - Source: tenzirStorageFolder, - Target: "/tmp", - }, - /* - { - Type: "bind", - Source: tenzirStorageFolder, - Target: "/var/log/tenzir/", - }, - { - Type: "bind", - Source: tenzirStorageFolder, - Target: "/var/cache/tenzir/", - }, - */ - }, - VolumeDriver: "local", - RestartPolicy: container.RestartPolicy{ - Name: "always", - }, - } - - if os.Getenv("SHUFFLE_DISABLE_SYSLOG") == "true" { - hostConfig.PortBindings = nat.PortMap{ - "5160/tcp": []nat.PortBinding{{HostPort: "5160"}}, - } - } - - if skipPipelineMount { - hostConfig.Mounts = []mount.Mount{} - } - - //networkingConfig := &network.NetworkingConfig{ - // EndpointsConfig: map[string]*network.EndpointSettings{ - // "tenzir-network": { - // IPAMConfig: &network.EndpointIPAMConfig{ - // IPv4Address: "192.168.102.100", - // }, - // }, - // }, - //} - - networkingConfig := &network.NetworkingConfig{ - EndpointsConfig: map[string]*network.EndpointSettings{ - "tenzir-network": { - IPAMConfig: nil, - Aliases: []string{"tenzir-node"}, - }, - }, - } - - // FIXME: Is this necessary? Seems to screw up networking: - // conflicting options: hostname and the network mode - /* - if isKubernetes != "true" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" { - hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) - } - */ - - resp, err := dockercli.ContainerCreate(ctx, config, hostConfig, networkingConfig, nil, containerName) - if err != nil { - if strings.Contains(err.Error(), "path does not exist") { - log.Printf("[ERROR] Not using permanent pipeline storage as storage folder %s does not exist. If you want permanent storage, create the %s folder then restart Orborus (1). Raw: %s", tenzirStorageFolder, tenzirStorageFolder, err) - skipPipelineMount = true - } else { - log.Printf("[ERROR] Failed to create Tenzir Node container: %v", err) - } - - return err - } - - if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" { - networkName := "shuffle_swarm_executions" - err = dockercli.NetworkConnect(ctx, networkName, resp.ID, nil) - if err != nil { - log.Printf("[ERROR] Error connecting tenzir container to network: %s", err) - } - } - - err = dockercli.ContainerStart(ctx, containerName, containerStartOptions) - if err != nil { - if strings.Contains(err.Error(), "path does not exist") { - log.Printf("[ERROR] Not using permanent pipeline storage as storage folder %s does not exist. If you want permanent storage, create the %s folder then restart Orborus (2). Raw: %s", tenzirStorageFolder, tenzirStorageFolder, err) - skipPipelineMount = true - } else { - log.Printf("[ERROR] Failed to START Tenzir Node container: %v", err) - } - - return err - } - - log.Printf("[INFO] Tenzir Node container started successfully. Waiting for it to become available..") - time.Sleep(20 * time.Second) - err = checkTenzirNode() - if err != nil { - log.Printf("[ERROR] Tenzir connection not available: %s. IF the URL seems wrong, set SHUFFLE_PIPELINE_URL=http://:5160", err) - return err - } - - log.Printf("[INFO] Successfully deployed Tenzir Node! Setting up default syslog listener on TCP/1514 AND UDP/1514") - - command := `load_tcp "0.0.0.0:1514" { read_syslog } | import` - _, err = createPipeline(command, "default-syslog-tcp-514") - if err != nil { - log.Printf("[ERROR] Failed to create tcp syslog pipeline: %s", err) - return nil - } - - command = `load_udp "0.0.0.0:1514", insert_newlines=true | read_syslog | import` - _, err = createPipeline(command, "default-syslog-udp-514") - if err != nil { - log.Printf("[ERROR] Failed to create udp syslog pipeline: %s", err) - return nil - } - - return nil -} - -func createNetworkIfNotExists(ctx context.Context, networkName, subnet, gateway string) error { - listOptions := network.ListOptions{} - networks, err := dockercli.NetworkList(ctx, listOptions) - if err != nil { - return err - } - - for _, network := range networks { - if network.Name == networkName { - // Network exists - return nil - } - } - - ipamConfig := &network.IPAM{ - Config: []network.IPAMConfig{ - { - Subnet: subnet, - Gateway: gateway, - }, - }, - } - - networkCreate := network.CreateOptions{ - //CheckDuplicate: true, - Driver: "bridge", - IPAM: ipamConfig, - } - - _, err = dockercli.NetworkCreate(ctx, networkName, networkCreate) - if err != nil { - return err - } - - return nil -} - -func checkTenzirNode() error { - if tenzirDisabled && os.Getenv("SHUFFLE_SKIP_PIPELINES") == "true" && os.Getenv("SHUFFLE_PIPELINE_ENABLED") == "false" { - return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES (2)") - } - - url := fmt.Sprintf("%s/api/v0/ping", pipelineUrl) - forwardMethod := "POST" - - client := http.Client{ - Timeout: 1 * time.Second, - } - req, err := http.NewRequest(forwardMethod, url, nil) - if err != nil { - log.Printf("[ERROR] Failed to create HTTP request: %s", err) - return err - } - - resp, err := client.Do(req) - if err == nil && resp.StatusCode == http.StatusOK { - return nil - } - - return fmt.Errorf("Tenzir node is not available due to: %s", err) -} - -func createPipeline(command, identifier string) (string, error) { - - //toBeDeleted := false - /* - // Pre-checked. No point here - pipelineId, err := searchPipeline(identifier) - if err != nil { - return "", err - } - */ - - url := fmt.Sprintf("%s/api/v0/pipeline/create", pipelineUrl) - forwardMethod := "POST" - - /* - if err != nil { - if strings.Contains(fmt.Sprintf("%s", err), "no existing pipeline found") { - log.Printf("[INFO] No existing pipeline found with id: %s. Creating a new one!", identifier) - } else { - log.Printf("[ERROR] Failed to search for existing pipeline but continuing anyway : %s", err) - } - } else { - log.Printf("[INFO] an existing pipeline found with ID: %s. it will be deleted", pipelineId) - toBeDeleted = true - } - */ - - // if strings.Contains(command, "shuffler.io") { - - // } else { - // var scheme string - // if strings.Contains(command, "http://") { - // scheme = "http://" - // } else if strings.Contains(command, "https://") { - // scheme = "https://" - // } - - // startIndex := strings.Index(command, scheme) - // if startIndex != -1 { - // endIndex := startIndex + len(scheme) - // endIndex += strings.Index(command[endIndex:], "/") - - // command = command[:startIndex] + baseUrl + command[endIndex:] - // } - // } - - //command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | sigma /var/lib/tenzir/rule.yaml" - //command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | import" - - // Make sure to escape them - //if strings.Contains(command, "/") { - // command = strings.ReplaceAll("\\\"", "", command) - // command = strings.ReplaceAll(command, "\"", "") - //} - - requestBody := map[string]interface{}{ - "definition": command, - "name": identifier, - "hidden": false, - "retry_delay": "500.0ms", - "unstoppable": true, - } - - requestBodyJSON, err := json.Marshal(requestBody) - if err != nil { - log.Printf("[ERROR] failed marshalling body: %s", err) - return "", err - } - - forwardData := bytes.NewBuffer(requestBodyJSON) - req, err := http.NewRequest( - forwardMethod, - url, - forwardData, - ) - - if err != nil { - log.Printf("[ERROR] Failed to create HTTP request: %s", err) - return "", err - } - - req.Header.Set("Content-Type", "application/json") - client := &http.Client{Timeout: 10 * time.Second} - resp, err := client.Do(req) - if err != nil { - log.Printf("[ERROR] Failed to send HTTP request: %s", err) - return "", err - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Printf("[ERROR] Failed reading response body: %s", err) - return "", err - } - - if strings.Contains(string(body), "error") { - log.Printf("[ERROR] Pipeline creation error resp (%d): %s", resp.StatusCode, string(body)) - } else { - log.Printf("[DEBUG] Pipeline creation debug (%d): %s", resp.StatusCode, string(body)) - } - - defer resp.Body.Close() - if resp.StatusCode != 200 { - log.Printf("[DEBUG] status code is %d instead of 200", resp.StatusCode) - return "", fmt.Errorf("got the status code %d instead of 200", resp.StatusCode) - } - - type PipelineResponse struct { - ID string `json:"id"` - Message string `json:"message"` - Severity string `json:"severity"` - } - - var response PipelineResponse - if err := json.Unmarshal(body, &response); err != nil { - log.Printf("[ERROR] Failed unmarshalling response: %s", err) - return "", err - } - - if response.ID == "" { - log.Printf("[ERROR] ID not found or empty in response. Severity: %#v, Message: %#v", response.Severity, response.Message) - return "", errors.New("Pipeline ID not found or empty in the response. See error logs.") - } - - return response.ID, nil -} - -func updatePipelineState(command, pipelineId, action string) (string, error) { - - url := fmt.Sprintf("%s/api/v0/pipeline/update", pipelineUrl) - forwardMethod := "POST" - requestBody := map[string]interface{}{ - "id": pipelineId, - "action": action, - - /* - "autostart": map[string]bool{ - "created": true, - "completed": false, - "failed": false, - }, - "autodelete": map[string]bool{ - "completed": false, - "failed": false, - "stopped": false, - }, - */ - } - - requestBodyJSON, err := json.Marshal(requestBody) - if err != nil { - return "", err - } - - log.Printf("[INFO] Updating pipeline %s with action %s to ensure it starts. Body: %s", pipelineId, action, string(requestBodyJSON)) - - forwardData := bytes.NewBuffer(requestBodyJSON) - req, err := http.NewRequest( - forwardMethod, - url, - forwardData, - ) - if err != nil { - log.Printf("[ERROR] Failed to update HTTP request: %s", err) - return "", err - } - - req.Header.Set("Content-Type", "application/json") - client := &http.Client{Timeout: 10 * time.Second} - resp, err := client.Do(req) - if err != nil { - log.Printf("[ERROR] Failed to send HTTP request: %s", err) - return "", err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("got the status code %d instead of 200", resp.StatusCode) - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - - var responseData struct { - Pipeline struct { - State string `json:"state"` - } `json:"pipeline"` - } - if err := json.Unmarshal(body, &responseData); err != nil { - return "", err - } - - return responseData.Pipeline.State, nil -} - -func deletePipeline(pipelineId string) error { - requestBody := map[string]string{ - "id": pipelineId, - } - - url := fmt.Sprintf("%s/api/v0/pipeline/delete", pipelineUrl) - forwardMethod := "POST" - - requestBodyJSON, err := json.Marshal(requestBody) - if err != nil { - log.Println("[ERROR] failed marshalling request body:", err) - return err - } - - forwardData := bytes.NewBuffer(requestBodyJSON) - - req, err := http.NewRequest( - forwardMethod, - url, - forwardData, - ) - if err != nil { - log.Printf("[ERROR] Failed to delete HTTP request: %s", err) - return err - } - req.Header.Set("Content-Type", "application/json") - - client := &http.Client{Timeout: 10 * time.Second} - resp, err := client.Do(req) - if err != nil { - log.Printf("[ERROR] Failed to send HTTP request: %s", err) - return err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - log.Printf("[DEBUG] The deletion of pipeline with ID: %s is unsucessful as status code is NOT 200 !!!", pipelineId) - return fmt.Errorf("got the status code %d instead of 200", resp.StatusCode) - } - - log.Printf("[INFO] Pipeline with ID: %s deleted successfully", pipelineId) - - pipelines = []shuffle.PipelineInfo{} - return nil -} - -// Lists the pipelines from the API exactly as they are. Definition is set up in Shuffle structs -func listPipelines() ([]shuffle.PipelineInfo, error) { - responseData := shuffle.PipelineInfoWrapper{} - - if tenzirDisabled { - return responseData.Pipelines, errors.New("Tenzir is disabled") - } - - var reqBody []byte - url := fmt.Sprintf("%s/api/v0/pipeline/list", pipelineUrl) - client := http.Client{ - Timeout: 2 * time.Second, - } - - req, err := http.NewRequest( - "POST", - url, - bytes.NewBuffer(reqBody), - ) - - if err != nil { - return responseData.Pipelines, err - } - - req.Header.Set("Content-Type", "application/json") - resp, err := client.Do(req) - if err != nil { - return responseData.Pipelines, err - } - - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return responseData.Pipelines, fmt.Errorf("Got the status code %d instead of 200 from Pipeline node", resp.StatusCode) - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return responseData.Pipelines, err - } - - if err := json.Unmarshal(body, &responseData); err != nil { - return responseData.Pipelines, err - } - - return responseData.Pipelines, nil -} - -func searchPipeline(identifier string) (string, error) { - allPipelines, err := listPipelines() - if err != nil { - return "", err - } - - for _, pipeline := range allPipelines { - if pipeline.Name == identifier { - return pipeline.ID, nil - } - } - - return "", errors.New("no existing pipeline found with name") -} - -func handleFileCategoryChange(ruleType string) error { - apiEndpoint := fmt.Sprintf("%s/api/v1/files/namespaces/%s", baseUrl, ruleType) - req, err := http.NewRequest("GET", apiEndpoint, nil) - if err != nil { - return err - } - - if len(pipelineApikey) == 0 { - //var auth = os.Getenv("AUTH") - //var org = os.Getenv("ORG") - - if len(auth) > 0 && len(org) > 0 { - pipelineApikey = auth - } else { - return errors.New("Shuffle API-key not set for Pipelines: SHUFFLE_PIPELINE_AUTH=") - } - } - - req.Header.Add("Authorization", "Bearer "+pipelineApikey) - if len(org) > 0 { - req.Header.Add("Org-Id", org) - } - - client := shuffle.GetExternalClient(apiEndpoint) - resp, err := client.Do(req) - if err != nil { - return err - } - - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("Received non-200 response '%d' from backend URL %s. ", resp.StatusCode, apiEndpoint) - } - - out, err := os.Create("files.zip") - if err != nil { - return err - } - - defer out.Close() - defer os.Remove("files.zip") - _, err = io.Copy(out, resp.Body) - if err != nil { - log.Printf("[ERROR] Failed to io.Copy ZIP file content: %s", err) - return err - } - - //log.Println("[DEBUG] ZIP file downloaded successfully.") - tenzirStorageFolder := os.Getenv("SHUFFLE_STORAGE_FOLDER") - if len(tenzirStorageFolder) == 0 { - tenzirStorageFolder = "/tmp/" - } - - tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/") - sigmaPath := fmt.Sprintf("%s/%s_rules", tenzirStorageFolder, ruleType) - err = extractZIP("files.zip", sigmaPath) - if err != nil { - log.Printf("[ERROR] Failed to extract ZIP file: %s", err) - return err - } - - log.Printf("[DEBUG] Detection files copied to '%s' successfully.", sigmaPath) - - return nil -} - -func extractZIP(zipFile, destDir string) error { - r, err := zip.OpenReader(zipFile) - if err != nil { - return err - } - - // FInd size of the zip - var totalSize uint64 - for _, f := range r.File { - totalSize += f.UncompressedSize64 - } - - log.Printf("[DEBUG] Total size of the ZIP file: %d bytes", totalSize) - defer r.Close() - if err := os.MkdirAll(destDir, 0755); err != nil { - return err - } - - log.Printf("[DEBUG] Total files to extract: %d", len(r.File)) - for _, f := range r.File { - // Fix path traversal - if strings.Contains(f.Name, "..") { - return fmt.Errorf("illegal file name: %s", f.Name) - } - - err := extractFile(f, destDir) - if err != nil { - return err - } - } - - return nil -} - -func extractFile(f *zip.File, destDir string) error { - rc, err := f.Open() - if err != nil { - return err - } - - defer rc.Close() - path := filepath.Join(destDir, f.Name) - out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) - if err != nil { - return err - } - - defer out.Close() - _, err = io.Copy(out, rc) - return err -} - -func copyToTenzir(srcPath, destPath string) error { - containerName := "tenzir-node" - - checkCmd := exec.Command("docker", "exec", containerName, "test", "-d", destPath) - if err := checkCmd.Run(); err == nil { - rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", "-rf", destPath) - if err := rmCmd.Run(); err != nil { - return fmt.Errorf("error removing existing directory in container: %v", err) - } - } - - cpCmd := exec.Command("docker", "cp", srcPath, fmt.Sprintf("%s:%s", containerName, destPath)) - var out bytes.Buffer - cpCmd.Stdout = &out - cpCmd.Stderr = &out - - err := cpCmd.Run() - if err != nil { - return fmt.Errorf("error copying files: %v, output: %s", err, out.String()) - } - - return nil -} - -func removeFileCategory(ruleType string) error { - tenzirStorageFolder := os.Getenv("SHUFFLE_STORAGE_FOLDER") - if len(tenzirStorageFolder) == 0 { - tenzirStorageFolder = "/tmp/" - } - - tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/") - - //sigmaPath := "/var/lib/tenzir/sigma_rules/*" - rulePath := fmt.Sprintf("%s/%s_rules", tenzirStorageFolder, ruleType) - - err := os.RemoveAll(rulePath) - if err != nil { - return fmt.Errorf("Error removing category files in %s: %v", rulePath, err) - } - - log.Printf("[INFO] Removed all local category data in %s", rulePath) - - return nil -} - -// curl https://get.tenzir.app | sh -func removeFile(fileName string) error { - containerName := "tenzir-node" - srcPath := fmt.Sprintf("/var/lib/tenzir/sigma_rules/%s", fileName) - - checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) - if err := checkSrcCmd.Run(); err != nil { - // If the file does not exist, simply return nil - if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { - log.Printf("[ERROR] No such file: %s, nothing to delete\n", srcPath) - return nil - } - return fmt.Errorf("error checking source file: %v", err) - } - - return removePath(containerName, srcPath) -} - -func removePath(containerName, path string) error { - // rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("rm -rf %s", path)) - rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", "-rf", path) - output, err := rmCmd.CombinedOutput() - if err != nil { - return fmt.Errorf("error removing path: %v, output: %s", err, output) - } - return nil -} - -func sendPipelineHealthStatus(sensorMode shuffle.SensorMode) (shuffle.LakeConfig, error) { - if sensorMode.Enabled { - return shuffle.LakeConfig{}, nil - } - - pipelinePayload := shuffle.LakeConfig{ - Enabled: false, - Pipelines: []shuffle.PipelineInfo{}, - } - - if tenzirDisabled { - return pipelinePayload, nil - } - - // To not spam down the list API too much - randint := rand.Intn(5) - if len(pipelines) == 0 || randint == 0 { - pipelineDef, err := listPipelines() - - if err == nil || len(pipelines) > 0 { - pipelines = pipelineDef - pipelinePayload.Pipelines = pipelines - } - } else { - pipelinePayload.Pipelines = pipelines - } - - err := deployTenzirNode() - if err != nil { - if (!strings.Contains(err.Error(), "SHUFFLE_SKIP_PIPELINES") && !strings.Contains(err.Error(), "Kubernetes not implemented for Tenzir node")) && !strings.Contains(err.Error(), "Tenzir Node is already running") && !strings.Contains(err.Error(), "docker daemon") { - log.Printf("[ERROR] Tenzir node connection problem: %s", err) - - } else { - //tenzirDisabled = true - if debug { - log.Printf("[WARNING] Disabling pipelines: %s. You will need to restart the Orborus to fix this.", err) - } - - } - - return pipelinePayload, err - } - - pipelinePayload.Enabled = true - - // No direct sending. - return pipelinePayload, nil -} - -func disableRule(fileName string) error { - containerName := "tenzir-node" - srcPath := fmt.Sprintf("/var/lib/tenzir/sigma_rules/%s", fileName) - destDir := "/var/lib/tenzir/disabled_rules" - destPath := fmt.Sprintf("%s/%s", destDir, fileName) - - // checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) - checkSrcCmd := exec.Command("docker", "exec", containerName, "test", "-f", srcPath) - if err := checkSrcCmd.Run(); err != nil { - if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { - fmt.Printf("File does not exist: %s\n", srcPath) - return nil // Nothing to disable - } - return fmt.Errorf("error checking source file: %v", err) - } - - // checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mkdir -p %s", destDir)) - checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mkdir", "-p", "--", destDir) - if err := checkDestDirCmd.Run(); err != nil { - return fmt.Errorf("error ensuring destination directory exists: %v", err) - } - - // moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mv %s %s", srcPath, destPath)) - moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mv", "--", srcPath, destPath) - if err := moveCmd.Run(); err != nil { - return fmt.Errorf("error moving file: %v", err) - } - - fmt.Printf("File %s moved to %s successfully.\n", fileName, destDir) - return nil -} - -func enableRule(fileName string) error { - containerName := "tenzir-node" - srcPath := fmt.Sprintf("/var/lib/tenzir/disabled_rules/%s", fileName) - destDir := "/var/lib/tenzir/sigma_rules" - destPath := fmt.Sprintf("%s/%s", destDir, fileName) - - // checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) - checkSrcCmd := exec.Command("docker", "exec", containerName, "test", "-f", srcPath) - if err := checkSrcCmd.Run(); err != nil { - if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { - fmt.Printf("File does not exist: %s\n", srcPath) - return nil // Nothing to enable - } - return fmt.Errorf("error checking source file: %v", err) - } - - // checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mkdir -p %s", destDir)) - checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mkdir", "-p", "--", destDir) - if err := checkDestDirCmd.Run(); err != nil { - return fmt.Errorf("error ensuring destination directory exists: %v", err) - } - // moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mv %s %s", srcPath, destPath)) - moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mv", "--", srcPath, destPath) - if err := moveCmd.Run(); err != nil { - return fmt.Errorf("error moving file: %v", err) - } - - fmt.Printf("[DEBUG] File %s moved to %s successfully.\n", fileName, destDir) - return nil -} - -// Is this ok to do with Docker? idk :) -func getRunningWorkers(ctx context.Context, workerTimeout int) int { - //log.Printf("[DEBUG] Getting running workers with API version %s", dockerApiVersion) - counter := 0 - if isKubernetes == "true" { - log.Printf("[INFO] Getting running workers in kubernetes") - - thresholdTime := time.Now().Add(time.Duration(-workerTimeout) * time.Second) - - clientset, _, err := shuffle.GetKubernetesClient() - if err != nil { - log.Printf("[ERROR] Failed getting kubernetes client: %s", err) - return 0 - } - - pods, podErr := clientset.CoreV1().Pods(kubernetesNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: "app.kubernetes.io/name=shuffle-worker", - }) - if podErr != nil { - log.Printf("[ERROR] Failed getting running workers: %s", podErr) - return 0 - } - - for _, pod := range pods.Items { - if pod.Status.Phase == "Running" && pod.CreationTimestamp.Time.After(thresholdTime) { - counter++ - } - } - - if counter > 0 { - log.Printf("[INFO] Found %d running workers in Orborus", counter) - } - } else { - - containers, err := dockercli.ContainerList(ctx, container.ListOptions{ - All: true, - }) - - // Automatically updates the version - if err != nil { - log.Printf("[ERROR] Error getting containers from Docker: %s", err) - - newVersionSplit := strings.Split(fmt.Sprintf("%s", err), "version is") - if len(newVersionSplit) > 1 { - //dockerApiVersion = strings.TrimSpace(newVersionSplit[1]) - log.Printf("[DEBUG] WANT to change the API version to default to %s?", strings.TrimSpace(newVersionSplit[1])) - } - - return maxConcurrency - } - - currenttime := time.Now().Unix() - - for _, container := range containers { - // Skip random containers. Only handle things related to Shuffle. - if !strings.Contains(container.Image, baseimagename) { - shuffleFound := false - for _, item := range container.Labels { - if item == "shuffle" { - shuffleFound = true - break - } - } - - // Check image name - if !shuffleFound { - continue - } - //} else { - // log.Printf("NAME: %s", container.Image) - } - - for _, name := range container.Names { - // FIXME - add name_version_uid_uid regex check as well - if !strings.HasPrefix(name, "/worker") { - continue - } - - //log.Printf("Time: %d - %d", currenttime-container.Created, int64(workerTimeout)) - if container.State == "running" && currenttime-container.Created < int64(workerTimeout) { - counter += 1 - break - } - } - } - } - return counter -} - -// FIXME - add this to remove exited workers -// Should it check what happened to the execution? idk -func zombiecheck(ctx context.Context, workerTimeout int, sensorMode shuffle.SensorMode) error { - if sensorMode.Enabled { - return nil - } - - isK8s := isKubernetes == "true" - - executionIds = []string{} - if swarmConfig == "run" || swarmConfig == "swarm" || isK8s { - //log.Printf("[DEBUG] Skipping Zombie check due to new execution model (swarm)") - return nil - } - - log.Println("[INFO] Looking for old containers to remove") - containers, err := dockercli.ContainerList(ctx, container.ListOptions{ - All: true, - }) - - if err != nil { - log.Printf("[ERROR] Failed creating Containerlist: %s", err) - return err - } - - containerNames := map[string]string{} - stopContainers := []string{} - removeContainers := []string{} - log.Printf("[INFO] Baseimage: %s, Workertimeout: %d", baseimagename, int64(workerTimeout)) - //baseString := `/bin/sh -c 'python app.py --log-level DEBUG'` - baseString := `python app.py` - for _, container := range containers { - // Skip random containers. Only handle things related to Shuffle. - if !strings.Contains(container.Image, baseimagename) && !strings.Contains(container.Command, baseString) && !strings.Contains(container.Command, "walkoff") && container.Command != "./worker" { - shuffleFound := false - for _, item := range container.Labels { - if item == "shuffle" { - shuffleFound = true - break - } - } - - // Check image name - if !shuffleFound { - //log.Printf("[DEBUG] Zombie container skip: %#v, %s", container.Labels, container.Image) - continue - } - //} else { - // log.Printf("NAME: %s", container.Image) - } else { - //log.Printf("Img: %s", container.Image) - //log.Printf("Names: %s", container.Names) - } - - for _, name := range container.Names { - // FIXME - add name_version_uid_uid regex check as well - if strings.HasPrefix(name, "/shuffle") && !strings.HasPrefix(name, "/shuffle-subflow") { - continue - } - - currenttime := time.Now().Unix() - //log.Printf("[INFO] (%s) NAME: %s. TIME: %d", container.State, name, currenttime-container.Created) - - // Need to check time here too because a container can be removed the same instant as its created - if container.State != "running" && currenttime-container.Created > int64(workerTimeout) { - removeContainers = append(removeContainers, container.ID) - containerNames[container.ID] = name - } - - // stopcontainer & removecontainer - //log.Printf("Time: %d - %d", currenttime-container.Created, int64(workerTimeout)) - if container.State == "running" && currenttime-container.Created > int64(workerTimeout) { - stopContainers = append(stopContainers, container.ID) - containerNames[container.ID] = name - } - } - } - - // FIXME - add killing of apps with same execution ID too - log.Printf("[INFO] Should STOP and remove %d containers.", len(stopContainers)) - var options container.StopOptions - for _, containername := range stopContainers { - log.Printf("[INFO] Stopping and removing container %s", containerNames[containername]) - go dockercli.ContainerStop(ctx, containername, options) - removeContainers = append(removeContainers, containername) - } - - removeOptions := container.RemoveOptions{ - RemoveVolumes: true, - Force: true, - } - - log.Printf("[INFO] Should REMOVE %d containers.", len(removeContainers)) - for _, containername := range removeContainers { - dockercli.ContainerRemove(ctx, containername, removeOptions) - } - - return nil -} - -func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, env []string) error { - parsedRequest := shuffle.OrborusExecutionRequest{ - ExecutionId: workflowExecution.ExecutionId, - Authorization: workflowExecution.Authorization, - BaseUrl: os.Getenv("BASE_URL"), - EnvironmentName: os.Getenv("ENVIRONMENT_NAME"), - Timezone: os.Getenv("TZ"), - Cleanup: os.Getenv("CLEANUP"), - HTTPProxy: os.Getenv("HTTP_PROXY"), - HTTPSProxy: os.Getenv("HTTPS_PROXY"), - ShufflePassProxyToApp: os.Getenv("SHUFFLE_PASS_APP_PROXY"), - WorkerServerUrl: os.Getenv("SHUFFLE_WORKER_SERVER_URL"), - } - - parsedBaseurl := baseUrl - if strings.Contains(baseUrl, ":") { - baseUrlSplit := strings.Split(baseUrl, ":") - if len(baseUrlSplit) >= 3 { - parsedBaseurl = strings.Join(baseUrlSplit[0:2], ":") - } - } - - data, err := json.Marshal(parsedRequest) - if err != nil { - log.Printf("[ERROR] Failed marshalling worker request: %s", err) - return err - } - - streamUrl := fmt.Sprintf("http://shuffle-workers:33333/api/v1/execute") - if containerId == "" || containerId == "shuffle-orborus" { - streamUrl = fmt.Sprintf("%s:33333/api/v1/execute", parsedBaseurl) - } - - identifier := "shuffle-workers" - if isKubernetes == "true" { - // FIXME: Do we need this to map the cluster? - //if shuffle.IsRunningInCluster() { - //log.Printf("[INFO] Running in Kubernetes cluster") - // try getting the k8s worker server url - //} - } - - if strings.Contains(streamUrl, "shuffler.io") || strings.Contains(streamUrl, "localhost") || strings.Contains(streamUrl, "127.0.0.1") || strings.Contains(streamUrl, "shuffle-backend") { - - // Specific to debugging - if len(workerServerUrl) == 0 { - if debug { - log.Printf("[INFO] Using default worker server url as previous is invalid: %s. Swapping to shuffle-workers:33333", streamUrl) - } - } - - streamUrl = fmt.Sprintf("http://shuffle-workers:33333/api/v1/execute") - } - - if len(workerServerUrl) > 0 { - // Check if a port is supplied or not - if strings.Contains(workerServerUrl, "/api/v1/execute") { - streamUrl = workerServerUrl - } else { - streamUrl = fmt.Sprintf("%s/api/v1/execute", workerServerUrl) - if !strings.Contains(workerServerUrl, ":") { - streamUrl = fmt.Sprintf("%s:33333/api/v1/execute", workerServerUrl) - } - } - } - - client := &http.Client{ - //Transport: &http.Transport{ - // TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - //}, - Timeout: time.Duration(120 * time.Second), - } - - if debug { - log.Printf("[DEBUG][%s] Worker request to be sent to URL: %s", workflowExecution.ExecutionId, streamUrl) - } - - req, err := http.NewRequest( - "POST", - streamUrl, - bytes.NewBuffer([]byte(data)), - ) - - if err != nil { - log.Printf("[ERROR] Failed creating worker request: %s", err) - if strings.Contains(fmt.Sprintf("%s", err), "connection refused") || strings.Contains(fmt.Sprintf("%s", err), "EOF") { - workerImage := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) - if len(newWorkerImage) > 0 { - workerImage = newWorkerImage - } - - if isKubernetes == "true" { - deployK8sWorker(workerImage, identifier, env) - } else { - deployServiceWorkers(workerImage) - } - - time.Sleep(time.Duration(10) * time.Second) - //err = sendWorkerRequest(executionRequest) - } - - return err - } - - newresp, err := client.Do(req) - if err != nil { - // Connection refused? - if !strings.Contains(fmt.Sprintf("%s", err), "timeout") { - log.Printf("[ERROR][%s] Error running worker request to %s (1): %s", workflowExecution.ExecutionId, streamUrl, err) - } - - if strings.Contains(fmt.Sprintf("%s", err), "connection refused") || strings.Contains(fmt.Sprintf("%s", err), "EOF") { - workerImage := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) - if len(newWorkerImage) > 0 { - workerImage = newWorkerImage - } - - if isKubernetes == "true" { - deployK8sWorker(workerImage, identifier, env) - } else { - deployServiceWorkers(workerImage) - } - - time.Sleep(time.Duration(10) * time.Second) - //err = sendWorkerRequest(executionRequest) - } - - return err - } - - defer newresp.Body.Close() - body, err := ioutil.ReadAll(newresp.Body) - if err != nil { - log.Printf("[ERROR] Failed reading body in worker request body to worker on %s: %s", streamUrl, err) - return err - } - window.AddEvent(time.Now()) - - if newresp.StatusCode != 200 { - log.Printf("[WARNING] POTENTIAL error running worker request (2) - status code is %d for %s, not 200. Body: %s", newresp.StatusCode, streamUrl, string(body)) - - // In case of old executions - if strings.Contains(strings.ToLower(string(body)), "bad status ") { - return nil - } - - if strings.Contains(strings.ToLower(string(body)), "no apps to handle") { - return nil - } - - return errors.New(fmt.Sprintf("Bad statuscode from worker: %d - expecting 200", newresp.StatusCode)) - } - - _ = body - - debugCommand := fmt.Sprintf("docker service logs shuffle-workers 2>&1 -f | grep %s", workflowExecution.ExecutionId) - if isKubernetes == "true" { - debugCommand = fmt.Sprintf("kubectl logs -n %s deployment/shuffle-workers | grep %s", kubernetesNamespace, workflowExecution.ExecutionId) - } - - log.Printf("[DEBUG][%s] Ran worker from requests. Worker URL: %s. DEBUGGING:\n%s", workflowExecution.ExecutionId, streamUrl, debugCommand) - return nil -} - -// 0x0elliot: -// let's never increase worker replicas. -// in our tests, workers replicas mattered a lot less. -// edge-case: subflows are helped with when worker replicas are higher. -func AutoScale(ctx context.Context) { - if os.Getenv("SHUFFLE_SCALE_REPLICAS") != "" { - return - } - - ticker := time.NewTicker(1 * time.Second) - coolDownPeriod := 10 * time.Second - queuePerMinuteInt = 20 - if os.Getenv("SHUFFLE_QUEUE_PER_MINUTE") != "" { - var err error - queuePerMinuteInt, err = strconv.Atoi(os.Getenv("SHUFFLE_QUEUE_PER_MINUTE")) - if err != nil { - log.Printf("[WARNING] Cannot convert %s to int. Using default value for it: %d", queuePerMinute, queuePerMinuteInt) - } - } - - lastScaleTime := time.Now() - currentWorkers := currentWokerCount(ctx, dockercli) - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - if time.Since(lastScaleTime) < (coolDownPeriod) { - continue - } - currentRequestCount := window.CountEvents(time.Now()) - requiredReplicas := 0 - if currentRequestCount >= queuePerMinuteInt*currentWorkers { - // FIXME: Hardcoded Max Replicas should be 6 - requiredReplicas = int(math.Min(float64(6), float64(currentRequestCount/queuePerMinuteInt)+1)) - } - - if requiredReplicas > 0 { - err := scaleService(ctx, dockercli, uint64(requiredReplicas)) - if err != nil { - log.Printf("[ERROR] Failed to scale service: %s", err) - } else { - lastScaleTime = time.Now() - currentWorkers = currentWokerCount(ctx, dockercli) - } - } - } - } -} - -func scaleService(ctx context.Context, client *dockerclient.Client, replicas uint64) error { - service, _, err := client.ServiceInspectWithRaw(ctx, "shuffle-workers", types.ServiceInspectOptions{}) - if err != nil { - return err - } - - if service.Spec.Mode.Replicated == nil { - return errors.New("Service cannot be replicated") - } - - if *service.Spec.Mode.Replicated.Replicas >= replicas { - return nil - } - - service.Spec.Mode.Replicated.Replicas = &replicas - - _, err = dockercli.ServiceUpdate(ctx, service.ID, service.Version, service.Spec, types.ServiceUpdateOptions{}) - if err != nil { - return err - } - - log.Printf("[INFO] Scaled shuffle-worker to %d replicas", replicas) - return nil -} - -func currentWokerCount(ctx context.Context, client *dockerclient.Client) int { - service, _, err := client.ServiceInspectWithRaw(ctx, "shuffle-workers", types.ServiceInspectOptions{}) - if err != nil { - return 0 - } - - if service.Spec.Mode.Replicated == nil { - return 0 - } - - return int(*service.Spec.Mode.Replicated.Replicas) -} - -func queueScaleFactor(numQueue int, queuePerMin int) float64 { - if numQueue > queuePerMin { - queuePressure := float64(numQueue) / float64(queuePerMin) - return 1.0 + math.Min(queuePressure-1.0, 1.0) - } - - return 1.0 -} - -func checkMemcached(ctx context.Context, dockercli *dockerclient.Client) (bool, error) { - containerName := "shuffle-cache" - continer, err := dockercli.ContainerInspect(context.Background(), containerName) - if err != nil { - if dockerclient.IsErrNotFound(err) { - return false, nil - } - return false, err - } - networkName := "shuffle_swarm_executions" - err = dockercli.NetworkConnect(ctx, networkName, containerName, nil) - if err != nil { - log.Printf("[WARNING] Failed connecting memcached container to network: %s", err) - } - - if continer.State.Running == false { - log.Printf("[INFO] Container %s exists but is not running. Attempting to start it.", containerName) - err = dockercli.ContainerStart(ctx, containerName, container.StartOptions{}) - if err != nil { - log.Printf("[ERROR] Failed to start container %s: %v", containerName, err) - return false, err - } - log.Printf("[INFO] Successfully started container %s.", containerName) - return true, nil - } - - return continer.State.Running, nil -} - -func deployMemcached(dockercli *dockerclient.Client) error { - if os.Getenv("SHUFFLE_MEMCACHED") != "" { - return errors.New("Memcached already running") - } - - defaultMem := "1024" - log.Printf("[INFO] Spanning a default memcached container to handle the distribution between cache across different workers. Default memory assigned %s", defaultMem) - - ctx := context.Background() - - memcachedImage := "docker.io/library/memcached:latest" - containerConfig := &container.Config{ - Image: memcachedImage, - Cmd: []string{"-m", defaultMem}, - } - - hostConfig := &container.HostConfig{ - PortBindings: nat.PortMap{ - "11211/tcp": []nat.PortBinding{{HostPort: "11211"}}, - }, - } - - _, _, err := dockercli.ImageInspectWithRaw(ctx, memcachedImage) - if dockerclient.IsErrNotFound(err) { - log.Printf("[DEBUG] Pulling image %s. This may take a while.", memcachedImage) - pullOptions := image.PullOptions{} - out, err := dockercli.ImagePull(ctx, memcachedImage, pullOptions) - if err != nil { - log.Printf("[ERROR] Failed to pull the memcached image: %s", err) - return err - } - defer out.Close() - - io.Copy(io.Discard, out) - } else if err != nil { - return err - } - - containerName := "shuffle-cache" - resp, err := dockercli.ContainerCreate(ctx, containerConfig, hostConfig, nil, nil, containerName) - if err != nil { - log.Printf("[ERROR] Error spanning memcached continer: %s", err) - return err - } - - if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" { - networkName := "shuffle_swarm_executions" - err = dockercli.NetworkConnect(ctx, networkName, resp.ID, nil) - if err != nil { - log.Printf("[ERROR] Error connecting tenzir container to network: %s", err) - } - } - - err = dockercli.ContainerStart(ctx, resp.ID, container.StartOptions{}) - if err != nil { - log.Printf("[ERROR] Error starting memcached continer: %s", err) - return err - } - - networkName := "shuffle_swarm_executions" - err = dockercli.NetworkConnect(ctx, networkName, resp.ID, nil) - if err != nil { - log.Printf("[ERROR] Error connecting memcached container to network: %s", err) - } - - log.Printf("[INFO] Memcached container started successfully at port 11211") - - return nil -} - -// How do we get the cpu usage? maybe just get the number of requests (much more useful for apps) -/* -func nodesResourceUsage(ctx context.Context, client *dockerclient.Client) error { - nodes, err := client.NodeList(ctx, types.NodeListOptions{}) - if err != nil { - return err - } - for _, node := range nodes { - res := node.Description.Resources - } - - return nil -} -*/ - -/* -func numberOfReplicas(ctx context.Context, queueLength int, config shuffle.ScalingConfig) (int, int) { - queueScaleFactor := queueScaleFactor(queueLength, config) - numReplicas := int(float64(queueLength) * queueScaleFactor) - serviceName := "shuffle-workers" - nodes, err := dockercli.NodeList(ctx, types.NodeListOptions{}) - if err != nil { - log.Printf("[ERROR] Cannot find any nodes in the swarm network") - } - - filterArgs := filters.NewArgs() - filterArgs.Add("service", serviceName) - filterArgs.Add("desired-state", "running") - - tasks, err := dockercli.TaskList(context.Background(), types.TaskListOptions{ - Filters: filterArgs, - }) - if err != nil { - log.Fatalf("[WARNING] Failed to list tasks for service %s: %s", serviceName, err) - } - - runningReplicas := len(tasks) - if numReplicas > runningReplicas*len(nodes) { - maxIncrease := config.MaxScaleUpStep - if numReplicas > (runningReplicas*len(nodes) + maxIncrease) { - numReplicas = runningReplicas + maxIncrease - } - } - - if numReplicas < config.MinReplicas { - numReplicas = config.MinReplicas - } - if numReplicas > config.MaxReplicas { - numReplicas = config.MaxReplicas - } - - return numReplicas, runningReplicas -} -*/ - -// TODO: Currently we use number of request made for the worker to run a execution as it is much -// easier to track in a window time frame. But this could be useful. -func collectMetrics(ctx context.Context, dockerClient *dockerclient.Client) (int, error) { - client := shuffle.GetExternalClient(baseUrl) - fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl) - req, err := http.NewRequest("GET", fullUrl, nil) - if err != nil { - log.Printf("[ERROR] Failed to send a request to %s: %s", fullUrl, err) - return 0, err - } - - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Org-Id", environment) - if len(auth) > 0 { - req.Header.Add("Authorization", auth) - } - - if len(org) > 0 { - req.Header.Add("Org", org) - } - - if len(orborusLabel) > 0 { - log.Printf("[DEBUG] Sending with Label '%s'", orborusLabel) - req.Header.Add("X-Orborus-Label", orborusLabel) - } - - if swarmConfig != "run" && swarmConfig != "swarm" { - req.Header.Add("X-Orborus-Runmode", "Default") - } else { - req.Header.Add("X-Orborus-Runmode", "Docker Swarm") - } - - resp, err := client.Do(req) - if err != nil { - return 0, err - } - - var executionRequests shuffle.ExecutionRequestWrapper - body, err := ioutil.ReadAll(resp.Body) - - json.Unmarshal(body, &executionRequests) - - return len(executionRequests.Data), nil -} - -func setBackendToSwarmNetwork(ctx context.Context) error { - containerId := "" - filterArgs := filters.NewArgs() - filterArgs.Add("name", "shuffle-backend") - - containers, err := dockercli.ContainerList(ctx, container.ListOptions{ - All: true, - Filters: filterArgs, - }) - if err != nil { - return err - } - if len(containers) == 0 { - return errors.New("No containers found with name shuffle-backend") - } - - containerId = containers[0].ID - networkName := "shuffle_swarm_executions" - err = dockercli.NetworkConnect(ctx, networkName, containerId, nil) - if err != nil { - log.Printf("[ERROR] Error connecting backend container to network: %s", err) - } - - return nil -} diff --git a/functions/onprem/orborus/orborus.yaml b/functions/onprem/orborus/orborus.yaml deleted file mode 100644 index 0ff50ef7..00000000 --- a/functions/onprem/orborus/orborus.yaml +++ /dev/null @@ -1,82 +0,0 @@ ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - namespace: default - name: pod-manager -rules: -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "create", "update", "delete"] -- apiGroups: ["batch"] - resources: ["jobs"] - verbs: ["create", "get", "list", "watch", "delete"] - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: pod-manager-binding - namespace: default -subjects: -- kind: ServiceAccount - name: default - namespace: default -roleRef: - kind: Role - name: pod-manager - apiGroup: rbac.authorization.k8s.io - ---- - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: orborus - annotations: - kompose.cmd: kompose convert -f docker-compose.yml - kompose.version: 1.26.0 (40646f47) - labels: - io.kompose.service: orborus -spec: - replicas: 1 - selector: - matchLabels: - io.kompose.service: orborus - strategy: {} - template: - metadata: - annotations: - kompose.cmd: kompose convert -f docker-compose.yml - kompose.version: 1.26.0 (40646f47) - creationTimestamp: null - labels: - io.kompose.network/shuffle: "true" - io.kompose.service: orborus - spec: - dnsPolicy: "Default" - containers: - - env: - - name: BASE_URL - value: "https://shuffler.io" - - name: SHUFFLE_SCALE_REPLICAS - value: "7" - - name: IS_KUBERNETES - value: "true" - - name: ENVIRONMENT_NAME - value: "environment test" - - name: ORG - value: "9c938e5b-d812-40d9-92f0-93783f43ec0d" - - name: AUTH - value: "3663a270-bb3a-4678-a365-d879601a1a0c" - - name: SHUFFLE_WORKER_IMAGE - value: "ghcr.io/shuffle/shuffle-worker:nightly" - - image: ghcr.io/shuffle/shuffle-orborus:nightly - imagePullPolicy: Always - name: shuffle-orborus - resources: {} - hostname: shuffle-orborus - restartPolicy: Always diff --git a/functions/onprem/orborus/proxy_server.py b/functions/onprem/orborus/proxy_server.py deleted file mode 100644 index 8020616c..00000000 --- a/functions/onprem/orborus/proxy_server.py +++ /dev/null @@ -1,54 +0,0 @@ -# -# curl -H "Org-id: Shuffle" --proxy "http://192.168.86.45:8081" http://192.168.86.45:5001/api/v1/workflows/queue - -import SocketServer -import SimpleHTTPServer -import requests -import json - -PORT = 8082 - -class MyProxy(SimpleHTTPServer.SimpleHTTPRequestHandler): - def do_GET(self): - url=self.path[:] - allheaders = {} - for item in ("%s" % self.headers).split("\n"): - headersplit = item.split(":") - if len(headersplit) == 2: - allheaders[headersplit[0]] = (headersplit[1][:-1]).strip() - - ret = requests.get(url, headers=allheaders) - print("RESP (%s) - %d - %s" % (url, ret.status_code, ret.text)) - - self.send_response(ret.status_code) - self.end_headers() - self.wfile.write(ret.text) - - def do_POST(self): - url=self.path[:] - allheaders = {} - for item in ("%s" % self.headers).split("\n"): - headersplit = item.split(":") - if len(headersplit) == 2: - allheaders[headersplit[0]] = (headersplit[1][:-1]).strip() - - length = int(self.headers.getheader('content-length')) - try: - message = json.loads(self.rfile.read(length)) - print("Got message: %s" % message) - ret = requests.post(url, headers=allheaders, json=message) - except: - message = self.rfile.read(length) - print("Got message: %s" % message) - ret = requests.post(url, headers=allheaders, data=message) - - print("RESP (%s) - %d - %s" % (url, ret.status_code, ret.text)) - - self.send_response(ret.status_code) - self.end_headers() - self.wfile.write(ret.text) - -httpd = SocketServer.ForkingTCPServer(('', PORT), MyProxy) -print("Now serving at %d" % PORT) -httpd.serve_forever() - diff --git a/functions/onprem/orborus/run.sh b/functions/onprem/orborus/run.sh deleted file mode 100755 index d71ac59c..00000000 --- a/functions/onprem/orborus/run.sh +++ /dev/null @@ -1,19 +0,0 @@ -#docker run \ -# --env DOCKER_API_VERSION=1.40 \ -# --env ENVIRONMENT_NAME="Shuffle" \ -# --env BASE_URL="http://192.168.86.45:5001" \ -# --env HTTP_PROXY="http://192.168.86.45:8082" \ -# --env HTTPS_PROXY="https://192.168.86.45:8082" \ -# --env SHUFFLE_PASS_WORKER_PROXY=true \ -# --env SHUFFLE_PASS_APP_PROXY=true \ -# -v /var/run/docker.sock:/var/run/docker.sock \ -# ghcr.io/frikky/shuffle-orborus:nightly - -docker run \ - --env DOCKER_API_VERSION=1.40 \ - --env ENVIRONMENT_NAME="Another env" \ - --env ORG="2e7b6a08-b63b-4fc2-bd70-718091509db1" \ - --env AUTH="env auth" \ - --env BASE_URL="https://shuffler.io" \ - -v /var/run/docker.sock:/var/run/docker.sock \ - ghcr.io/frikky/shuffle-orborus:nightly