diff --git a/.env b/.env index 5f507244..f2d7b6fc 100755 --- a/.env +++ b/.env @@ -28,12 +28,12 @@ SHUFFLE_APP_HOTLOAD_FOLDER=./shuffle-apps SHUFFLE_APP_HOTLOAD_LOCATION=./shuffle-apps SHUFFLE_FILE_LOCATION=./shuffle-files -# Encryption modifier. This HAS to be set to encrypt any authentication being used in Shuffle. This is put together with other relevant values to ensure multiple parts are needed to decrypt. +# Encryption modifier. This HAS to be set to encrypt any authentication being used in Shuffle. This is put together with other relevant values to ensure multiple parts are needed to decrypt. # If this key is lost or changed, you will have to reauthenticate all apps. SHUFFLE_ENCRYPTION_MODIFIER= # Other configs -BASE_URL=http://shuffle-backend:5001 +BASE_URL=http://shuffle-backend:5001 SSO_REDIRECT_URL=http://localhost:3001 BACKEND_HOSTNAME=shuffle-backend BACKEND_PORT=5001 @@ -46,7 +46,7 @@ OUTER_HOSTNAME=shuffle-backend DB_LOCATION=./shuffle-database DOCKER_API_VERSION=1.40 -# Orborus/Proxy configurations +# Orborus/Proxy configurations HTTP_PROXY= HTTPS_PROXY= SHUFFLE_PASS_WORKER_PROXY=TRUE @@ -58,10 +58,10 @@ TZ=Europe/Amsterdam # Used to FIND the containername. cgroup v2: issue 501 ORBORUS_CONTAINER_NAME= # Used for setting up a startup delay for Orborus -SHUFFLE_ORBORUS_STARTUP_DELAY= -SHUFFLE_SKIPSSL_VERIFY=true +SHUFFLE_ORBORUS_STARTUP_DELAY= +SHUFFLE_SKIPSSL_VERIFY=true # Used for controlling if the environment should run in kubernetes or not -IS_KUBERNETES=false +IS_KUBERNETES=false #SHUFFLE_BASE_IMAGE_NAME=shuffle #SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io @@ -72,9 +72,9 @@ SHUFFLE_BASE_IMAGE_REPOSITORY=frikky # where they don't want to update http, subflow and shuffle tools again SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY=true -# The eth0 interface inside a container corresponds -# to the virtual Ethernet interface that connects -# the container to the docker0 +# The eth0 interface inside a container corresponds +# to the virtual Ethernet interface that connects +# the container to the docker0 SHUFFLE_SWARM_BRIDGE_DEFAULT_INTERFACE=eth0 # 1500 by default SHUFFLE_SWARM_BRIDGE_DEFAULT_MTU=1500 @@ -84,7 +84,7 @@ SHUFFLE_MEMCACHED= SHUFFLE_CONTAINER_AUTO_CLEANUP=true # The amount of concurrent executions Orborus can handle. This is a soft limit, but it's recommended to keep it low. SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=5 -SHUFFLE_HEALTHCHECK_DISABLED=false +SHUFFLE_HEALTHCHECK_DISABLED=false SHUFFLE_ELASTIC=true SHUFFLE_LOGS_DISABLED=true SHUFFLE_CHAT_DISABLED=false @@ -112,8 +112,9 @@ SHUFFLE_OPENSEARCH_USERNAME="admin" SHUFFLE_OPENSEARCH_PASSWORD="StrongShufflePassword321!" # In use for the first time setup of OpenSearch + backend of Shuffle OPENSEARCH_INITIAL_ADMIN_PASSWORD="StrongShufflePassword321!" # In use for the first time setup of OpenSearch -#Tenzir related +#Tenzir related SHUFFLE_TENZIR_URL= +SHUFFLE_PROTECTED_CLEANUP_DISABLED=true DEBUG_MODE=false diff --git a/.github/workflows/quick-testing.yml b/.github/workflows/quick-testing.yml index cf35967a..20cf6471 100644 --- a/.github/workflows/quick-testing.yml +++ b/.github/workflows/quick-testing.yml @@ -15,142 +15,145 @@ jobs: os: [ubuntu-latest] architecture: [x64, arm64] steps: - - name: Checkout code - uses: actions/checkout@v2 + - name: Checkout code + uses: actions/checkout@v2 - - name: Install Docker Compose - run: | - sudo apt-get update - sudo apt-get install -y docker-compose - docker-compose --version + - name: Shai-Hulud 2.0 Detector + uses: gensecaihq/Shai-Hulud-2.0-Detector@v1.0.0 - - name: Set up opensearch directory - run: chmod -R 777 shuffle-database + - name: Install Docker Compose + run: | + sudo apt-get update + sudo apt-get install -y docker-compose + docker-compose --version - - name: Build the stack - run: docker compose up -d + - name: Set up opensearch directory + run: chmod -R 777 shuffle-database - - name: Wait for 30 seconds - run: sleep 30 + - name: Build the stack + run: docker compose up -d - - name: Check for restarting containers in a loop and fixing perms again - run: | - # echo "Changing permissions on shuffle-database directory again" - # chmod -R 777 shuffle-database + - name: Wait for 30 seconds + run: sleep 30 - ATTEMPTS=30 # Total time = ATTEMPTS * 5 seconds = 30 seconds - for i in $(seq 1 $ATTEMPTS); do - RESTARTING_CONTAINERS=$(docker ps --filter "status=restarting" --format "{{.Names}}") - if [ -n "$RESTARTING_CONTAINERS" ]; then - echo "The following containers are restarting:" - echo "$RESTARTING_CONTAINERS" + - name: Check for restarting containers in a loop and fixing perms again + run: | + # echo "Changing permissions on shuffle-database directory again" + # chmod -R 777 shuffle-database + + ATTEMPTS=30 # Total time = ATTEMPTS * 5 seconds = 30 seconds + for i in $(seq 1 $ATTEMPTS); do + RESTARTING_CONTAINERS=$(docker ps --filter "status=restarting" --format "{{.Names}}") + if [ -n "$RESTARTING_CONTAINERS" ]; then + echo "The following containers are restarting:" + echo "$RESTARTING_CONTAINERS" + exit 1 + fi + echo "No containers are restarting. Attempt $i/$ATTEMPTS." + sleep 1 + done + echo "No containers were found in a restarting state after $ATTEMPTS checks." + + - name: Check if the response from the frontend contains the word "Shuffle" + run: | + RESPONSE=$(curl -s http://localhost:3001) + if echo "$RESPONSE" | grep -q "Shuffle"; then + echo "The word 'Shuffle' was found in the response." + else + echo "The word 'Shuffle' was not found in the response." exit 1 fi - echo "No containers are restarting. Attempt $i/$ATTEMPTS." - sleep 1 - done - echo "No containers were found in a restarting state after $ATTEMPTS checks." + - name: Register a user and check the status code + run: | + MAX_RETRIES=30 + RETRY_INTERVAL=10 + CONTAINER_NAME="shuffle-backend" - - name: Check if the response from the frontend contains the word "Shuffle" - run: | - RESPONSE=$(curl -s http://localhost:3001) - if echo "$RESPONSE" | grep -q "Shuffle"; then - echo "The word 'Shuffle' was found in the response." - else - echo "The word 'Shuffle' was not found in the response." + for (( i=1; i<=$MAX_RETRIES; i++ )) + do + STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" 'http://localhost:3001/api/v1/register' \ + -H 'Accept: */*' \ + -H 'Accept-Language: en-US,en;q=0.9' \ + -H 'Connection: keep-alive' \ + -H 'Content-Type: application/json' \ + --data-raw '{"username":"demo@demo.io","password":"supercoolpassword"}') + + if [ "$STATUS_CODE" -eq 200 ]; then + echo "User registration was successful with status code 200." + exit 0 + elif [ "$STATUS_CODE" -ne 502 ]; then + echo "User registration failed with status code $STATUS_CODE." + exit 1 + fi + + echo "Received status code $STATUS_CODE. Retrying in $RETRY_INTERVAL seconds... ($i/$MAX_RETRIES)" + echo "Fetching last 30 lines of logs from container $CONTAINER_NAME..." + + logs_output=$(docker logs --tail 30 "$CONTAINER_NAME") + echo "$logs_output" + + echo "Fetching last 30 lines of logs from container shuffle-opensearch..." + + opensearch_logs=$(docker logs --tail 30 shuffle-opensearch) + echo "$opensearch_logs" + + sleep $RETRY_INTERVAL + done + + echo "User registration failed after $MAX_RETRIES attempts." exit 1 - fi - - name: Register a user and check the status code - run: | - MAX_RETRIES=30 - RETRY_INTERVAL=10 - CONTAINER_NAME="shuffle-backend" - for (( i=1; i<=$MAX_RETRIES; i++ )) - do - STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" 'http://localhost:3001/api/v1/register' \ - -H 'Accept: */*' \ - -H 'Accept-Language: en-US,en;q=0.9' \ - -H 'Connection: keep-alive' \ - -H 'Content-Type: application/json' \ - --data-raw '{"username":"demo@demo.io","password":"supercoolpassword"}') + - name: Run Selenium testing for frontend + run: | + cd $GITHUB_WORKSPACE/frontend + # write some log to see the current directory + chmod +x frontend-testing.sh + ./frontend-testing.sh - if [ "$STATUS_CODE" -eq 200 ]; then - echo "User registration was successful with status code 200." + - name: Get the API key and run a health check + id: health_check + run: | + RESPONSE=$(curl -s -k -u admin:StrongShufflePassword321! 'https://localhost:9200/users/_search') + echo "Raw Response: $RESPONSE" + + API_KEY=$(echo "$RESPONSE" | jq -r '.hits.hits[0]._source.apikey') + if [ -n "$API_KEY" ] && [ "$API_KEY" != "null" ]; then + echo "Admin API key: $API_KEY" + echo "API_KEY=$API_KEY" >> $GITHUB_ENV + else + echo "Failed to retrieve the API key for the admin user." + exit 1 + fi + + echo "Waiting 1 minute before sending the health API request..." + sleep 60 + + echo "Checking health API..." + HEALTH_RESPONSE=$(curl -s 'http://localhost:3001/api/v1/health?force=true' \ + -H "Authorization: Bearer $API_KEY") + + echo "Health API Response: $HEALTH_RESPONSE" + + WORKFLOWS_CREATE=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.create') + WORKFLOWS_RUN=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run') + WORKFLOWS_RUN_FINISHED=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run_finished') + WORKFLOWS_RUN_STATUS=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run_status') + WORKFLOWS_DELETE=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.delete') + + echo "WORKFLOWS_CREATE: $WORKFLOWS_CREATE" + echo "WORKFLOWS_RUN: $WORKFLOWS_RUN" + echo "WORKFLOWS_RUN_FINISHED: $WORKFLOWS_RUN_FINISHED" + echo "WORKFLOWS_RUN_STATUS: $WORKFLOWS_RUN_STATUS" + echo "WORKFLOWS_DELETE: $WORKFLOWS_DELETE" + + if [ "$WORKFLOWS_CREATE" = "true" ] && [ "$WORKFLOWS_RUN" = "true" ] && [ "$WORKFLOWS_RUN_FINISHED" = "true" ] && [ "$WORKFLOWS_RUN_STATUS" = "FINISHED" ] && [ "$WORKFLOWS_DELETE" = "true" ]; then + echo "Health endpoint check was successful." exit 0 - elif [ "$STATUS_CODE" -ne 502 ]; then - echo "User registration failed with status code $STATUS_CODE." + else + echo "Health check failed." exit 1 fi - echo "Received status code $STATUS_CODE. Retrying in $RETRY_INTERVAL seconds... ($i/$MAX_RETRIES)" - echo "Fetching last 30 lines of logs from container $CONTAINER_NAME..." - - logs_output=$(docker logs --tail 30 "$CONTAINER_NAME") - echo "$logs_output" - - echo "Fetching last 30 lines of logs from container shuffle-opensearch..." - - opensearch_logs=$(docker logs --tail 30 shuffle-opensearch) - echo "$opensearch_logs" - - sleep $RETRY_INTERVAL - done - - echo "User registration failed after $MAX_RETRIES attempts." - exit 1 - - - name: Run Selenium testing for frontend - run: | - cd $GITHUB_WORKSPACE/frontend - # write some log to see the current directory - chmod +x frontend-testing.sh - ./frontend-testing.sh - - - name: Get the API key and run a health check - id: health_check - run: | - RESPONSE=$(curl -s -k -u admin:StrongShufflePassword321! 'https://localhost:9200/users/_search') - echo "Raw Response: $RESPONSE" - - API_KEY=$(echo "$RESPONSE" | jq -r '.hits.hits[0]._source.apikey') - if [ -n "$API_KEY" ] && [ "$API_KEY" != "null" ]; then - echo "Admin API key: $API_KEY" - echo "API_KEY=$API_KEY" >> $GITHUB_ENV - else - echo "Failed to retrieve the API key for the admin user." - exit 1 - fi - - echo "Waiting 1 minute before sending the health API request..." - sleep 60 - - echo "Checking health API..." - HEALTH_RESPONSE=$(curl -s 'http://localhost:3001/api/v1/health?force=true' \ - -H "Authorization: Bearer $API_KEY") - - echo "Health API Response: $HEALTH_RESPONSE" - - WORKFLOWS_CREATE=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.create') - WORKFLOWS_RUN=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run') - WORKFLOWS_RUN_FINISHED=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run_finished') - WORKFLOWS_RUN_STATUS=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run_status') - WORKFLOWS_DELETE=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.delete') - - echo "WORKFLOWS_CREATE: $WORKFLOWS_CREATE" - echo "WORKFLOWS_RUN: $WORKFLOWS_RUN" - echo "WORKFLOWS_RUN_FINISHED: $WORKFLOWS_RUN_FINISHED" - echo "WORKFLOWS_RUN_STATUS: $WORKFLOWS_RUN_STATUS" - echo "WORKFLOWS_DELETE: $WORKFLOWS_DELETE" - - if [ "$WORKFLOWS_CREATE" = "true" ] && [ "$WORKFLOWS_RUN" = "true" ] && [ "$WORKFLOWS_RUN_FINISHED" = "true" ] && [ "$WORKFLOWS_RUN_STATUS" = "FINISHED" ] && [ "$WORKFLOWS_DELETE" = "true" ]; then - echo "Health endpoint check was successful." - exit 0 - else - echo "Health check failed." - exit 1 - fi - notify: needs: build if: failure() @@ -163,7 +166,7 @@ jobs: Repository: ${{ github.repository }} Branch: ${{ github.ref_name }} Workflow URL: $WORKFLOW_URL" - + curl -s -X POST https://api.twilio.com/2010-04-01/Accounts/${{ secrets.TWILIO_ACCOUNT_SID }}/Messages.json \ --data-urlencode "To=${{ secrets.TWILIO_TO_NUMBER }}" \ --data-urlencode "From=${{ secrets.TWILIO_FROM_NUMBER }}" \ diff --git a/backend/Dockerfile b/backend/Dockerfile index 5ccc3de8..2ee3bcf0 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -24,7 +24,7 @@ RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o shufflebackend . # Certificate build - gets required certs FROM alpine:latest as certs -RUN apk add --update ca-certificates +RUN apk add --update --no-scripts ca-certificates # Sets up the final image FROM alpine:3.22.2 diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 42a139aa..cac11ac2 100755 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -21,7 +21,6 @@ import ( "github.com/docker/docker/api/types" //"github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/image" - "github.com/docker/docker/client" newdockerclient "github.com/fsouza/go-dockerclient" "github.com/go-git/go-billy/v5" @@ -35,12 +34,13 @@ import ( "os" "strings" + "time" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "time" // "k8s.io/client-go/tools/clientcmd" // "k8s.io/client-go/util/homedir" ) @@ -206,7 +206,8 @@ func fixTags(tags []string) []string { // Custom Docker image builder wrapper in memory func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string, downloadIfFail bool) error { ctx := context.Background() - client, err := client.NewEnvClient() + // client, err := client.NewEnvClient() + client, _, err := shuffle.GetDockerClient() defer client.Close() if err != nil { log.Printf("Unable to create docker client: %s", err) @@ -493,7 +494,8 @@ func buildImage(tags []string, dockerfileLocation string) error { } ctx := context.Background() - client, err := client.NewEnvClient() + // client, err := client.NewEnvClient() + client, _, err := shuffle.GetDockerClient() defer client.Close() if err != nil { log.Printf("Unable to create docker client: %s", err) @@ -581,7 +583,8 @@ func imageCheckBuilder(images []string) error { return nil ctx := context.Background() - client, err := client.NewEnvClient() + // client, err := client.NewEnvClient() + client, _, err := shuffle.GetDockerClient() if err != nil { log.Printf("Unable to create docker client: %s", err) return err @@ -670,7 +673,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { } //log.Printf("[DEBUG] Image to load: %s", version.Name) - dockercli, err := client.NewEnvClient() + dockercli, _, err := shuffle.GetDockerClient() if err != nil { log.Printf("[WARNING] Unable to create docker client: %s", err) resp.WriteHeader(422) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 8692fb7f..36d955c1 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -26,15 +26,15 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.54 + github.com/shuffle/shuffle-shared v0.9.69 github.com/shuffle/singul v0.0.20 - golang.org/x/crypto v0.40.0 + golang.org/x/crypto v0.45.0 google.golang.org/api v0.236.0 google.golang.org/grpc v1.72.2 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.33.1 - k8s.io/apimachinery v0.33.1 - k8s.io/client-go v0.33.1 + k8s.io/api v0.34.2 + k8s.io/apimachinery v0.34.2 + k8s.io/client-go v0.34.2 ) require ( @@ -70,13 +70,13 @@ require ( github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frikky/schemaless v0.0.24 // indirect - github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/frikky/schemaless v0.0.25 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect github.com/go-logr/logr v1.4.2 // indirect @@ -87,8 +87,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.6.9 // indirect - github.com/google/go-cmp v0.7.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-github/v28 v28.1.1 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.9 // indirect @@ -110,9 +109,10 @@ require ( github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/openai/openai-go/v3 v3.8.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/osteele/liquid v1.7.0 // indirect @@ -132,6 +132,10 @@ require ( github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/zeebo/errs v1.4.0 // indirect @@ -145,13 +149,15 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect go.opentelemetry.io/otel/trace v1.36.0 // indirect go.uber.org/atomic v1.9.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/net v0.41.0 // indirect + golang.org/x/net v0.47.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.34.0 // indirect - golang.org/x/term v0.33.0 // indirect - golang.org/x/text v0.27.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/term v0.37.0 // indirect + golang.org/x/text v0.31.0 // indirect golang.org/x/time v0.11.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect @@ -163,10 +169,10 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect - k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect - sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index ea6056ba..9790290b 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -138,8 +138,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.24 h1:1bNg96gm82yLbRlDS0haQH/96X3GlCAEuTCSasXIJZE= -github.com/frikky/schemaless v0.0.24/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= +github.com/frikky/schemaless v0.0.25 h1:qXjrKT54LWl3tzDFIUmevZ86F7FiusPpQKfa40xxNl8= +github.com/frikky/schemaless v0.0.25/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= github.com/fsouza/go-dockerclient v1.12.1 h1:FMoLq+Zhv9Oz/rFmu6JWkImfr6CBgZOPcL+bHW4gS0o= github.com/fsouza/go-dockerclient v1.12.1/go.mod h1:OqsgJJcpCwqyM3JED7TdfM9QVWS5O7jSYwXxYKmOooY= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= @@ -344,8 +344,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU= github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0= -github.com/shuffle/shuffle-shared v0.9.53 h1:ykziiRA82FZX0YiuErSmFLQMxvAFi6yUajgRjd3J5xE= -github.com/shuffle/shuffle-shared v0.9.53/go.mod h1:g03P/ZM8pUwBbdxW0l9A1siMMuemEQshgDpO9hF23eE= +github.com/shuffle/shuffle-shared v0.9.60 h1:8NiovcsSsVX8i0UHmbFBANjUQ/u+4PDDpJXg9JfMAUo= +github.com/shuffle/shuffle-shared v0.9.60/go.mod h1:vK6t1WY5Nfg5vOAk6taT788jIGKs6/4iN1d8Argyn4o= github.com/shuffle/singul v0.0.20 h1:Lz+K4l2GJQ5W6o8ePr/2qlIcGuZuMC7W8Wj/JFw1XDI= github.com/shuffle/singul v0.0.20/go.mod h1:qY3ZmwaNwmRApLWMTqZaNEMXO6X8bjr/zY44xDU62hQ= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 5f54465a..ba05049a 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3,7 +3,7 @@ package main import ( uuid "github.com/satori/go.uuid" "github.com/shuffle/shuffle-shared" - "github.com/shuffle/singul/pkg" + singul "github.com/shuffle/singul/pkg" "net/http/pprof" @@ -44,11 +44,12 @@ import ( "github.com/go-git/go-git/v5/storage/memory" // Random + "sort" + xj "github.com/basgys/goxml2json" newscheduler "github.com/carlescere/scheduler" "golang.org/x/crypto/bcrypt" "gopkg.in/yaml.v3" - "sort" // Web "github.com/gorilla/mux" @@ -1312,7 +1313,9 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { // Should run calculations if len(org.SSOConfig.OpenIdAuthorization) > 0 { baseSSOUrl = shuffle.GetOpenIdUrl(request, *org) - + if err != nil { + log.Printf("[ERROR] Failed getting OpenID URL for org %s: %s", org.Name, err) + } break } @@ -5433,7 +5436,7 @@ func initHandlers() { r.HandleFunc("/api/v1/users/getsettings", shuffle.HandleSettings).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/users/getusers", shuffle.HandleGetUsers).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/users/updateuser", shuffle.HandleUpdateUser).Methods("PUT", "OPTIONS") - r.HandleFunc("/api/v1/users/{userID}/remove", shuffle.HandleDeleteUsersAccount).Methods("DELETE", "OPTIONS") + // r.HandleFunc("/api/v1/users/{userID}/remove", shuffle.HandleDeleteUsersAccount).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/users/{user}", shuffle.DeleteUser).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/users/passwordchange", shuffle.HandlePasswordChange).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/users/{key}/get2fa", shuffle.HandleGet2fa).Methods("GET", "OPTIONS") @@ -5657,7 +5660,7 @@ func initHandlers() { // Docker orborus specific - downloads an image r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "GET", "OPTIONS") - r.HandleFunc("/api/v1/login_sso", shuffle.HandleSSO).Methods("GET", "POST", "OPTIONS") + r.HandleFunc("/api/v1/login_sso", shuffle.HandleSAML).Methods("GET", "POST", "OPTIONS") r.HandleFunc("/api/v1/login_openid", shuffle.HandleOpenId).Methods("GET", "POST", "OPTIONS") // Important for email, IDS etc. Create this by: diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 0c015470..934bad62 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -11,6 +11,7 @@ import ( "io" "io/ioutil" "log" + //"math/rand" "net/http" "net/url" @@ -21,7 +22,6 @@ import ( "time" "github.com/docker/docker/api/types/image" - dockerclient "github.com/docker/docker/client" "github.com/h2non/filetype" uuid "github.com/satori/go.uuid" @@ -2997,6 +2997,8 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. "email", "shuffle-ai", "shuffle-subflow", + "yara", + "sigma", } // It's here to prevent getting them in every iteration @@ -3393,6 +3395,8 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. fmt.Sprintf("/apps"), orgId, false, + "HIGH", + "APP_BUID", ) } @@ -3527,7 +3531,8 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { // As it's not even Docker if tmpBody.ForceUpdate { - dockercli, err := dockerclient.NewEnvClient() + // dockercli, err := dockerclient.NewEnvClient() + dockercli, _, err := shuffle.GetDockerClient() if err == nil { appSdk := os.Getenv("SHUFFLE_APP_SDK_VERSION") diff --git a/docker-compose.yml b/docker-compose.yml index 0839bc4c..ef0abd31 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,7 +50,7 @@ services: - HTTPS_PROXY=${HTTPS_PROXY} - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY} - - SHUFFLE_APP_REPLICAS=${SHUFFLE_APP_REPLICAS} + # - SHUFFLE_APP_REPLICAS=${SHUFFLE_APP_REPLICAS} - SHUFFLE_STATS_DISABLED=true - SHUFFLE_LOGS_DISABLED=true - SHUFFLE_SWARM_CONFIG=run @@ -80,7 +80,7 @@ services: soft: -1 hard: -1 nofile: - soft: 65536 + soft: 65536 hard: 65536 volumes: - shuffle-database:/usr/share/opensearch/data:z @@ -127,7 +127,7 @@ services: #docker-socket-proxy: # image: tecnativa/docker-socket-proxy - # container_name: docker-socket-proxy + # container_name: docker-socket-proxy # hostname: docker-socket-proxy # privileged: true # environment: @@ -167,4 +167,4 @@ networks: driver: bridge # driver_opts: # com.docker.network.driver.mtu: 1460 - # uncomment to set MTU for swarm mode. MTU should be whatever is your host's preferred MTU is: https://shuffler.io/docs/troubleshooting#TLS_timeout_error/Timeout_Errors/EOF_Errors + # uncomment to set MTU for swarm mode. MTU should be whatever is your host's preferred MTU is: https://shuffler.io/docs/troubleshooting#TLS_timeout_error/Timeout_Errors/EOF_Errors diff --git a/frontend/package.json b/frontend/package.json index 8cd277bc..229e5635 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -26,10 +26,13 @@ "cpx": "^1.5.0", "create-react-app": "^5.0.1", "cytoscape": "^3.29.2", + "cytoscape-bubblesets": "^4.1.0", "cytoscape-clipboard": "^2.2.1", + "cytoscape-cose-bilkent": "^4.1.0", "cytoscape-cxtmenu": "^3.4.0", "cytoscape-edgehandles": "^3.5.1", "cytoscape-grid-guide": "~2.3.3", + "cytoscape-layers": "^3.1.0", "cytoscape-node-html-label": "^1.2.2", "cytoscape-panzoom": "^2.5.3", "cytoscape-undo-redo": "^1.3.3", diff --git a/frontend/src/components/LeftSideBar.jsx b/frontend/src/components/LeftSideBar.jsx index 5d908414..5245c9bc 100644 --- a/frontend/src/components/LeftSideBar.jsx +++ b/frontend/src/components/LeftSideBar.jsx @@ -127,6 +127,8 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => { setCurrentSelectedTheme(userdata?.theme); } }, [userdata]); + + const CustomPopper = (props) => { return ( @@ -1949,7 +1951,7 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => { } - {!isCloud ? ( + {!isCloud && activeOrgData?.old_org? (
{ if (!content || typeof content !== 'string' || content.trim().length === 0) { return content; } - + try { // Check if content looks like JSON (starts with { or [) const trimmedContent = content.trim(); @@ -165,7 +165,7 @@ const CodeEditor = (props) => { return IndentJsonLikeString(content, 2); } } - + // Return original content if it doesn't look like JSON return content; } catch (error) { @@ -179,7 +179,7 @@ const CodeEditor = (props) => { const [localcodedata, setlocalcodedata] = React.useState(codedata === undefined || codedata === null || codedata.length === 0 ? "" : codedata); // const {codelang, setcodelang} = props - const {themeMode, supportEmail} = useContext(Context) + const { themeMode, supportEmail } = useContext(Context) const theme = getTheme(themeMode) const [validation, setValidation] = React.useState(false); @@ -370,9 +370,9 @@ const CodeEditor = (props) => { setMainVariables(tmpVariables) } - const handleKeyDown = (event) => { - if ((event.metaKey || event.ctrlKey) && event.key === 'Enter') { - event.preventDefault() + const handleKeyDown = (event) => { + if ((event.metaKey || event.ctrlKey) && event.key === 'Enter') { + event.preventDefault() const tryItButton = document.getElementById("try-it-button") if (tryItButton !== undefined && tryItButton !== null) { tryItButton.click() @@ -382,7 +382,7 @@ const CodeEditor = (props) => { // Remove the original useEffect for actionlist since we'll update on action/trigger changes useEffect(() => { - document.addEventListener("keydown", handleKeyDown) + document.addEventListener("keydown", handleKeyDown) updateAvailableVariables(actionlist) }, []) @@ -972,7 +972,11 @@ const CodeEditor = (props) => { // Resolve the child paths/variables first, // So that "$a.b.c" is handled before "$a.b" and then "$a" // Replace "$parent.child" before "$parent" so "$parent.child" doesn't become "parentValue.child" (which can't be matched later). - found.sort((a, b) => b.length - a.length) + console.log("FOUND PROD BEFORE SORT:", found) + found.sort((a, b) => { + return Number(b.length) - Number(a.length) + }) + console.log("FOUND PROD AFTER SORT:", found) //console.log("FOUND: ", found) try { @@ -983,19 +987,19 @@ const CodeEditor = (props) => { var valuefound = false - // First check if it's a workflow variable - if (actionlist !== undefined && actionlist.length > 0) { - const workflowVar = actionlist?.find(item => - item.type === "workflow_variable" && + // First check if it's a workflow variable + if (actionlist !== undefined && actionlist.length > 0) { + const workflowVar = actionlist?.find(item => + item.type === "workflow_variable" && `$${item.autocomplete.toLowerCase()}` === fixedVariable.toLowerCase() ) - + if (workflowVar && workflowVar.example) { valuefound = true try { // Try to parse the example value if it's stored as a JSON string - if (typeof workflowVar.example === "string" && - (workflowVar.example.startsWith("[") || workflowVar.example.startsWith("{"))) { + if (typeof workflowVar.example === "string" && + (workflowVar.example.startsWith("[") || workflowVar.example.startsWith("{"))) { const parsedExample = JSON.parse(workflowVar.example) input = input.replace(found[i], JSON.stringify(parsedExample), -1) } else { @@ -1029,7 +1033,7 @@ const CodeEditor = (props) => { foundlocation = j break } - + // FIXME: There is something wrong here with: // $variable.# // vs @@ -1069,7 +1073,7 @@ const CodeEditor = (props) => { const replacedSlice = input.slice(foundlocation, input.length).replace(fixedVariable, newvalue, 1) input = input.slice(0, foundlocation) + replacedSlice } - } + } } catch (e) { console.log("Replace error: ", e) } @@ -1082,26 +1086,26 @@ const CodeEditor = (props) => { var shouldbreak = false for (var k = 0; k < actionlist.length; k++) { var parsedPaths = [] - - // Handle both workflow variables and regular actions - if (actionlist[k].type === "workflow_variable") { - // Try to parse the value if it's a string that could be JSON - if (typeof actionlist[k].value === "string") { - try { - const parsedValue = JSON.parse(actionlist[k].value) - if (typeof parsedValue === "object") { - parsedPaths = GetParsedPaths(parsedValue, ""); - } - } catch (e) { - // Not valid JSON, use the value directly - parsedPaths = GetParsedPaths(actionlist[k].value, ""); + + // Handle both workflow variables and regular actions + if (actionlist[k].type === "workflow_variable") { + // Try to parse the value if it's a string that could be JSON + if (typeof actionlist[k].value === "string") { + try { + const parsedValue = JSON.parse(actionlist[k].value) + if (typeof parsedValue === "object") { + parsedPaths = GetParsedPaths(parsedValue, ""); } - } else if (typeof actionlist[k].value === "object") { + } catch (e) { + // Not valid JSON, use the value directly parsedPaths = GetParsedPaths(actionlist[k].value, ""); } - } else if (typeof actionlist[k].example === "object") { - parsedPaths = GetParsedPaths(actionlist[k].example, ""); - } + } else if (typeof actionlist[k].value === "object") { + parsedPaths = GetParsedPaths(actionlist[k].value, ""); + } + } else if (typeof actionlist[k].example === "object") { + parsedPaths = GetParsedPaths(actionlist[k].example, ""); + } for (var key in parsedPaths) { const fullpath = "$" + actionlist[k].autocomplete.toLowerCase() + parsedPaths[key].autocomplete.toLowerCase() @@ -1115,22 +1119,22 @@ const CodeEditor = (props) => { var new_input = "" try { - const sourceData = actionlist[k].type === "workflow_variable" ? - (() => { - // Try to parse the value if it's a JSON string - if (typeof actionlist[k].value === "string") { - try { - return JSON.parse(actionlist[k].value); - } catch (e) { - // If parsing fails, return the original string value - return actionlist[k].value; + const sourceData = actionlist[k].type === "workflow_variable" ? + (() => { + // Try to parse the value if it's a JSON string + if (typeof actionlist[k].value === "string") { + try { + return JSON.parse(actionlist[k].value); + } catch (e) { + // If parsing fails, return the original string value + return actionlist[k].value; + } } - } - return actionlist[k].value; - })() : - actionlist[k].example; - - new_input = FindJsonPath(fullpath, sourceData) + return actionlist[k].value; + })() : + actionlist[k].example; + + new_input = FindJsonPath(fullpath, sourceData) } catch (e) { console.log("ERR IN INPUT: ", e) } @@ -1437,85 +1441,85 @@ const CodeEditor = (props) => { return ( { - e.preventDefault() - // Check if inside div.ace_content - if (e.srcElement.className === "ace_content") { - // Input on the correct line. Each line is: - // Show some tooltip at mouse cursor that shows "Insert Action" + style={{ + position: "absolute", + zIndex: 15000, + }} + onDrag={(e) => { + e.preventDefault() + // Check if inside div.ace_content + if (e.srcElement.className === "ace_content") { + // Input on the correct line. Each line is: + // Show some tooltip at mouse cursor that shows "Insert Action" - // Append the text to the DOM - } else { - //console.log("PAGEX: ", e.pageX, e.pageY) - //console.log("OffsetX: ", e.offsetX, e.offsetY) - //console.log("E: ", e) - } + // Append the text to the DOM + } else { + //console.log("PAGEX: ", e.pageX, e.pageY) + //console.log("OffsetX: ", e.offsetX, e.offsetY) + //console.log("E: ", e) + } - if (!dragging) { - setDragging(true) - } - }} - onStop={(e) => { - if (e.srcElement.className === "ace_content") { - console.log("DRAG STOP IN CONTENT!", e.srcElement.className) + if (!dragging) { + setDragging(true) + } + }} + onStop={(e) => { + if (e.srcElement.className === "ace_content") { + console.log("DRAG STOP IN CONTENT!", e.srcElement.className) - const usedposition = e.offsetY - if (usedposition === undefined || usedposition === null) { - toast.info(`Error: LayerY is undefined or null. Please contact ${supportEmail}`) - return - } + const usedposition = e.offsetY + if (usedposition === undefined || usedposition === null) { + toast.info(`Error: LayerY is undefined or null. Please contact ${supportEmail}`) + return + } - if (usedposition === 0) { - usedposition = 1 - } + if (usedposition === 0) { + usedposition = 1 + } - const lineheight = 15 - const codedatasplit = localcodedata.split('\n') - if (codedatasplit === undefined || codedatasplit === null || codedatasplit.length === 0) { - return - } + const lineheight = 15 + const codedatasplit = localcodedata.split('\n') + if (codedatasplit === undefined || codedatasplit === null || codedatasplit.length === 0) { + return + } - // Int - const lineposition = parseInt(usedposition/lineheight) + // Int + const lineposition = parseInt(usedposition / lineheight) - // Find the correct line - if (lineposition > codedatasplit.length) { - codedatasplit[codedatasplit.length-1] += value - } else { - codedatasplit[lineposition] += value - } + // Find the correct line + if (lineposition > codedatasplit.length) { + codedatasplit[codedatasplit.length - 1] += value + } else { + codedatasplit[lineposition] += value + } - //e.srcElement.layerY - setlocalcodedata(codedatasplit.join('\n')) - } + //e.srcElement.layerY + setlocalcodedata(codedatasplit.join('\n')) + } - setDragging(false) - }} - dragging={dragging} - position={{ - x: 0, - y: 0, - }} - onMouseHover={() => { - setHovering(true) - }} - onMouseLeave={() => { - setHovering(false) - }} + setDragging(false) + }} + dragging={dragging} + position={{ + x: 0, + y: 0, + }} + onMouseHover={() => { + setHovering(true) + }} + onMouseLeave={() => { + setHovering(false) + }} > -
{value} @@ -1525,56 +1529,56 @@ const CodeEditor = (props) => { } const IndentJsonLikeString = (input, indentSize = 2) => { - const indent = ' '.repeat(indentSize); - let level = 0; - let inString = false; - let escapeNext = false; - let result = ''; + const indent = ' '.repeat(indentSize); + let level = 0; + let inString = false; + let escapeNext = false; + let result = ''; - for (let i = 0; i < input.length; i++) { - let char = input[i]; + for (let i = 0; i < input.length; i++) { + let char = input[i]; - if (escapeNext) { - result += char; - escapeNext = false; - continue; + if (escapeNext) { + result += char; + escapeNext = false; + continue; + } + + if (char === '\\') { + escapeNext = true; + result += char; + continue; + } + + if (char === '"') { + inString = !inString; + result += char; + continue; + } + + if (!inString) { + if (char === '{' || char === '[') { + result += char + '\n' + indent.repeat(++level); + continue; + } else if (char === '}' || char === ']') { + result += '\n' + indent.repeat(--level) + char; + continue; + } else if (char === ',') { + result += char + '\n' + indent.repeat(level); + continue; + } else if (char === ':') { + result += ': '; + continue; + } else if (char === ' ' || char === '\t' || char === '\n' || char === '\r') { + // Skip whitespace characters when not in string + continue; + } + } + + result += char; } - if (char === '\\') { - escapeNext = true; - result += char; - continue; - } - - if (char === '"') { - inString = !inString; - result += char; - continue; - } - - if (!inString) { - if (char === '{' || char === '[') { - result += char + '\n' + indent.repeat(++level); - continue; - } else if (char === '}' || char === ']') { - result += '\n' + indent.repeat(--level) + char; - continue; - } else if (char === ',') { - result += char + '\n' + indent.repeat(level); - continue; - } else if (char === ':') { - result += ': '; - continue; - } else if (char === ' ' || char === '\t' || char === '\n' || char === '\r') { - // Skip whitespace characters when not in string - continue; - } - } - - result += char; - } - - return result; + return result; } @@ -1585,16 +1589,16 @@ const CodeEditor = (props) => { return ( -
+
{ setExpanded(!expanded) }}> - + {innerdata?.name}
@@ -1622,8 +1626,8 @@ const CodeEditor = (props) => { aria-labelledby="draggable-dialog-title" // disableBackdropClick={true} disableEnforceFocus={true} - style={{ - pointerEvents: "none", + style={{ + pointerEvents: "none", zIndex: activeDialog === "codeeditor" ? 1200 : 1100, }} hideBackdrop={true} @@ -1673,7 +1677,7 @@ const CodeEditor = (props) => { : null} - {fullScreenModeEnabled ? null : + {fullScreenModeEnabled ? null : {
- {sourceDataOpen ? + {sourceDataOpen ?
Source Data @@ -1781,7 +1785,7 @@ const CodeEditor = (props) => { }; const handleActionHover = (inside, actionId) => { - }; + }; const handleMouseover = () => { if (innerdata.type === "Execution Argument") { @@ -1802,20 +1806,20 @@ const CodeEditor = (props) => { var parsedPaths = []; if (innerdata.type === "workflow_variable") { // Try to parse the value if it's a string that could be JSON - if (typeof innerdata.value === "string") { + if (typeof innerdata.value === "string") { try { - const parsedValue = JSON.parse(innerdata.value) - if (typeof parsedValue === "object") { - parsedPaths = GetParsedPaths(parsedValue, ""); - } + const parsedValue = JSON.parse(innerdata.value) + if (typeof parsedValue === "object") { + parsedPaths = GetParsedPaths(parsedValue, ""); + } } catch (e) { - // Not valid JSON, use the value directly - parsedPaths = GetParsedPaths(innerdata.value, ""); + // Not valid JSON, use the value directly + parsedPaths = GetParsedPaths(innerdata.value, ""); } - } else if (typeof innerdata.value === "object") { + } else if (typeof innerdata.value === "object") { parsedPaths = GetParsedPaths(innerdata.value, ""); - } - } else if (typeof innerdata.example === "object") { + } + } else if (typeof innerdata.example === "object") { parsedPaths = GetParsedPaths(innerdata.example, ""); } @@ -1830,7 +1834,7 @@ const CodeEditor = (props) => { ) })}
- : null} + : null}
{isFileEditor ? @@ -1858,7 +1862,7 @@ const CodeEditor = (props) => { height: 50, width: 50, right: 25, - top: 90, + top: 90, zIndex: 5000, }} disabled={localcodedata === undefined || localcodedata === null || localcodedata.length === 0} @@ -1902,7 +1906,7 @@ const CodeEditor = (props) => { {isFileEditor || isWorkflowEditor ? null :
- {userdata !== undefined && userdata !== null && userdata.support === true ? + {userdata !== undefined && userdata !== null && userdata.support === true ? - : null} + : null}
} -
{ onDrop={(e) => { console.log("DROP: ", e) }} - > + > {(availableVariables !== undefined && availableVariables !== null && availableVariables.length > 0) || isFileEditor || isWorkflowEditor ? ( { backgroundColor: "rgba(40,40,40,1)", zIndex: activeDialog === "codeeditor" ? 1200 : 1100, - + }} onLoad={(editor) => { highlight_variables(localcodedata) @@ -2465,12 +2469,12 @@ const CodeEditor = (props) => {
{isFileEditor || isWorkflowEditor ? null : -
@@ -2486,59 +2490,59 @@ const CodeEditor = (props) => { }} >
- {actionId === null && triggerId === null ? -
- {`Condition ${selectedEdge?.conditions?.findIndex(cond => cond.condition.id === conditionId) + 1 || "0"}`} - {/* Source node image */} - {selectedEdge?.source ? - Source - : null - } + {actionId === null && triggerId === null ? +
+ {`Condition ${selectedEdge?.conditions?.findIndex(cond => cond.condition.id === conditionId) + 1 || "0"}`} + {/* Source node image */} + {selectedEdge?.source ? + Source + : null + } - {/* Add arrow icon */} - { - selectedEdge && Object.keys(selectedEdge).length > 0 ? - 0 ? + : null - } + } - {/* Destination node image */} - {selectedEdge?.target ? - Destination - : null - } -
- : - - {selectedAction.name === "execute_python" || selectedAction.name === "execute_bash" ? - "Code to run" : - triggerId ? + {/* Destination node image */} + {selectedEdge?.target ? + Destination + : null + } +
+ : + + {selectedAction.name === "execute_python" || selectedAction.name === "execute_bash" ? + "Code to run" : + triggerId ? `Output: ${triggerName?.replaceAll("_", " ").slice(0, 1).toUpperCase() + triggerName?.replaceAll("_", " ").slice(1)} (${triggerField})` : `Output: ${appName?.replaceAll("_", " ").slice(0, 1).toUpperCase() + appName?.replaceAll("_", " ").slice(1)} (${fieldName})` } @@ -2570,7 +2574,7 @@ const CodeEditor = (props) => { backdropFilter: "blur(8px)", boxShadow: "0 4px 6px rgba(0, 0, 0, 0.1), 0 1px 3px rgba(0, 0, 0, 0.08)", transition: "all 0.2s ease", - paddingRight: 20, + paddingRight: 20, color: "#FF8544", borderRadius: theme.palette?.borderRadius, "&:hover": { @@ -2592,18 +2596,18 @@ const CodeEditor = (props) => { - {selectedAction === undefined ? Try it : selectedAction.name === "execute_python" ? "Run Python Code" : selectedAction.name === "execute_bash" ? "Run Bash" : "Try it"} - - Ctrl + - + {selectedAction === undefined ? Try it : selectedAction.name === "execute_python" ? "Run Python Code" : selectedAction.name === "execute_bash" ? "Run Bash" : "Try it"} + + Ctrl + + } @@ -2713,7 +2717,7 @@ const CodeEditor = (props) => { Output is based on the last VALID run of the node(s) you are referencing. Refresh the page to get updated Variable values.  {selectedAction?.name === "execute_python" ? "For Python: exit() to stop a python script ANYWHERE." - : null} + : null} No test output yet. @@ -2735,7 +2739,7 @@ const CodeEditor = (props) => {
-
+
} - {!isCloud ? ( + {!isCloud && currentOrg?.old_org ? (
+ +## GKE testing +For testing on GKE, you can use the following command: + +```bash +PROJECT_NAME="project" +CLUSTER_NAME="cluster" + +# feel free to change the region and machine type +gcloud container clusters create "$CLUSTER_NAME" \ + --project "$PROJECT_NAME" \ + --region "asia-south2" \ + --release-channel "regular" \ + --machine-type "e2-standard-2" \ + --num-nodes "1" \ + --node-locations "asia-south2-b,asia-south2-a,asia-south2-c" + +# given that your gcloud cli is connected with kubectl +kubectl config set-context $CLUSTER_NAME + +# apply shuffle k8s helm chart +helm install shuffle oci://ghcr.io/shuffle/charts/shuffle \ + --version 2.1.1 \ + --namespace shuffle \ + --create-namespace + +# verify +kubectl get pods -n shuffle +``` diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index f8c8aeff..6e1b6b8f 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -1558,15 +1558,6 @@ app: ## annotations: {} ## @param app.serviceAccount.automountServiceAccountToken Automount service account token for the app service account - ## NOTE: You likely want to allow access to cluster-proxies, e.g: - ## extraEgress: - ## - to: - ## - namespaceSelector: - ## matchLabels: - ## kubernetes.io/metadata.name: istio-system - ## podSelector: - ## matchLabels: - ## istio: pilot ## automountServiceAccountToken: true ## @param app.serviceAccount.imagePullSecrets Add image pull secrets to the app service account diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 22036ec1..d8966001 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,9 +10,9 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.53 - k8s.io/api v0.33.1 - k8s.io/apimachinery v0.33.1 + github.com/shuffle/shuffle-shared v0.9.69 + k8s.io/api v0.34.2 + k8s.io/apimachinery v0.34.2 ) require ( @@ -52,14 +52,14 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/kin-openapi v0.42.0 // indirect - github.com/frikky/schemaless v0.0.24 // indirect - github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/frikky/schemaless v0.0.25 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.6.2 // indirect @@ -73,8 +73,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.6.9 // indirect - github.com/google/go-cmp v0.7.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-github/v28 v28.1.1 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.9 // indirect @@ -89,8 +88,9 @@ require ( github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/sequential v0.6.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/openai/openai-go/v3 v3.8.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/osteele/liquid v1.7.0 // indirect @@ -106,8 +106,12 @@ require ( github.com/shuffle/opensearch-go/v4 v4.0.0 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/zeebo/errs v1.4.0 // indirect @@ -121,14 +125,16 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect go.opentelemetry.io/otel/trace v1.36.0 // indirect go.opentelemetry.io/proto/otlp v1.5.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.40.0 // indirect - golang.org/x/net v0.41.0 // indirect + golang.org/x/crypto v0.45.0 // indirect + golang.org/x/net v0.47.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.34.0 // indirect - golang.org/x/term v0.33.0 // indirect - golang.org/x/text v0.27.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/term v0.37.0 // indirect + golang.org/x/text v0.31.0 // indirect golang.org/x/time v0.11.0 // indirect google.golang.org/api v0.236.0 // indirect google.golang.org/appengine v1.6.8 // indirect @@ -142,12 +148,12 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/client-go v0.33.1 // indirect + k8s.io/client-go v0.34.2 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect - k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect - sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 5d274a17..75f8854d 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -128,8 +128,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.24 h1:1bNg96gm82yLbRlDS0haQH/96X3GlCAEuTCSasXIJZE= -github.com/frikky/schemaless v0.0.24/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= +github.com/frikky/schemaless v0.0.25 h1:qXjrKT54LWl3tzDFIUmevZ86F7FiusPpQKfa40xxNl8= +github.com/frikky/schemaless v0.0.25/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -309,8 +309,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU= github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0= -github.com/shuffle/shuffle-shared v0.9.53 h1:ykziiRA82FZX0YiuErSmFLQMxvAFi6yUajgRjd3J5xE= -github.com/shuffle/shuffle-shared v0.9.53/go.mod h1:g03P/ZM8pUwBbdxW0l9A1siMMuemEQshgDpO9hF23eE= +github.com/shuffle/shuffle-shared v0.9.61 h1:osyJqgRx68m9l+kssB1fpHA4pmkNXK4ZAVVEHL5dW3Q= +github.com/shuffle/shuffle-shared v0.9.61/go.mod h1:vK6t1WY5Nfg5vOAk6taT788jIGKs6/4iN1d8Argyn4o= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index b5bcf486..9a70a774 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -136,11 +136,34 @@ var window = shuffle.NewTimeWindow(1 * time.Minute) func init() { var err error - dockercli, err = dockerclient.NewEnvClient() + // dockercli, err = dockerclient.NewEnvClient() + dockercli, dockerApiVersion, err = shuffle.GetDockerClient() if err != nil { log.Printf("Unable to create docker client: %s", err) } + if os.Getenv("SHUFFLE_EC2_INSTANCE") == "true" { + log.Printf("[INFO] Detected AWS EC2 instance. Setting up Docker Swarm with AWS optimizations.") + containers, err := dockercli.ContainerList(context.Background(), container.ListOptions{}) + if err == nil { + for _, container := range containers { + if strings.Contains(container.Image, "shuffle-orborus") { + if len(container.Names) != 0 { + if strings.Contains(container.Names[0], "shuffle-orborus") { + containerName = container.Names[0] + containerName = strings.TrimPrefix(containerName, "/") + os.Setenv("ORBORUS_CONTAINER_NAME", containerName) + log.Printf("[DEBUG] Found orborus container name: %s", containerName) + break + } + } + } + } + } else { + log.Printf("[ERROR] Failed to find orborus container: %s", err) + } + } + getThisContainerId() if len(pipelineApikey) == 0 { @@ -1426,6 +1449,9 @@ func deployK8sWorker(image string, identifier string, env []string) error { } replicaNumberInt32 := int32(replicaNumber) + // worker makes authenticated requests to the k8s api to create app deployments. + // Therefore, it needs to have access to the service account token. + automountServiceAccountToken := true deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -1445,9 +1471,10 @@ func deployK8sWorker(image string, identifier string, env []string) error { Containers: []corev1.Container{ containerAttachment, }, - DNSPolicy: corev1.DNSClusterFirst, - ServiceAccountName: workerServiceAccountName, - SecurityContext: podSecurityContext, + DNSPolicy: corev1.DNSClusterFirst, + ServiceAccountName: workerServiceAccountName, + AutomountServiceAccountToken: &automountServiceAccountToken, + SecurityContext: podSecurityContext, }, }, }, @@ -2411,6 +2438,7 @@ func main() { } if swarmConfig == "run" || swarmConfig == "swarm" || isKubernetes == "true" { + if isKubernetes != "true" { checkSwarmService(ctx) } @@ -2745,6 +2773,8 @@ func main() { fmt.Sprintf("/detections/Sigma"), org, true, + "LOW", + "TENZIR_START", ) if err != nil { @@ -2755,6 +2785,10 @@ func main() { } } else { + if debug { + log.Printf("[DEBUG] Passing execution ID request to normal queue: %#v", incRequest.ExecutionId) + } + newrequests = append(newrequests, incRequest) } } @@ -4289,6 +4323,10 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, Timeout: time.Duration(120 * time.Second), } + if debug { + log.Printf("[DEBUG][%s] Worker request to be sent to URL: %s", workflowExecution.ExecutionId, streamUrl) + } + req, err := http.NewRequest( "POST", streamUrl, diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 4dbcd59c..314250a0 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -11,11 +11,11 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.53 + github.com/shuffle/shuffle-shared v0.9.69 github.com/shuffle/singul v0.0.20 - k8s.io/api v0.33.1 - k8s.io/apimachinery v0.33.1 - k8s.io/client-go v0.33.1 + k8s.io/api v0.34.2 + k8s.io/apimachinery v0.34.2 + k8s.io/client-go v0.34.2 ) require ( @@ -54,14 +54,14 @@ require ( github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/kin-openapi v0.42.0 // indirect - github.com/frikky/schemaless v0.0.24 // indirect - github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/frikky/schemaless v0.0.25 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.6.2 // indirect @@ -75,8 +75,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.6.9 // indirect - github.com/google/go-cmp v0.7.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-github/v28 v28.1.1 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.9 // indirect @@ -91,8 +90,9 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/openai/openai-go/v3 v3.8.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/osteele/liquid v1.7.0 // indirect @@ -110,6 +110,10 @@ require ( github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/zeebo/errs v1.4.0 // indirect @@ -121,14 +125,16 @@ require ( go.opentelemetry.io/otel/sdk v1.36.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect go.opentelemetry.io/otel/trace v1.36.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.40.0 // indirect - golang.org/x/net v0.41.0 // indirect + golang.org/x/crypto v0.45.0 // indirect + golang.org/x/net v0.47.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.34.0 // indirect - golang.org/x/term v0.33.0 // indirect - golang.org/x/text v0.27.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/term v0.37.0 // indirect + golang.org/x/text v0.31.0 // indirect golang.org/x/time v0.11.0 // indirect google.golang.org/api v0.236.0 // indirect google.golang.org/appengine v1.6.8 // indirect @@ -143,10 +149,10 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect - k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect - sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index 81a8f2d2..6f6f3181 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -128,8 +128,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.24 h1:1bNg96gm82yLbRlDS0haQH/96X3GlCAEuTCSasXIJZE= -github.com/frikky/schemaless v0.0.24/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= +github.com/frikky/schemaless v0.0.25 h1:qXjrKT54LWl3tzDFIUmevZ86F7FiusPpQKfa40xxNl8= +github.com/frikky/schemaless v0.0.25/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -311,8 +311,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU= github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0= -github.com/shuffle/shuffle-shared v0.9.53 h1:ykziiRA82FZX0YiuErSmFLQMxvAFi6yUajgRjd3J5xE= -github.com/shuffle/shuffle-shared v0.9.53/go.mod h1:g03P/ZM8pUwBbdxW0l9A1siMMuemEQshgDpO9hF23eE= +github.com/shuffle/shuffle-shared v0.9.60 h1:8NiovcsSsVX8i0UHmbFBANjUQ/u+4PDDpJXg9JfMAUo= +github.com/shuffle/shuffle-shared v0.9.60/go.mod h1:vK6t1WY5Nfg5vOAk6taT788jIGKs6/4iN1d8Argyn4o= github.com/shuffle/singul v0.0.20 h1:Lz+K4l2GJQ5W6o8ePr/2qlIcGuZuMC7W8Wj/JFw1XDI= github.com/shuffle/singul v0.0.20/go.mod h1:qY3ZmwaNwmRApLWMTqZaNEMXO6X8bjr/zY44xDU62hQ= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 09c95187..abf793fc 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -625,6 +625,8 @@ func deployk8sApp(image string, identifier string, env []string) error { } replicaNumberInt32 := int32(replicaNumber) + // apps do not need access the k8s api. + automountServiceAccountToken := false deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -656,9 +658,10 @@ func deployk8sApp(image string, identifier string, env []string) error { Resources: buildResourcesFromEnv(), }, }, - DNSPolicy: corev1.DNSClusterFirst, - ServiceAccountName: appServiceAccountName, - SecurityContext: podSecurityContext, + DNSPolicy: corev1.DNSClusterFirst, + ServiceAccountName: appServiceAccountName, + AutomountServiceAccountToken: &automountServiceAccountToken, + SecurityContext: podSecurityContext, }, }, }, @@ -1243,7 +1246,8 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont func removeContainer(containername string) error { ctx := context.Background() - cli, err := dockerclient.NewEnvClient() + // cli, err := dockerclient.NewEnvClient() + cli, _, err := shuffle.GetDockerClient() if err != nil { log.Printf("[DEBUG] Unable to create docker client: %s", err) return err @@ -1341,7 +1345,8 @@ func getWorkerURLs() ([]string, error) { } // Create a new Docker client - cli, err := dockerclient.NewEnvClient() + // cli, err := dockerclient.NewEnvClient() + cli, _, err := shuffle.GetDockerClient() if err != nil { log.Println("[ERROR] Failed to create Docker client:", err) return workerUrls, err @@ -1458,7 +1463,8 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { var err error if isKubernetes != "true" { - dockercli, err = dockerclient.NewEnvClient() + // dockercli, err = dockerclient.NewEnvClient() + dockercli, _, err = shuffle.GetDockerClient() if err != nil { log.Printf("[ERROR] Unable to create docker client (3): %s", err) return @@ -3578,7 +3584,8 @@ func findAppInfo(image, name string, redeploy bool) (int, error) { //Filters: if exposedPort == -1 || redeploy { - dockercli, err := dockerclient.NewEnvClient() + // dockercli, err := dockerclient.NewEnvClient() + dockercli, _, err := shuffle.GetDockerClient() if err != nil { log.Printf("[ERROR] Unable to create docker client (2): %s", err) return -1, err @@ -3690,7 +3697,8 @@ func findAppInfo(image, name string, redeploy bool) (int, error) { if exposedPort >= 0 { //log.Printf("[INFO] Found service %s on port %d - no need to deploy another", name, exposedPort) } else { - dockercli, err := dockerclient.NewEnvClient() + // dockercli, err := dockerclient.NewEnvClient() + dockercli, _, err := shuffle.GetDockerClient() if err != nil { log.Printf("[ERROR] Unable to create docker client (2): %s", err) return -1, err @@ -3768,7 +3776,8 @@ func findAppInfoKubernetes(image, name string, env []string) error { // Backups in case networks are removed func initSwarmNetwork() error { ctx := context.Background() - dockercli, err := dockerclient.NewEnvClient() + // dockercli, err := dockerclient.NewEnvClient() + dockercli, _, err := shuffle.GetDockerClient() if err != nil { log.Printf("[ERROR] Unable to create docker client (2): %s", err) return err @@ -4074,7 +4083,8 @@ func baseDeploy() { //var err error if isKubernetes != "true" { - cli, err := dockerclient.NewEnvClient() + // cli, err := dockerclient.NewEnvClient() + cli, _, err := shuffle.GetDockerClient() if err != nil { log.Printf("[ERROR] Unable to create docker client (3): %s", err) return @@ -4815,7 +4825,8 @@ func handleDownloadImage(resp http.ResponseWriter, request *http.Request) { return } - client, err := dockerclient.NewEnvClient() + // client, err := dockerclient.NewEnvClient() + client, _, err := shuffle.GetDockerClient() if err != nil { log.Printf("[ERROR] Unable to create docker client (4): %s", err) resp.WriteHeader(401) @@ -4979,7 +4990,13 @@ func AutoScaleApps(ctx context.Context, client *dockerclient.Client, maxExecutio } func scaleApps(ctx context.Context, client *dockerclient.Client, replicas uint64) error { - client, err := dockerclient.NewEnvClient() + // client, err := dockerclient.NewEnvClient() + client, _, err := shuffle.GetDockerClient() + if err != nil { + log.Printf("[ERROR] Unable to create docker client (scaleApps): %s", err) + return err + } + services, err := client.ServiceList(ctx, types.ServiceListOptions{}) if err != nil { log.Printf("[ERROR] Failed to find services in the swarm: %s", err) @@ -5057,7 +5074,8 @@ func numberOfApps(ctx context.Context, dockercli *dockerclient.Client) int { } if dockercli == nil { - dockercli, err = dockerclient.NewEnvClient() + // dockercli, err = dockerclient.NewEnvClient() + dockercli, _, err = shuffle.GetDockerClient() if err != nil { log.Printf("[ERROR] Unable to create docker client (5): %s", err) return 0 @@ -5144,7 +5162,13 @@ func IsServiceRunning(ctx context.Context, cli *dockerclient.Client) bool { } func numberOfWorkers(ctx context.Context, cli *dockerclient.Client) int { - cli, err := dockerclient.NewEnvClient() + // cli, err := dockerclient.NewEnvClient() + cli, _, err := shuffle.GetDockerClient() + if err != nil { + log.Printf("[ERROR] Unable to create docker client (5): %s", err) + return 0 + } + service, _, err := cli.ServiceInspectWithRaw(ctx, "shuffle-workers", types.ServiceInspectOptions{}) if err != nil { return 0