diff --git a/.env b/.env index 5f507244..f2d7b6fc 100755 --- a/.env +++ b/.env @@ -28,12 +28,12 @@ SHUFFLE_APP_HOTLOAD_FOLDER=./shuffle-apps SHUFFLE_APP_HOTLOAD_LOCATION=./shuffle-apps SHUFFLE_FILE_LOCATION=./shuffle-files -# Encryption modifier. This HAS to be set to encrypt any authentication being used in Shuffle. This is put together with other relevant values to ensure multiple parts are needed to decrypt. +# Encryption modifier. This HAS to be set to encrypt any authentication being used in Shuffle. This is put together with other relevant values to ensure multiple parts are needed to decrypt. # If this key is lost or changed, you will have to reauthenticate all apps. SHUFFLE_ENCRYPTION_MODIFIER= # Other configs -BASE_URL=http://shuffle-backend:5001 +BASE_URL=http://shuffle-backend:5001 SSO_REDIRECT_URL=http://localhost:3001 BACKEND_HOSTNAME=shuffle-backend BACKEND_PORT=5001 @@ -46,7 +46,7 @@ OUTER_HOSTNAME=shuffle-backend DB_LOCATION=./shuffle-database DOCKER_API_VERSION=1.40 -# Orborus/Proxy configurations +# Orborus/Proxy configurations HTTP_PROXY= HTTPS_PROXY= SHUFFLE_PASS_WORKER_PROXY=TRUE @@ -58,10 +58,10 @@ TZ=Europe/Amsterdam # Used to FIND the containername. cgroup v2: issue 501 ORBORUS_CONTAINER_NAME= # Used for setting up a startup delay for Orborus -SHUFFLE_ORBORUS_STARTUP_DELAY= -SHUFFLE_SKIPSSL_VERIFY=true +SHUFFLE_ORBORUS_STARTUP_DELAY= +SHUFFLE_SKIPSSL_VERIFY=true # Used for controlling if the environment should run in kubernetes or not -IS_KUBERNETES=false +IS_KUBERNETES=false #SHUFFLE_BASE_IMAGE_NAME=shuffle #SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io @@ -72,9 +72,9 @@ SHUFFLE_BASE_IMAGE_REPOSITORY=frikky # where they don't want to update http, subflow and shuffle tools again SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY=true -# The eth0 interface inside a container corresponds -# to the virtual Ethernet interface that connects -# the container to the docker0 +# The eth0 interface inside a container corresponds +# to the virtual Ethernet interface that connects +# the container to the docker0 SHUFFLE_SWARM_BRIDGE_DEFAULT_INTERFACE=eth0 # 1500 by default SHUFFLE_SWARM_BRIDGE_DEFAULT_MTU=1500 @@ -84,7 +84,7 @@ SHUFFLE_MEMCACHED= SHUFFLE_CONTAINER_AUTO_CLEANUP=true # The amount of concurrent executions Orborus can handle. This is a soft limit, but it's recommended to keep it low. SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=5 -SHUFFLE_HEALTHCHECK_DISABLED=false +SHUFFLE_HEALTHCHECK_DISABLED=false SHUFFLE_ELASTIC=true SHUFFLE_LOGS_DISABLED=true SHUFFLE_CHAT_DISABLED=false @@ -112,8 +112,9 @@ SHUFFLE_OPENSEARCH_USERNAME="admin" SHUFFLE_OPENSEARCH_PASSWORD="StrongShufflePassword321!" # In use for the first time setup of OpenSearch + backend of Shuffle OPENSEARCH_INITIAL_ADMIN_PASSWORD="StrongShufflePassword321!" # In use for the first time setup of OpenSearch -#Tenzir related +#Tenzir related SHUFFLE_TENZIR_URL= +SHUFFLE_PROTECTED_CLEANUP_DISABLED=true DEBUG_MODE=false diff --git a/.github/workflows/quick-testing.yml b/.github/workflows/quick-testing.yml index cf35967a..20cf6471 100644 --- a/.github/workflows/quick-testing.yml +++ b/.github/workflows/quick-testing.yml @@ -15,142 +15,145 @@ jobs: os: [ubuntu-latest] architecture: [x64, arm64] steps: - - name: Checkout code - uses: actions/checkout@v2 + - name: Checkout code + uses: actions/checkout@v2 - - name: Install Docker Compose - run: | - sudo apt-get update - sudo apt-get install -y docker-compose - docker-compose --version + - name: Shai-Hulud 2.0 Detector + uses: gensecaihq/Shai-Hulud-2.0-Detector@v1.0.0 - - name: Set up opensearch directory - run: chmod -R 777 shuffle-database + - name: Install Docker Compose + run: | + sudo apt-get update + sudo apt-get install -y docker-compose + docker-compose --version - - name: Build the stack - run: docker compose up -d + - name: Set up opensearch directory + run: chmod -R 777 shuffle-database - - name: Wait for 30 seconds - run: sleep 30 + - name: Build the stack + run: docker compose up -d - - name: Check for restarting containers in a loop and fixing perms again - run: | - # echo "Changing permissions on shuffle-database directory again" - # chmod -R 777 shuffle-database + - name: Wait for 30 seconds + run: sleep 30 - ATTEMPTS=30 # Total time = ATTEMPTS * 5 seconds = 30 seconds - for i in $(seq 1 $ATTEMPTS); do - RESTARTING_CONTAINERS=$(docker ps --filter "status=restarting" --format "{{.Names}}") - if [ -n "$RESTARTING_CONTAINERS" ]; then - echo "The following containers are restarting:" - echo "$RESTARTING_CONTAINERS" + - name: Check for restarting containers in a loop and fixing perms again + run: | + # echo "Changing permissions on shuffle-database directory again" + # chmod -R 777 shuffle-database + + ATTEMPTS=30 # Total time = ATTEMPTS * 5 seconds = 30 seconds + for i in $(seq 1 $ATTEMPTS); do + RESTARTING_CONTAINERS=$(docker ps --filter "status=restarting" --format "{{.Names}}") + if [ -n "$RESTARTING_CONTAINERS" ]; then + echo "The following containers are restarting:" + echo "$RESTARTING_CONTAINERS" + exit 1 + fi + echo "No containers are restarting. Attempt $i/$ATTEMPTS." + sleep 1 + done + echo "No containers were found in a restarting state after $ATTEMPTS checks." + + - name: Check if the response from the frontend contains the word "Shuffle" + run: | + RESPONSE=$(curl -s http://localhost:3001) + if echo "$RESPONSE" | grep -q "Shuffle"; then + echo "The word 'Shuffle' was found in the response." + else + echo "The word 'Shuffle' was not found in the response." exit 1 fi - echo "No containers are restarting. Attempt $i/$ATTEMPTS." - sleep 1 - done - echo "No containers were found in a restarting state after $ATTEMPTS checks." + - name: Register a user and check the status code + run: | + MAX_RETRIES=30 + RETRY_INTERVAL=10 + CONTAINER_NAME="shuffle-backend" - - name: Check if the response from the frontend contains the word "Shuffle" - run: | - RESPONSE=$(curl -s http://localhost:3001) - if echo "$RESPONSE" | grep -q "Shuffle"; then - echo "The word 'Shuffle' was found in the response." - else - echo "The word 'Shuffle' was not found in the response." + for (( i=1; i<=$MAX_RETRIES; i++ )) + do + STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" 'http://localhost:3001/api/v1/register' \ + -H 'Accept: */*' \ + -H 'Accept-Language: en-US,en;q=0.9' \ + -H 'Connection: keep-alive' \ + -H 'Content-Type: application/json' \ + --data-raw '{"username":"demo@demo.io","password":"supercoolpassword"}') + + if [ "$STATUS_CODE" -eq 200 ]; then + echo "User registration was successful with status code 200." + exit 0 + elif [ "$STATUS_CODE" -ne 502 ]; then + echo "User registration failed with status code $STATUS_CODE." + exit 1 + fi + + echo "Received status code $STATUS_CODE. Retrying in $RETRY_INTERVAL seconds... ($i/$MAX_RETRIES)" + echo "Fetching last 30 lines of logs from container $CONTAINER_NAME..." + + logs_output=$(docker logs --tail 30 "$CONTAINER_NAME") + echo "$logs_output" + + echo "Fetching last 30 lines of logs from container shuffle-opensearch..." + + opensearch_logs=$(docker logs --tail 30 shuffle-opensearch) + echo "$opensearch_logs" + + sleep $RETRY_INTERVAL + done + + echo "User registration failed after $MAX_RETRIES attempts." exit 1 - fi - - name: Register a user and check the status code - run: | - MAX_RETRIES=30 - RETRY_INTERVAL=10 - CONTAINER_NAME="shuffle-backend" - for (( i=1; i<=$MAX_RETRIES; i++ )) - do - STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" 'http://localhost:3001/api/v1/register' \ - -H 'Accept: */*' \ - -H 'Accept-Language: en-US,en;q=0.9' \ - -H 'Connection: keep-alive' \ - -H 'Content-Type: application/json' \ - --data-raw '{"username":"demo@demo.io","password":"supercoolpassword"}') + - name: Run Selenium testing for frontend + run: | + cd $GITHUB_WORKSPACE/frontend + # write some log to see the current directory + chmod +x frontend-testing.sh + ./frontend-testing.sh - if [ "$STATUS_CODE" -eq 200 ]; then - echo "User registration was successful with status code 200." + - name: Get the API key and run a health check + id: health_check + run: | + RESPONSE=$(curl -s -k -u admin:StrongShufflePassword321! 'https://localhost:9200/users/_search') + echo "Raw Response: $RESPONSE" + + API_KEY=$(echo "$RESPONSE" | jq -r '.hits.hits[0]._source.apikey') + if [ -n "$API_KEY" ] && [ "$API_KEY" != "null" ]; then + echo "Admin API key: $API_KEY" + echo "API_KEY=$API_KEY" >> $GITHUB_ENV + else + echo "Failed to retrieve the API key for the admin user." + exit 1 + fi + + echo "Waiting 1 minute before sending the health API request..." + sleep 60 + + echo "Checking health API..." + HEALTH_RESPONSE=$(curl -s 'http://localhost:3001/api/v1/health?force=true' \ + -H "Authorization: Bearer $API_KEY") + + echo "Health API Response: $HEALTH_RESPONSE" + + WORKFLOWS_CREATE=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.create') + WORKFLOWS_RUN=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run') + WORKFLOWS_RUN_FINISHED=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run_finished') + WORKFLOWS_RUN_STATUS=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run_status') + WORKFLOWS_DELETE=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.delete') + + echo "WORKFLOWS_CREATE: $WORKFLOWS_CREATE" + echo "WORKFLOWS_RUN: $WORKFLOWS_RUN" + echo "WORKFLOWS_RUN_FINISHED: $WORKFLOWS_RUN_FINISHED" + echo "WORKFLOWS_RUN_STATUS: $WORKFLOWS_RUN_STATUS" + echo "WORKFLOWS_DELETE: $WORKFLOWS_DELETE" + + if [ "$WORKFLOWS_CREATE" = "true" ] && [ "$WORKFLOWS_RUN" = "true" ] && [ "$WORKFLOWS_RUN_FINISHED" = "true" ] && [ "$WORKFLOWS_RUN_STATUS" = "FINISHED" ] && [ "$WORKFLOWS_DELETE" = "true" ]; then + echo "Health endpoint check was successful." exit 0 - elif [ "$STATUS_CODE" -ne 502 ]; then - echo "User registration failed with status code $STATUS_CODE." + else + echo "Health check failed." exit 1 fi - echo "Received status code $STATUS_CODE. Retrying in $RETRY_INTERVAL seconds... ($i/$MAX_RETRIES)" - echo "Fetching last 30 lines of logs from container $CONTAINER_NAME..." - - logs_output=$(docker logs --tail 30 "$CONTAINER_NAME") - echo "$logs_output" - - echo "Fetching last 30 lines of logs from container shuffle-opensearch..." - - opensearch_logs=$(docker logs --tail 30 shuffle-opensearch) - echo "$opensearch_logs" - - sleep $RETRY_INTERVAL - done - - echo "User registration failed after $MAX_RETRIES attempts." - exit 1 - - - name: Run Selenium testing for frontend - run: | - cd $GITHUB_WORKSPACE/frontend - # write some log to see the current directory - chmod +x frontend-testing.sh - ./frontend-testing.sh - - - name: Get the API key and run a health check - id: health_check - run: | - RESPONSE=$(curl -s -k -u admin:StrongShufflePassword321! 'https://localhost:9200/users/_search') - echo "Raw Response: $RESPONSE" - - API_KEY=$(echo "$RESPONSE" | jq -r '.hits.hits[0]._source.apikey') - if [ -n "$API_KEY" ] && [ "$API_KEY" != "null" ]; then - echo "Admin API key: $API_KEY" - echo "API_KEY=$API_KEY" >> $GITHUB_ENV - else - echo "Failed to retrieve the API key for the admin user." - exit 1 - fi - - echo "Waiting 1 minute before sending the health API request..." - sleep 60 - - echo "Checking health API..." - HEALTH_RESPONSE=$(curl -s 'http://localhost:3001/api/v1/health?force=true' \ - -H "Authorization: Bearer $API_KEY") - - echo "Health API Response: $HEALTH_RESPONSE" - - WORKFLOWS_CREATE=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.create') - WORKFLOWS_RUN=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run') - WORKFLOWS_RUN_FINISHED=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run_finished') - WORKFLOWS_RUN_STATUS=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run_status') - WORKFLOWS_DELETE=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.delete') - - echo "WORKFLOWS_CREATE: $WORKFLOWS_CREATE" - echo "WORKFLOWS_RUN: $WORKFLOWS_RUN" - echo "WORKFLOWS_RUN_FINISHED: $WORKFLOWS_RUN_FINISHED" - echo "WORKFLOWS_RUN_STATUS: $WORKFLOWS_RUN_STATUS" - echo "WORKFLOWS_DELETE: $WORKFLOWS_DELETE" - - if [ "$WORKFLOWS_CREATE" = "true" ] && [ "$WORKFLOWS_RUN" = "true" ] && [ "$WORKFLOWS_RUN_FINISHED" = "true" ] && [ "$WORKFLOWS_RUN_STATUS" = "FINISHED" ] && [ "$WORKFLOWS_DELETE" = "true" ]; then - echo "Health endpoint check was successful." - exit 0 - else - echo "Health check failed." - exit 1 - fi - notify: needs: build if: failure() @@ -163,7 +166,7 @@ jobs: Repository: ${{ github.repository }} Branch: ${{ github.ref_name }} Workflow URL: $WORKFLOW_URL" - + curl -s -X POST https://api.twilio.com/2010-04-01/Accounts/${{ secrets.TWILIO_ACCOUNT_SID }}/Messages.json \ --data-urlencode "To=${{ secrets.TWILIO_TO_NUMBER }}" \ --data-urlencode "From=${{ secrets.TWILIO_FROM_NUMBER }}" \ diff --git a/backend/Dockerfile b/backend/Dockerfile index 5ccc3de8..2ee3bcf0 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -24,7 +24,7 @@ RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o shufflebackend . # Certificate build - gets required certs FROM alpine:latest as certs -RUN apk add --update ca-certificates +RUN apk add --update --no-scripts ca-certificates # Sets up the final image FROM alpine:3.22.2 diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 42a139aa..cac11ac2 100755 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -21,7 +21,6 @@ import ( "github.com/docker/docker/api/types" //"github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/image" - "github.com/docker/docker/client" newdockerclient "github.com/fsouza/go-dockerclient" "github.com/go-git/go-billy/v5" @@ -35,12 +34,13 @@ import ( "os" "strings" + "time" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "time" // "k8s.io/client-go/tools/clientcmd" // "k8s.io/client-go/util/homedir" ) @@ -206,7 +206,8 @@ func fixTags(tags []string) []string { // Custom Docker image builder wrapper in memory func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string, downloadIfFail bool) error { ctx := context.Background() - client, err := client.NewEnvClient() + // client, err := client.NewEnvClient() + client, _, err := shuffle.GetDockerClient() defer client.Close() if err != nil { log.Printf("Unable to create docker client: %s", err) @@ -493,7 +494,8 @@ func buildImage(tags []string, dockerfileLocation string) error { } ctx := context.Background() - client, err := client.NewEnvClient() + // client, err := client.NewEnvClient() + client, _, err := shuffle.GetDockerClient() defer client.Close() if err != nil { log.Printf("Unable to create docker client: %s", err) @@ -581,7 +583,8 @@ func imageCheckBuilder(images []string) error { return nil ctx := context.Background() - client, err := client.NewEnvClient() + // client, err := client.NewEnvClient() + client, _, err := shuffle.GetDockerClient() if err != nil { log.Printf("Unable to create docker client: %s", err) return err @@ -670,7 +673,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { } //log.Printf("[DEBUG] Image to load: %s", version.Name) - dockercli, err := client.NewEnvClient() + dockercli, _, err := shuffle.GetDockerClient() if err != nil { log.Printf("[WARNING] Unable to create docker client: %s", err) resp.WriteHeader(422) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 8692fb7f..36d955c1 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -26,15 +26,15 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.54 + github.com/shuffle/shuffle-shared v0.9.69 github.com/shuffle/singul v0.0.20 - golang.org/x/crypto v0.40.0 + golang.org/x/crypto v0.45.0 google.golang.org/api v0.236.0 google.golang.org/grpc v1.72.2 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.33.1 - k8s.io/apimachinery v0.33.1 - k8s.io/client-go v0.33.1 + k8s.io/api v0.34.2 + k8s.io/apimachinery v0.34.2 + k8s.io/client-go v0.34.2 ) require ( @@ -70,13 +70,13 @@ require ( github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frikky/schemaless v0.0.24 // indirect - github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/frikky/schemaless v0.0.25 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect github.com/go-logr/logr v1.4.2 // indirect @@ -87,8 +87,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.6.9 // indirect - github.com/google/go-cmp v0.7.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-github/v28 v28.1.1 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.9 // indirect @@ -110,9 +109,10 @@ require ( github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/openai/openai-go/v3 v3.8.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/osteele/liquid v1.7.0 // indirect @@ -132,6 +132,10 @@ require ( github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/zeebo/errs v1.4.0 // indirect @@ -145,13 +149,15 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect go.opentelemetry.io/otel/trace v1.36.0 // indirect go.uber.org/atomic v1.9.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/net v0.41.0 // indirect + golang.org/x/net v0.47.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.34.0 // indirect - golang.org/x/term v0.33.0 // indirect - golang.org/x/text v0.27.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/term v0.37.0 // indirect + golang.org/x/text v0.31.0 // indirect golang.org/x/time v0.11.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect @@ -163,10 +169,10 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect - k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect - sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index ea6056ba..9790290b 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -138,8 +138,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.24 h1:1bNg96gm82yLbRlDS0haQH/96X3GlCAEuTCSasXIJZE= -github.com/frikky/schemaless v0.0.24/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= +github.com/frikky/schemaless v0.0.25 h1:qXjrKT54LWl3tzDFIUmevZ86F7FiusPpQKfa40xxNl8= +github.com/frikky/schemaless v0.0.25/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= github.com/fsouza/go-dockerclient v1.12.1 h1:FMoLq+Zhv9Oz/rFmu6JWkImfr6CBgZOPcL+bHW4gS0o= github.com/fsouza/go-dockerclient v1.12.1/go.mod h1:OqsgJJcpCwqyM3JED7TdfM9QVWS5O7jSYwXxYKmOooY= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= @@ -344,8 +344,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU= github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0= -github.com/shuffle/shuffle-shared v0.9.53 h1:ykziiRA82FZX0YiuErSmFLQMxvAFi6yUajgRjd3J5xE= -github.com/shuffle/shuffle-shared v0.9.53/go.mod h1:g03P/ZM8pUwBbdxW0l9A1siMMuemEQshgDpO9hF23eE= +github.com/shuffle/shuffle-shared v0.9.60 h1:8NiovcsSsVX8i0UHmbFBANjUQ/u+4PDDpJXg9JfMAUo= +github.com/shuffle/shuffle-shared v0.9.60/go.mod h1:vK6t1WY5Nfg5vOAk6taT788jIGKs6/4iN1d8Argyn4o= github.com/shuffle/singul v0.0.20 h1:Lz+K4l2GJQ5W6o8ePr/2qlIcGuZuMC7W8Wj/JFw1XDI= github.com/shuffle/singul v0.0.20/go.mod h1:qY3ZmwaNwmRApLWMTqZaNEMXO6X8bjr/zY44xDU62hQ= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 5f54465a..ba05049a 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3,7 +3,7 @@ package main import ( uuid "github.com/satori/go.uuid" "github.com/shuffle/shuffle-shared" - "github.com/shuffle/singul/pkg" + singul "github.com/shuffle/singul/pkg" "net/http/pprof" @@ -44,11 +44,12 @@ import ( "github.com/go-git/go-git/v5/storage/memory" // Random + "sort" + xj "github.com/basgys/goxml2json" newscheduler "github.com/carlescere/scheduler" "golang.org/x/crypto/bcrypt" "gopkg.in/yaml.v3" - "sort" // Web "github.com/gorilla/mux" @@ -1312,7 +1313,9 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { // Should run calculations if len(org.SSOConfig.OpenIdAuthorization) > 0 { baseSSOUrl = shuffle.GetOpenIdUrl(request, *org) - + if err != nil { + log.Printf("[ERROR] Failed getting OpenID URL for org %s: %s", org.Name, err) + } break } @@ -5433,7 +5436,7 @@ func initHandlers() { r.HandleFunc("/api/v1/users/getsettings", shuffle.HandleSettings).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/users/getusers", shuffle.HandleGetUsers).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/users/updateuser", shuffle.HandleUpdateUser).Methods("PUT", "OPTIONS") - r.HandleFunc("/api/v1/users/{userID}/remove", shuffle.HandleDeleteUsersAccount).Methods("DELETE", "OPTIONS") + // r.HandleFunc("/api/v1/users/{userID}/remove", shuffle.HandleDeleteUsersAccount).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/users/{user}", shuffle.DeleteUser).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/users/passwordchange", shuffle.HandlePasswordChange).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/users/{key}/get2fa", shuffle.HandleGet2fa).Methods("GET", "OPTIONS") @@ -5657,7 +5660,7 @@ func initHandlers() { // Docker orborus specific - downloads an image r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "GET", "OPTIONS") - r.HandleFunc("/api/v1/login_sso", shuffle.HandleSSO).Methods("GET", "POST", "OPTIONS") + r.HandleFunc("/api/v1/login_sso", shuffle.HandleSAML).Methods("GET", "POST", "OPTIONS") r.HandleFunc("/api/v1/login_openid", shuffle.HandleOpenId).Methods("GET", "POST", "OPTIONS") // Important for email, IDS etc. Create this by: diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 0c015470..934bad62 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -11,6 +11,7 @@ import ( "io" "io/ioutil" "log" + //"math/rand" "net/http" "net/url" @@ -21,7 +22,6 @@ import ( "time" "github.com/docker/docker/api/types/image" - dockerclient "github.com/docker/docker/client" "github.com/h2non/filetype" uuid "github.com/satori/go.uuid" @@ -2997,6 +2997,8 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. "email", "shuffle-ai", "shuffle-subflow", + "yara", + "sigma", } // It's here to prevent getting them in every iteration @@ -3393,6 +3395,8 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. fmt.Sprintf("/apps"), orgId, false, + "HIGH", + "APP_BUID", ) } @@ -3527,7 +3531,8 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { // As it's not even Docker if tmpBody.ForceUpdate { - dockercli, err := dockerclient.NewEnvClient() + // dockercli, err := dockerclient.NewEnvClient() + dockercli, _, err := shuffle.GetDockerClient() if err == nil { appSdk := os.Getenv("SHUFFLE_APP_SDK_VERSION") diff --git a/docker-compose.yml b/docker-compose.yml index 0839bc4c..ef0abd31 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,7 +50,7 @@ services: - HTTPS_PROXY=${HTTPS_PROXY} - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY} - - SHUFFLE_APP_REPLICAS=${SHUFFLE_APP_REPLICAS} + # - SHUFFLE_APP_REPLICAS=${SHUFFLE_APP_REPLICAS} - SHUFFLE_STATS_DISABLED=true - SHUFFLE_LOGS_DISABLED=true - SHUFFLE_SWARM_CONFIG=run @@ -80,7 +80,7 @@ services: soft: -1 hard: -1 nofile: - soft: 65536 + soft: 65536 hard: 65536 volumes: - shuffle-database:/usr/share/opensearch/data:z @@ -127,7 +127,7 @@ services: #docker-socket-proxy: # image: tecnativa/docker-socket-proxy - # container_name: docker-socket-proxy + # container_name: docker-socket-proxy # hostname: docker-socket-proxy # privileged: true # environment: @@ -167,4 +167,4 @@ networks: driver: bridge # driver_opts: # com.docker.network.driver.mtu: 1460 - # uncomment to set MTU for swarm mode. MTU should be whatever is your host's preferred MTU is: https://shuffler.io/docs/troubleshooting#TLS_timeout_error/Timeout_Errors/EOF_Errors + # uncomment to set MTU for swarm mode. MTU should be whatever is your host's preferred MTU is: https://shuffler.io/docs/troubleshooting#TLS_timeout_error/Timeout_Errors/EOF_Errors diff --git a/frontend/package.json b/frontend/package.json index 8cd277bc..229e5635 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -26,10 +26,13 @@ "cpx": "^1.5.0", "create-react-app": "^5.0.1", "cytoscape": "^3.29.2", + "cytoscape-bubblesets": "^4.1.0", "cytoscape-clipboard": "^2.2.1", + "cytoscape-cose-bilkent": "^4.1.0", "cytoscape-cxtmenu": "^3.4.0", "cytoscape-edgehandles": "^3.5.1", "cytoscape-grid-guide": "~2.3.3", + "cytoscape-layers": "^3.1.0", "cytoscape-node-html-label": "^1.2.2", "cytoscape-panzoom": "^2.5.3", "cytoscape-undo-redo": "^1.3.3", diff --git a/frontend/src/components/LeftSideBar.jsx b/frontend/src/components/LeftSideBar.jsx index 5d908414..5245c9bc 100644 --- a/frontend/src/components/LeftSideBar.jsx +++ b/frontend/src/components/LeftSideBar.jsx @@ -127,6 +127,8 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => { setCurrentSelectedTheme(userdata?.theme); } }, [userdata]); + + const CustomPopper = (props) => { return ( @@ -1949,7 +1951,7 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => { } - {!isCloud ? ( + {!isCloud && activeOrgData?.old_org? (