diff --git a/.env b/.env index 48bd17bc..65a00b0b 100755 --- a/.env +++ b/.env @@ -69,6 +69,10 @@ IS_KUBERNETES=false SHUFFLE_BASE_IMAGE_REPOSITORY=frikky #SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-1.4.0" +# For environments using their own docker registry +# where they don't want to update http, subflow and shuffle tools again +SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY=true + # The eth0 interface inside a container corresponds # to the virtual Ethernet interface that connects # the container to the docker0 diff --git a/.github/workflows/nightly-release.yaml b/.github/workflows/nightly-release.yaml new file mode 100644 index 00000000..ef888ced --- /dev/null +++ b/.github/workflows/nightly-release.yaml @@ -0,0 +1,85 @@ +name: Nightly Release +on: + release: + types: [published] + branches: + - 2.0.0 + +jobs: + main: + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + include: + - app: frontend + path: frontend + experimental: true + - app: backend + path: backend + experimental: true + - app: app_sdk + path: backend/app_sdk + experimental: true + - app: orborus + path: functions/onprem/orborus + experimental: true + - app: worker + path: functions/onprem/worker + experimental: true + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set version + id: set_version + run: | + if [[ ${{ github.event_name }} == 'release' ]]; then + echo "VERSION=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT + else + echo "VERSION=nightly-untagged-latest" >> $GITHUB_OUTPUT + fi + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: "amd64,arm64,arm" + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to Ghcr + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Ghcr Build and push + id: docker_build + uses: docker/build-push-action@v4 + env: + BUILDX_NO_DEFAULT_LOAD: true + with: + logout: false + context: ${{ matrix.path }}/ + file: ${{ matrix.path }}/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + tags: | + ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }} + ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }} + frikky/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }} + frikky/shuffle:${{ matrix.app }} + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file diff --git a/README.md b/README.md index c1a214e2..1ffc0c29 100755 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Shuffle Automation [![CodeQL](https://github.com/Shuffle/Shuffle/actions/workflows/codeql-analysis.yml/badge.svg?branch=launch)](https://github.com/Shuffle/Shuffle/actions/workflows/codeql-analysis.yml) [![Autobuild](https://github.com/Shuffle/Shuffle/actions/workflows/dockerbuild.yaml/badge.svg?branch=launch)](https://github.com/Shuffle/Shuffle/actions/workflows/dockerbuild.yaml) +[![Deploy to Google Cloud](https://deploy.cloud.run/button.svg)](https://deploy.cloud.run/?git_repo=https://github.com/0x0elliot/Shuffle&ref=2.0.0)

diff --git a/app.json b/app.json new file mode 100644 index 00000000..e8cd75ca --- /dev/null +++ b/app.json @@ -0,0 +1,9 @@ +{ + "name": "Shuffle", + "description": "Security Automation Platform", + "repository": "https://github.com/0x0elliot/Shuffle", + "ref": "2.0.0", + "scripts": { + "postclone": "chmod +x startup.sh && ./startup.sh" + } + } \ No newline at end of file diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 7ce1309c..6b3ba779 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5112,7 +5112,8 @@ func initHandlers() { r.HandleFunc("/api/v1/apps/{appId}", shuffle.UpdateWorkflowAppConfig).Methods("PATCH", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}", shuffle.DeleteWorkflowApp).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}/config", shuffle.GetWorkflowAppConfig).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "POST", "OPTIONS") + r.HandleFunc("/api/v1/apps/{appName}/run_hotload", handleSingleAppHotloadRequest).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/get_existing", LoadSpecificApps).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/download_remote", LoadSpecificApps).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/validate", validateAppInput).Methods("POST", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 922bdced..3b3e18c8 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2817,6 +2817,70 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } +func handleSingleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { + cors := shuffle.HandleCors(resp, request) + if cors { + return + } + ctx := context.Background() + cacheKey := fmt.Sprintf("workflowapps-sorted-1000") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-0") + shuffle.DeleteCache(ctx, cacheKey) + // Just need to be logged in + // FIXME - should have some permissions? + user, err := shuffle.HandleApiAuthentication(resp, request) + if err != nil { + log.Printf("Api authentication failed in app hotload: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + if user.Role != "admin" { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Must be admin to hotload apps"}`)) + return + } + location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER") + if len(location) == 0 { + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "SHUFFLE_APP_HOTLOAD_FOLDER not specified in .env"}`))) + return + } + requestUrlFields := strings.Split(request.URL.String(), "/") + var appName string + if requestUrlFields[1] == "api" { + if len(requestUrlFields) <= 4 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + appName = requestUrlFields[4] + if strings.Contains(appName, "?") { + appName = strings.Split(appName, "?")[0] + } + } + location = location + "/" + appName + log.Printf("[INFO] Starting hotloading from %s", location) + err = handleAppHotload(ctx, location, true) + if err != nil { + log.Printf("[WARNING] Failed app hotload: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + cacheKey = fmt.Sprintf("workflowapps-sorted-100") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-1000") + shuffle.DeleteCache(ctx, cacheKey) + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) +} + func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { cors := shuffle.HandleCors(resp, request) if cors { diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 06ae0659..68333f20 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -4,13 +4,13 @@ go 1.22.0 toolchain go1.22.2 -replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared +// replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared require ( github.com/docker/docker v27.0.2+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.6.74 + github.com/shuffle/shuffle-shared v0.6.79 k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 ) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 26c030ce..ab979114 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -966,6 +966,10 @@ func deployK8sWorker(image string, identifier string, env []string) error { env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL"))) } + if len(os.Getenv("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY")) > 0 { + env = append(env, fmt.Sprintf("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY=%s", os.Getenv("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY"))) + } + clientset, _, err := shuffle.GetKubernetesClient() if err != nil { log.Printf("[ERROR] Error getting kubernetes client:", err) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index d652c1f9..a05de7fc 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -423,8 +423,27 @@ func deployk8sApp(image string, identifier string, env []string) error { // value = strings.ReplaceAll(value, "_", "-") value := identifier + baseDeployMode := false + + // check if autoDeploy contains a value + // that is equal to the image being deployed. + for _, value := range autoDeploy { + if value == image { + baseDeployMode = true + } + } + + autoDeployOverride := os.Getenv("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY") == "true" + + localRegistry := "" + // Checking if app is generated or not - localRegistry := os.Getenv("REGISTRY_URL") + if !baseDeployMode && !autoDeployOverride { + localRegistry = os.Getenv("REGISTRY_URL") + } else { + log.Printf("[DEBUG] Detected baseDeploy image (%s) and ghcr override. Resorting to using ghcr instead of registry", image) + } + /* appDetails := strings.Split(image, ":")[1] appDetailsSplit := strings.Split(appDetails, "_") @@ -445,15 +464,15 @@ func deployk8sApp(image string, identifier string, env []string) error { } */ - if len(localRegistry) == 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0 { + if (len(localRegistry) == 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0) && !(baseDeployMode && autoDeployOverride) { localRegistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY") } - if len(localRegistry) > 0 && strings.Count(image, "/") <= 2 { + if (len(localRegistry) > 0 && strings.Count(image, "/") <= 2) && !(baseDeployMode && autoDeployOverride) { log.Printf("[DEBUG] Using REGISTRY_URL %s", localRegistry) image = fmt.Sprintf("%s/%s", localRegistry, image) } else { - if strings.Count(image, "/") <= 2 { + if strings.Count(image, "/") <= 2 && !strings.HasPrefix(image, "frikky/shuffle:") { image = fmt.Sprintf("frikky/shuffle:%s", image) } } diff --git a/startup.sh b/startup.sh new file mode 100644 index 00000000..4fac7596 --- /dev/null +++ b/startup.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# Update and install dependencies +apt-get update +apt-get install -y docker.io docker-compose curl git + +# Start and enable Docker +systemctl start docker +systemctl enable docker + +# Clone Shuffle repository +git clone --branch 2.0.0 https://github.com/Shuffle/Shuffle.git +cd Shuffle + +# Setup directories and permissions +mkdir shuffle-database && chmod -R 777 shuffle-database + +# Start services +docker-compose up -d + +# Wait for initial startup +echo "Waiting 30 seconds for initial startup..." +sleep 30 + +# Check for restarting containers +echo "Checking for restarting containers..." +ATTEMPTS=30 +for i in $(seq 1 $ATTEMPTS); do + RESTARTING_CONTAINERS=$(docker ps --filter "status=restarting" --format "{{.Names}}") + if [ -n "$RESTARTING_CONTAINERS" ]; then + echo "The following containers are restarting:" + echo "$RESTARTING_CONTAINERS" + exit 1 + fi + echo "No containers are restarting. Attempt $i/$ATTEMPTS." + sleep 1 +done +echo "No containers were found in a restarting state after $ATTEMPTS checks." + +# Check frontend response +echo "Checking frontend response..." +RESPONSE=$(curl -s http://localhost:3001) +if echo "$RESPONSE" | grep -q "Shuffle"; then + echo "The word 'Shuffle' was found in the response." +else + echo "The word 'Shuffle' was not found in the response." + exit 1 +fi + +# Register user +echo "Attempting to register user..." +MAX_RETRIES=30 +RETRY_INTERVAL=10 +CONTAINER_NAME="shuffle-backend" + +for (( i=1; i<=$MAX_RETRIES; i++ )) +do + STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" 'http://localhost:3001/api/v1/register' \ + -H 'Accept: */*' \ + -H 'Accept-Language: en-US,en;q=0.9' \ + -H 'Connection: keep-alive' \ + -H 'Content-Type: application/json' \ + --data-raw '{"username":"demo@demo.io","password":"supercoolpassword"}') + + if [ "$STATUS_CODE" -eq 200 ]; then + echo "User registration was successful with status code 200." + exit 0 + elif [ "$STATUS_CODE" -ne 502 ]; then + echo "User registration failed with status code $STATUS_CODE." + exit 1 + fi + + echo "Received status code $STATUS_CODE. Retrying in $RETRY_INTERVAL seconds... ($i/$MAX_RETRIES)" + echo "Fetching last 30 lines of logs from container $CONTAINER_NAME..." + docker logs --tail 30 "$CONTAINER_NAME" + echo "Fetching last 30 lines of logs from container shuffle-opensearch..." + docker logs --tail 30 shuffle-opensearch + + sleep $RETRY_INTERVAL +done + +echo "User registration failed after $MAX_RETRIES attempts." +exit 1 \ No newline at end of file