Merge branch '2.0.0' of https://github.com/shuffle/shuffle into 2.0.0

.env

@@ -69,6 +69,10 @@ IS_KUBERNETES=false
 SHUFFLE_BASE_IMAGE_REPOSITORY=frikky
 #SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-1.4.0"
 
+# For environments using their own docker registry
+# where they don't want to update http, subflow and shuffle tools again
+SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY=true
+
 # The eth0 interface inside a container corresponds 
 # to the virtual Ethernet interface that connects 
 # the container to the docker0 

.github/workflows/nightly-release.yaml

@@ -0,0 +1,85 @@
+name: Nightly Release
+on:
+  release:
+    types: [published]
+    branches:
+      - 2.0.0
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - app: frontend
+            path: frontend
+            experimental: true
+          - app: backend
+            path: backend
+            experimental: true
+          - app: app_sdk
+            path: backend/app_sdk
+            experimental: true
+          - app: orborus
+            path: functions/onprem/orborus
+            experimental: true
+          - app: worker
+            path: functions/onprem/worker
+            experimental: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      
+      - name: Set version
+        id: set_version
+        run: |
+          if [[ ${{ github.event_name }} == 'release' ]]; then
+            echo "VERSION=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
+          else
+            echo "VERSION=nightly-untagged-latest" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: "amd64,arm64,arm"
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to Ghcr
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Ghcr Build and push
+        id: docker_build
+        uses: docker/build-push-action@v4
+        env:
+          BUILDX_NO_DEFAULT_LOAD: true
+        with:
+          logout: false
+          context: ${{ matrix.path }}/
+          file: ${{ matrix.path }}/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+          tags: |
+            ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }}
+            frikky/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }}
+            frikky/shuffle:${{ matrix.app }}
+
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

README.md

@@ -6,6 +6,7 @@ Shuffle Automation
 
 [![CodeQL](https://github.com/Shuffle/Shuffle/actions/workflows/codeql-analysis.yml/badge.svg?branch=launch)](https://github.com/Shuffle/Shuffle/actions/workflows/codeql-analysis.yml)
 [![Autobuild](https://github.com/Shuffle/Shuffle/actions/workflows/dockerbuild.yaml/badge.svg?branch=launch)](https://github.com/Shuffle/Shuffle/actions/workflows/dockerbuild.yaml)
+[![Deploy to Google Cloud](https://deploy.cloud.run/button.svg)](https://deploy.cloud.run/?git_repo=https://github.com/0x0elliot/Shuffle&ref=2.0.0)
 
 </h1><h4 align="center">
 

app.json

@@ -0,0 +1,9 @@
+{
+    "name": "Shuffle",
+    "description": "Security Automation Platform",
+    "repository": "https://github.com/0x0elliot/Shuffle",
+    "ref": "2.0.0",
+    "scripts": {
+      "postclone": "chmod +x startup.sh && ./startup.sh"
+    }
+  }

backend/go-app/main.go

@@ -5112,7 +5112,8 @@ func initHandlers() {
 	r.HandleFunc("/api/v1/apps/{appId}", shuffle.UpdateWorkflowAppConfig).Methods("PATCH", "OPTIONS")
 	r.HandleFunc("/api/v1/apps/{appId}", shuffle.DeleteWorkflowApp).Methods("DELETE", "OPTIONS")
 	r.HandleFunc("/api/v1/apps/{appId}/config", shuffle.GetWorkflowAppConfig).Methods("GET", "OPTIONS")
-	r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "OPTIONS")
+	r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "POST", "OPTIONS")
+	r.HandleFunc("/api/v1/apps/{appName}/run_hotload", handleSingleAppHotloadRequest).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/apps/get_existing", LoadSpecificApps).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/apps/download_remote", LoadSpecificApps).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/apps/validate", validateAppInput).Methods("POST", "OPTIONS")

backend/go-app/walkoff.go

@@ -2817,6 +2817,70 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) {
 	resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
 }
 
+func handleSingleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) {
+	cors := shuffle.HandleCors(resp, request)
+	if cors {
+		return
+	}
+	ctx := context.Background()
+	cacheKey := fmt.Sprintf("workflowapps-sorted-1000")
+	shuffle.DeleteCache(ctx, cacheKey)
+	cacheKey = fmt.Sprintf("workflowapps-sorted-500")
+	shuffle.DeleteCache(ctx, cacheKey)
+	cacheKey = fmt.Sprintf("workflowapps-sorted-0")
+	shuffle.DeleteCache(ctx, cacheKey)
+	// Just need to be logged in
+	// FIXME - should have some permissions?
+	user, err := shuffle.HandleApiAuthentication(resp, request)
+	if err != nil {
+		log.Printf("Api authentication failed in app hotload: %s", err)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false}`))
+		return
+	}
+	if user.Role != "admin" {
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "Must be admin to hotload apps"}`))
+		return
+	}
+	location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER")
+	if len(location) == 0 {
+		resp.WriteHeader(500)
+		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "SHUFFLE_APP_HOTLOAD_FOLDER not specified in .env"}`)))
+		return
+	}
+	requestUrlFields := strings.Split(request.URL.String(), "/")
+	var appName string
+	if requestUrlFields[1] == "api" {
+		if len(requestUrlFields) <= 4 {
+			resp.WriteHeader(401)
+			resp.Write([]byte(`{"success": false}`))
+			return
+		}
+		appName = requestUrlFields[4]
+		if strings.Contains(appName, "?") {
+			appName = strings.Split(appName, "?")[0]
+		}
+	}
+	location = location + "/" + appName
+	log.Printf("[INFO] Starting hotloading from %s", location)
+	err = handleAppHotload(ctx, location, true)
+	if err != nil {
+		log.Printf("[WARNING] Failed app hotload: %s", err)
+		resp.WriteHeader(500)
+		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
+		return
+	}
+	cacheKey = fmt.Sprintf("workflowapps-sorted-100")
+	shuffle.DeleteCache(ctx, cacheKey)
+	cacheKey = fmt.Sprintf("workflowapps-sorted-500")
+	shuffle.DeleteCache(ctx, cacheKey)
+	cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
+	shuffle.DeleteCache(ctx, cacheKey)
+	resp.WriteHeader(200)
+	resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
+}
+
 func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) {
 	cors := shuffle.HandleCors(resp, request)
 	if cors {

functions/onprem/orborus/go.mod

@@ -4,13 +4,13 @@ go 1.22.0
 
 toolchain go1.22.2
 
-replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared
+// replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared
 
 require (
 	github.com/docker/docker v27.0.2+incompatible
 	github.com/docker/go-connections v0.5.0
 	github.com/satori/go.uuid v1.2.0
-	github.com/shuffle/shuffle-shared v0.6.74
+	github.com/shuffle/shuffle-shared v0.6.79
 	k8s.io/api v0.30.2
 	k8s.io/apimachinery v0.30.2
 )

functions/onprem/orborus/orborus.go

@@ -966,6 +966,10 @@ func deployK8sWorker(image string, identifier string, env []string) error {
 		env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL")))
 	}
 
+	if len(os.Getenv("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY")) > 0 {
+		env = append(env, fmt.Sprintf("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY=%s", os.Getenv("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY")))
+	}
+
 	clientset, _, err := shuffle.GetKubernetesClient()
 	if err != nil {
 		log.Printf("[ERROR] Error getting kubernetes client:", err)

functions/onprem/worker/worker.go

@@ -423,8 +423,27 @@ func deployk8sApp(image string, identifier string, env []string) error {
 	// value = strings.ReplaceAll(value, "_", "-")
 	value := identifier
 
+	baseDeployMode := false
+
+	// check if autoDeploy contains a value 
+	// that is equal to the image being deployed.
+	for _, value := range autoDeploy {
+		if value == image {
+			baseDeployMode = true
+		}
+	}
+
+	autoDeployOverride := os.Getenv("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY") == "true"
+
+	localRegistry := ""
+
 	// Checking if app is generated or not
-	localRegistry := os.Getenv("REGISTRY_URL")
+	if !baseDeployMode && !autoDeployOverride {
+		localRegistry = os.Getenv("REGISTRY_URL")
+	} else {
+		log.Printf("[DEBUG] Detected baseDeploy image (%s) and ghcr override. Resorting to using ghcr instead of registry", image)
+	}
+
 	/*
 		appDetails := strings.Split(image, ":")[1]
 		appDetailsSplit := strings.Split(appDetails, "_")
@@ -445,15 +464,15 @@ func deployk8sApp(image string, identifier string, env []string) error {
 		}
 	*/
 
-	if len(localRegistry) == 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0 {
+	if (len(localRegistry) == 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0) && !(baseDeployMode && autoDeployOverride) {
 		localRegistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")
 	}
 
-	if len(localRegistry) > 0 && strings.Count(image, "/") <= 2 {
+	if (len(localRegistry) > 0 && strings.Count(image, "/") <= 2) && !(baseDeployMode && autoDeployOverride) {
 		log.Printf("[DEBUG] Using REGISTRY_URL %s", localRegistry)
 		image = fmt.Sprintf("%s/%s", localRegistry, image)
 	} else {
-		if strings.Count(image, "/") <= 2 {
+		if strings.Count(image, "/") <= 2 && !strings.HasPrefix(image, "frikky/shuffle:") {
 			image = fmt.Sprintf("frikky/shuffle:%s", image)
 		}
 	}

startup.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+# Update and install dependencies
+apt-get update
+apt-get install -y docker.io docker-compose curl git
+
+# Start and enable Docker
+systemctl start docker
+systemctl enable docker
+
+# Clone Shuffle repository
+git clone --branch 2.0.0 https://github.com/Shuffle/Shuffle.git
+cd Shuffle
+
+# Setup directories and permissions
+mkdir shuffle-database && chmod -R 777 shuffle-database
+
+# Start services
+docker-compose up -d
+
+# Wait for initial startup
+echo "Waiting 30 seconds for initial startup..."
+sleep 30
+
+# Check for restarting containers
+echo "Checking for restarting containers..."
+ATTEMPTS=30
+for i in $(seq 1 $ATTEMPTS); do
+  RESTARTING_CONTAINERS=$(docker ps --filter "status=restarting" --format "{{.Names}}")
+  if [ -n "$RESTARTING_CONTAINERS" ]; then
+    echo "The following containers are restarting:"
+    echo "$RESTARTING_CONTAINERS"
+    exit 1
+  fi
+  echo "No containers are restarting. Attempt $i/$ATTEMPTS."
+  sleep 1
+done
+echo "No containers were found in a restarting state after $ATTEMPTS checks."
+
+# Check frontend response
+echo "Checking frontend response..."
+RESPONSE=$(curl -s http://localhost:3001)
+if echo "$RESPONSE" | grep -q "Shuffle"; then
+  echo "The word 'Shuffle' was found in the response."
+else
+  echo "The word 'Shuffle' was not found in the response."
+  exit 1
+fi
+
+# Register user
+echo "Attempting to register user..."
+MAX_RETRIES=30
+RETRY_INTERVAL=10
+CONTAINER_NAME="shuffle-backend"
+
+for (( i=1; i<=$MAX_RETRIES; i++ ))
+do
+  STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" 'http://localhost:3001/api/v1/register' \
+    -H 'Accept: */*' \
+    -H 'Accept-Language: en-US,en;q=0.9' \
+    -H 'Connection: keep-alive' \
+    -H 'Content-Type: application/json' \
+    --data-raw '{"username":"demo@demo.io","password":"supercoolpassword"}')
+    
+  if [ "$STATUS_CODE" -eq 200 ]; then
+    echo "User registration was successful with status code 200."
+    exit 0
+  elif [ "$STATUS_CODE" -ne 502 ]; then
+    echo "User registration failed with status code $STATUS_CODE."
+    exit 1
+  fi
+  
+  echo "Received status code $STATUS_CODE. Retrying in $RETRY_INTERVAL seconds... ($i/$MAX_RETRIES)"
+  echo "Fetching last 30 lines of logs from container $CONTAINER_NAME..."
+  docker logs --tail 30 "$CONTAINER_NAME"
+  echo "Fetching last 30 lines of logs from container shuffle-opensearch..."
+  docker logs --tail 30 shuffle-opensearch
+  
+  sleep $RETRY_INTERVAL
+done
+
+echo "User registration failed after $MAX_RETRIES attempts."
+exit 1