diff --git a/.env b/.env
index 48bd17bc..65a00b0b 100755
--- a/.env
+++ b/.env
@@ -69,6 +69,10 @@ IS_KUBERNETES=false
SHUFFLE_BASE_IMAGE_REPOSITORY=frikky
#SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-1.4.0"
+# For environments using their own docker registry
+# where they don't want to update http, subflow and shuffle tools again
+SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY=true
+
# The eth0 interface inside a container corresponds
# to the virtual Ethernet interface that connects
# the container to the docker0
diff --git a/.github/workflows/nightly-release.yaml b/.github/workflows/nightly-release.yaml
new file mode 100644
index 00000000..ef888ced
--- /dev/null
+++ b/.github/workflows/nightly-release.yaml
@@ -0,0 +1,85 @@
+name: Nightly Release
+on:
+ release:
+ types: [published]
+ branches:
+ - 2.0.0
+
+jobs:
+ main:
+ runs-on: ubuntu-latest
+ continue-on-error: ${{ matrix.experimental }}
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - app: frontend
+ path: frontend
+ experimental: true
+ - app: backend
+ path: backend
+ experimental: true
+ - app: app_sdk
+ path: backend/app_sdk
+ experimental: true
+ - app: orborus
+ path: functions/onprem/orborus
+ experimental: true
+ - app: worker
+ path: functions/onprem/worker
+ experimental: true
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+
+ - name: Set version
+ id: set_version
+ run: |
+ if [[ ${{ github.event_name }} == 'release' ]]; then
+ echo "VERSION=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
+ else
+ echo "VERSION=nightly-untagged-latest" >> $GITHUB_OUTPUT
+ fi
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v3
+ with:
+ platforms: "amd64,arm64,arm"
+
+ - name: Login to DockerHub
+ uses: docker/login-action@v3
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ - name: Login to Ghcr
+ uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Ghcr Build and push
+ id: docker_build
+ uses: docker/build-push-action@v4
+ env:
+ BUILDX_NO_DEFAULT_LOAD: true
+ with:
+ logout: false
+ context: ${{ matrix.path }}/
+ file: ${{ matrix.path }}/Dockerfile
+ platforms: linux/amd64,linux/arm64
+ push: true
+ cache-from: type=local,src=/tmp/.buildx-cache
+ cache-to: type=local,dest=/tmp/.buildx-cache
+ tags: |
+ ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }}
+ ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }}
+ frikky/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }}
+ frikky/shuffle:${{ matrix.app }}
+
+ - name: Image digest
+ run: echo ${{ steps.docker_build.outputs.digest }}
\ No newline at end of file
diff --git a/README.md b/README.md
index c1a214e2..1ffc0c29 100755
--- a/README.md
+++ b/README.md
@@ -6,6 +6,7 @@ Shuffle Automation
[](https://github.com/Shuffle/Shuffle/actions/workflows/codeql-analysis.yml)
[](https://github.com/Shuffle/Shuffle/actions/workflows/dockerbuild.yaml)
+[](https://deploy.cloud.run/?git_repo=https://github.com/0x0elliot/Shuffle&ref=2.0.0)
diff --git a/app.json b/app.json
new file mode 100644
index 00000000..e8cd75ca
--- /dev/null
+++ b/app.json
@@ -0,0 +1,9 @@
+{
+ "name": "Shuffle",
+ "description": "Security Automation Platform",
+ "repository": "https://github.com/0x0elliot/Shuffle",
+ "ref": "2.0.0",
+ "scripts": {
+ "postclone": "chmod +x startup.sh && ./startup.sh"
+ }
+ }
\ No newline at end of file
diff --git a/backend/go-app/main.go b/backend/go-app/main.go
index 7ce1309c..6b3ba779 100755
--- a/backend/go-app/main.go
+++ b/backend/go-app/main.go
@@ -5112,7 +5112,8 @@ func initHandlers() {
r.HandleFunc("/api/v1/apps/{appId}", shuffle.UpdateWorkflowAppConfig).Methods("PATCH", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appId}", shuffle.DeleteWorkflowApp).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appId}/config", shuffle.GetWorkflowAppConfig).Methods("GET", "OPTIONS")
- r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "OPTIONS")
+ r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "POST", "OPTIONS")
+ r.HandleFunc("/api/v1/apps/{appName}/run_hotload", handleSingleAppHotloadRequest).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/get_existing", LoadSpecificApps).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/download_remote", LoadSpecificApps).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/validate", validateAppInput).Methods("POST", "OPTIONS")
diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go
index 922bdced..3b3e18c8 100755
--- a/backend/go-app/walkoff.go
+++ b/backend/go-app/walkoff.go
@@ -2817,6 +2817,70 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) {
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
+func handleSingleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) {
+ cors := shuffle.HandleCors(resp, request)
+ if cors {
+ return
+ }
+ ctx := context.Background()
+ cacheKey := fmt.Sprintf("workflowapps-sorted-1000")
+ shuffle.DeleteCache(ctx, cacheKey)
+ cacheKey = fmt.Sprintf("workflowapps-sorted-500")
+ shuffle.DeleteCache(ctx, cacheKey)
+ cacheKey = fmt.Sprintf("workflowapps-sorted-0")
+ shuffle.DeleteCache(ctx, cacheKey)
+ // Just need to be logged in
+ // FIXME - should have some permissions?
+ user, err := shuffle.HandleApiAuthentication(resp, request)
+ if err != nil {
+ log.Printf("Api authentication failed in app hotload: %s", err)
+ resp.WriteHeader(401)
+ resp.Write([]byte(`{"success": false}`))
+ return
+ }
+ if user.Role != "admin" {
+ resp.WriteHeader(401)
+ resp.Write([]byte(`{"success": false, "reason": "Must be admin to hotload apps"}`))
+ return
+ }
+ location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER")
+ if len(location) == 0 {
+ resp.WriteHeader(500)
+ resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "SHUFFLE_APP_HOTLOAD_FOLDER not specified in .env"}`)))
+ return
+ }
+ requestUrlFields := strings.Split(request.URL.String(), "/")
+ var appName string
+ if requestUrlFields[1] == "api" {
+ if len(requestUrlFields) <= 4 {
+ resp.WriteHeader(401)
+ resp.Write([]byte(`{"success": false}`))
+ return
+ }
+ appName = requestUrlFields[4]
+ if strings.Contains(appName, "?") {
+ appName = strings.Split(appName, "?")[0]
+ }
+ }
+ location = location + "/" + appName
+ log.Printf("[INFO] Starting hotloading from %s", location)
+ err = handleAppHotload(ctx, location, true)
+ if err != nil {
+ log.Printf("[WARNING] Failed app hotload: %s", err)
+ resp.WriteHeader(500)
+ resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
+ return
+ }
+ cacheKey = fmt.Sprintf("workflowapps-sorted-100")
+ shuffle.DeleteCache(ctx, cacheKey)
+ cacheKey = fmt.Sprintf("workflowapps-sorted-500")
+ shuffle.DeleteCache(ctx, cacheKey)
+ cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
+ shuffle.DeleteCache(ctx, cacheKey)
+ resp.WriteHeader(200)
+ resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
+}
+
func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod
index 06ae0659..68333f20 100644
--- a/functions/onprem/orborus/go.mod
+++ b/functions/onprem/orborus/go.mod
@@ -4,13 +4,13 @@ go 1.22.0
toolchain go1.22.2
-replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared
+// replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared
require (
github.com/docker/docker v27.0.2+incompatible
github.com/docker/go-connections v0.5.0
github.com/satori/go.uuid v1.2.0
- github.com/shuffle/shuffle-shared v0.6.74
+ github.com/shuffle/shuffle-shared v0.6.79
k8s.io/api v0.30.2
k8s.io/apimachinery v0.30.2
)
diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go
index 26c030ce..ab979114 100755
--- a/functions/onprem/orborus/orborus.go
+++ b/functions/onprem/orborus/orborus.go
@@ -966,6 +966,10 @@ func deployK8sWorker(image string, identifier string, env []string) error {
env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL")))
}
+ if len(os.Getenv("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY")) > 0 {
+ env = append(env, fmt.Sprintf("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY=%s", os.Getenv("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY")))
+ }
+
clientset, _, err := shuffle.GetKubernetesClient()
if err != nil {
log.Printf("[ERROR] Error getting kubernetes client:", err)
diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go
index d652c1f9..a05de7fc 100644
--- a/functions/onprem/worker/worker.go
+++ b/functions/onprem/worker/worker.go
@@ -423,8 +423,27 @@ func deployk8sApp(image string, identifier string, env []string) error {
// value = strings.ReplaceAll(value, "_", "-")
value := identifier
+ baseDeployMode := false
+
+ // check if autoDeploy contains a value
+ // that is equal to the image being deployed.
+ for _, value := range autoDeploy {
+ if value == image {
+ baseDeployMode = true
+ }
+ }
+
+ autoDeployOverride := os.Getenv("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY") == "true"
+
+ localRegistry := ""
+
// Checking if app is generated or not
- localRegistry := os.Getenv("REGISTRY_URL")
+ if !baseDeployMode && !autoDeployOverride {
+ localRegistry = os.Getenv("REGISTRY_URL")
+ } else {
+ log.Printf("[DEBUG] Detected baseDeploy image (%s) and ghcr override. Resorting to using ghcr instead of registry", image)
+ }
+
/*
appDetails := strings.Split(image, ":")[1]
appDetailsSplit := strings.Split(appDetails, "_")
@@ -445,15 +464,15 @@ func deployk8sApp(image string, identifier string, env []string) error {
}
*/
- if len(localRegistry) == 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0 {
+ if (len(localRegistry) == 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0) && !(baseDeployMode && autoDeployOverride) {
localRegistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")
}
- if len(localRegistry) > 0 && strings.Count(image, "/") <= 2 {
+ if (len(localRegistry) > 0 && strings.Count(image, "/") <= 2) && !(baseDeployMode && autoDeployOverride) {
log.Printf("[DEBUG] Using REGISTRY_URL %s", localRegistry)
image = fmt.Sprintf("%s/%s", localRegistry, image)
} else {
- if strings.Count(image, "/") <= 2 {
+ if strings.Count(image, "/") <= 2 && !strings.HasPrefix(image, "frikky/shuffle:") {
image = fmt.Sprintf("frikky/shuffle:%s", image)
}
}
diff --git a/startup.sh b/startup.sh
new file mode 100644
index 00000000..4fac7596
--- /dev/null
+++ b/startup.sh
@@ -0,0 +1,83 @@
+#!/bin/bash
+
+# Update and install dependencies
+apt-get update
+apt-get install -y docker.io docker-compose curl git
+
+# Start and enable Docker
+systemctl start docker
+systemctl enable docker
+
+# Clone Shuffle repository
+git clone --branch 2.0.0 https://github.com/Shuffle/Shuffle.git
+cd Shuffle
+
+# Setup directories and permissions
+mkdir shuffle-database && chmod -R 777 shuffle-database
+
+# Start services
+docker-compose up -d
+
+# Wait for initial startup
+echo "Waiting 30 seconds for initial startup..."
+sleep 30
+
+# Check for restarting containers
+echo "Checking for restarting containers..."
+ATTEMPTS=30
+for i in $(seq 1 $ATTEMPTS); do
+ RESTARTING_CONTAINERS=$(docker ps --filter "status=restarting" --format "{{.Names}}")
+ if [ -n "$RESTARTING_CONTAINERS" ]; then
+ echo "The following containers are restarting:"
+ echo "$RESTARTING_CONTAINERS"
+ exit 1
+ fi
+ echo "No containers are restarting. Attempt $i/$ATTEMPTS."
+ sleep 1
+done
+echo "No containers were found in a restarting state after $ATTEMPTS checks."
+
+# Check frontend response
+echo "Checking frontend response..."
+RESPONSE=$(curl -s http://localhost:3001)
+if echo "$RESPONSE" | grep -q "Shuffle"; then
+ echo "The word 'Shuffle' was found in the response."
+else
+ echo "The word 'Shuffle' was not found in the response."
+ exit 1
+fi
+
+# Register user
+echo "Attempting to register user..."
+MAX_RETRIES=30
+RETRY_INTERVAL=10
+CONTAINER_NAME="shuffle-backend"
+
+for (( i=1; i<=$MAX_RETRIES; i++ ))
+do
+ STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" 'http://localhost:3001/api/v1/register' \
+ -H 'Accept: */*' \
+ -H 'Accept-Language: en-US,en;q=0.9' \
+ -H 'Connection: keep-alive' \
+ -H 'Content-Type: application/json' \
+ --data-raw '{"username":"demo@demo.io","password":"supercoolpassword"}')
+
+ if [ "$STATUS_CODE" -eq 200 ]; then
+ echo "User registration was successful with status code 200."
+ exit 0
+ elif [ "$STATUS_CODE" -ne 502 ]; then
+ echo "User registration failed with status code $STATUS_CODE."
+ exit 1
+ fi
+
+ echo "Received status code $STATUS_CODE. Retrying in $RETRY_INTERVAL seconds... ($i/$MAX_RETRIES)"
+ echo "Fetching last 30 lines of logs from container $CONTAINER_NAME..."
+ docker logs --tail 30 "$CONTAINER_NAME"
+ echo "Fetching last 30 lines of logs from container shuffle-opensearch..."
+ docker logs --tail 30 shuffle-opensearch
+
+ sleep $RETRY_INTERVAL
+done
+
+echo "User registration failed after $MAX_RETRIES attempts."
+exit 1
\ No newline at end of file