marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch '2.0.0' of https://github.com/shuffle/shuffle into 2.0.0

Browse Source
This commit is contained in:
Frikky
2024-11-06 21:47:19 +01:00
parent bc395990d5 e3b8719762
commit e63b2871ee
10 changed files with 277 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+4
View File
@@ -69,6 +69,10 @@ IS_KUBERNETES=false
SHUFFLE_BASE_IMAGE_REPOSITORY=frikky
#SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-1.4.0"
# For environments using their own docker registry
# where they don't want to update http, subflow and shuffle tools again
SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY=true
# The eth0 interface inside a container corresponds
# to the virtual Ethernet interface that connects
# the container to the docker0
.github/workflows/nightly-release.yaml
+85
View File
@@ -0,0 +1,85 @@
name: Nightly Release
on:
release:
types: [published]
branches:
- 2.0.0
jobs:
main:
runs-on: ubuntu-latest
continue-on-error: ${{ matrix.experimental }}
strategy:
fail-fast: false
matrix:
include:
- app: frontend
path: frontend
experimental: true
- app: backend
path: backend
experimental: true
- app: app_sdk
path: backend/app_sdk
experimental: true
- app: orborus
path: functions/onprem/orborus
experimental: true
- app: worker
path: functions/onprem/worker
experimental: true
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set version
id: set_version
run: |
if [[ ${{ github.event_name }} == 'release' ]]; then
echo "VERSION=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
else
echo "VERSION=nightly-untagged-latest" >> $GITHUB_OUTPUT
fi
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: "amd64,arm64,arm"
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to Ghcr
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Ghcr Build and push
id: docker_build
uses: docker/build-push-action@v4
env:
BUILDX_NO_DEFAULT_LOAD: true
with:
logout: false
context: ${{ matrix.path }}/
file: ${{ matrix.path }}/Dockerfile
platforms: linux/amd64,linux/arm64
push: true
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache
tags: |
ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }}
${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }}
frikky/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }}
frikky/shuffle:${{ matrix.app }}
- name: Image digest
run: echo ${{ steps.docker_build.outputs.digest }}
README.md
+1
View File
@@ -6,6 +6,7 @@ Shuffle Automation
[![CodeQL](https://github.com/Shuffle/Shuffle/actions/workflows/codeql-analysis.yml/badge.svg?branch=launch)](https://github.com/Shuffle/Shuffle/actions/workflows/codeql-analysis.yml)
[![Autobuild](https://github.com/Shuffle/Shuffle/actions/workflows/dockerbuild.yaml/badge.svg?branch=launch)](https://github.com/Shuffle/Shuffle/actions/workflows/dockerbuild.yaml)
[![Deploy to Google Cloud](https://deploy.cloud.run/button.svg)](https://deploy.cloud.run/?git_repo=https://github.com/0x0elliot/Shuffle&ref=2.0.0)
</h1><h4 align="center">
app.json
+9
View File
@@ -0,0 +1,9 @@
{
"name": "Shuffle",
"description": "Security Automation Platform",
"repository": "https://github.com/0x0elliot/Shuffle",
"ref": "2.0.0",
"scripts": {
"postclone": "chmod +x startup.sh && ./startup.sh"
}
}
backend/go-app/main.go
+2 -1
View File
@@ -5112,7 +5112,8 @@ func initHandlers() {
r.HandleFunc("/api/v1/apps/{appId}", shuffle.UpdateWorkflowAppConfig).Methods("PATCH", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appId}", shuffle.DeleteWorkflowApp).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appId}/config", shuffle.GetWorkflowAppConfig).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appName}/run_hotload", handleSingleAppHotloadRequest).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/get_existing", LoadSpecificApps).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/download_remote", LoadSpecificApps).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/validate", validateAppInput).Methods("POST", "OPTIONS")
backend/go-app/walkoff.go
+64
View File
@@ -2817,6 +2817,70 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) {
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func handleSingleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
ctx := context.Background()
cacheKey := fmt.Sprintf("workflowapps-sorted-1000")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-500")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-0")
shuffle.DeleteCache(ctx, cacheKey)
// Just need to be logged in
// FIXME - should have some permissions?
user, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in app hotload: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Must be admin to hotload apps"}`))
return
}
location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER")
if len(location) == 0 {
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "SHUFFLE_APP_HOTLOAD_FOLDER not specified in .env"}`)))
return
}
requestUrlFields := strings.Split(request.URL.String(), "/")
var appName string
if requestUrlFields[1] == "api" {
if len(requestUrlFields) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
appName = requestUrlFields[4]
if strings.Contains(appName, "?") {
appName = strings.Split(appName, "?")[0]
}
}
location = location + "/" + appName
log.Printf("[INFO] Starting hotloading from %s", location)
err = handleAppHotload(ctx, location, true)
if err != nil {
log.Printf("[WARNING] Failed app hotload: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
cacheKey = fmt.Sprintf("workflowapps-sorted-100")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-500")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
shuffle.DeleteCache(ctx, cacheKey)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
functions/onprem/orborus/go.mod
+2 -2
View File
@@ -4,13 +4,13 @@ go 1.22.0
toolchain go1.22.2
replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared
// replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared
require (
github.com/docker/docker v27.0.2+incompatible
github.com/docker/go-connections v0.5.0
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.6.74
github.com/shuffle/shuffle-shared v0.6.79
k8s.io/api v0.30.2
k8s.io/apimachinery v0.30.2
)
functions/onprem/orborus/orborus.go
+4
View File
@@ -966,6 +966,10 @@ func deployK8sWorker(image string, identifier string, env []string) error {
env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL")))
}
if len(os.Getenv("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY")) > 0 {
env = append(env, fmt.Sprintf("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY=%s", os.Getenv("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY")))
}
clientset, _, err := shuffle.GetKubernetesClient()
if err != nil {
log.Printf("[ERROR] Error getting kubernetes client:", err)
functions/onprem/worker/worker.go
+23 -4
View File
@@ -423,8 +423,27 @@ func deployk8sApp(image string, identifier string, env []string) error {
// value = strings.ReplaceAll(value, "_", "-")
value := identifier
baseDeployMode := false
// check if autoDeploy contains a value
// that is equal to the image being deployed.
for _, value := range autoDeploy {
if value == image {
baseDeployMode = true
}
}
autoDeployOverride := os.Getenv("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY") == "true"
localRegistry := ""
// Checking if app is generated or not
localRegistry := os.Getenv("REGISTRY_URL")
if !baseDeployMode && !autoDeployOverride {
localRegistry = os.Getenv("REGISTRY_URL")
} else {
log.Printf("[DEBUG] Detected baseDeploy image (%s) and ghcr override. Resorting to using ghcr instead of registry", image)
}
/*
appDetails := strings.Split(image, ":")[1]
appDetailsSplit := strings.Split(appDetails, "_")
@@ -445,15 +464,15 @@ func deployk8sApp(image string, identifier string, env []string) error {
}
*/
if len(localRegistry) == 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0 {
if (len(localRegistry) == 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0) && !(baseDeployMode && autoDeployOverride) {
localRegistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")
}
if len(localRegistry) > 0 && strings.Count(image, "/") <= 2 {
if (len(localRegistry) > 0 && strings.Count(image, "/") <= 2) && !(baseDeployMode && autoDeployOverride) {
log.Printf("[DEBUG] Using REGISTRY_URL %s", localRegistry)
image = fmt.Sprintf("%s/%s", localRegistry, image)
} else {
if strings.Count(image, "/") <= 2 {
if strings.Count(image, "/") <= 2 && !strings.HasPrefix(image, "frikky/shuffle:") {
image = fmt.Sprintf("frikky/shuffle:%s", image)
}
}
startup.sh
+83
View File
@@ -0,0 +1,83 @@
#!/bin/bash
# Update and install dependencies
apt-get update
apt-get install -y docker.io docker-compose curl git
# Start and enable Docker
systemctl start docker
systemctl enable docker
# Clone Shuffle repository
git clone --branch 2.0.0 https://github.com/Shuffle/Shuffle.git
cd Shuffle
# Setup directories and permissions
mkdir shuffle-database && chmod -R 777 shuffle-database
# Start services
docker-compose up -d
# Wait for initial startup
echo "Waiting 30 seconds for initial startup..."
sleep 30
# Check for restarting containers
echo "Checking for restarting containers..."
ATTEMPTS=30
for i in $(seq 1 $ATTEMPTS); do
RESTARTING_CONTAINERS=$(docker ps --filter "status=restarting" --format "{{.Names}}")
if [ -n "$RESTARTING_CONTAINERS" ]; then
echo "The following containers are restarting:"
echo "$RESTARTING_CONTAINERS"
exit 1
fi
echo "No containers are restarting. Attempt $i/$ATTEMPTS."
sleep 1
done
echo "No containers were found in a restarting state after $ATTEMPTS checks."
# Check frontend response
echo "Checking frontend response..."
RESPONSE=$(curl -s http://localhost:3001)
if echo "$RESPONSE" | grep -q "Shuffle"; then
echo "The word 'Shuffle' was found in the response."
else
echo "The word 'Shuffle' was not found in the response."
exit 1
fi
# Register user
echo "Attempting to register user..."
MAX_RETRIES=30
RETRY_INTERVAL=10
CONTAINER_NAME="shuffle-backend"
for (( i=1; i<=$MAX_RETRIES; i++ ))
do
STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" 'http://localhost:3001/api/v1/register' \
-H 'Accept: */*' \
-H 'Accept-Language: en-US,en;q=0.9' \
-H 'Connection: keep-alive' \
-H 'Content-Type: application/json' \
--data-raw '{"username":"demo@demo.io","password":"supercoolpassword"}')
if [ "$STATUS_CODE" -eq 200 ]; then
echo "User registration was successful with status code 200."
exit 0
elif [ "$STATUS_CODE" -ne 502 ]; then
echo "User registration failed with status code $STATUS_CODE."
exit 1
fi
echo "Received status code $STATUS_CODE. Retrying in $RETRY_INTERVAL seconds... ($i/$MAX_RETRIES)"
echo "Fetching last 30 lines of logs from container $CONTAINER_NAME..."
docker logs --tail 30 "$CONTAINER_NAME"
echo "Fetching last 30 lines of logs from container shuffle-opensearch..."
docker logs --tail 30 shuffle-opensearch
sleep $RETRY_INTERVAL
done
echo "User registration failed after $MAX_RETRIES attempts."
exit 1