From d995f73913565e9a9d009dc7ddb6abdf899b4fa6 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 23 Oct 2024 16:40:26 +0530 Subject: [PATCH 01/18] feat: adding click to deploy features --- README.md | 1 + app.json | 15 ++++++++++ cloudbuild.yaml | 78 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+) create mode 100644 app.json create mode 100644 cloudbuild.yaml diff --git a/README.md b/README.md index c1a214e2..ff08d91e 100755 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Shuffle Automation [![CodeQL](https://github.com/Shuffle/Shuffle/actions/workflows/codeql-analysis.yml/badge.svg?branch=launch)](https://github.com/Shuffle/Shuffle/actions/workflows/codeql-analysis.yml) [![Autobuild](https://github.com/Shuffle/Shuffle/actions/workflows/dockerbuild.yaml/badge.svg?branch=launch)](https://github.com/Shuffle/Shuffle/actions/workflows/dockerbuild.yaml) +[![Deploy to Google Cloud](https://deploy.cloud.run/button.svg)](https://deploy.cloud.run/?git_repo=https://github.com/Shuffle/Shuffle&ref=2.0.0)

diff --git a/app.json b/app.json new file mode 100644 index 00000000..e31846b1 --- /dev/null +++ b/app.json @@ -0,0 +1,15 @@ +{ + "name": "Shuffle", + "description": "Security Automation Platform", + "repository": "https://github.com/shuffle/shuffle", + "env": { + "SERVICE_NAME": { + "description": "Name of the Cloud Run service", + "value": "shuffle-app" + }, + "REGION": { + "description": "GCP region to deploy to", + "value": "us-central1" + } + } + } \ No newline at end of file diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 100644 index 00000000..111178cf --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1,78 @@ +# cloudbuild.yaml +steps: + # Create and set permissions for database directory + - name: 'gcr.io/cloud-builders/docker' + entrypoint: 'bash' + args: + - '-c' + - | + mkdir -p shuffle-database && chmod -R 777 shuffle-database + + # Build images using docker-compose + - name: 'docker/compose' + args: ['build'] + + # Push images to Container Registry + - name: 'docker/compose' + args: ['push'] + + # Start services and perform health checks + - name: 'docker/compose' + entrypoint: 'bash' + args: + - '-c' + - | + # Start services + docker-compose up -d + + # Initial wait for services to start + echo "Waiting 30 seconds for initial startup..." + sleep 30 + + # Check for restarting containers + ATTEMPTS=30 + for i in $(seq 1 $ATTEMPTS); do + RESTARTING_CONTAINERS=$(docker ps --filter "status=restarting" --format "{{.Names}}") + if [ -n "$RESTARTING_CONTAINERS" ]; then + echo "The following containers are restarting:" + echo "$RESTARTING_CONTAINERS" + docker-compose logs + exit 1 + fi + echo "No containers are restarting. Attempt $i/$ATTEMPTS." + sleep 1 + done + echo "No containers were found in a restarting state after $ATTEMPTS checks." + + # Health check via HTTP endpoint + echo "Performing HTTP health check..." + RESPONSE=$(curl -s http://localhost:3001) + if echo "$RESPONSE" | grep -q "Shuffle"; then + echo "Health check passed: 'Shuffle' found in response." + else + echo "Health check failed: 'Shuffle' not found in response." + echo "Response received:" + echo "$RESPONSE" + docker-compose logs + exit 1 + fi + + # Deploy to Cloud Run if all checks pass + - name: 'gcr.io/cloud-builders/gcloud' + args: + - 'run' + - 'deploy' + - '${_SERVICE_NAME}' + - '--image' + - 'gcr.io/$PROJECT_ID/${_SERVICE_NAME}' + - '--platform' + - 'managed' + - '--region' + - '${_REGION}' + - '--allow-unauthenticated' + +# Timeouts and options +timeout: '1800s' +options: + machineType: 'E2_HIGHCPU_8' + logging: CLOUD_LOGGING_ONLY \ No newline at end of file From db3dd124b557fe057ffd4e31473467395834c4ba Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 23 Oct 2024 17:01:21 +0530 Subject: [PATCH 02/18] feat: shifting to VMs! --- app.json | 14 +++------ cloudbuild.yaml | 78 ------------------------------------------------- 2 files changed, 4 insertions(+), 88 deletions(-) delete mode 100644 cloudbuild.yaml diff --git a/app.json b/app.json index e31846b1..f2eef82d 100644 --- a/app.json +++ b/app.json @@ -1,15 +1,9 @@ { "name": "Shuffle", "description": "Security Automation Platform", - "repository": "https://github.com/shuffle/shuffle", - "env": { - "SERVICE_NAME": { - "description": "Name of the Cloud Run service", - "value": "shuffle-app" - }, - "REGION": { - "description": "GCP region to deploy to", - "value": "us-central1" - } + "repository": "https://github.com/Shuffle/Shuffle", + "ref": "2.0.0", + "scripts": { + "postclone": "chmod +x startup.sh && ./startup.sh" } } \ No newline at end of file diff --git a/cloudbuild.yaml b/cloudbuild.yaml deleted file mode 100644 index 111178cf..00000000 --- a/cloudbuild.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# cloudbuild.yaml -steps: - # Create and set permissions for database directory - - name: 'gcr.io/cloud-builders/docker' - entrypoint: 'bash' - args: - - '-c' - - | - mkdir -p shuffle-database && chmod -R 777 shuffle-database - - # Build images using docker-compose - - name: 'docker/compose' - args: ['build'] - - # Push images to Container Registry - - name: 'docker/compose' - args: ['push'] - - # Start services and perform health checks - - name: 'docker/compose' - entrypoint: 'bash' - args: - - '-c' - - | - # Start services - docker-compose up -d - - # Initial wait for services to start - echo "Waiting 30 seconds for initial startup..." - sleep 30 - - # Check for restarting containers - ATTEMPTS=30 - for i in $(seq 1 $ATTEMPTS); do - RESTARTING_CONTAINERS=$(docker ps --filter "status=restarting" --format "{{.Names}}") - if [ -n "$RESTARTING_CONTAINERS" ]; then - echo "The following containers are restarting:" - echo "$RESTARTING_CONTAINERS" - docker-compose logs - exit 1 - fi - echo "No containers are restarting. Attempt $i/$ATTEMPTS." - sleep 1 - done - echo "No containers were found in a restarting state after $ATTEMPTS checks." - - # Health check via HTTP endpoint - echo "Performing HTTP health check..." - RESPONSE=$(curl -s http://localhost:3001) - if echo "$RESPONSE" | grep -q "Shuffle"; then - echo "Health check passed: 'Shuffle' found in response." - else - echo "Health check failed: 'Shuffle' not found in response." - echo "Response received:" - echo "$RESPONSE" - docker-compose logs - exit 1 - fi - - # Deploy to Cloud Run if all checks pass - - name: 'gcr.io/cloud-builders/gcloud' - args: - - 'run' - - 'deploy' - - '${_SERVICE_NAME}' - - '--image' - - 'gcr.io/$PROJECT_ID/${_SERVICE_NAME}' - - '--platform' - - 'managed' - - '--region' - - '${_REGION}' - - '--allow-unauthenticated' - -# Timeouts and options -timeout: '1800s' -options: - machineType: 'E2_HIGHCPU_8' - logging: CLOUD_LOGGING_ONLY \ No newline at end of file From 6eed80f90d16070ab5438f5d83474b96400ba6f0 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 23 Oct 2024 17:01:42 +0530 Subject: [PATCH 03/18] feat: shifting to VMs! (1) --- startup.sh | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 startup.sh diff --git a/startup.sh b/startup.sh new file mode 100644 index 00000000..4fac7596 --- /dev/null +++ b/startup.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# Update and install dependencies +apt-get update +apt-get install -y docker.io docker-compose curl git + +# Start and enable Docker +systemctl start docker +systemctl enable docker + +# Clone Shuffle repository +git clone --branch 2.0.0 https://github.com/Shuffle/Shuffle.git +cd Shuffle + +# Setup directories and permissions +mkdir shuffle-database && chmod -R 777 shuffle-database + +# Start services +docker-compose up -d + +# Wait for initial startup +echo "Waiting 30 seconds for initial startup..." +sleep 30 + +# Check for restarting containers +echo "Checking for restarting containers..." +ATTEMPTS=30 +for i in $(seq 1 $ATTEMPTS); do + RESTARTING_CONTAINERS=$(docker ps --filter "status=restarting" --format "{{.Names}}") + if [ -n "$RESTARTING_CONTAINERS" ]; then + echo "The following containers are restarting:" + echo "$RESTARTING_CONTAINERS" + exit 1 + fi + echo "No containers are restarting. Attempt $i/$ATTEMPTS." + sleep 1 +done +echo "No containers were found in a restarting state after $ATTEMPTS checks." + +# Check frontend response +echo "Checking frontend response..." +RESPONSE=$(curl -s http://localhost:3001) +if echo "$RESPONSE" | grep -q "Shuffle"; then + echo "The word 'Shuffle' was found in the response." +else + echo "The word 'Shuffle' was not found in the response." + exit 1 +fi + +# Register user +echo "Attempting to register user..." +MAX_RETRIES=30 +RETRY_INTERVAL=10 +CONTAINER_NAME="shuffle-backend" + +for (( i=1; i<=$MAX_RETRIES; i++ )) +do + STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" 'http://localhost:3001/api/v1/register' \ + -H 'Accept: */*' \ + -H 'Accept-Language: en-US,en;q=0.9' \ + -H 'Connection: keep-alive' \ + -H 'Content-Type: application/json' \ + --data-raw '{"username":"demo@demo.io","password":"supercoolpassword"}') + + if [ "$STATUS_CODE" -eq 200 ]; then + echo "User registration was successful with status code 200." + exit 0 + elif [ "$STATUS_CODE" -ne 502 ]; then + echo "User registration failed with status code $STATUS_CODE." + exit 1 + fi + + echo "Received status code $STATUS_CODE. Retrying in $RETRY_INTERVAL seconds... ($i/$MAX_RETRIES)" + echo "Fetching last 30 lines of logs from container $CONTAINER_NAME..." + docker logs --tail 30 "$CONTAINER_NAME" + echo "Fetching last 30 lines of logs from container shuffle-opensearch..." + docker logs --tail 30 shuffle-opensearch + + sleep $RETRY_INTERVAL +done + +echo "User registration failed after $MAX_RETRIES attempts." +exit 1 \ No newline at end of file From 27a7eee97e304da9c3e8f8e7ccdb42195d23a279 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 23 Oct 2024 17:04:27 +0530 Subject: [PATCH 04/18] feat: shifting to VMs! (2) --- app.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.json b/app.json index f2eef82d..e8cd75ca 100644 --- a/app.json +++ b/app.json @@ -1,7 +1,7 @@ { "name": "Shuffle", "description": "Security Automation Platform", - "repository": "https://github.com/Shuffle/Shuffle", + "repository": "https://github.com/0x0elliot/Shuffle", "ref": "2.0.0", "scripts": { "postclone": "chmod +x startup.sh && ./startup.sh" From 54877b2aff1a0a9d30f8dfb9773b8591e82c7f8d Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Tue, 29 Oct 2024 18:43:02 +0530 Subject: [PATCH 05/18] fix: adding correct test repo in click to deploy --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ff08d91e..1ffc0c29 100755 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Shuffle Automation [![CodeQL](https://github.com/Shuffle/Shuffle/actions/workflows/codeql-analysis.yml/badge.svg?branch=launch)](https://github.com/Shuffle/Shuffle/actions/workflows/codeql-analysis.yml) [![Autobuild](https://github.com/Shuffle/Shuffle/actions/workflows/dockerbuild.yaml/badge.svg?branch=launch)](https://github.com/Shuffle/Shuffle/actions/workflows/dockerbuild.yaml) -[![Deploy to Google Cloud](https://deploy.cloud.run/button.svg)](https://deploy.cloud.run/?git_repo=https://github.com/Shuffle/Shuffle&ref=2.0.0) +[![Deploy to Google Cloud](https://deploy.cloud.run/button.svg)](https://deploy.cloud.run/?git_repo=https://github.com/0x0elliot/Shuffle&ref=2.0.0)

From d8c5fa220f5ce9c3bc2e3ba33bfc6aaf48b06b14 Mon Sep 17 00:00:00 2001 From: LuisThuillier Date: Wed, 30 Oct 2024 18:37:59 +0100 Subject: [PATCH 06/18] Adding single app hotload & fix POST as an option to both hotloads --- backend/go-app/main.go | 3 +- backend/go-app/walkoff.go | 64 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 7ce1309c..26de3dc9 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5112,7 +5112,8 @@ func initHandlers() { r.HandleFunc("/api/v1/apps/{appId}", shuffle.UpdateWorkflowAppConfig).Methods("PATCH", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}", shuffle.DeleteWorkflowApp).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}/config", shuffle.GetWorkflowAppConfig).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/apps/{appName}/run_hotload", handleSingleAppHotloadRequest).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/get_existing", LoadSpecificApps).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/download_remote", LoadSpecificApps).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/validate", validateAppInput).Methods("POST", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 7b8008ce..0f3c7b68 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2802,6 +2802,70 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } +func handleSingleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { + cors := shuffle.HandleCors(resp, request) + if cors { + return + } + ctx := context.Background() + cacheKey := fmt.Sprintf("workflowapps-sorted-1000") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-0") + shuffle.DeleteCache(ctx, cacheKey) + // Just need to be logged in + // FIXME - should have some permissions? + user, err := shuffle.HandleApiAuthentication(resp, request) + if err != nil { + log.Printf("Api authentication failed in app hotload: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + if user.Role != "admin" { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Must be admin to hotload apps"}`)) + return + } + location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER") + if len(location) == 0 { + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "SHUFFLE_APP_HOTLOAD_FOLDER not specified in .env"}`))) + return + } + requestUrlFields := strings.Split(request.URL.String(), "/") + var appName string + if requestUrlFields[1] == "api" { + if len(requestUrlFields) <= 4 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + appName = requestUrlFields[4] + if strings.Contains(appName, "?") { + appName = strings.Split(appName, "?")[0] + } + } + location = location + "/" + appName + log.Printf("[INFO] Starting hotloading from %s", location) + err = handleAppHotload(ctx, location, true) + if err != nil { + log.Printf("[WARNING] Failed app hotload: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + cacheKey = fmt.Sprintf("workflowapps-sorted-100") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-1000") + shuffle.DeleteCache(ctx, cacheKey) + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) +} + func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { cors := shuffle.HandleCors(resp, request) if cors { From 1bebc7e0544e315f5169a5aa93c4f2f03b4b5f07 Mon Sep 17 00:00:00 2001 From: LuisThuillier Date: Fri, 1 Nov 2024 08:06:25 +0100 Subject: [PATCH 07/18] Add GET and POST to not break break backwards compatibility. --- backend/go-app/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 26de3dc9..6b3ba779 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5112,7 +5112,7 @@ func initHandlers() { r.HandleFunc("/api/v1/apps/{appId}", shuffle.UpdateWorkflowAppConfig).Methods("PATCH", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}", shuffle.DeleteWorkflowApp).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}/config", shuffle.GetWorkflowAppConfig).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "POST", "OPTIONS") r.HandleFunc("/api/v1/apps/{appName}/run_hotload", handleSingleAppHotloadRequest).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/get_existing", LoadSpecificApps).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/download_remote", LoadSpecificApps).Methods("POST", "OPTIONS") From 4993a4e2c3ada8eb45511a15391ae8fb3aa225eb Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Sat, 2 Nov 2024 18:50:44 +0530 Subject: [PATCH 08/18] ci: adding a nightly release ci --- .github/workflows/nightly-release.yaml | 85 ++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/nightly-release.yaml diff --git a/.github/workflows/nightly-release.yaml b/.github/workflows/nightly-release.yaml new file mode 100644 index 00000000..3c62e710 --- /dev/null +++ b/.github/workflows/nightly-release.yaml @@ -0,0 +1,85 @@ +name: dockerbuild +on: + release: + types: [published] + branches: + - 2.0.0 + +jobs: + main: + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + include: + - app: frontend + path: frontend + experimental: true + - app: backend + path: backend + experimental: true + - app: app_sdk + path: backend/app_sdk + experimental: true + - app: orborus + path: functions/onprem/orborus + experimental: true + - app: worker + path: functions/onprem/worker + experimental: true + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set version + id: set_version + run: | + if [[ ${{ github.event_name }} == 'release' ]]; then + echo "VERSION=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT + else + echo "VERSION=nightly-untagged-latest" >> $GITHUB_OUTPUT + fi + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: "amd64,arm64,arm" + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to Ghcr + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Ghcr Build and push + id: docker_build + uses: docker/build-push-action@v4 + env: + BUILDX_NO_DEFAULT_LOAD: true + with: + logout: false + context: ${{ matrix.path }}/ + file: ${{ matrix.path }}/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + tags: | + ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }} + ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }} + frikky/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }} + frikky/shuffle:${{ matrix.app }} + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file From 2f925acb431706599470f1d4b018e3cf00bc10d9 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Sat, 2 Nov 2024 19:01:08 +0530 Subject: [PATCH 09/18] ci: using the right names --- .github/workflows/nightly-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly-release.yaml b/.github/workflows/nightly-release.yaml index 3c62e710..ef888ced 100644 --- a/.github/workflows/nightly-release.yaml +++ b/.github/workflows/nightly-release.yaml @@ -1,4 +1,4 @@ -name: dockerbuild +name: Nightly Release on: release: types: [published] From becb75fc79bde47a123446ebb88df41831561cc7 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Sun, 3 Nov 2024 16:28:34 +0530 Subject: [PATCH 10/18] fix: resort to the original registry during baseDeploy --- functions/onprem/worker/worker.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index d652c1f9..b911ef98 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -423,8 +423,24 @@ func deployk8sApp(image string, identifier string, env []string) error { // value = strings.ReplaceAll(value, "_", "-") value := identifier + baseDeployMode := false + + // check if autoDeploy contains a value + // that is equal to the image being deployed. + for _, value := range autoDeploy { + if value == image { + log.Printf("[DEBUG] Detected baseDeploy image. Resorting to not using the registry") + baseDeployMode = true + } + } + + localRegistry := "" + // Checking if app is generated or not - localRegistry := os.Getenv("REGISTRY_URL") + if !baseDeployMode { + localRegistry := os.Getenv("REGISTRY_URL") + } + /* appDetails := strings.Split(image, ":")[1] appDetailsSplit := strings.Split(appDetails, "_") @@ -445,11 +461,11 @@ func deployk8sApp(image string, identifier string, env []string) error { } */ - if len(localRegistry) == 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0 { + if (len(localRegistry) == 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0) && !baseDeployMode { localRegistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY") } - if len(localRegistry) > 0 && strings.Count(image, "/") <= 2 { + if (len(localRegistry) > 0 && strings.Count(image, "/") <= 2) { log.Printf("[DEBUG] Using REGISTRY_URL %s", localRegistry) image = fmt.Sprintf("%s/%s", localRegistry, image) } else { From 6c42e1c1abd69ca0e9bbb181901a032ba1357840 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:47:56 +0530 Subject: [PATCH 11/18] fix: use dockerhub on override variable --- .env | 4 ++++ functions/onprem/orborus/orborus.go | 4 ++++ functions/onprem/worker/worker.go | 7 +++++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 48bd17bc..f125bc4e 100755 --- a/.env +++ b/.env @@ -69,6 +69,10 @@ IS_KUBERNETES=false SHUFFLE_BASE_IMAGE_REPOSITORY=frikky #SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-1.4.0" +# For environments using their own docker registry +# where they don't want to update http, subflow and shuffle tools again +SHUFFLE_USE_DOCKERHUB_FOR_AUTODEPLOY=true + # The eth0 interface inside a container corresponds # to the virtual Ethernet interface that connects # the container to the docker0 diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 857e216d..790180da 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -966,6 +966,10 @@ func deployK8sWorker(image string, identifier string, env []string) error { env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL"))) } + if len(os.Getenv("SHUFFLE_USE_DOCKERHUB_FOR_AUTODEPLOY")) > 0 { + env = append(env, fmt.Sprintf("SHUFFLE_USE_DOCKERHUB_FOR_AUTODEPLOY=%s", os.Getenv("SHUFFLE_USE_DOCKERHUB_FOR_AUTODEPLOY"))) + } + clientset, _, err := shuffle.GetKubernetesClient() if err != nil { log.Printf("[ERROR] Error getting kubernetes client:", err) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index b911ef98..a0452b4e 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -429,16 +429,19 @@ func deployk8sApp(image string, identifier string, env []string) error { // that is equal to the image being deployed. for _, value := range autoDeploy { if value == image { - log.Printf("[DEBUG] Detected baseDeploy image. Resorting to not using the registry") baseDeployMode = true } } + autoDeployOverride := os.Getenv("SHUFFLE_USE_DOCKERHUB_FOR_AUTODEPLOY") == "true" + localRegistry := "" // Checking if app is generated or not - if !baseDeployMode { + if !baseDeployMode && !autoDeployOverride { localRegistry := os.Getenv("REGISTRY_URL") + } else { + log.Printf("[DEBUG] Detected baseDeploy image (%s) and dockerhub override. Resorting to using dockerhub instead of registry", image) } /* From 2281d1470c562c67430565025cfa0dbd0cc17970 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:55:01 +0530 Subject: [PATCH 12/18] chore: updating shuffle-shared to v0.6.79 --- functions/onprem/orborus/go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 06ae0659..68333f20 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -4,13 +4,13 @@ go 1.22.0 toolchain go1.22.2 -replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared +// replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared require ( github.com/docker/docker v27.0.2+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.6.74 + github.com/shuffle/shuffle-shared v0.6.79 k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 ) From fd82941caa73d9c80fb4f455366c3e4026eab683 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:55:37 +0530 Subject: [PATCH 13/18] fix: typo in variable --- functions/onprem/worker/worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index a0452b4e..e4821486 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -439,7 +439,7 @@ func deployk8sApp(image string, identifier string, env []string) error { // Checking if app is generated or not if !baseDeployMode && !autoDeployOverride { - localRegistry := os.Getenv("REGISTRY_URL") + localRegistry = os.Getenv("REGISTRY_URL") } else { log.Printf("[DEBUG] Detected baseDeploy image (%s) and dockerhub override. Resorting to using dockerhub instead of registry", image) } From 681e8ffbf7edfaedbb41402480ce0bae36d970bc Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Mon, 4 Nov 2024 13:43:26 +0530 Subject: [PATCH 14/18] fix: image prefix related bug --- functions/onprem/worker/worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index e4821486..a05005da 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -472,7 +472,7 @@ func deployk8sApp(image string, identifier string, env []string) error { log.Printf("[DEBUG] Using REGISTRY_URL %s", localRegistry) image = fmt.Sprintf("%s/%s", localRegistry, image) } else { - if strings.Count(image, "/") <= 2 { + if strings.Count(image, "/") <= 2 && !strings.HasPrefix(image, "frikky/shuffle:") { image = fmt.Sprintf("frikky/shuffle:%s", image) } } From 8c499bddba312bd3ebbc1f837bb9b19fa8730262 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Mon, 4 Nov 2024 13:45:42 +0530 Subject: [PATCH 15/18] fix: better environment variable --- .env | 2 +- functions/onprem/orborus/orborus.go | 2 +- functions/onprem/worker/worker.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env b/.env index f125bc4e..65a00b0b 100755 --- a/.env +++ b/.env @@ -71,7 +71,7 @@ SHUFFLE_BASE_IMAGE_REPOSITORY=frikky # For environments using their own docker registry # where they don't want to update http, subflow and shuffle tools again -SHUFFLE_USE_DOCKERHUB_FOR_AUTODEPLOY=true +SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY=true # The eth0 interface inside a container corresponds # to the virtual Ethernet interface that connects diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 790180da..5e3a92ef 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -966,7 +966,7 @@ func deployK8sWorker(image string, identifier string, env []string) error { env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL"))) } - if len(os.Getenv("SHUFFLE_USE_DOCKERHUB_FOR_AUTODEPLOY")) > 0 { + if len(os.Getenv("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY")) > 0 { env = append(env, fmt.Sprintf("SHUFFLE_USE_DOCKERHUB_FOR_AUTODEPLOY=%s", os.Getenv("SHUFFLE_USE_DOCKERHUB_FOR_AUTODEPLOY"))) } diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index a05005da..37689e11 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -433,7 +433,7 @@ func deployk8sApp(image string, identifier string, env []string) error { } } - autoDeployOverride := os.Getenv("SHUFFLE_USE_DOCKERHUB_FOR_AUTODEPLOY") == "true" + autoDeployOverride := os.Getenv("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY") == "true" localRegistry := "" From 294f4de9b8ca69ecd0ee019748fced549040b870 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Mon, 4 Nov 2024 13:48:23 +0530 Subject: [PATCH 16/18] fix: cleaning up booleans --- functions/onprem/worker/worker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 37689e11..80ddc475 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -464,11 +464,11 @@ func deployk8sApp(image string, identifier string, env []string) error { } */ - if (len(localRegistry) == 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0) && !baseDeployMode { + if (len(localRegistry) == 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0) && !(baseDeployMode && autoDeployOverride) { localRegistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY") } - if (len(localRegistry) > 0 && strings.Count(image, "/") <= 2) { + if (len(localRegistry) > 0 && strings.Count(image, "/") <= 2) && !(baseDeployMode && autoDeployOverride) { log.Printf("[DEBUG] Using REGISTRY_URL %s", localRegistry) image = fmt.Sprintf("%s/%s", localRegistry, image) } else { From 5799c48fbea1763b9309d1f52ed1a7c02e5c338b Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Mon, 4 Nov 2024 13:51:30 +0530 Subject: [PATCH 17/18] fix: cleaning up typos in printf --- functions/onprem/worker/worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 80ddc475..a05de7fc 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -441,7 +441,7 @@ func deployk8sApp(image string, identifier string, env []string) error { if !baseDeployMode && !autoDeployOverride { localRegistry = os.Getenv("REGISTRY_URL") } else { - log.Printf("[DEBUG] Detected baseDeploy image (%s) and dockerhub override. Resorting to using dockerhub instead of registry", image) + log.Printf("[DEBUG] Detected baseDeploy image (%s) and ghcr override. Resorting to using ghcr instead of registry", image) } /* From e3b87197625c52b91006ce1aed8c746af7ad0f69 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Tue, 5 Nov 2024 11:49:13 +0530 Subject: [PATCH 18/18] fix: env rename --- functions/onprem/orborus/orborus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 5e3a92ef..ae9c1fbc 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -967,7 +967,7 @@ func deployK8sWorker(image string, identifier string, env []string) error { } if len(os.Getenv("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_USE_DOCKERHUB_FOR_AUTODEPLOY=%s", os.Getenv("SHUFFLE_USE_DOCKERHUB_FOR_AUTODEPLOY"))) + env = append(env, fmt.Sprintf("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY=%s", os.Getenv("SHUFFLE_USE_GCHR_OVERRIDE_FOR_AUTODEPLOY"))) } clientset, _, err := shuffle.GetKubernetesClient()