From 6c7cbc279044b4c2a89b46301cbf74663ebf0316 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Mon, 2 Dec 2024 23:23:12 +0530 Subject: [PATCH 1/8] removed debug logs which are not important --- functions/onprem/orborus/orborus.go | 1 - functions/onprem/worker/worker.go | 12 +++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index a9af0dd9..1a9d5b99 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2624,7 +2624,6 @@ func deployTenzirNode() error { err := checkTenzirNode() if err == nil { - log.Printf("[INFO] Tenzir Node is already running") return nil } diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 1bf6249d..ae6034b3 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -104,11 +104,11 @@ var window = shuffle.NewTimeWindow(10 * time.Second) // Images to be autodeployed in the latest version of Shuffle. var autoDeploy = map[string]string{ - "http:1.4.0": "frikky/shuffle:http_1.4.0", - "http:1.3.0": "frikky/shuffle:http_1.3.0", - "shuffle-tools:1.2.0": "frikky/shuffle:shuffle-tools_1.2.0", - "shuffle-subflow:1.0.0": "frikky/shuffle:shuffle-subflow_1.0.0", - "shuffle-subflow:1.1.0": "frikky/shuffle:shuffle-subflow_1.1.0", + "http:1.4.0": "frikky/shuffle:http_1.4.0", + "http:1.3.0": "frikky/shuffle:http_1.3.0", + "shuffle-tools:1.2.0": "frikky/shuffle:shuffle-tools_1.2.0", + "shuffle-subflow:1.0.0": "frikky/shuffle:shuffle-subflow_1.0.0", + "shuffle-subflow:1.1.0": "frikky/shuffle:shuffle-subflow_1.1.0", "shuffle-tools-fork:1.0.0": "frikky/shuffle:shuffle-tools-fork_1.0.0", } @@ -4179,8 +4179,6 @@ func AutoScaleApps(ctx context.Context, client *dockerclient.Client, maxExecutio j := numberOfApps(ctx, client) workers := numberOfWorkers(ctx, client) execPerMin := maxExecutionsPerMinute / workers - log.Printf("[DEBUG] Running with %d workers\n\n\n\n\n", workers) - if count >= execPerMin { log.Printf("[DEBUG] Too many executions per minute (%d). Scaling down to %d", count, execPerMin) scaleApps(ctx, client, uint64(j+1)) From 2b2c2cdf30edd0eb7977a8f0df7a35e05efd49d4 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Tue, 3 Dec 2024 21:37:16 +0530 Subject: [PATCH 2/8] [fix]: Memcached image pulling error and network error --- functions/onprem/orborus/orborus.go | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 1a9d5b99..27b441c1 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -3960,6 +3960,11 @@ func checkMemcached(ctx context.Context, dockercli *dockerclient.Client) (bool, } return false, err } + networkName := "shuffle_swarm_executions" + err = dockercli.NetworkConnect(ctx, networkName, containerName, nil) + if err != nil { + log.Printf("[WARNING] Failed connecting memcached container to network: %s", err) + } if continer.State.Running == false { log.Printf("[INFO] Container %s exists but is not running. Attempting to start it.", containerName) @@ -3997,9 +4002,23 @@ func deployMemcached(dockercli *dockerclient.Client) error { }, } - dockercli.ImagePull(ctx, memcachedImage, image.PullOptions{}) - containerName := "shuffle-cache" + _, _, err := dockercli.ImageInspectWithRaw(ctx, memcachedImage) + if dockerclient.IsErrNotFound(err) { + log.Printf("[DEBUG] Pulling image %s. This may take a while.", memcachedImage) + pullOptions := image.PullOptions{} + out, err := dockercli.ImagePull(ctx, memcachedImage, pullOptions) + if err != nil { + log.Printf("[ERROR] Failed to pull the memcached image: %s", err) + return err + } + defer out.Close() + io.Copy(io.Discard, out) + } else if err != nil { + return err + } + + containerName := "shuffle-cache" resp, err := dockercli.ContainerCreate(ctx, containerConfig, hostConfig, nil, nil, containerName) if err != nil { log.Printf("[ERROR] Error spanning memcached continer: %s", err) From 61ad552fc555df2197a1cb318aa75790cb2a4e4b Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Tue, 3 Dec 2024 23:57:36 +0530 Subject: [PATCH 3/8] [fix]: network attachment failer for memcached --- functions/onprem/orborus/orborus.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 27b441c1..a947298a 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -4024,11 +4024,21 @@ func deployMemcached(dockercli *dockerclient.Client) error { log.Printf("[ERROR] Error spanning memcached continer: %s", err) return err } + + if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" { + networkName := "shuffle_swarm_executions" + err = dockercli.NetworkConnect(ctx, networkName, resp.ID, nil) + if err != nil { + log.Printf("[ERROR] Error connecting tenzir container to network: %s", err) + } + } + err = dockercli.ContainerStart(ctx, resp.ID, container.StartOptions{}) if err != nil { log.Printf("[ERROR] Error starting memcached continer: %s", err) return err } + networkName := "shuffle_swarm_executions" err = dockercli.NetworkConnect(ctx, networkName, resp.ID, nil) if err != nil { From 879695d4f2b88a6284940e1a97e2aeaf43aa6633 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 4 Dec 2024 01:09:37 +0530 Subject: [PATCH 4/8] latest realise for orborus and worker --- functions/onprem/orborus/orborus.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index a947298a..4f7a30a3 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -333,19 +333,6 @@ func deployServiceWorkers(image string) { } ctx := context.Background() - isMemcachedRunning, err := checkMemcached(ctx, dockercli) - if err != nil { - log.Printf("[ERROR] Failed checking memcached: %s", err) - } - if isMemcachedRunning == false { - log.Printf("[ERROR] Memcached is not running. Will try to deploy it.") - deployMemcached(dockercli) - } - - ip := "shuffle-cache" - - os.Setenv("SHUFFLE_MEMCACHED", fmt.Sprintf("%s:11211", ip)) - // Looks for and cleans up all existing items in swarm we can't re-use (Shuffle only) // frikky@debian:~/git/shuffle/functions/onprem/worker$ docker service create --replicas 5 --name shuffle-workers --env SHUFFLE_SWARM_CONFIG=run --publish published=33333,target=33333 ghcr.io/shuffle/shuffle-worker:nightly @@ -457,6 +444,19 @@ func deployServiceWorkers(image string) { } } + isMemcachedRunning, err := checkMemcached(ctx, dockercli) + if err != nil { + log.Printf("[ERROR] Failed checking memcached: %s", err) + } + if isMemcachedRunning == false { + log.Printf("[ERROR] Memcached is not running. Will try to deploy it.") + deployMemcached(dockercli) + } + + ip := "shuffle-cache" + + os.Setenv("SHUFFLE_MEMCACHED", fmt.Sprintf("%s:11211", ip)) + defaultNetworkAttach := false if containerId != "" { log.Printf("[DEBUG] Should connect orborus container to worker network as it's running in Docker with name %#v!", containerId) From 3010594761d54c2720fa3378c5741959d004c452 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 4 Dec 2024 01:13:09 +0530 Subject: [PATCH 5/8] new workflow for latest realise of orborus and worker --- .github/workflows/latest-release.yaml | 76 +++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 .github/workflows/latest-release.yaml diff --git a/.github/workflows/latest-release.yaml b/.github/workflows/latest-release.yaml new file mode 100644 index 00000000..4812489a --- /dev/null +++ b/.github/workflows/latest-release.yaml @@ -0,0 +1,76 @@ +name: dockerbuild + +on: + push: + branches: + - 2.0.0 + paths: + - "**" + - "!.github/**" + - "!**.md" + - "!docker-compose.yml" +jobs: + main: + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + include: + - app: orborus + path: functions/onprem/orborus + version: lastest + experimental: true + - app: worker + path: functions/onprem/worker + version: lastest + experimental: true + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: "amd64,arm64,arm" + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to Ghcr + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Ghcr Build and push + id: docker_build + uses: docker/build-push-action@v4 + env: + BUILDX_NO_DEFAULT_LOAD: true + with: + logout: false + context: ${{ matrix.path }}/ + file: ${{ matrix.path }}/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + tags: | + ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ matrix.version }} + ghcr.io/shuffle/shuffle-${{ matrix.app }}:nightly + ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ matrix.version }} + ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:nightly + frikky/shuffle-${{ matrix.app }}:${{ matrix.version }} + frikky/shuffle-${{ matrix.app }}:nightly + frikky/shuffle:${{ matrix.app }} + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} From a8389a9d09d31a5baf632db0b866d4b5381b8474 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 4 Dec 2024 11:07:41 +0530 Subject: [PATCH 6/8] build latest for worker and orborus --- .github/workflows/dockerbuild.yaml | 4 +- .github/workflows/latest-release.yaml | 76 --------------------------- 2 files changed, 2 insertions(+), 78 deletions(-) delete mode 100644 .github/workflows/latest-release.yaml diff --git a/.github/workflows/dockerbuild.yaml b/.github/workflows/dockerbuild.yaml index ef4848e0..aa794810 100644 --- a/.github/workflows/dockerbuild.yaml +++ b/.github/workflows/dockerbuild.yaml @@ -27,11 +27,11 @@ jobs: experimental: true - app: orborus path: functions/onprem/orborus - version: nightly + version: latest experimental: true - app: worker path: functions/onprem/worker - version: nightly + version: latest experimental: true steps: - name: Checkout diff --git a/.github/workflows/latest-release.yaml b/.github/workflows/latest-release.yaml deleted file mode 100644 index 4812489a..00000000 --- a/.github/workflows/latest-release.yaml +++ /dev/null @@ -1,76 +0,0 @@ -name: dockerbuild - -on: - push: - branches: - - 2.0.0 - paths: - - "**" - - "!.github/**" - - "!**.md" - - "!docker-compose.yml" -jobs: - main: - runs-on: ubuntu-latest - continue-on-error: ${{ matrix.experimental }} - strategy: - fail-fast: false - matrix: - include: - - app: orborus - path: functions/onprem/orborus - version: lastest - experimental: true - - app: worker - path: functions/onprem/worker - version: lastest - experimental: true - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: "amd64,arm64,arm" - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Login to Ghcr - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Ghcr Build and push - id: docker_build - uses: docker/build-push-action@v4 - env: - BUILDX_NO_DEFAULT_LOAD: true - with: - logout: false - context: ${{ matrix.path }}/ - file: ${{ matrix.path }}/Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - tags: | - ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ matrix.version }} - ghcr.io/shuffle/shuffle-${{ matrix.app }}:nightly - ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ matrix.version }} - ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:nightly - frikky/shuffle-${{ matrix.app }}:${{ matrix.version }} - frikky/shuffle-${{ matrix.app }}:nightly - frikky/shuffle:${{ matrix.app }} - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} From d9cd55351f9dd21816aaf69e2763d1bbf9cdb684 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 4 Dec 2024 11:11:49 +0530 Subject: [PATCH 7/8] latest build for orborus and worker --- functions/onprem/orborus/orborus.go | 4 +++- functions/onprem/worker/worker.go | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 4f7a30a3..b6e7680e 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2150,7 +2150,9 @@ func main() { log.Printf("[DEBUG] Starting iteration on environment %#v (default = Shuffle). Got statuscode %d from backend on first request", environment, newresp.StatusCode) } - go AutoScale(ctx) + if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" && os.Getenv("SHUFFLE_SCALE_REPLICAS") == "" { + go AutoScale(ctx) + } hasStarted = true } diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index ae6034b3..189a619e 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -4129,8 +4129,9 @@ func runWebserver(listener net.Listener) { log.Printf("[DEBUG] SHUFFLE_APP_EXECUTIONS_PER_MINUTE set to value %s. Trying to overwrite default (%d)", os.Getenv("SHUFFLE_APP_EXECUTIONS_PER_MINUTE"), maxExecutionsPerMinute) } - go AutoScaleApps(ctx, dockercli, maxExecutionsPerMinute) - + if strings.ToLower(os.Getenv("SHUFFLE_SWARM_CONFIG")) == "run" || strings.ToLower(os.Getenv("SHUFFLE_APP_REPLICAS")) == "" { + go AutoScaleApps(ctx, dockercli, maxExecutionsPerMinute) + } if strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" { r.HandleFunc("/debug/pprof/", pprof.Index) r.HandleFunc("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP) From 0773a1d6a11d8a959a07dd50d4ed6bf010daac7f Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 4 Dec 2024 17:50:07 +0530 Subject: [PATCH 8/8] remove tenzir verbosity --- functions/onprem/orborus/orborus.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index b6e7680e..582e555c 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2931,9 +2931,7 @@ func checkTenzirNode() error { return nil } - log.Printf("[DEBUG] Failed to verify Tenzir node on %s: %s", url, err) - - return fmt.Errorf("Tenzir node is not available") + return fmt.Errorf("Tenzir node is not available due to: %s", err) } func createPipeline(command, identifier string) (string, error) {