From 6b18e90e3bc2c52b28c377b8f07019454fcf86d2 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 22 May 2024 20:23:38 +0530 Subject: [PATCH 01/14] init: quick testing --- .github/workflows/quick-testing.yml | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/quick-testing.yml diff --git a/.github/workflows/quick-testing.yml b/.github/workflows/quick-testing.yml new file mode 100644 index 00000000..f36c6241 --- /dev/null +++ b/.github/workflows/quick-testing.yml @@ -0,0 +1,31 @@ +name: Docker Compose Up and Ping + +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Docker + run: | + curl -fsSL https://get.docker.com | sh + sudo systemctl start docker + + - name: Set up Docker Compose + run: | + curl -L "https://github.com/docker/compose/releases/download/v2.10.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose + docker-compose version + + - name: Start Docker Compose + run: docker-compose up -d + + - name: Wait for services to start + run: sleep 20 + + - name: Ping localhost + run: curl --fail http://localhost:3001 || exit 1 From 6119655d318111b2272012df313c4131544fbd4f Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 22 May 2024 20:29:57 +0530 Subject: [PATCH 02/14] fix: trying out docker-compose in actions --- .github/workflows/quick-testing.yml | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/.github/workflows/quick-testing.yml b/.github/workflows/quick-testing.yml index f36c6241..f09cf801 100644 --- a/.github/workflows/quick-testing.yml +++ b/.github/workflows/quick-testing.yml @@ -10,22 +10,8 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - - name: Set up Docker - run: | - curl -fsSL https://get.docker.com | sh - sudo systemctl start docker - - - name: Set up Docker Compose - run: | - curl -L "https://github.com/docker/compose/releases/download/v2.10.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose - chmod +x /usr/local/bin/docker-compose - docker-compose version - - - name: Start Docker Compose + - name: Build the stack run: docker-compose up -d - - name: Wait for services to start - run: sleep 20 - - - name: Ping localhost - run: curl --fail http://localhost:3001 || exit 1 + - name: In 30 seconds, check docker ps + run: sleep 30 && docker ps \ No newline at end of file From 8f3e663f20a20a3ee162da5eb874d7024ce25f04 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 22 May 2024 20:54:18 +0530 Subject: [PATCH 03/14] fix: making testing end-to-end --- .github/workflows/quick-testing.yml | 83 ++++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 2 deletions(-) diff --git a/.github/workflows/quick-testing.yml b/.github/workflows/quick-testing.yml index f09cf801..b183b50d 100644 --- a/.github/workflows/quick-testing.yml +++ b/.github/workflows/quick-testing.yml @@ -13,5 +13,84 @@ jobs: - name: Build the stack run: docker-compose up -d - - name: In 30 seconds, check docker ps - run: sleep 30 && docker ps \ No newline at end of file + - name: Wait for 30 seconds + run: sleep 30 + + - name: Check for restarting containers in a loop + run: | + ATTEMPTS=30 # Total time = ATTEMPTS * 5 seconds = 30 seconds + for i in $(seq 1 $ATTEMPTS); do + RESTARTING_CONTAINERS=$(docker ps --filter "status=restarting" --format "{{.Names}}") + if [ -n "$RESTARTING_CONTAINERS" ]; then + echo "The following containers are restarting:" + echo "$RESTARTING_CONTAINERS" + exit 1 + fi + echo "No containers are restarting. Attempt $i/$ATTEMPTS." + sleep 1 + done + echo "No containers were found in a restarting state after $ATTEMPTS checks." + + - name: Check if the response from the web service includes "Shuffle" + run: | + RESPONSE=$(curl -s http://localhost:3001) + if echo "$RESPONSE" | grep -q "Shuffle"; then + echo "The word 'Shuffle' was found in the response." + else + echo "The word 'Shuffle' was not found in the response." + exit 1 + fi + + - name: Register a user and check the status code + run: | + STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" 'http://localhost:3001/api/v1/register' \ + -H 'Accept: */*' \ + -H 'Accept-Language: en-US,en;q=0.9' \ + -H 'Connection: keep-alive' \ + -H 'Content-Type: application/json' \ + --data-raw '{"username":"demo@demo.io","password":"supercoolpassword"}') + if [ "$STATUS_CODE" -eq 200 ]; then + echo "User registration was successful with status code 200." + else + echo "User registration failed with status code $STATUS_CODE." + exit 1 + fi + + - name: Get the API key and run a health check + run: | + RESPONSE=$(curl -s -k -u admin:StrongShufflePassword321! 'https://localhost:9200/users/_search') + API_KEY=$(echo "$RESPONSE" | jq -r '.hits.hits[0]._source.apikey') + if [ -n "$API_KEY" ]; then + echo "Admin API key: $API_KEY" + else + echo "Failed to retrieve the API key for the admin user." + exit 1 + fi + + HEALTH_RESPONSE=$(curl -s 'http://localhost:3001/api/v1/health?force=true' \ + -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7' \ + -H 'Accept-Language: en-US,en;q=0.9' \ + -H 'Connection: keep-alive' \ + -H 'Sec-Fetch-Dest: document' \ + -H 'Sec-Fetch-Mode: navigate' \ + -H 'Sec-Fetch-Site: none' \ + -H "Authorization: Bearer $API_KEY" \ + -H 'Sec-Fetch-User: ?1' \ + -H 'Upgrade-Insecure-Requests: 1' \ + -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36' \ + -H 'sec-ch-ua: "Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"' \ + -H 'sec-ch-ua-mobile: ?0' \ + -H 'sec-ch-ua-platform: "macOS"') + + WORKFLOWS_CREATE=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.create') + WORKFLOWS_RUN=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run') + WORKFLOWS_RUN_FINISHED=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run_finished') + WORKFLOWS_RUN_STATUS=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run_status') + WORKFLOWS_DELETE=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.delete') + + if [ "$WORKFLOWS_CREATE" = "true" ] && [ "$WORKFLOWS_RUN" = "true" ] && [ "$WORKFLOWS_RUN_FINISHED" = "true" ] && [ "$WORKFLOWS_RUN_STATUS" = "FINISHED" ] && [ "$WORKFLOWS_DELETE" = "true" ]; then + echo "Health endpoint check was successful." + else + echo "Health endpoint check failed. Response did not meet expected criteria." + exit 1 + fi From b2cc556b6f0e6e78382b8958447036f012055f33 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 22 May 2024 20:58:59 +0530 Subject: [PATCH 04/14] fix: tackling 502s --- .github/workflows/quick-testing.yml | 38 ++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/.github/workflows/quick-testing.yml b/.github/workflows/quick-testing.yml index b183b50d..b8e73daa 100644 --- a/.github/workflows/quick-testing.yml +++ b/.github/workflows/quick-testing.yml @@ -43,18 +43,32 @@ jobs: - name: Register a user and check the status code run: | - STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" 'http://localhost:3001/api/v1/register' \ - -H 'Accept: */*' \ - -H 'Accept-Language: en-US,en;q=0.9' \ - -H 'Connection: keep-alive' \ - -H 'Content-Type: application/json' \ - --data-raw '{"username":"demo@demo.io","password":"supercoolpassword"}') - if [ "$STATUS_CODE" -eq 200 ]; then - echo "User registration was successful with status code 200." - else - echo "User registration failed with status code $STATUS_CODE." - exit 1 - fi + MAX_RETRIES=30 + RETRY_INTERVAL=2 + + for (( i=1; i<=$MAX_RETRIES; i++ )) + do + STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" 'http://localhost:3001/api/v1/register' \ + -H 'Accept: */*' \ + -H 'Accept-Language: en-US,en;q=0.9' \ + -H 'Connection: keep-alive' \ + -H 'Content-Type: application/json' \ + --data-raw '{"username":"demo@demo.io","password":"supercoolpassword"}') + + if [ "$STATUS_CODE" -eq 200 ]; then + echo "User registration was successful with status code 200." + exit 0 + elif [ "$STATUS_CODE" -ne 502 ]; then + echo "User registration failed with status code $STATUS_CODE." + exit 1 + fi + + echo "Received status code 502. Retrying in $RETRY_INTERVAL seconds... ($i/$MAX_RETRIES)" + sleep $RETRY_INTERVAL + done + + echo "User registration failed after $MAX_RETRIES attempts." + exit 1 - name: Get the API key and run a health check run: | From c57117e298728b16e549eb073bb09bbbaf7dd59d Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 22 May 2024 21:04:07 +0530 Subject: [PATCH 05/14] fix: debugging 502s in pinging --- .github/workflows/quick-testing.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/quick-testing.yml b/.github/workflows/quick-testing.yml index b8e73daa..881f2474 100644 --- a/.github/workflows/quick-testing.yml +++ b/.github/workflows/quick-testing.yml @@ -44,7 +44,8 @@ jobs: - name: Register a user and check the status code run: | MAX_RETRIES=30 - RETRY_INTERVAL=2 + RETRY_INTERVAL=5 + CONTAINER_NAME="shuffle-backend" for (( i=1; i<=$MAX_RETRIES; i++ )) do @@ -60,6 +61,8 @@ jobs: exit 0 elif [ "$STATUS_CODE" -ne 502 ]; then echo "User registration failed with status code $STATUS_CODE." + echo "Fetching last 30 lines of logs from container $CONTAINER_NAME..." + docker logs --tail 30 "$CONTAINER_NAME" exit 1 fi From 728f5d1d16dc49232471bedafe3998fde545077d Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 22 May 2024 21:10:08 +0530 Subject: [PATCH 06/14] fix: debugging 502s in pinging (1) --- .github/workflows/quick-testing.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/quick-testing.yml b/.github/workflows/quick-testing.yml index 881f2474..2160022c 100644 --- a/.github/workflows/quick-testing.yml +++ b/.github/workflows/quick-testing.yml @@ -44,7 +44,7 @@ jobs: - name: Register a user and check the status code run: | MAX_RETRIES=30 - RETRY_INTERVAL=5 + RETRY_INTERVAL=10 CONTAINER_NAME="shuffle-backend" for (( i=1; i<=$MAX_RETRIES; i++ )) @@ -62,7 +62,8 @@ jobs: elif [ "$STATUS_CODE" -ne 502 ]; then echo "User registration failed with status code $STATUS_CODE." echo "Fetching last 30 lines of logs from container $CONTAINER_NAME..." - docker logs --tail 30 "$CONTAINER_NAME" + logs_output=$(docker logs --tail 30 "$CONTAINER_NAME") + echo "$logs_output" exit 1 fi From 00438d7b9ced29068783e891e524208f2bdc1372 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 22 May 2024 21:12:37 +0530 Subject: [PATCH 07/14] fix: debugging 502s in pinging (2) --- .github/workflows/quick-testing.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/quick-testing.yml b/.github/workflows/quick-testing.yml index 2160022c..ae194436 100644 --- a/.github/workflows/quick-testing.yml +++ b/.github/workflows/quick-testing.yml @@ -61,13 +61,13 @@ jobs: exit 0 elif [ "$STATUS_CODE" -ne 502 ]; then echo "User registration failed with status code $STATUS_CODE." - echo "Fetching last 30 lines of logs from container $CONTAINER_NAME..." - logs_output=$(docker logs --tail 30 "$CONTAINER_NAME") - echo "$logs_output" exit 1 fi - echo "Received status code 502. Retrying in $RETRY_INTERVAL seconds... ($i/$MAX_RETRIES)" + echo "Received status code $STATUS_CODE. Retrying in $RETRY_INTERVAL seconds... ($i/$MAX_RETRIES)" + echo "Fetching last 30 lines of logs from container $CONTAINER_NAME..." + logs_output=$(docker logs --tail 30 "$CONTAINER_NAME") + echo "$logs_output" sleep $RETRY_INTERVAL done From 800b108c325a6b786e7de8d9183872e00ac746d7 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 22 May 2024 21:15:35 +0530 Subject: [PATCH 08/14] fix: debugging 502s in pinging (3) --- .github/workflows/quick-testing.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/quick-testing.yml b/.github/workflows/quick-testing.yml index ae194436..e86d3fce 100644 --- a/.github/workflows/quick-testing.yml +++ b/.github/workflows/quick-testing.yml @@ -65,9 +65,17 @@ jobs: fi echo "Received status code $STATUS_CODE. Retrying in $RETRY_INTERVAL seconds... ($i/$MAX_RETRIES)" + echo "Fetching last 30 lines of logs from container $CONTAINER_NAME..." + logs_output=$(docker logs --tail 30 "$CONTAINER_NAME") echo "$logs_output" + + echo "Fetching last 30 lines of logs from container shuffle-opensearch..." + + opensearch_logs=$(docker logs --tail 30 shuffle-opensearch) + echo "$opensearch_logs" + sleep $RETRY_INTERVAL done From 10b05a883e6ae7ca62402b1d7f02d4380d8d8352 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 22 May 2024 21:19:33 +0530 Subject: [PATCH 09/14] fix: debugging 502s in pinging (4) --- .github/workflows/quick-testing.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/quick-testing.yml b/.github/workflows/quick-testing.yml index e86d3fce..9dbf950e 100644 --- a/.github/workflows/quick-testing.yml +++ b/.github/workflows/quick-testing.yml @@ -9,6 +9,9 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 + + - name: Set up opensearch directory + run: mkdir shuffle-database && chmod -R 755 shuffle-database - name: Build the stack run: docker-compose up -d @@ -31,7 +34,7 @@ jobs: done echo "No containers were found in a restarting state after $ATTEMPTS checks." - - name: Check if the response from the web service includes "Shuffle" + - name: Check if the response from the frontend contains the word "Shuffle" run: | RESPONSE=$(curl -s http://localhost:3001) if echo "$RESPONSE" | grep -q "Shuffle"; then @@ -65,7 +68,6 @@ jobs: fi echo "Received status code $STATUS_CODE. Retrying in $RETRY_INTERVAL seconds... ($i/$MAX_RETRIES)" - echo "Fetching last 30 lines of logs from container $CONTAINER_NAME..." logs_output=$(docker logs --tail 30 "$CONTAINER_NAME") From 279950883b43188ccbfd7a3cde83b68d4ca85351 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 22 May 2024 21:22:17 +0530 Subject: [PATCH 10/14] fix: debugging 502s in pinging (5) --- .github/workflows/quick-testing.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/quick-testing.yml b/.github/workflows/quick-testing.yml index 9dbf950e..cbdfd1dd 100644 --- a/.github/workflows/quick-testing.yml +++ b/.github/workflows/quick-testing.yml @@ -11,7 +11,7 @@ jobs: uses: actions/checkout@v2 - name: Set up opensearch directory - run: mkdir shuffle-database && chmod -R 755 shuffle-database + run: mkdir shuffle-database && chmod -R 1000:1000 shuffle-database - name: Build the stack run: docker-compose up -d @@ -19,8 +19,9 @@ jobs: - name: Wait for 30 seconds run: sleep 30 - - name: Check for restarting containers in a loop + - name: Check for restarting containers in a loop and fixing perms again once run: | + chmod -R 1000:1000 shuffle-database ATTEMPTS=30 # Total time = ATTEMPTS * 5 seconds = 30 seconds for i in $(seq 1 $ATTEMPTS); do RESTARTING_CONTAINERS=$(docker ps --filter "status=restarting" --format "{{.Names}}") From d3c344e39b7897b707acf781ab635ed77fa02801 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 22 May 2024 21:24:53 +0530 Subject: [PATCH 11/14] fix: debugging 502s in pinging (6) --- .github/workflows/quick-testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/quick-testing.yml b/.github/workflows/quick-testing.yml index cbdfd1dd..6c53238d 100644 --- a/.github/workflows/quick-testing.yml +++ b/.github/workflows/quick-testing.yml @@ -11,7 +11,7 @@ jobs: uses: actions/checkout@v2 - name: Set up opensearch directory - run: mkdir shuffle-database && chmod -R 1000:1000 shuffle-database + run: mkdir shuffle-database && chown -R 1000:1000 shuffle-database - name: Build the stack run: docker-compose up -d @@ -21,7 +21,7 @@ jobs: - name: Check for restarting containers in a loop and fixing perms again once run: | - chmod -R 1000:1000 shuffle-database + chown -R 1000:1000 shuffle-database ATTEMPTS=30 # Total time = ATTEMPTS * 5 seconds = 30 seconds for i in $(seq 1 $ATTEMPTS); do RESTARTING_CONTAINERS=$(docker ps --filter "status=restarting" --format "{{.Names}}") From a0c1db71a35919479716c5a9047c8283332ead5d Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 22 May 2024 21:31:14 +0530 Subject: [PATCH 12/14] fix: debugging 502s in pinging (7) --- .github/workflows/quick-testing.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/quick-testing.yml b/.github/workflows/quick-testing.yml index 6c53238d..af8924d3 100644 --- a/.github/workflows/quick-testing.yml +++ b/.github/workflows/quick-testing.yml @@ -11,7 +11,7 @@ jobs: uses: actions/checkout@v2 - name: Set up opensearch directory - run: mkdir shuffle-database && chown -R 1000:1000 shuffle-database + run: mkdir shuffle-database && chmod -R 755 shuffle-database - name: Build the stack run: docker-compose up -d @@ -19,9 +19,11 @@ jobs: - name: Wait for 30 seconds run: sleep 30 - - name: Check for restarting containers in a loop and fixing perms again once + - name: Check for restarting containers in a loop and fixing perms again run: | - chown -R 1000:1000 shuffle-database + echo "Changing permissions on shuffle-database directory again" + chmod -R 755 shuffle-database + ATTEMPTS=30 # Total time = ATTEMPTS * 5 seconds = 30 seconds for i in $(seq 1 $ATTEMPTS); do RESTARTING_CONTAINERS=$(docker ps --filter "status=restarting" --format "{{.Names}}") From f01773a98b99777b14b0fb1e4fa45572191dc60b Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 22 May 2024 21:35:50 +0530 Subject: [PATCH 13/14] fix: debugging 502s in pinging (8) --- .github/workflows/quick-testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/quick-testing.yml b/.github/workflows/quick-testing.yml index af8924d3..997116f7 100644 --- a/.github/workflows/quick-testing.yml +++ b/.github/workflows/quick-testing.yml @@ -11,7 +11,7 @@ jobs: uses: actions/checkout@v2 - name: Set up opensearch directory - run: mkdir shuffle-database && chmod -R 755 shuffle-database + run: mkdir shuffle-database && chmod -R 777 shuffle-database - name: Build the stack run: docker-compose up -d @@ -22,7 +22,7 @@ jobs: - name: Check for restarting containers in a loop and fixing perms again run: | echo "Changing permissions on shuffle-database directory again" - chmod -R 755 shuffle-database + chmod -R 777 shuffle-database ATTEMPTS=30 # Total time = ATTEMPTS * 5 seconds = 30 seconds for i in $(seq 1 $ATTEMPTS); do From fa2ed689c318c12a828f789496fc5b78dc489ca6 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 22 May 2024 21:38:22 +0530 Subject: [PATCH 14/14] fix: debugging 502s in pinging (9) --- .github/workflows/quick-testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/quick-testing.yml b/.github/workflows/quick-testing.yml index 997116f7..a04ea03b 100644 --- a/.github/workflows/quick-testing.yml +++ b/.github/workflows/quick-testing.yml @@ -21,8 +21,8 @@ jobs: - name: Check for restarting containers in a loop and fixing perms again run: | - echo "Changing permissions on shuffle-database directory again" - chmod -R 777 shuffle-database + # echo "Changing permissions on shuffle-database directory again" + # chmod -R 777 shuffle-database ATTEMPTS=30 # Total time = ATTEMPTS * 5 seconds = 30 seconds for i in $(seq 1 $ATTEMPTS); do