From 9aa2687af3c175bf53fa472a7db44c896c9c7bb6 Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 7 Dec 2023 22:48:00 +0100 Subject: [PATCH 01/13] Update dockerbuild.yaml --- .github/workflows/dockerbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerbuild.yaml b/.github/workflows/dockerbuild.yaml index ca750fa5..a1fee859 100644 --- a/.github/workflows/dockerbuild.yaml +++ b/.github/workflows/dockerbuild.yaml @@ -3,7 +3,7 @@ name: dockerbuild on: push: branches: - - 1.4.0 + - main paths: - "**" - "!.github/**" From 305e67acbdb7500cbdec841693b6083504f71b98 Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 7 Dec 2023 22:48:11 +0100 Subject: [PATCH 02/13] Update dockerbuild.yaml --- .github/workflows/dockerbuild.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerbuild.yaml b/.github/workflows/dockerbuild.yaml index a1fee859..b3c546f1 100644 --- a/.github/workflows/dockerbuild.yaml +++ b/.github/workflows/dockerbuild.yaml @@ -77,9 +77,9 @@ jobs: cache-to: type=local,dest=/tmp/.buildx-cache tags: | ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ matrix.version }} - ghcr.io/shuffle/shuffle-${{ matrix.app }}:nightly + ghcr.io/shuffle/shuffle-${{ matrix.app }}:latest ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ matrix.version }} - ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:nightly + ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:latest - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} From 5a419d018de19c039ef38e6b6c4b80fa5954e833 Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 7 Dec 2023 22:48:34 +0100 Subject: [PATCH 03/13] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 188da59c..e8c2f9c1 100755 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ Contributing guidelines are outlined [here](https://github.com/frikky/Shuffle/bl + ## License All modular information related to Shuffle will be under MIT (anyone can use it for whatever purpose), with Shuffle itself using AGPLv3. From 575fc36c0ed804e262077c497cc47b0f65bffe0b Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 7 Dec 2023 22:49:17 +0100 Subject: [PATCH 04/13] Update docker-compose.yml --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 727f3e65..2b4dadf2 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - image: ghcr.io/shuffle/shuffle-frontend:nightly + image: ghcr.io/shuffle/shuffle-frontend:latest container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -15,7 +15,7 @@ services: depends_on: - backend backend: - image: ghcr.io/shuffle/shuffle-backend:nightly + image: ghcr.io/shuffle/shuffle-backend:latest container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -34,7 +34,7 @@ services: - SHUFFLE_FILE_LOCATION=/shuffle-files restart: unless-stopped orborus: - image: ghcr.io/shuffle/shuffle-orborus:nightly + image: ghcr.io/shuffle/shuffle-orborus:latest container_name: shuffle-orborus hostname: shuffle-orborus networks: From b1fb3a7c26c2430721dd71be2c4d85a640d77de1 Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 7 Dec 2023 22:49:50 +0100 Subject: [PATCH 05/13] Update app_base.py --- backend/app_sdk/app_base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 003f4459..f99c9a3b 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -145,6 +145,7 @@ def escape(a): a = str(a) return json_escape(a) + @shuffle_filters.register def neat_json(a): try: From e5f651dd1f42c15c21c4f09a1cc40621d9a1e5ef Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 7 Dec 2023 23:20:20 +0100 Subject: [PATCH 06/13] whoa --- .github/workflows/dockerbuild.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/dockerbuild.yaml b/.github/workflows/dockerbuild.yaml index b3c546f1..3ae6630d 100644 --- a/.github/workflows/dockerbuild.yaml +++ b/.github/workflows/dockerbuild.yaml @@ -78,6 +78,8 @@ jobs: tags: | ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ matrix.version }} ghcr.io/shuffle/shuffle-${{ matrix.app }}:latest + frikky/shuffle-${{ matrix.app }}:${{ matrix.version }} + frikky/shuffle-${{ matrix.app }}:latest ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ matrix.version }} ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:latest From e667de55006d640f3b9a48965e5f42307f4cc507 Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 7 Dec 2023 23:20:52 +0100 Subject: [PATCH 07/13] Rebuild with correct dockerhub location for sdk --- backend/go-app/main.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 571e8422..0123d3a8 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -11,9 +11,6 @@ import ( "crypto/md5" "strconv" - //"crypto/tls" - //"crypto/x509" - //"encoding/base64" "encoding/hex" "encoding/json" "errors" From ccc2ede226384f2af13d307692128d2178de8efa Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 7 Dec 2023 23:39:42 +0100 Subject: [PATCH 08/13] Dockerbuild fix --- .github/workflows/dockerbuild.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerbuild.yaml b/.github/workflows/dockerbuild.yaml index 3ae6630d..ed4c109f 100644 --- a/.github/workflows/dockerbuild.yaml +++ b/.github/workflows/dockerbuild.yaml @@ -78,10 +78,11 @@ jobs: tags: | ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ matrix.version }} ghcr.io/shuffle/shuffle-${{ matrix.app }}:latest - frikky/shuffle-${{ matrix.app }}:${{ matrix.version }} - frikky/shuffle-${{ matrix.app }}:latest ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ matrix.version }} ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:latest + frikky/shuffle-${{ matrix.app }}:${{ matrix.version }} + frikky/shuffle-${{ matrix.app }}:latest + frikky/shuffle:${{ matrix.app }} - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} From 92fdd115e165110fe3b439aabe524ae623dca081 Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 8 Dec 2023 00:46:54 +0100 Subject: [PATCH 09/13] Increased context timeout for orborus --- frontend/src/views/LoginPage.jsx | 2 +- functions/onprem/orborus/orborus.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/views/LoginPage.jsx b/frontend/src/views/LoginPage.jsx index 5958b2fc..5ca12681 100755 --- a/frontend/src/views/LoginPage.jsx +++ b/frontend/src/views/LoginPage.jsx @@ -333,7 +333,7 @@ const LoginDialog = (props) => { 1. Make sure shuffle-database folder has correct access, and that you have a minimum of 2Gb of RAM available:{" "}

- sudo chown 1000:1000 -R shuffle-database + sudo chown -R 1000:1000 shuffle-database Date: Mon, 11 Dec 2023 14:08:03 +0100 Subject: [PATCH 10/13] Fixed some orborus + worker things for rebuild --- functions/onprem/orborus/go.mod | 2 +- functions/onprem/orborus/go.sum | 2 ++ functions/onprem/worker/go.mod | 2 +- functions/onprem/worker/go.sum | 2 ++ 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 4d2d9450..a621444e 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -9,7 +9,7 @@ require ( github.com/mackerelio/go-osstat v0.2.3 github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil v3.21.11+incompatible - github.com/shuffle/shuffle-shared v0.5.29 + github.com/shuffle/shuffle-shared v0.5.31 k8s.io/api v0.28.1 k8s.io/apimachinery v0.28.1 k8s.io/client-go v0.28.1 diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index fa8b721c..d62bb66d 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -280,6 +280,8 @@ github.com/shuffle/shuffle-shared v0.4.96 h1:iaIB/HP9eKpw9DMMJZhSLDbKdHJt075kFYL github.com/shuffle/shuffle-shared v0.4.96/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/shuffle/shuffle-shared v0.5.29 h1:n4vThl7v3mFVXbrIW71XREFdmZZo7mOBAWxnsdiNjDk= github.com/shuffle/shuffle-shared v0.5.29/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= +github.com/shuffle/shuffle-shared v0.5.31 h1:OV4IIfKWWFW66WjGvyXOmmsSz3p8pW9L1ge1mDo8ftM= +github.com/shuffle/shuffle-shared v0.5.31/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 9e0f8e1e..b847ac38 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -11,7 +11,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.5.29 + github.com/shuffle/shuffle-shared v0.5.31 k8s.io/api v0.28.3 k8s.io/apimachinery v0.28.3 k8s.io/client-go v0.28.3 diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index 5afdce76..821c92f0 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -286,6 +286,8 @@ github.com/shuffle/shuffle-shared v0.4.57 h1:o+mMPRY4ourkE3R0qdi80jg6RlCtvAJ/VVr github.com/shuffle/shuffle-shared v0.4.57/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/shuffle/shuffle-shared v0.5.29 h1:n4vThl7v3mFVXbrIW71XREFdmZZo7mOBAWxnsdiNjDk= github.com/shuffle/shuffle-shared v0.5.29/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= +github.com/shuffle/shuffle-shared v0.5.31 h1:OV4IIfKWWFW66WjGvyXOmmsSz3p8pW9L1ge1mDo8ftM= +github.com/shuffle/shuffle-shared v0.5.31/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= From f862a1358f2695007fe177aaf89299bb8456fac6 Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 12 Dec 2023 11:36:26 +0100 Subject: [PATCH 11/13] Update .env --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 73fbdf63..4f361299 100755 --- a/.env +++ b/.env @@ -59,7 +59,7 @@ IS_KUBERNETES=false # Used for controlling if the environment should run in SHUFFLE_BASE_IMAGE_NAME=shuffle SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io -SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-1.1.0" +SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-1.3.1" # The eth0 interface inside a container corresponds # to the virtual Ethernet interface that connects From ae0fb2e76f9f013052b1abc9f8c721e761760ba0 Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 20 Dec 2023 16:53:45 +0100 Subject: [PATCH 12/13] Added SHUFFLE_STATS_DISABLED to true by default to not overload Docker --- docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2b4dadf2..63124856 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,7 +43,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock environment: - SHUFFLE_APP_SDK_TIMEOUT=300 - - SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=5 # The amount of concurrent executions Orborus can handle. + - SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=7 # The amount of concurrent executions Orborus can handle. #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:5001 @@ -55,6 +55,7 @@ services: - HTTPS_PROXY=${HTTPS_PROXY} - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY} + - SHUFFLE_STATS_DISABLED=true restart: unless-stopped security_opt: - seccomp:unconfined From 001fa3952d37cf9a01abe2f263ea9b90cb28a674 Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 12 Jan 2024 12:20:37 +0100 Subject: [PATCH 13/13] Rename kubernetes.md to README.md --- functions/kubernetes/{kubernetes.md => README.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename functions/kubernetes/{kubernetes.md => README.md} (100%) diff --git a/functions/kubernetes/kubernetes.md b/functions/kubernetes/README.md similarity index 100% rename from functions/kubernetes/kubernetes.md rename to functions/kubernetes/README.md