From 093df3d5fbecf438a0128c751674ec6826baa18d Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 31 Mar 2023 02:14:04 +0200 Subject: [PATCH] Added way more robust swarm system with caching utilization at scale --- .env | 5 +++++ backend/go-app/go.mod | 2 +- frontend/src/components/Oauth2Auth.jsx | 13 +++++++------ functions/onprem/orborus/go.mod | 4 ++-- functions/onprem/orborus/go.sum | 4 ++++ functions/onprem/worker/build.sh | 2 +- functions/onprem/worker/go.mod | 2 +- functions/onprem/worker/go.sum | 2 ++ functions/onprem/worker/worker.go | 2 +- 9 files changed, 24 insertions(+), 12 deletions(-) diff --git a/.env b/.env index 35edd59a..eafa2765 100644 --- a/.env +++ b/.env @@ -57,6 +57,11 @@ SHUFFLE_BASE_IMAGE_NAME=shuffle SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-1.1.0" +## shuffle_memcached (for distributed caching) +## shuffle_SWARM_CONFIG (run vs not run) +## shuffle_Scale_Replicas (workers/node) +## shuffle_App_Replicas (apps/node) + # Used for auto-cleanup of containers. REALLY important at scale. SHUFFLE_CONTAINER_AUTO_CLEANUP=false SHUFFLE_ELASTIC=true diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 67547cb4..b4809e89 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -19,7 +19,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.3.74 + github.com/shuffle/shuffle-shared v0.3.98 golang.org/x/crypto v0.3.0 google.golang.org/api v0.103.0 google.golang.org/appengine v1.6.7 diff --git a/frontend/src/components/Oauth2Auth.jsx b/frontend/src/components/Oauth2Auth.jsx index 98b3d296..6157e270 100644 --- a/frontend/src/components/Oauth2Auth.jsx +++ b/frontend/src/components/Oauth2Auth.jsx @@ -113,10 +113,9 @@ const AuthenticationOauth2 = (props) => { const [buttonClicked, setButtonClicked] = React.useState(false); const [offlineAccess, setOfflineAccess] = React.useState(true); - const allscopes = - authenticationType.scope !== undefined ? authenticationType.scope : []; + const allscopes = authenticationType.scope !== undefined ? authenticationType.scope : []; + - console.log("ALLSCOPES: ", allscopes) const [selectedScopes, setSelectedScopes] = React.useState(allscopes.length === 1 ? [allscopes[0]] : []) const [manuallyConfigure, setManuallyConfigure] = React.useState( defaultConfigSet ? false : true @@ -247,8 +246,7 @@ const AuthenticationOauth2 = (props) => { "AI02egeCQh1Zskm1QAJaaR6dzjR97V2F", "", "https://api.atlassian.com", - ["read:jira-work", "write:jira-work", "read:servicedesk:jira-service-management", "write:servicedesk:jira-service-management", "read:request:jira-service-management", "write:request:jira-service-management"], - + ["read:jira-work", "write:jira-work", "read:servicedesk:jira-service-management", "write:servicedesk:jira-service-management", "read:request:jira-service-management", "write:request:jira-service-management", "offline_access"], admin_consent, ) } @@ -266,8 +264,11 @@ const AuthenticationOauth2 = (props) => { var resources = ""; if (scopes !== undefined && (scopes !== null) & (scopes.length > 0)) { + console.log("IN scope 1") if (offlineAccess === true && !scopes.includes("offline_access")) { - if (authenticationType.redirect_uri.includes("microsoft")) { + + console.log("IN scope 2") + if (!authenticationType.redirect_uri.includes("google")) { console.log("Appending offline access") scopes.push("offline_access") } diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index ae8790b6..e9662c6f 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -2,13 +2,13 @@ module orborus go 1.19 -//replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared require ( github.com/docker/docker v23.0.0+incompatible github.com/mackerelio/go-osstat v0.2.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.3.74 + github.com/shuffle/shuffle-shared v0.3.75 ) require ( diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 45b35fe2..5bee1dea 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -200,6 +200,10 @@ github.com/shuffle/shuffle-shared v0.3.52 h1:d9OycFpuWxrcgHdP2vplKAkY8n+oK5vW02v github.com/shuffle/shuffle-shared v0.3.52/go.mod h1:jQrYySmvp/0De5ftrAaY6xwwr7TMfqBmBxQ2AX9yrjQ= github.com/shuffle/shuffle-shared v0.3.66 h1:M8iK88pk42vE5Up05t8QJZXWjNMt3XXC0vA93QVmPkE= github.com/shuffle/shuffle-shared v0.3.66/go.mod h1:jQrYySmvp/0De5ftrAaY6xwwr7TMfqBmBxQ2AX9yrjQ= +github.com/shuffle/shuffle-shared v0.3.74 h1:i7M1Gug9j2Wa02WuSxKDbXoLvBWKW5Pxf/EpFy6v38Y= +github.com/shuffle/shuffle-shared v0.3.74/go.mod h1:jQrYySmvp/0De5ftrAaY6xwwr7TMfqBmBxQ2AX9yrjQ= +github.com/shuffle/shuffle-shared v0.3.75 h1:ALXJSn13kcRbxfax1p/d1hvh1LL6Rx8iBhw2UJ5OuAQ= +github.com/shuffle/shuffle-shared v0.3.75/go.mod h1:jQrYySmvp/0De5ftrAaY6xwwr7TMfqBmBxQ2AX9yrjQ= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index e9563ff8..b12f5a7e 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=1.1.4 +VERSION=1.1.5 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index e3119e6a..4c648d81 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -11,7 +11,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.3.74 + github.com/shuffle/shuffle-shared v0.3.75 ) require ( diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index c14f3eb8..1facca8a 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -214,6 +214,8 @@ github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shuffle/shuffle-shared v0.3.51 h1:+JPEGw6R4a320who+SrGP/VqBxZdKPdcLw/07cO7d6c= github.com/shuffle/shuffle-shared v0.3.51/go.mod h1:jQrYySmvp/0De5ftrAaY6xwwr7TMfqBmBxQ2AX9yrjQ= +github.com/shuffle/shuffle-shared v0.3.74 h1:i7M1Gug9j2Wa02WuSxKDbXoLvBWKW5Pxf/EpFy6v38Y= +github.com/shuffle/shuffle-shared v0.3.74/go.mod h1:jQrYySmvp/0De5ftrAaY6xwwr7TMfqBmBxQ2AX9yrjQ= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index ee3c2766..a4067ff3 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -2118,7 +2118,7 @@ func main() { } // Checks if a subflow is child of the startnode, as sub-subflows aren't working properly yet - childNodes := shuffle.FindChildNodes(workflowExecution, workflowExecution.Start) + childNodes := shuffle.FindChildNodes(workflowExecution, workflowExecution.Start, []string{}, []string{}) log.Printf("[DEBUG] Looking for subflow in %#v to check execution pattern as child of %s", childNodes, workflowExecution.Start) subflowFound := false for _, childNode := range childNodes {