marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adding new changes for worker scale cross-server

Browse Source
This commit is contained in:
frikky
2021-11-14 23:37:14 +01:00
parent a572608184
commit b5c977ae4e
11 changed files with 96 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
functions/onprem/orborus/build.sh
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=shuffle-orborus
VERSION=0.9.32
VERSION=0.9.34
echo "Running docker build with $NAME:$VERSION"
#docker rmi frikky/shuffle:$NAME --force
functions/onprem/orborus/orborus.go
+52 -3
View File
@@ -149,7 +149,7 @@ func cleanupExistingNodes(ctx context.Context) error {
return err
}
log.Printf("\n\nFound %d contaienrs", len(services))
//log.Printf("\n\nFound %d contaienrs", len(services))
for _, service := range services {
log.Printf("[INFO] Service: %#v", service.Spec.Annotations.Name)
@@ -222,7 +222,7 @@ func deployServiceWorkers(image string) {
//containerName := fmt.Sprintf("shuffle-worker-%s", parsedUuid)
replicas := uint64(2)
replicas := uint64(1)
scaleReplicas := os.Getenv("SHUFFLE_SCALE_REPLICAS")
if len(scaleReplicas) > 0 {
tmpInt, err := strconv.Atoi(scaleReplicas)
@@ -236,14 +236,38 @@ func deployServiceWorkers(image string) {
}
innerContainerName := fmt.Sprintf("shuffle-workers")
parsedConcurrent := uint64(50)
cnt, _ := findActiveSwarmNodes()
nodeCount := uint64(1)
if cnt > 0 {
nodeCount = uint64(cnt)
}
if cnt == 0 {
cnt = 1
}
log.Printf("[DEBUG] Found %d node(s) to replicate over. Defaulting to 1 IF we can't auto-discover them.", cnt)
replicatedJobs := uint64(replicas * nodeCount)
log.Printf("[DEBUG] Deploying %d containers for worker with swarm to each node. Service name: %s. Image: %s", replicas, innerContainerName, image)
// FIXME: May not need ingress ports. Could use internal services and DNS of swarm itself
// https://github.com/moby/moby/blob/e2f740de442bac52b280bc485a3ca5b31567d938/api/types/swarm/service.go#L46
serviceSpec := swarm.ServiceSpec{
Annotations: swarm.Annotations{
Name: innerContainerName,
Labels: map[string]string{},
},
Mode: swarm.ServiceMode{
Replicated: &swarm.ReplicatedService{
Replicas: &replicatedJobs,
},
ReplicatedJob: &swarm.ReplicatedJob{
MaxConcurrent: &parsedConcurrent,
},
},
Networks: []swarm.NetworkAttachmentConfig{
swarm.NetworkAttachmentConfig{
Target: networkName,
@@ -269,6 +293,7 @@ func deployServiceWorkers(image string) {
Env: []string{
fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", os.Getenv("SHUFFLE_SWARM_CONFIG")),
fmt.Sprintf("SHUFFLE_SWARM_NETWORK_NAME=%s", networkName),
fmt.Sprintf("SHUFFLE_APP_REPLICAS=%d", cnt),
},
Mounts: []mount.Mount{
mount.Mount{
@@ -306,7 +331,7 @@ func deployServiceWorkers(image string) {
)
if err == nil {
log.Printf("[DEBUG] Successfully deployed workers with %d replicas", replicas)
log.Printf("[DEBUG] Successfully deployed workers with %d replica(s)", replicas)
//time.Sleep(time.Duration(10) * time.Second)
//log.Printf("[DEBUG] Servicecreate request: %#v %#v", service, err)
} else {
@@ -574,6 +599,30 @@ func getStats() {
fmt.Printf("\n")
}
func findActiveSwarmNodes() (int64, error) {
ctx := context.Background()
nodes, err := dockercli.NodeList(ctx, types.NodeListOptions{})
if err != nil {
return 0, err
}
nodeCount := int64(0)
for _, node := range nodes {
//log.Printf("ID: %s - %#v", node.ID, node.Status.State)
if node.Status.State == "ready" {
nodeCount += 1
}
}
return nodeCount, nil
/*
containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
All: true,
})
*/
}
// Initial loop etc
func main() {
log.Println("[INFO] Setting up execution environment")
functions/onprem/worker/build.sh
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=shuffle-worker
VERSION=0.9.32
VERSION=0.9.34
echo "Running docker build with $NAME:$VERSION"
#CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin .
functions/onprem/worker/go.mod
+1 -1
View File
@@ -10,6 +10,6 @@ require (
github.com/docker/go-connections v0.4.0 // indirect
github.com/gorilla/mux v1.8.0
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/shuffle/shuffle-shared v0.1.33
github.com/shuffle/shuffle-shared v0.1.35
go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
)
functions/onprem/worker/go.sum
+4
View File
@@ -572,6 +572,8 @@ github.com/shuffle/shuffle-shared v0.1.32 h1:VzYecAkUbydIe5ufe5z6T+3PaaKWJASVznh
github.com/shuffle/shuffle-shared v0.1.32/go.mod h1:0QrK51T12CpCj/be8hXduj/RtDnoeaZ3rfogELZE2IU=
github.com/shuffle/shuffle-shared v0.1.33 h1:1U0yKWNfW7K7EKOj2aqSmd20UIA+nJeIurGEfx29ffU=
github.com/shuffle/shuffle-shared v0.1.33/go.mod h1:0QrK51T12CpCj/be8hXduj/RtDnoeaZ3rfogELZE2IU=
github.com/shuffle/shuffle-shared v0.1.35 h1:CoCur/G+TaM2xiLgDCVdVxPhFffNK/4YRWTtzRBprvg=
github.com/shuffle/shuffle-shared v0.1.35/go.mod h1:2ndjLm4ZOvY6arGFwOgGnkQ457Ke7gka9HDF/EkdIxQ=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
@@ -582,6 +584,8 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
functions/onprem/worker/worker.go
+30 -1
View File
@@ -16,6 +16,7 @@ import (
"net/http"
"net/url"
"os"
"strconv"
"strings"
"time"
@@ -2228,13 +2229,40 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
networkName = swarmNetworkName
}
replicatedJobs := uint64(1)
// Sent from Orborus
// Should be equal to
scaleReplicas := os.Getenv("SHUFFLE_APP_REPLICAS")
if len(scaleReplicas) > 0 {
tmpInt, err := strconv.Atoi(scaleReplicas)
if err != nil {
log.Printf("[ERROR] %s is not a valid number for replication", scaleReplicas)
} else {
replicatedJobs = uint64(tmpInt)
}
log.Printf("[DEBUG] SHUFFLE_APP_REPLICAS set to value %#v. Trying to overwrite default (%d/node)", scaleReplicas, replicatedJobs)
}
log.Printf("[DEBUG] Deploying app with name %s with image %s", name, image)
parsedConcurrent := uint64(50)
containerName := fmt.Sprintf(strings.Replace(name, ".", "-", -1))
serviceSpec := swarm.ServiceSpec{
Annotations: swarm.Annotations{
Name: containerName,
Labels: map[string]string{},
},
Mode: swarm.ServiceMode{
Replicated: &swarm.ReplicatedService{
// Max total
Replicas: &replicatedJobs,
},
ReplicatedJob: &swarm.ReplicatedJob{
MaxConcurrent: &parsedConcurrent,
},
},
Networks: []swarm.NetworkAttachmentConfig{
swarm.NetworkAttachmentConfig{
Target: networkName,
@@ -2269,7 +2297,8 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
Condition: swarm.RestartPolicyConditionNone,
},
Placement: &swarm.Placement{
MaxReplicas: 5,
// Max per node
MaxReplicas: 1,
},
},
}