marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed Worker X Orborus interaction for image downloads to be similar

Browse Source
This commit is contained in:
Frikky
2024-09-30 21:31:21 +02:00
parent 5226f79551
commit 14c61e6b36
5 changed files with 82 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
functions/onprem/orborus/go.mod
+2 -2
View File
@@ -4,13 +4,13 @@ go 1.22.0
toolchain go1.22.2
replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared
//replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared
require (
github.com/docker/docker v27.0.2+incompatible
github.com/docker/go-connections v0.5.0
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.6.63
github.com/shuffle/shuffle-shared v0.6.71
k8s.io/api v0.30.2
k8s.io/apimachinery v0.30.2
k8s.io/client-go v0.30.2
functions/onprem/orborus/go.sum
+2
View File
@@ -273,6 +273,8 @@ github.com/shuffle/shuffle-shared v0.6.61 h1:+9CCLeZLiAVDgNRTkZxnIgz+FZ7UrEHez2B
github.com/shuffle/shuffle-shared v0.6.61/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus=
github.com/shuffle/shuffle-shared v0.6.63 h1:eNQMpVhe/mAMxl61W9Wj6/Z4PrtPeEnbjvMtDdT1mqw=
github.com/shuffle/shuffle-shared v0.6.63/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus=
github.com/shuffle/shuffle-shared v0.6.71 h1:ZivcCTYmGQilxOjm0tBTNXP3caPMRH6f5ohEzfnh03I=
github.com/shuffle/shuffle-shared v0.6.71/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/skeema/knownhosts v1.2.1 h1:SHWdIUa82uGZz+F+47k8SY4QhhI291cXCpopT1lK2AQ=
github.com/skeema/knownhosts v1.2.1/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo=
functions/onprem/orborus/orborus.go
+75 -16
View File
@@ -724,46 +724,105 @@ func buildEnvVars(envMap map[string]string) []corev1.EnvVar {
}
func handleBackendImageDownload(ctx context.Context, images string) error {
// Replicate images with lowercase, as the name may be wrong
// Most of the time lowercase is correct. Swapping to have that first
originalImages := images
images = strings.ToLower(images) + "," + originalImages
log.Printf("[DEBUG] Should remove existing image (s): %s", images)
// Remove the image
handled := []string{}
log.Printf("[DEBUG] Should remove existing image (s): %s", images)
removeOptions := image.RemoveOptions{}
newImages := []string{}
for _, image := range strings.Split(images, ",") {
image = strings.TrimSpace(image)
if shuffle.ArrayContains(handled, image) {
continue
}
handled = append(handled, image)
if !strings.Contains(image, "/") {
image = fmt.Sprintf("frikky/shuffle:%s", image)
}
newImages = append(newImages, image)
// There is no real point in actual removal. This may however be a good idea, as Worker will force download the new one anyway
resp, err := dockercli.ImageRemove(ctx, image, removeOptions)
if err != nil {
log.Printf("[ERROR] Failed removing image: %s", err)
log.Printf("[ERROR] Failed removing image: %s. Resp: %#v", err, resp)
// Goroutining images that don't already exist, as they are most likely not the correct one
go shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image)
} else {
log.Printf("[DEBUG] Removed image: %s", resp)
}
log.Printf("[DEBUG] Removed image: %s", image)
err = shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image)
if err != nil {
log.Printf("[ERROR] Failed downloading image: %s", err)
} else {
log.Printf("[DEBUG] Downloaded image: %s", image)
//break
err = shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image)
if err != nil {
log.Printf("[ERROR] Failed downloading image: %s", err)
} else {
log.Printf("[DEBUG] Downloaded image: %s", image)
//break
}
}
}
if swarmConfig == "run" || swarmConfig == "swarm" {
log.Printf("[DEBUG] Should update service with new image after updating(s): %s. \n\nNOT IMPLEMENTED: Contact support@shuffler.io for support.\n\n", images)
log.Printf("[DEBUG] Should update service with new image after updating(s): %s. \n\nBETA REPLACEMENT IMPLEMENTATION: Contact support@shuffler.io for support.", strings.Join(newImages, "\n"))
// 1. Download the image
// 2. Find the existing service using the image
// 3. Update the service with the new image in a rolling restart
// Find the existing service
serviceListOptions := types.ServiceListOptions{}
services, err := dockercli.ServiceList(
ctx,
serviceListOptions,
)
if err != nil {
log.Printf("[ERROR] Failed finding containers: %s", err)
} else {
log.Printf("[DEBUG] Found %d services", len(services))
for _, service := range services {
log.Printf("Imagename: %s", service.Spec.TaskTemplate.ContainerSpec.Image)
for _, image := range newImages {
if !strings.Contains(service.Spec.TaskTemplate.ContainerSpec.Image, image) {
continue
}
log.Printf("[DEBUG] Found service for image %#v: %#v", service.Spec.Annotations.Name)
// Update the service to run with the new image
//docker service update --image username/imagename:latest servicename --force
serviceUpdateOptions := types.ServiceUpdateOptions{}
resp, err := dockercli.ServiceUpdate(
ctx,
service.ID,
service.Version,
service.Spec,
serviceUpdateOptions,
)
if err != nil {
log.Printf("[ERROR] Failed updating service %s with the new image %s: %s. Resp: %#v", service.Spec.Annotations.Name, image, err, resp)
} else {
log.Printf("[DEBUG] Updated service %s with the new image %s. Resp: %#v", service.Spec.Annotations.Name, image, resp)
if !strings.Contains(fmt.Sprintf("%s", resp), "error") {
break
}
}
}
}
}
}
return nil
@@ -2047,11 +2106,11 @@ func main() {
toBeRemoved.Data = append(toBeRemoved.Data, incRequest)
}
} else if incRequest.Type == "DOCKER_IMAGE_DOWNLOAD" {
log.Printf("[INFO] Should delete -> download new image %#v", incRequest.ExecutionArgument)
log.Printf("[INFO] Should delete -> download new images: %#v", incRequest.ExecutionArgument)
if len(incRequest.ExecutionArgument) > 0 {
// FIXME: Wait X seconds before running this as the image build may not be done yet. This is shitty, but may be ok to do in Orborus. Easy fix for the future: Just let it run through jobs 5-10 times before actually picking it up
time.Sleep(time.Duration(25) * time.Second)
//time.Sleep(time.Duration(25) * time.Second)
err = handleBackendImageDownload(ctx, incRequest.ExecutionArgument)
if err != nil {
functions/onprem/worker/go.mod
+1 -1
View File
@@ -6,7 +6,7 @@ require (
github.com/docker/docker v26.1.0+incompatible
github.com/gorilla/mux v1.8.1
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.6.63
github.com/shuffle/shuffle-shared v0.6.71
k8s.io/api v0.30.2
k8s.io/apimachinery v0.30.2
k8s.io/client-go v0.30.2
functions/onprem/worker/go.sum
+2
View File
@@ -403,6 +403,8 @@ github.com/shuffle/shuffle-shared v0.6.27 h1:q4qZD6bGZFIvZ5Y10unGr3N3rZ7OryWyvva
github.com/shuffle/shuffle-shared v0.6.27/go.mod h1:rWkh1eWdIx7OqQzJ1+JzF3Hck1X/Ty1WkUtjLrp+CU4=
github.com/shuffle/shuffle-shared v0.6.63 h1:eNQMpVhe/mAMxl61W9Wj6/Z4PrtPeEnbjvMtDdT1mqw=
github.com/shuffle/shuffle-shared v0.6.63/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus=
github.com/shuffle/shuffle-shared v0.6.71 h1:ZivcCTYmGQilxOjm0tBTNXP3caPMRH6f5ohEzfnh03I=
github.com/shuffle/shuffle-shared v0.6.71/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=