diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 33c8a332..957ec5a3 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -4,13 +4,13 @@ go 1.22.0 toolchain go1.22.2 -replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared require ( github.com/docker/docker v27.0.2+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.6.63 + github.com/shuffle/shuffle-shared v0.6.71 k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 k8s.io/client-go v0.30.2 diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 532623cd..53a114e7 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -273,6 +273,8 @@ github.com/shuffle/shuffle-shared v0.6.61 h1:+9CCLeZLiAVDgNRTkZxnIgz+FZ7UrEHez2B github.com/shuffle/shuffle-shared v0.6.61/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus= github.com/shuffle/shuffle-shared v0.6.63 h1:eNQMpVhe/mAMxl61W9Wj6/Z4PrtPeEnbjvMtDdT1mqw= github.com/shuffle/shuffle-shared v0.6.63/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus= +github.com/shuffle/shuffle-shared v0.6.71 h1:ZivcCTYmGQilxOjm0tBTNXP3caPMRH6f5ohEzfnh03I= +github.com/shuffle/shuffle-shared v0.6.71/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/skeema/knownhosts v1.2.1 h1:SHWdIUa82uGZz+F+47k8SY4QhhI291cXCpopT1lK2AQ= github.com/skeema/knownhosts v1.2.1/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 7572e806..c42fb166 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -724,46 +724,105 @@ func buildEnvVars(envMap map[string]string) []corev1.EnvVar { } func handleBackendImageDownload(ctx context.Context, images string) error { - // Replicate images with lowercase, as the name may be wrong // Most of the time lowercase is correct. Swapping to have that first originalImages := images images = strings.ToLower(images) + "," + originalImages - log.Printf("[DEBUG] Should remove existing image (s): %s", images) - // Remove the image + handled := []string{} + log.Printf("[DEBUG] Should remove existing image (s): %s", images) removeOptions := image.RemoveOptions{} + + newImages := []string{} for _, image := range strings.Split(images, ",") { image = strings.TrimSpace(image) + if shuffle.ArrayContains(handled, image) { + continue + } + + handled = append(handled, image) if !strings.Contains(image, "/") { image = fmt.Sprintf("frikky/shuffle:%s", image) } + newImages = append(newImages, image) + // There is no real point in actual removal. This may however be a good idea, as Worker will force download the new one anyway resp, err := dockercli.ImageRemove(ctx, image, removeOptions) if err != nil { - log.Printf("[ERROR] Failed removing image: %s", err) + log.Printf("[ERROR] Failed removing image: %s. Resp: %#v", err, resp) + + // Goroutining images that don't already exist, as they are most likely not the correct one + go shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image) } else { - log.Printf("[DEBUG] Removed image: %s", resp) - } + log.Printf("[DEBUG] Removed image: %s", image) - - err = shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image) - if err != nil { - log.Printf("[ERROR] Failed downloading image: %s", err) - } else { - log.Printf("[DEBUG] Downloaded image: %s", image) - //break + err = shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image) + if err != nil { + log.Printf("[ERROR] Failed downloading image: %s", err) + } else { + log.Printf("[DEBUG] Downloaded image: %s", image) + //break + } } } if swarmConfig == "run" || swarmConfig == "swarm" { - log.Printf("[DEBUG] Should update service with new image after updating(s): %s. \n\nNOT IMPLEMENTED: Contact support@shuffler.io for support.\n\n", images) + log.Printf("[DEBUG] Should update service with new image after updating(s): %s. \n\nBETA REPLACEMENT IMPLEMENTATION: Contact support@shuffler.io for support.", strings.Join(newImages, "\n")) // 1. Download the image // 2. Find the existing service using the image // 3. Update the service with the new image in a rolling restart + + // Find the existing service + serviceListOptions := types.ServiceListOptions{} + services, err := dockercli.ServiceList( + ctx, + serviceListOptions, + ) + + if err != nil { + log.Printf("[ERROR] Failed finding containers: %s", err) + } else { + log.Printf("[DEBUG] Found %d services", len(services)) + + for _, service := range services { + + log.Printf("Imagename: %s", service.Spec.TaskTemplate.ContainerSpec.Image) + + for _, image := range newImages { + if !strings.Contains(service.Spec.TaskTemplate.ContainerSpec.Image, image) { + continue + } + + log.Printf("[DEBUG] Found service for image %#v: %#v", service.Spec.Annotations.Name) + + // Update the service to run with the new image + //docker service update --image username/imagename:latest servicename --force + serviceUpdateOptions := types.ServiceUpdateOptions{} + resp, err := dockercli.ServiceUpdate( + ctx, + service.ID, + service.Version, + service.Spec, + serviceUpdateOptions, + ) + + if err != nil { + log.Printf("[ERROR] Failed updating service %s with the new image %s: %s. Resp: %#v", service.Spec.Annotations.Name, image, err, resp) + } else { + log.Printf("[DEBUG] Updated service %s with the new image %s. Resp: %#v", service.Spec.Annotations.Name, image, resp) + + if !strings.Contains(fmt.Sprintf("%s", resp), "error") { + break + } + } + } + } + + } + } return nil @@ -2047,11 +2106,11 @@ func main() { toBeRemoved.Data = append(toBeRemoved.Data, incRequest) } } else if incRequest.Type == "DOCKER_IMAGE_DOWNLOAD" { - log.Printf("[INFO] Should delete -> download new image %#v", incRequest.ExecutionArgument) + log.Printf("[INFO] Should delete -> download new images: %#v", incRequest.ExecutionArgument) if len(incRequest.ExecutionArgument) > 0 { // FIXME: Wait X seconds before running this as the image build may not be done yet. This is shitty, but may be ok to do in Orborus. Easy fix for the future: Just let it run through jobs 5-10 times before actually picking it up - time.Sleep(time.Duration(25) * time.Second) + //time.Sleep(time.Duration(25) * time.Second) err = handleBackendImageDownload(ctx, incRequest.ExecutionArgument) if err != nil { diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 700a01dc..29c9466e 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -6,7 +6,7 @@ require ( github.com/docker/docker v26.1.0+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.6.63 + github.com/shuffle/shuffle-shared v0.6.71 k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 k8s.io/client-go v0.30.2 diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index 602a53a4..ce38d1e5 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -403,6 +403,8 @@ github.com/shuffle/shuffle-shared v0.6.27 h1:q4qZD6bGZFIvZ5Y10unGr3N3rZ7OryWyvva github.com/shuffle/shuffle-shared v0.6.27/go.mod h1:rWkh1eWdIx7OqQzJ1+JzF3Hck1X/Ty1WkUtjLrp+CU4= github.com/shuffle/shuffle-shared v0.6.63 h1:eNQMpVhe/mAMxl61W9Wj6/Z4PrtPeEnbjvMtDdT1mqw= github.com/shuffle/shuffle-shared v0.6.63/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus= +github.com/shuffle/shuffle-shared v0.6.71 h1:ZivcCTYmGQilxOjm0tBTNXP3caPMRH6f5ohEzfnh03I= +github.com/shuffle/shuffle-shared v0.6.71/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=