From bc8b1c2c6ec7a646442b5a0fe47c9b7981de215a Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 3 Apr 2025 01:53:55 +0200 Subject: [PATCH] Improved the image download mechanism for self-built apps onprem --- backend/go-app/docker.go | 64 +++++++++++++++++++++-------- backend/go-app/go.mod | 2 +- backend/go-app/walkoff.go | 2 +- functions/onprem/orborus/go.mod | 2 +- functions/onprem/orborus/go.sum | 4 +- functions/onprem/orborus/orborus.go | 4 +- functions/onprem/worker/go.mod | 2 +- functions/onprem/worker/go.sum | 4 +- 8 files changed, 56 insertions(+), 28 deletions(-) diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index e56455eb..9f0c72e2 100755 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -604,11 +604,27 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { // return //} - body, err := ioutil.ReadAll(request.Body) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`)) - return + var err error + body := []byte{} + //log.Printf("IMAGE REQUEST BODY: %#v", request.Body) + if request.Body == nil || request.Body == http.NoBody { + // Check for the image query, otherwise we skip everything + imageQuery := request.URL.Query().Get("image") + if len(imageQuery) == 0 { + resp.WriteHeader(400) + resp.Write([]byte(`{"success": false, "reason": "No image query found"}`)) + return + } + + body = []byte(fmt.Sprintf(`{"name": "%s"}`, imageQuery)) + + } else { + body, err = ioutil.ReadAll(request.Body) + if err != nil { + resp.WriteHeader(400) + resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`)) + return + } } // This has to be done in a weird way because Datastore doesn't @@ -630,21 +646,24 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { return } - ctx := context.Background() - images, err := dockercli.ImageList(ctx, image.ListOptions{ - All: true, - }) - img := image.Summary{} - tagFound := "" - img2 := image.Summary{} + tagFound := "" tagFound2 := "" - alternativeNameSplit := strings.Split(version.Name, "/") - alternativeName := version.Name - if len(alternativeNameSplit) == 3 { - alternativeName = strings.Join(alternativeNameSplit[1:3], "/") + // Old way of doing it + //alternativeNameSplit := strings.Split(version.Name, "/") + //alternativeName := version.Name + //if len(alternativeNameSplit) == 3 { + // alternativeName = strings.Join(alternativeNameSplit[1:3], "/") + //} + + appname, baseAppname, appnameSplit2, err := shuffle.GetAppNameSplit(version) + if err != nil { + log.Printf("[ERROR] Failed getting appname split: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't get the right docker image name"}`))) + return } if len(version.Name) == 0 { @@ -655,11 +674,20 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { } - log.Printf("[INFO] Trying to download image: '%s'. Alt name: %#v", version.Name, alternativeName) + log.Printf("[INFO] Trying to download image: '%s'. Appname: '%s'. BaseAppname: '%s', Split2: %s", version.Name, appname, baseAppname, appnameSplit2) + + alternativeName := appname + ctx := context.Background() + images, err := dockercli.ImageList(ctx, image.ListOptions{ + All: true, + }) for _, image := range images { for _, tag := range image.RepoTags { - //log.Printf("[DEBUG] Tag: %s", tag) + if strings.Contains(tag, "") { + continue + } + if strings.ToLower(tag) == strings.ToLower(version.Name) { img = image tagFound = tag diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 6aa6d2e0..e697498e 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -20,7 +20,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.33 + github.com/shuffle/shuffle-shared v0.8.34 golang.org/x/crypto v0.36.0 google.golang.org/api v0.176.1 google.golang.org/grpc v1.68.1 diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index b3aa3537..8b8fb878 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -974,7 +974,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { if err != nil { log.Printf("[ERROR] Failed to list child workflows: %s", err) } else { - log.Printf("\n\n[DEBUG] Found %d child workflows for workflow %s\n\n", len(childWorkflows), workflow.ID) + //log.Printf("\n\n[DEBUG] Found %d child workflows for workflow %s\n\n", len(childWorkflows), workflow.ID) // Find cookies and append them to request.Header to replicate current request as closely as possible for _, childWorkflow := range childWorkflows { diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 0c3ada48..cd7ec3c5 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v27.5.0+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.33 + github.com/shuffle/shuffle-shared v0.8.34 k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 ) diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 6b5c6bd0..7b2f951d 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -301,8 +301,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.33 h1:XEiaUf2ykAPku9lX7JUXFDRBSropqiT0L2p4RdqVawE= -github.com/shuffle/shuffle-shared v0.8.33/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.34 h1:yITGbBEaORKNo6yybrLMoHdHnZsnvq2tAZiileaGa0U= +github.com/shuffle/shuffle-shared v0.8.34/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index a945dd80..7c976475 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -742,7 +742,7 @@ func handleBackendImageDownload(ctx context.Context, images string) error { // Remove the image handled := []string{} - log.Printf("[DEBUG] Removing existing image (s): %s. Waiting 30 seconds before starting to ensure backend has the latest images built and ready to distribute.", images) + //log.Printf("[DEBUG] Removing existing image (s): %s", images) newImages := []string{} for _, curimage := range strings.Split(images, ",") { curimage = strings.TrimSpace(curimage) @@ -770,7 +770,7 @@ func handleBackendImageDownload(ctx context.Context, images string) error { log.Printf("[DEBUG] Removed image: %s", curimage) } } else { - log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig) + //log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig) } err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, curimage) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index eacebda6..7fda9ba2 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -8,7 +8,7 @@ require ( github.com/docker/docker v27.5.0+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.33 + github.com/shuffle/shuffle-shared v0.8.34 k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 k8s.io/client-go v0.30.2 diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index 360e7230..6878ebfb 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -294,8 +294,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.33 h1:XEiaUf2ykAPku9lX7JUXFDRBSropqiT0L2p4RdqVawE= -github.com/shuffle/shuffle-shared v0.8.33/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.34 h1:yITGbBEaORKNo6yybrLMoHdHnZsnvq2tAZiileaGa0U= +github.com/shuffle/shuffle-shared v0.8.34/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=