From b204296e719ff1ffdde79b2cde0f08ee88eae388 Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 3 Apr 2025 19:19:40 +0200 Subject: [PATCH] Fixed a problem with docker mounts always being read-write and not mounting properly into containers --- backend/go-app/main.go | 8 ++++ frontend/src/views/AppCreator.jsx | 4 +- functions/onprem/orborus/go.mod | 2 +- functions/onprem/orborus/orborus.go | 15 +++++-- functions/onprem/worker/worker.go | 65 ++++++++++++++++++++--------- 5 files changed, 69 insertions(+), 25 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index dda3cfd2..dbef20e4 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3544,6 +3544,10 @@ func handleCloudJob(job shuffle.CloudSyncJob) error { } backendPort := os.Getenv("BACKEND_PORT") + if backendPort == "" { + backendPort = "5001" + } + redirectDomain := fmt.Sprintf("localhost:%s", backendPort) redirectUrl := fmt.Sprintf("http://%s/api/v1/triggers/outlook/register", redirectDomain) outlookClient, _, err := shuffle.GetOutlookClient(ctx, "", hook.OauthToken, redirectUrl) @@ -4201,6 +4205,10 @@ func runInitEs(ctx context.Context) { log.Printf("[INFO] Running schedule for cleaning up or re-running unfinished workflows in %d environments.", len(environments)) backendPort := os.Getenv("BACKEND_PORT") + if backendPort == "" { + backendPort = "5001" + } + for _, environment := range environments { // Allowed without PROXY management as it's localhost // client := shuffle.GetExternalClient(syncUrl) diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 50106831..74fe9ae6 100755 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -2037,7 +2037,7 @@ const AppCreator = (defaultprops) => { for (let actionkey in actions) { var item = JSON.parse(JSON.stringify(actions[actionkey])) if (item.errors.length > 0) { - toast("Saving with error in action " + item.name); + //toast("Saving with error in action " + item.name); } if (item.name === undefined && item.description !== undefined) { @@ -3858,7 +3858,7 @@ const AppCreator = (defaultprops) => { if (currentAction.url === "" && actions !== undefined && actions !== null && actions.length > 0) { for (var i = 0; i < actions.length; i++) { if (actions[i].name.toLowerCase() === e.target.value.toLowerCase()) { - toast("Action with name " + e.target.value + " already exists. If you keep this, it will be overwritten.") + //toast("Action with name " + e.target.value + " already exists. If you keep this, it will be overwritten.") break } } diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index cd7ec3c5..0ffdc2a6 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -4,7 +4,7 @@ go 1.23.0 toolchain go1.23.6 -//replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared require ( github.com/docker/docker v27.5.0+incompatible diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index f1fc5f29..e773b1fd 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -744,6 +744,8 @@ func handleBackendImageDownload(ctx context.Context, images string) error { handled := []string{} //log.Printf("[DEBUG] Removing existing image (s): %s", images) newImages := []string{} + + successful := []string{} for _, curimage := range strings.Split(images, ",") { curimage = strings.TrimSpace(curimage) if shuffle.ArrayContains(handled, curimage) { @@ -775,12 +777,19 @@ func handleBackendImageDownload(ctx context.Context, images string) error { err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, curimage) if err != nil { - log.Printf("[ERROR] Failed downloading image: %s", err) + //log.Printf("[ERROR] Failed downloading image: %s", err) } else { - log.Printf("[DEBUG] Downloaded image: %s", curimage) + //log.Printf("[DEBUG] Downloaded image: %s", curimage) + successful = append(successful, curimage) } } + if len(successful) == 0 { + log.Printf("[ERROR] Failed downloading image copies: %s. This means the app may not have been updated.", strings.Join(handled, ", ")) + } else { + log.Printf("[DEBUG] Successfully downloaded image copies: %s", strings.Join(successful, ", ")) + } + if swarmConfig == "run" || swarmConfig == "swarm" { log.Printf("[DEBUG] Should update service with new image after updating(s): %s. \n\nBETA REPLACEMENT IMPLEMENTATION: Contact support@shuffler.io for support.", strings.Join(newImages, "\n")) @@ -1919,7 +1928,7 @@ func main() { } // Handle Cleanup - made it cleanup by default - if strings.ToLower(os.Getenv("SHUFFLE_CONTAINER_AUTO_CLEANUP")) != "false" { + if strings.ToLower(os.Getenv("SHUFFLE_CONTAINER_AUTO_CLEANUP")) != "false" && os.Getenv("CLEANUP") == "" { cleanupEnv = "true" } diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index d20e3578..d52275d1 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -894,19 +894,27 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] if len(volumeBindString) > 0 { volumeBindSplit := strings.Split(volumeBindString, ",") for _, volumeBind := range volumeBindSplit { - if strings.Contains(volumeBind, ":") { - volumeBinds = append(volumeBinds, volumeBind) - } else { - log.Printf("[ERROR] Volume bind '%s' is invalid.", volumeBind) + if volumeBind == "srcfolder=dstfolder" || volumeBind == "srcfolder:dstfolder" || volumeBind == "/srcfolder:/dstfolder" { + log.Printf("[DEBUG] Volume bind '%s' is invalid and is used for visualization.", volumeBind) + continue } + + if !strings.HasPrefix(volumeBind, "/") { + log.Printf("[ERROR] Volume bind '%s' is invalid. Use absolute paths.", volumeBind) + continue + } + + if !strings.Contains(volumeBind, ":") { + log.Printf("[ERROR] Volume bind '%s' is invalid. Use absolute paths with colon inbetween them (/srcpath:dstpath/", volumeBind) + continue + } + + volumeBinds = append(volumeBinds, volumeBind) } } // Add more volume binds if possible if len(volumeBinds) > 0 { - log.Printf("[DEBUG] Setting up binds for container. Got %d volume binds.", len(volumeBinds)) - - //hostConfig.Binds = volumeBinds // Only use mounts, not direct binds hostConfig.Binds = []string{} @@ -917,15 +925,27 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] continue } - log.Printf("[DEBUG] Appending bind %s to app container", bind) + log.Printf("[DEBUG] Appending bind %s to App container", bind) bindSplit := strings.Split(bind, ":") sourceFolder := bindSplit[0] destinationFolder := bindSplit[1] - hostConfig.Mounts = append(hostConfig.Mounts, mount.Mount{ + + readOnly := false + if len(bindSplit) > 2 { + mode := bindSplit[2] + if mode == "ro" { + readOnly = true + } + } + + builtMount := mount.Mount{ Type: mount.TypeBind, Source: sourceFolder, Target: destinationFolder, - }) + ReadOnly: readOnly, + } + + hostConfig.Mounts = append(hostConfig.Mounts, builtMount) } } @@ -1053,17 +1073,24 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont err = cli.ContainerStart(ctx, cont.ID, container.StartOptions{}) if err != nil { if strings.Contains(fmt.Sprintf("%s", err), "cannot join network") || strings.Contains(fmt.Sprintf("%s", err), "No such container") { + // Remove the "CREATED" one from the previous: + removeErr := cli.ContainerRemove(ctx, cont.ID, container.RemoveOptions{}) + if removeErr != nil { + log.Printf("[ERROR] Failed to remove container %s: %s", cont.ID, removeErr) + } + + log.Printf("[WARNING] Failed deploying App on first attempt: %s. Removing some HostConfig configs.", err) parsedUuid := uuid.NewV4() identifier = fmt.Sprintf("%s-%s-nonetwork", identifier, parsedUuid) - hostConfig = &container.HostConfig{ - LogConfig: container.LogConfig{ - Type: "json-file", - Config: map[string]string{ - "max-size": "10m", - }, + + hostConfig.NetworkMode = container.NetworkMode("") + hostConfig.LogConfig = container.LogConfig{ + Type: "json-file", + Config: map[string]string{ + "max-size": "10m", }, - Resources: container.Resources{}, } + hostConfig.Resources = container.Resources{} cont, err = cli.ContainerCreate( context.Background(), @@ -1085,12 +1112,12 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont return err } - log.Printf("[DEBUG] Running secondary check without network with worker") + //log.Printf("[DEBUG] Running secondary check without network with worker") err = cli.ContainerStart(ctx, cont.ID, container.StartOptions{}) } if err != nil { - log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err) + log.Printf("[ERROR] Failed to start container (2) in runtime location %s: %s", environment, err) cacheErr := shuffle.DeleteCache(ctx, actionExecId) if cacheErr != nil {