diff --git a/.env b/.env index ecc19573..07deaf4a 100644 --- a/.env +++ b/.env @@ -65,6 +65,12 @@ SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-1.1.0" ## shuffle_Scale_Replicas (workers/node) ## shuffle_App_Replicas (apps/node) +SHUFFLE_SWARM_BRIDGE_DEFAULT_MTU=1500 # 1500 by default +# The eth0 interface inside a container corresponds +# to the virtual Ethernet interface that connects +# the container to the docker0 +SHUFFLE_SWARM_BRIDGE_DEFAULT_INTERFACE=eth0 + # Used for auto-cleanup of containers. REALLY important at scale. SHUFFLE_CONTAINER_AUTO_CLEANUP=false SHUFFLE_ELASTIC=true diff --git a/docker-compose.yml b/docker-compose.yml index a6732013..04d07033 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -113,4 +113,10 @@ services: networks: shuffle: driver: bridge - #driver: overlay + + # uncomment to set MTU for swarm mode. + # MTU should be whatever is your host's preferred MTU is. + # Refer to this doc to figure out what your host's MTU is: + # https://shuffler.io/docs/troubleshooting#TLS_timeout_error/Timeout_Errors/EOF_Errors + # driver_opts: + # com.docker.network.driver.mtu: 1460 diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 4a425dd2..1944b2b4 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -53,6 +53,8 @@ var concurrencyEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY") var appSdkVersion = os.Getenv("SHUFFLE_APP_SDK_VERSION") var workerVersion = os.Getenv("SHUFFLE_WORKER_VERSION") var newWorkerImage = os.Getenv("SHUFFLE_WORKER_IMAGE") +var dockerSwarmBridgeMTU = os.Getenv("SHUFFLE_SWARM_BRIDGE_DEFAULT_MTU") +var dockerSwarmBridgeInterface = os.Getenv("SHUFFLE_SWARM_BRIDGE_DEFAULT_INTERFACE") // var baseimagename = "docker.pkg.github.com/shuffle/shuffle" // var baseimagename = "ghcr.io/frikky" @@ -208,6 +210,49 @@ func deployServiceWorkers(image string) { // frikky@debian:~/git/shuffle/functions/onprem/worker$ docker service create --replicas 5 --name shuffle-workers --env SHUFFLE_SWARM_CONFIG=run --publish published=33333,target=33333 ghcr.io/shuffle/shuffle-worker:nightly + // Get a list of network interfaces + interfaces, err := net.Interfaces() + if err != nil { + log.Printf("[ERROR] Failed to get network interfaces: %s", err) + } + + mtu, err := strconv.Atoi(dockerSwarmBridgeMTU) // by default + bridgeName := dockerSwarmBridgeInterface + + if bridgeName == "" { + bridgeName = "eth0" + } + + if err != nil { + log.Printf("[ERROR] Failed to convert the default MTU to int: %s. Using 1500 instead", err) + mtu = 1500 + } + + // Check if there is at least one interface + if len(interfaces) < 2 { + // this assumes that the machine should have at least 2 network + // interfaces. If not, we will use the default MTU. + // interface 1 is the loopback interface + // interface 2 is eth0, The eth0 interface inside a + // Docker container corresponds to the virtual Ethernet + // interface that connects the container to the docker0 + log.Printf("[ERROR] Failed to get enough network interfaces") + } else { + // Get the preferred interface + for _, iface := range interfaces { + if strings.Contains(iface.Name, bridgeName) { + targetInterface := iface + mtu = targetInterface.MTU + log.Printf("[INFO] Using MTU %d from interface %s", mtu, targetInterface.Name) + break + } + } + } + + // Create the network options with the specified MTU + options := make(map[string]string) + options["com.docker.network.driver.mtu"] = fmt.Sprintf("%d", mtu) + ingressOptions := types.NetworkCreate{ Driver: "overlay", Attachable: false, @@ -223,7 +268,7 @@ func deployServiceWorkers(image string) { }, } - _, err := dockercli.NetworkCreate( + _, err = dockercli.NetworkCreate( ctx, "ingress", ingressOptions, @@ -242,6 +287,7 @@ func deployServiceWorkers(image string) { networkCreateOptions := types.NetworkCreate{ Driver: "overlay", + Options: options, Attachable: true, Ingress: false, IPAM: &network.IPAM{