From fddca0146a5745b34a27d2823d1186ef804e7b12 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Fri, 23 Jun 2023 05:23:37 +0530 Subject: [PATCH 1/6] fixing #1129: Automatically syncing up swarm network MTU --- functions/onprem/orborus/orborus.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 1c6ed66f..14cb8dac 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -212,6 +212,27 @@ func deployServiceWorkers(image string) { networkName = swarmNetworkName } + // Get a list of network interfaces + interfaces, err := net.Interfaces() + if err != nil { + panic(err) + } + + // Check if there is at least one interface + if len(interfaces) < 2 { + panic("Insufficient network interfaces") + } + + // Get the second interface + targetInterface := interfaces[1] + mtu := targetInterface.MTU + + // Print the interface and MTU information + fmt.Printf("Target Interface: %s, MTU: %d\n", targetInterface.Name, mtu) + // Create the network options with the specified MTU + options := make(map[string]string) + options["com.docker.network.driver.mtu"] = fmt.Sprintf("%d", mtu) + ingressOptions := types.NetworkCreate{ Driver: "overlay", Attachable: false, @@ -227,7 +248,7 @@ func deployServiceWorkers(image string) { }, } - _, err := dockercli.NetworkCreate( + _, err = dockercli.NetworkCreate( ctx, "ingress", ingressOptions, @@ -241,6 +262,7 @@ func deployServiceWorkers(image string) { // Specific subnet? networkCreateOptions := types.NetworkCreate{ Driver: "overlay", + Options: options, Attachable: true, Ingress: false, IPAM: &network.IPAM{ From 071933c9568944aded3a3d8699a2e0b28b77f18b Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Fri, 30 Jun 2023 00:30:24 +0530 Subject: [PATCH 2/6] Following up on review --- functions/onprem/orborus/orborus.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 14cb8dac..af540e93 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -214,21 +214,25 @@ func deployServiceWorkers(image string) { // Get a list of network interfaces interfaces, err := net.Interfaces() + mtu := 1500 // default docker MTU if err != nil { - panic(err) + log.Printf("[ERROR] Failed to get network interfaces: %s", err) } // Check if there is at least one interface if len(interfaces) < 2 { - panic("Insufficient network interfaces") - } + log.Printf("[ERROR] Failed to get enough network interfaces") + } else { + // Get the preferred interface + for _, iface := range interfaces { + if strings.Contains(iface.Name, "eth0") { + targetInterface := iface + mtu = targetInterface.MTU + break + } + } + } - // Get the second interface - targetInterface := interfaces[1] - mtu := targetInterface.MTU - - // Print the interface and MTU information - fmt.Printf("Target Interface: %s, MTU: %d\n", targetInterface.Name, mtu) // Create the network options with the specified MTU options := make(map[string]string) options["com.docker.network.driver.mtu"] = fmt.Sprintf("%d", mtu) From eaf0ddf57a3ca75e34c889e41f8a47a15e71be10 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Fri, 30 Jun 2023 21:20:52 +0530 Subject: [PATCH 3/6] Making the code more reliable and configurable --- .env | 6 ++++++ functions/onprem/orborus/orborus.go | 19 +++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/.env b/.env index ecc19573..07deaf4a 100644 --- a/.env +++ b/.env @@ -65,6 +65,12 @@ SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-1.1.0" ## shuffle_Scale_Replicas (workers/node) ## shuffle_App_Replicas (apps/node) +SHUFFLE_SWARM_BRIDGE_DEFAULT_MTU=1500 # 1500 by default +# The eth0 interface inside a container corresponds +# to the virtual Ethernet interface that connects +# the container to the docker0 +SHUFFLE_SWARM_BRIDGE_DEFAULT_INTERFACE=eth0 + # Used for auto-cleanup of containers. REALLY important at scale. SHUFFLE_CONTAINER_AUTO_CLEANUP=false SHUFFLE_ELASTIC=true diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index af540e93..ef9c0264 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -53,6 +53,8 @@ var concurrencyEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY") var appSdkVersion = os.Getenv("SHUFFLE_APP_SDK_VERSION") var workerVersion = os.Getenv("SHUFFLE_WORKER_VERSION") var newWorkerImage = os.Getenv("SHUFFLE_WORKER_IMAGE") +var dockerSwarmBridgeMTU = os.Getenv("SHUFFLE_SWARM_BRIDGE_DEFAULT_MTU") +var dockerSwarmBridgeInterface = os.Getenv("SHUFFLE_SWARM_BRIDGE_DEFAULT_INTERFACE") // var baseimagename = "docker.pkg.github.com/shuffle/shuffle" // var baseimagename = "ghcr.io/frikky" @@ -214,18 +216,31 @@ func deployServiceWorkers(image string) { // Get a list of network interfaces interfaces, err := net.Interfaces() - mtu := 1500 // default docker MTU if err != nil { log.Printf("[ERROR] Failed to get network interfaces: %s", err) } + mtu, err := strconv.Atoi(dockerSwarmBridgeMTU) // by default + bridgeName := dockerSwarmBridgeInterface + + if err != nil { + log.Printf("[ERROR] Failed to convert the default MTU to int: %s. Using 1500 instead", err) + mtu = 1500 + } + // Check if there is at least one interface if len(interfaces) < 2 { + // this assumes that the machine should have at least 2 network + // interfaces. If not, we will use the default MTU. + // interface 1 is the loopback interface + // interface 2 is eth0, The eth0 interface inside a + // Docker container corresponds to the virtual Ethernet + // interface that connects the container to the docker0 log.Printf("[ERROR] Failed to get enough network interfaces") } else { // Get the preferred interface for _, iface := range interfaces { - if strings.Contains(iface.Name, "eth0") { + if strings.Contains(iface.Name, bridgeName) { targetInterface := iface mtu = targetInterface.MTU break From b12eb06494dfae3a0e6491a030d93f327d84b64b Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Fri, 30 Jun 2023 21:35:58 +0530 Subject: [PATCH 4/6] Leaving a template in docker-compose.yml --- docker-compose.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 18c96895..9e66eb53 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -113,4 +113,10 @@ services: networks: shuffle: driver: bridge - #driver: overlay + + # uncomment to set MTU for swarm mode. + # MTU should be whatever is your host's preferred MTU is. + # Refer to this doc to figure out what your host's MTU is: + # https://shuffler.io/docs/troubleshooting#TLS_timeout_error/Timeout_Errors/EOF_Errors + # driver_opts: + # com.docker.network.driver.mtu: 1460 From 91e4c94282bc4764fbf66e6564f23d4d0f6400df Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Sat, 1 Jul 2023 05:46:19 +0530 Subject: [PATCH 5/6] Covering up empty bridgeName edge case --- functions/onprem/orborus/orborus.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index ef9c0264..dba6cbc9 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -223,6 +223,10 @@ func deployServiceWorkers(image string) { mtu, err := strconv.Atoi(dockerSwarmBridgeMTU) // by default bridgeName := dockerSwarmBridgeInterface + if bridgeName == "" { + bridgeName = "eth0" + } + if err != nil { log.Printf("[ERROR] Failed to convert the default MTU to int: %s. Using 1500 instead", err) mtu = 1500 From 9ed3fe03deb0234879c010c278b94f32be64323d Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Sat, 1 Jul 2023 05:46:57 +0530 Subject: [PATCH 6/6] Improving Logging for auto MTU pick up --- functions/onprem/orborus/orborus.go | 1 + 1 file changed, 1 insertion(+) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index dba6cbc9..1c1f40be 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -247,6 +247,7 @@ func deployServiceWorkers(image string) { if strings.Contains(iface.Name, bridgeName) { targetInterface := iface mtu = targetInterface.MTU + log.Printf("[INFO] Using MTU %d from interface %s", mtu, targetInterface.Name) break } }