Merge pull request #1146 from 0x0elliot/0x0elliot/fix-swarm-timeout

Fix swarm timeout between orborus and worker hosted on separate nodes

.env

@@ -65,6 +65,12 @@ SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-1.1.0"
 ## shuffle_Scale_Replicas (workers/node)
 ## shuffle_App_Replicas 	(apps/node)
 
+SHUFFLE_SWARM_BRIDGE_DEFAULT_MTU=1500 # 1500 by default
+# The eth0 interface inside a container corresponds 
+# to the virtual Ethernet interface that connects 
+# the container to the docker0 
+SHUFFLE_SWARM_BRIDGE_DEFAULT_INTERFACE=eth0 
+
 # Used for auto-cleanup of containers. REALLY important at scale.
 SHUFFLE_CONTAINER_AUTO_CLEANUP=false
 SHUFFLE_ELASTIC=true

docker-compose.yml

@@ -113,4 +113,10 @@ services:
 networks:
   shuffle:
     driver: bridge
-    #driver: overlay 
+
+    # uncomment to set MTU for swarm mode.
+    # MTU should be whatever is your host's preferred MTU is.
+    # Refer to this doc to figure out what your host's MTU is:
+    # https://shuffler.io/docs/troubleshooting#TLS_timeout_error/Timeout_Errors/EOF_Errors 
+    # driver_opts:
+    #   com.docker.network.driver.mtu: 1460

functions/onprem/orborus/orborus.go

@@ -53,6 +53,8 @@ var concurrencyEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY")
 var appSdkVersion = os.Getenv("SHUFFLE_APP_SDK_VERSION")
 var workerVersion = os.Getenv("SHUFFLE_WORKER_VERSION")
 var newWorkerImage = os.Getenv("SHUFFLE_WORKER_IMAGE")
+var dockerSwarmBridgeMTU = os.Getenv("SHUFFLE_SWARM_BRIDGE_DEFAULT_MTU")
+var dockerSwarmBridgeInterface = os.Getenv("SHUFFLE_SWARM_BRIDGE_DEFAULT_INTERFACE")
 
 // var baseimagename = "docker.pkg.github.com/shuffle/shuffle"
 // var baseimagename = "ghcr.io/frikky"
@@ -208,6 +210,49 @@ func deployServiceWorkers(image string) {
 
 		// frikky@debian:~/git/shuffle/functions/onprem/worker$ docker service create --replicas 5 --name shuffle-workers --env SHUFFLE_SWARM_CONFIG=run --publish published=33333,target=33333 ghcr.io/shuffle/shuffle-worker:nightly
 
+		// Get a list of network interfaces
+		interfaces, err := net.Interfaces()
+		if err != nil {
+			log.Printf("[ERROR] Failed to get network interfaces: %s", err)
+		}
+
+		mtu, err := strconv.Atoi(dockerSwarmBridgeMTU) // by default
+		bridgeName := dockerSwarmBridgeInterface
+
+		if bridgeName == "" {
+			bridgeName = "eth0"
+		}
+
+		if err != nil {
+			log.Printf("[ERROR] Failed to convert the default MTU to int: %s. Using 1500 instead", err)
+			mtu = 1500
+		}
+
+		// Check if there is at least one interface
+		if len(interfaces) < 2 {
+			// this assumes that the machine should have at least 2 network
+			// interfaces. If not, we will use the default MTU.
+			// interface 1 is the loopback interface
+			// interface 2 is eth0, The eth0 interface inside a 
+			// Docker container corresponds to the virtual Ethernet
+			// interface that connects the container to the docker0
+			log.Printf("[ERROR] Failed to get enough network interfaces")
+		} else {		
+			// Get the preferred interface
+			for _, iface := range interfaces {
+				if strings.Contains(iface.Name, bridgeName) {
+					targetInterface := iface
+					mtu = targetInterface.MTU
+					log.Printf("[INFO] Using MTU %d from interface %s", mtu, targetInterface.Name)
+					break
+				}
+			}
+		}		
+
+		// Create the network options with the specified MTU
+		options := make(map[string]string)
+		options["com.docker.network.driver.mtu"] = fmt.Sprintf("%d", mtu)
+
 		ingressOptions := types.NetworkCreate{
 			Driver:     "overlay",
 			Attachable: false,
@@ -223,7 +268,7 @@ func deployServiceWorkers(image string) {
 			},
 		}
 
-		_, err := dockercli.NetworkCreate(
+		_, err = dockercli.NetworkCreate(
 			ctx,
 			"ingress",
 			ingressOptions,
@@ -242,6 +287,7 @@ func deployServiceWorkers(image string) {
 
 		networkCreateOptions := types.NetworkCreate{
 			Driver:     "overlay",
+			Options:    options,
 			Attachable: true,
 			Ingress:    false,
 			IPAM: &network.IPAM{