diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 8e32a56d..c194ecdd 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.2.2+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.99 + github.com/shuffle/shuffle-shared v0.9.2 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 ) diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 13ae08b5..3944b4e5 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -142,8 +142,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.16 h1:4d2ZktB9xGsAusbbKliOI8TuriSrdIMzD/6ToY3wkz8= -github.com/frikky/schemaless v0.0.16/go.mod h1:jT48kTcmr1q3o8i+8qe7g+eCsbwaz2Q9CjOJevQQzQs= +github.com/frikky/schemaless v0.0.17 h1:Tlg7td64r/EGAfQAL8LIlB8mOEnSRAgfJ8TERxzJSvM= +github.com/frikky/schemaless v0.0.17/go.mod h1:jT48kTcmr1q3o8i+8qe7g+eCsbwaz2Q9CjOJevQQzQs= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -324,8 +324,8 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.84 h1:ElIMQYjKBVOiadbiGkSzt/lPU5xqaQwRxQvk9wx/xYM= -github.com/shuffle/shuffle-shared v0.8.84/go.mod h1:RdfNxqCPI+zU4jQKy3E/p4Io2injm7LpSKQUCDHNtLk= +github.com/shuffle/shuffle-shared v0.9.2 h1:xl/dTNKWh9mol2XPaGXCUlUTqLuwrwLxlEuv7AINwkk= +github.com/shuffle/shuffle-shared v0.9.2/go.mod h1:PFLG4eaxz8zQc9Lbc0Ok/nG8rNsaBkZ0TNar+VU+J/g= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index ce7318d1..bf232b9d 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -90,7 +90,7 @@ var baseimageregistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY") //var baseimagetagsuffix = os.Getenv("SHUFFLE_BASE_IMAGE_TAG_SUFFIX") -// Used for cloud with auth +// Used for cloud with auth. Onprem in certain cases too. var auth = os.Getenv("AUTH") var org = os.Getenv("ORG") diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 7b39554b..6cdc092d 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.2.2+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.99 + github.com/shuffle/shuffle-shared v0.9.2 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 k8s.io/client-go v0.33.1 diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index 0ef2d3a6..b345e99e 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -142,8 +142,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.16 h1:4d2ZktB9xGsAusbbKliOI8TuriSrdIMzD/6ToY3wkz8= -github.com/frikky/schemaless v0.0.16/go.mod h1:jT48kTcmr1q3o8i+8qe7g+eCsbwaz2Q9CjOJevQQzQs= +github.com/frikky/schemaless v0.0.17 h1:Tlg7td64r/EGAfQAL8LIlB8mOEnSRAgfJ8TERxzJSvM= +github.com/frikky/schemaless v0.0.17/go.mod h1:jT48kTcmr1q3o8i+8qe7g+eCsbwaz2Q9CjOJevQQzQs= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -326,8 +326,8 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.84 h1:ElIMQYjKBVOiadbiGkSzt/lPU5xqaQwRxQvk9wx/xYM= -github.com/shuffle/shuffle-shared v0.8.84/go.mod h1:RdfNxqCPI+zU4jQKy3E/p4Io2injm7LpSKQUCDHNtLk= +github.com/shuffle/shuffle-shared v0.9.2 h1:xl/dTNKWh9mol2XPaGXCUlUTqLuwrwLxlEuv7AINwkk= +github.com/shuffle/shuffle-shared v0.9.2/go.mod h1:PFLG4eaxz8zQc9Lbc0Ok/nG8rNsaBkZ0TNar+VU+J/g= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 86f72e00..ba81eeab 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -683,6 +683,10 @@ func deployk8sApp(image string, identifier string, env []string) error { return err } + // Giving the service time to start before we contineu anything + log.Printf("[DEBUG] Waiting 20 seconds before moving on to let app '%s' start properly. Service: %s (k8s)", name, image) + time.Sleep(20 * time.Second) + return nil } @@ -3135,7 +3139,7 @@ func findActiveSwarmNodes(dockercli *dockerclient.Client) (int64, error) { } /*** STARTREMOVE ***/ -func deploySwarmService(dockercli *dockerclient.Client, name, image string, deployport int) error { +func deploySwarmService(dockercli *dockerclient.Client, name, image string, deployport int, retry bool) error { log.Printf("[DEBUG] Deploying service for %s to swarm on port %d", name, deployport) //containerName := fmt.Sprintf("shuffle-worker-%s", parsedUuid) @@ -3296,6 +3300,21 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl _ = service if err != nil { + + if strings.Contains(fmt.Sprintf("%s", err), "network") && strings.Contains(fmt.Sprintf("%s", err), "not found") { + log.Printf("[DEBUG] Network %s not found. Trying to initialize it.", networkName) + networkErr := initSwarmNetwork() + if networkErr != nil { + log.Printf("[ERROR] Failed initializing swarm network: %s", err) + //return err + } + + // Retry deploying the service + if !retry { + return deploySwarmService(dockercli, name, image, deployport, true) + } + } + log.Printf("[DEBUG] Failed deploying %s with image %s: %s", name, image, err) return err } @@ -3351,13 +3370,108 @@ func findAppInfoKubernetes(image, name string, env []string) error { return err } -func findAppInfo(image, name string) (int, error) { +// Backups in case networks are removed +func initSwarmNetwork() error { + ctx := context.Background() dockercli, err := dockerclient.NewEnvClient() if err != nil { log.Printf("[ERROR] Unable to create docker client (2): %s", err) - return -1, err + return err } + // Create the network options with the specified MTU + options := make(map[string]string) + mtu := 1500 + options["com.docker.network.driver.mtu"] = fmt.Sprintf("%d", mtu) + + + ingressOptions := network.CreateOptions{ + Driver: "overlay", + Attachable: false, + Ingress: true, + IPAM: &network.IPAM{ + Driver: "default", + Config: []network.IPAMConfig{ + network.IPAMConfig{ + Subnet: "10.225.225.0/24", + Gateway: "10.225.225.1", + }, + }, + }, + } + + _, err = dockercli.NetworkCreate( + ctx, + "ingress", + ingressOptions, + ) + + if err != nil { + log.Printf("[WARNING] Ingress network may already exist: %s", err) + } + + //docker network create --driver=overlay workers + // Specific subnet? + networkName := "shuffle_swarm_executions" + if len(swarmNetworkName) > 0 { + networkName = swarmNetworkName + } + + networkCreateOptions := network.CreateOptions{ + Driver: "overlay", + Options: options, + Attachable: true, + Ingress: false, + IPAM: &network.IPAM{ + Driver: "default", + Config: []network.IPAMConfig{ + network.IPAMConfig{ + Subnet: "10.224.224.0/24", + Gateway: "10.224.224.1", + }, + }, + }, + } + _, err = dockercli.NetworkCreate( + ctx, + networkName, + networkCreateOptions, + ) + + if err != nil { + log.Printf("[WARNING] Swarm Executions network may already exist: %s", err) + } + + networkName = "shuffle-executions" + networkCreateOptions = network.CreateOptions{ + Driver: "overlay", + Options: options, + Attachable: true, + Ingress: false, + IPAM: &network.IPAM{ + Driver: "default", + Config: []network.IPAMConfig{ + network.IPAMConfig{ + Subnet: "10.223.223.0/24", + Gateway: "10.223.223.1", + }, + }, + }, + } + _, err = dockercli.NetworkCreate( + ctx, + networkName, + networkCreateOptions, + ) + + if err != nil { + log.Printf("[WARNING] Swarm Executions network may already exist: %s", err) + } + + return nil +} + +func findAppInfo(image, name string) (int, error) { highest := baseport exposedPort := -1 @@ -3379,6 +3493,12 @@ func findAppInfo(image, name string) (int, error) { //Filters: if exposedPort == -1 { + dockercli, err := dockerclient.NewEnvClient() + if err != nil { + log.Printf("[ERROR] Unable to create docker client (2): %s", err) + return -1, err + } + serviceListOptions := types.ServiceListOptions{} services, err := dockercli.ServiceList( context.Background(), @@ -3443,26 +3563,29 @@ func findAppInfo(image, name string) (int, error) { if exposedPort >= 0 { //log.Printf("[INFO] Found service %s on port %d - no need to deploy another", name, exposedPort) } else { + dockercli, err := dockerclient.NewEnvClient() + if err != nil { + log.Printf("[ERROR] Unable to create docker client (2): %s", err) + return -1, err + } + // Increment by 1 for highest port if highest <= baseport { highest = baseport } highest += 1 - err = deploySwarmService(dockercli, name, image, highest) + err = deploySwarmService(dockercli, name, image, highest, false) if err != nil { log.Printf("[WARNING] NOT Found service: %s. error: %s", name, err) return highest, err } else { - log.Printf("[DEBUG] Deployed app with name %s", name) + log.Printf("[DEBUG] Waiting 20 seconds before moving on to let app '%s' start properly. Service: %s", name, image) + time.Sleep(time.Duration(20) * time.Second) } exposedPort = highest - - if appsInitialized { - log.Printf("[DEBUG] Waiting 30 seconds before moving on to let app start") - time.Sleep(time.Duration(30) * time.Second) - } + //return exposedPort, errors.New("Deployed app %s") } return exposedPort, nil @@ -3652,7 +3775,7 @@ func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, func baseDeploy() { var cli *dockerclient.Client - var err error + //var err error if isKubernetes != "true" { cli, err := dockerclient.NewEnvClient() @@ -3714,8 +3837,7 @@ func baseDeploy() { //deployApp(cli, value, identifier, env, workflowExecution, action) log.Printf("[DEBUG] Deploying app with identifier %s to ensure basic apps are available from the get-go", identifier) - err = deployApp(cli, value, identifier, env, workflowExecution, action) - _ = err + go deployApp(cli, value, identifier, env, workflowExecution, action) //err := deployApp(cli, value, identifier, env, workflowExecution, action) //if err != nil { // log.Printf("[DEBUG] Failed deploying app %s: %s", value, err)