Fixed base of new execution model with worker, orborus etc
This commit is contained in:
frikky
@@ -11,7 +11,7 @@ RUN go get github.com/docker/docker/api/types && \
|
||||
go get github.com/mackerelio/go-osstat/cpu && \
|
||||
go get github.com/mackerelio/go-osstat/memory && \
|
||||
go get github.com/satori/go.uuid && \
|
||||
go get github.com/frikky/shuffle-shared
|
||||
go get github.com/shuffle/shuffle-shared
|
||||
RUN go build
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o orborus .
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
NAME=shuffle-orborus
|
||||
VERSION=0.9.28
|
||||
VERSION=0.9.30
|
||||
|
||||
echo "Running docker build with $NAME:$VERSION"
|
||||
#docker rmi frikky/shuffle:$NAME --force
|
||||
|
||||
@@ -4,10 +4,9 @@ go 1.15
|
||||
|
||||
require (
|
||||
github.com/containerd/containerd v1.5.7 // indirect
|
||||
github.com/docker/docker v20.10.9+incompatible
|
||||
github.com/docker/docker v20.10.10+incompatible
|
||||
github.com/docker/go-connections v0.4.0 // indirect
|
||||
github.com/mackerelio/go-osstat v0.2.1
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
github.com/shuffle/shuffle-shared v0.1.19
|
||||
go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
|
||||
github.com/shuffle/shuffle-shared v0.1.24
|
||||
)
|
||||
|
||||
@@ -63,7 +63,6 @@ github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB
|
||||
github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
|
||||
github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
|
||||
github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
|
||||
github.com/Microsoft/go-winio v0.4.17 h1:iT12IBVClFevaf8PuVyi3UmZOVh4OqnaLxDTW2O6j3w=
|
||||
github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
|
||||
github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
|
||||
github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
|
||||
@@ -242,8 +241,8 @@ github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible
|
||||
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
|
||||
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
|
||||
github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
|
||||
github.com/docker/docker v20.10.9+incompatible h1:JlsVnETOjM2RLQa0Cc1XCIspUdXW3Zenq9P54uXBm6k=
|
||||
github.com/docker/docker v20.10.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
|
||||
github.com/docker/docker v20.10.10+incompatible h1:GKkP0T7U4ks6X3lmmHKC2QDprnpRJor2Z5a8m62R9ZM=
|
||||
github.com/docker/docker v20.10.10+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
|
||||
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
|
||||
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
|
||||
github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
|
||||
@@ -424,7 +423,6 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
|
||||
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
@@ -563,13 +561,12 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
|
||||
github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
|
||||
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
|
||||
github.com/shuffle/shuffle-shared v0.1.19 h1:bZmwdC3gKPFxtKoGEjHGY8C96+xB+cBdHaz6glP8SwA=
|
||||
github.com/shuffle/shuffle-shared v0.1.19/go.mod h1:0QrK51T12CpCj/be8hXduj/RtDnoeaZ3rfogELZE2IU=
|
||||
github.com/shuffle/shuffle-shared v0.1.24 h1:5kJmwN175x8RiF1Ao2bD8WQyc+8ZuOHbrHeJ8H3nO2s=
|
||||
github.com/shuffle/shuffle-shared v0.1.24/go.mod h1:0QrK51T12CpCj/be8hXduj/RtDnoeaZ3rfogELZE2IU=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
|
||||
github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
|
||||
@@ -656,8 +653,6 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
|
||||
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
|
||||
go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8=
|
||||
go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg=
|
||||
golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
@@ -695,7 +690,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
|
||||
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
|
||||
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI=
|
||||
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||
@@ -706,7 +700,6 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY=
|
||||
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -911,12 +904,10 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f
|
||||
golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963 h1:K+NlvTLy0oONtRtkl1jRD9xIhnItbG2PiE7YOdjPb+k=
|
||||
golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
@@ -26,6 +27,7 @@ import (
|
||||
"github.com/docker/docker/api/types"
|
||||
"github.com/docker/docker/api/types/container"
|
||||
"github.com/docker/docker/api/types/mount"
|
||||
//"github.com/docker/docker/api/types/network"
|
||||
"github.com/docker/docker/api/types/swarm"
|
||||
//"github.com/docker/docker/api/types/filters"
|
||||
dockerclient "github.com/docker/docker/client"
|
||||
@@ -63,6 +65,7 @@ var timezone = os.Getenv("TZ")
|
||||
var containerName = os.Getenv("ORBORUS_CONTAINER_NAME")
|
||||
var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG")
|
||||
var executionIds = []string{}
|
||||
var dockerized bool
|
||||
|
||||
var dockercli *dockerclient.Client
|
||||
var containerId string
|
||||
@@ -81,6 +84,7 @@ func init() {
|
||||
// form id of current running container
|
||||
func getThisContainerId() {
|
||||
fCol := ""
|
||||
dockerized = true
|
||||
|
||||
// some adjusting based on current running mode
|
||||
switch runningMode {
|
||||
@@ -98,6 +102,7 @@ func getThisContainerId() {
|
||||
|
||||
default:
|
||||
fCol = "3" // for backward-compatibility with production
|
||||
dockerized = false
|
||||
log.Printf("[WARNING] RUNNING_MODE not set - defaulting to Docker (NOT Kubernetes).")
|
||||
}
|
||||
|
||||
@@ -132,49 +137,52 @@ func getThisContainerId() {
|
||||
log.Printf(`[INFO] Started with containerId "%s"`, containerId)
|
||||
}
|
||||
|
||||
// Deploys the internal worker whenever something happens
|
||||
// https://docs.docker.com/engine/api/sdk/examples/
|
||||
func deployWorker(image string, identifier string, env []string, executionRequest shuffle.ExecutionRequest) {
|
||||
// Binds is the actual "-v" volume.
|
||||
// Max 20% CPU every second
|
||||
|
||||
//CPUQuota: 25000,
|
||||
//CPUPeriod: 100000,
|
||||
//CPUShares: 256,
|
||||
hostConfig := &container.HostConfig{
|
||||
LogConfig: container.LogConfig{
|
||||
Type: "json-file",
|
||||
Config: map[string]string{},
|
||||
},
|
||||
Resources: container.Resources{},
|
||||
Binds: []string{
|
||||
"/var/run/docker.sock:/var/run/docker.sock:rw",
|
||||
},
|
||||
NetworkMode: container.NetworkMode(fmt.Sprintf("container:%s", containerId)),
|
||||
}
|
||||
|
||||
if cleanupEnv == "true" {
|
||||
hostConfig.AutoRemove = true
|
||||
}
|
||||
|
||||
config := &container.Config{
|
||||
Image: image,
|
||||
Env: env,
|
||||
}
|
||||
|
||||
//var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG")
|
||||
parsedUuid := uuid.NewV4()
|
||||
func deployServiceWorkers(image string) {
|
||||
log.Printf("[DEBUG] Validating deployment of workers as services IF swarmConfig = run")
|
||||
if swarmConfig == "run" {
|
||||
// frikky@debian:~/git/shuffle/functions/onprem/worker$ docker service create --replicas 5 --name shuffle-workers --env SHUFFLE_SWARM_CONFIG=run --publish published=33333,target=33333 ghcr.io/frikky/shuffle-worker:nightly
|
||||
networkName := "shuffle-executions"
|
||||
ctx := context.Background()
|
||||
|
||||
log.Printf("[DEBUG] Deploying containers with swarm")
|
||||
//docker network create --driver=overlay workers
|
||||
networkCreateOptions := types.NetworkCreate{
|
||||
Driver: "overlay",
|
||||
}
|
||||
_, err := dockercli.NetworkCreate(
|
||||
ctx,
|
||||
networkName,
|
||||
networkCreateOptions,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
if strings.Contains(fmt.Sprintf("%s", err), "already exists") {
|
||||
|
||||
} else {
|
||||
log.Printf("[DEBUG] Failed to create network %s for workers: %s. This is not critical, and containers will still be added", networkName, err)
|
||||
}
|
||||
}
|
||||
//serviceOptions := types.ServiceCreateOptions{}
|
||||
//service, err := dockercli.ServiceCreate(
|
||||
// context.Background(),
|
||||
// serviceSpec,
|
||||
// serviceOptions,
|
||||
//)
|
||||
|
||||
log.Printf("[DEBUG] Deploying containers for worker with swarm")
|
||||
//containerName := fmt.Sprintf("shuffle-worker-%s", parsedUuid)
|
||||
containerName := fmt.Sprintf("shuffle-workers")
|
||||
innerContainerName := fmt.Sprintf("shuffle-workers")
|
||||
replicas := uint64(2)
|
||||
|
||||
serviceSpec := swarm.ServiceSpec{
|
||||
Annotations: swarm.Annotations{
|
||||
Name: containerName,
|
||||
Name: innerContainerName,
|
||||
Labels: map[string]string{},
|
||||
},
|
||||
Networks: []swarm.NetworkAttachmentConfig{
|
||||
swarm.NetworkAttachmentConfig{
|
||||
Target: networkName,
|
||||
},
|
||||
},
|
||||
EndpointSpec: &swarm.EndpointSpec{
|
||||
Ports: []swarm.PortConfig{
|
||||
swarm.PortConfig{
|
||||
@@ -207,7 +215,7 @@ func deployWorker(image string, identifier string, env []string, executionReques
|
||||
Condition: swarm.RestartPolicyConditionNone,
|
||||
},
|
||||
Placement: &swarm.Placement{
|
||||
MaxReplicas: 1,
|
||||
MaxReplicas: replicas,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -217,26 +225,75 @@ func deployWorker(image string, identifier string, env []string, executionReques
|
||||
}
|
||||
|
||||
serviceOptions := types.ServiceCreateOptions{}
|
||||
service, err := dockercli.ServiceCreate(
|
||||
context.Background(),
|
||||
_, err = dockercli.ServiceCreate(
|
||||
ctx,
|
||||
serviceSpec,
|
||||
serviceOptions,
|
||||
)
|
||||
|
||||
if err == nil {
|
||||
log.Printf("[DEBUG] Waiting 10 seconds for workers to come awake")
|
||||
time.Sleep(time.Duration(10) * time.Second)
|
||||
}
|
||||
|
||||
log.Printf("Servicecreate request: %#v %#v", service, err)
|
||||
|
||||
err = sendWorkerRequest(executionRequest)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed worker request: %s", err)
|
||||
log.Printf("[DEBUG] Successfully deployed workers with %d replicas", replicas)
|
||||
//time.Sleep(time.Duration(10) * time.Second)
|
||||
//log.Printf("[DEBUG] Servicecreate request: %#v %#v", service, err)
|
||||
} else {
|
||||
log.Printf("[DEBUG] Started worker from request: %s - %#v - %s", containerName, service, err)
|
||||
if !strings.Contains(fmt.Sprintf("%s", err), "Already Exists") {
|
||||
log.Printf("[ERROR] Failed making service: %s", err)
|
||||
}
|
||||
}
|
||||
return
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// Deploys the internal worker whenever something happens
|
||||
// https://docs.docker.com/engine/api/sdk/examples/
|
||||
func deployWorker(image string, identifier string, env []string, executionRequest shuffle.ExecutionRequest) error {
|
||||
// Binds is the actual "-v" volume.
|
||||
// Max 20% CPU every second
|
||||
|
||||
//CPUQuota: 25000,
|
||||
//CPUPeriod: 100000,
|
||||
//CPUShares: 256,
|
||||
hostConfig := &container.HostConfig{
|
||||
LogConfig: container.LogConfig{
|
||||
Type: "json-file",
|
||||
Config: map[string]string{},
|
||||
},
|
||||
Resources: container.Resources{},
|
||||
Binds: []string{
|
||||
"/var/run/docker.sock:/var/run/docker.sock:rw",
|
||||
},
|
||||
}
|
||||
|
||||
if len(containerId) > 0 && dockerized == true {
|
||||
hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId))
|
||||
}
|
||||
|
||||
if cleanupEnv == "true" {
|
||||
hostConfig.AutoRemove = true
|
||||
}
|
||||
|
||||
config := &container.Config{
|
||||
Image: image,
|
||||
Env: env,
|
||||
}
|
||||
|
||||
//var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG")
|
||||
parsedUuid := uuid.NewV4()
|
||||
if swarmConfig == "run" {
|
||||
err := sendWorkerRequest(executionRequest)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed worker request for %s: %s", executionRequest.ExecutionId, err)
|
||||
|
||||
if strings.Contains(fmt.Sprintf("%s", err), "connection refused") {
|
||||
workerImage := fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion)
|
||||
go deployServiceWorkers(workerImage)
|
||||
}
|
||||
return err
|
||||
} else {
|
||||
log.Printf("[DEBUG] Started worker from request with name: %s", executionRequest.ExecutionId)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
//log.Printf("[INFO] Identifier: %s", identifier)
|
||||
@@ -264,11 +321,11 @@ func deployWorker(image string, identifier string, env []string, executionReques
|
||||
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Container create error(2): %s", err)
|
||||
return
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
log.Printf("[ERROR] Container create error: %s", err)
|
||||
return
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -276,7 +333,7 @@ func deployWorker(image string, identifier string, env []string, executionReques
|
||||
err = dockercli.ContainerStart(context.Background(), cont.ID, containerStartOptions)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err)
|
||||
return
|
||||
return err
|
||||
|
||||
//stats, err := cli.ContainerInspect(context.Background(), containerName)
|
||||
//if err != nil {
|
||||
@@ -303,7 +360,7 @@ func deployWorker(image string, identifier string, env []string, executionReques
|
||||
log.Printf("[INFO] Container %s was created under environment %s", cont.ID, environment)
|
||||
}
|
||||
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
func stopWorker(containername string) error {
|
||||
@@ -486,6 +543,8 @@ func main() {
|
||||
//workerImage := fmt.Sprintf("%s/%s:worker%s", baseimageregistry, baseimagename, baseimagetagsuffix)
|
||||
workerImage := fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion)
|
||||
|
||||
go deployServiceWorkers(workerImage)
|
||||
|
||||
log.Printf("[INFO] Finished configuring docker environment")
|
||||
|
||||
// FIXME - time limit
|
||||
@@ -638,7 +697,6 @@ func main() {
|
||||
continue
|
||||
} else {
|
||||
//log.Printf("[INFO] Adding to be ran %s", execution.ExecutionId)
|
||||
executionIds = append(executionIds, execution.ExecutionId)
|
||||
}
|
||||
|
||||
// Now, how do I execute this one?
|
||||
@@ -665,11 +723,15 @@ func main() {
|
||||
env = append(env, fmt.Sprintf("DOCKER_API_VERSION=%s", dockerApiVersion))
|
||||
}
|
||||
|
||||
go deployWorker(workerImage, containerName, env, execution)
|
||||
|
||||
log.Printf("[INFO] ExecutionID %s was deployed and to be removed from queue.", execution.ExecutionId)
|
||||
err = deployWorker(workerImage, containerName, env, execution)
|
||||
zombiecounter += 1
|
||||
toBeRemoved.Data = append(toBeRemoved.Data, execution)
|
||||
if err == nil {
|
||||
log.Printf("[INFO] ExecutionID %s was deployed and to be removed from queue.", execution.ExecutionId)
|
||||
toBeRemoved.Data = append(toBeRemoved.Data, execution)
|
||||
executionIds = append(executionIds, execution.ExecutionId)
|
||||
} else {
|
||||
log.Printf("[WARNING] Execution ID %s failed to deploy: %s", execution.ExecutionId, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Removes handled workflows (worker is made)
|
||||
@@ -876,20 +938,8 @@ func zombiecheck(ctx context.Context, workerTimeout int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type ExecutionRequest struct {
|
||||
ExecutionId string `json:"execution_id"`
|
||||
Authorization string `json:"authorization"`
|
||||
HTTPProxy string `json:"http_proxy"`
|
||||
HTTPSProxy string `json:"https_proxy"`
|
||||
BaseUrl string `json:"base_url"`
|
||||
EnvironmentName string `json:"environment_name"`
|
||||
Timezone string `json:"timezone"`
|
||||
Cleanup string `json:"cleanup"`
|
||||
ShufflePassProxyToApp string `json:"shuffle_pass_proxy_to_app"`
|
||||
}
|
||||
|
||||
func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest) error {
|
||||
parsedRequest := ExecutionRequest{
|
||||
parsedRequest := shuffle.OrborusExecutionRequest{
|
||||
ExecutionId: workflowExecution.ExecutionId,
|
||||
Authorization: workflowExecution.Authorization,
|
||||
BaseUrl: os.Getenv("BASE_URL"),
|
||||
@@ -925,16 +975,21 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest) error {
|
||||
|
||||
client := &http.Client{}
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed creating finishing request: %s", err)
|
||||
log.Printf("[ERROR] Failed creating worker request: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
newresp, err := client.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Error running finishing request: %s", err)
|
||||
log.Printf("[ERROR] Error running worker request: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if newresp.StatusCode != 200 {
|
||||
log.Printf("[ERROR] Error running request - status code is %d, not 200", newresp.StatusCode)
|
||||
return errors.New(fmt.Sprintf("Bad statuscode: %d - expecting 200", newresp.StatusCode))
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(newresp.Body)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed reading body: %s", err)
|
||||
|
||||
Reference in New Issue
Block a user