diff --git a/backend/Dockerfile b/backend/Dockerfile index 14d9137c..d203ed78 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -24,7 +24,7 @@ RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o webapp . FROM alpine:latest as certs RUN apk --update add ca-certificates -from scratch +FROM alpine:3.12 COPY --from=builder /app/ /app COPY --from=builder /app_sdk/ /app_sdk diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index f9dadcad..0f99e856 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,9 +1,10 @@ #!/bin/bash NAME=app_sdk -VERSION=0.2.0 +VERSION=0.6.0 -docker rmi frikky/shuffle:$NAME --force -docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION +docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force +docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -docker push frikky/shuffle:$NAME -docker push frikky/$NAME:$VERSION +#docker push frikky/shuffle:$NAME +#docker push frikky/$NAME:$VERSION +docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 865d5867..717f04b7 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -11,11 +11,12 @@ import ( "fmt" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" - network "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" - natting "github.com/docker/go-connections/nat" "github.com/go-git/go-billy/v5" + network "github.com/docker/docker/api/types/network" + natting "github.com/docker/go-connections/nat" + "io" "io/ioutil" "log" @@ -180,9 +181,10 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin // Dockerfile is inside the TAR itself. Not local context // docker build --build-arg http_proxy=http://my.proxy.url buildOptions := types.ImageBuildOptions{ - Remove: true, - Tags: tags, - BuildArgs: map[string]*string{}, + Remove: true, + Tags: tags, + BuildArgs: map[string]*string{}, + NetworkMode: "host", } httpProxy := os.Getenv("HTTP_PROXY") @@ -242,9 +244,10 @@ func buildImage(tags []string, dockerfileFolder string) error { dockerFileTarReader := bytes.NewReader(buf.Bytes()) buildOptions := types.ImageBuildOptions{ - Remove: true, - Tags: tags, - BuildArgs: map[string]*string{}, + Remove: true, + Tags: tags, + BuildArgs: map[string]*string{}, + NetworkMode: "host", } httpProxy := os.Getenv("HTTP_PROXY") diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 1792a9bc..b50ba5b1 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -666,7 +666,7 @@ func handleApiAuthentication(resp http.ResponseWriter, request *http.Request) (U if len(Userdata.Username) > 0 { return Userdata, nil } else { - return Userdata, errors.New(fmt.Sprintf("User is invalid - no username found: %#v", Userdata)) + return Userdata, errors.New(fmt.Sprintf("User is invalid - no username found")) } } diff --git a/docker-compose.yml b/docker-compose.yml index 164eeca9..603deb69 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - build: ./frontend + #build: ./frontend image: frikky/shuffle:frontend container_name: shuffle-frontend hostname: shuffle-frontend @@ -16,19 +16,19 @@ services: depends_on: - backend backend: - build: ./backend + #build: ./backend image: frikky/shuffle:backend container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: - ports: + ports: - "${BACKEND_PORT}:5001" networks: - shuffle volumes: - /var/run/docker.sock:/var/run/docker.sock - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps - environment: + environment: - DATASTORE_EMULATOR_HOST=shuffle-database:8000 - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - ORG_ID=${ORG_ID} @@ -43,13 +43,13 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: frikky/shuffle:orborus + image: docker.pkg.github.com/frikky/shuffle/orborus:0.6.0 container_name: shuffle-orborus hostname: shuffle-orborus networks: - shuffle - volumes: - - /var/run/docker.sock:/var/run/docker.sock + volumes: + - /var/run/docker.sock:/var/run/docker.sock environment: - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} diff --git a/functions/onprem/orborus/Dockerfile b/functions/onprem/orborus/Dockerfile index 1a15a2e6..e5bc379e 100644 --- a/functions/onprem/orborus/Dockerfile +++ b/functions/onprem/orborus/Dockerfile @@ -2,13 +2,14 @@ from golang as builder RUN mkdir /app WORKDIR /app -COPY orborus.go /app/orborus.go - RUN go get github.com/docker/docker/api/types github.com/docker/docker/api/types/container github.com/docker/docker/client +COPY orborus.go /app/orborus.go + RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o orborus . -from scratch +FROM alpine:3.12 +RUN apk add --no-cache bash COPY --from=builder /app/ / CMD ["./orborus"] diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 4dd3ad47..285d6772 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,10 +1,10 @@ NAME=orborus -VERSION=0.1.0 +VERSION=0.6.0 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t frikky/$NAME:$VERSION -docker push frikky/$NAME:$VERSION -docker push frikky/shuffle:$NAME -#docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION +#docker push frikky/$NAME:$VERSION +#docker push frikky/shuffle:$NAME +docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index fbee4052..fce0608d 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -14,31 +14,31 @@ import ( "log" "net/http" "os" + "os/exec" "strings" "time" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" - network "github.com/docker/docker/api/types/network" dockerclient "github.com/docker/docker/client" //network "github.com/docker/docker/api/types/network" //natting "github.com/docker/go-connections/nat" ) -var baseUrl = os.Getenv("BASE_URL") -var baseimagename = "frikky/shuffle" -var shuffleNetwork = "" // Filled in init if found - -var dockerApiVersion = os.Getenv("DOCKER_API_VERSION") -var environment = os.Getenv("ENVIRONMENT_NAME") -var orgId = os.Getenv("ORG_ID") - // Starts jobs in bulk, so this could be increased var sleepTime = 3 -// Timeout if somethinc rashes -//var workerTimeout = 600 +// Timeout if something rashes var workerTimeout = 300 +var appSdkVersion = "0.6.0" +var workerVersion = "0.6.0" +var baseimagename = "docker.pkg.github.com/frikky/shuffle" + +var orgId = os.Getenv("ORG_ID") +var baseUrl = os.Getenv("BASE_URL") +var environment = os.Getenv("ENVIRONMENT_NAME") +var dockerApiVersion = os.Getenv("DOCKER_API_VERSION") +var runningMode = strings.ToLower(os.Getenv("RUNNING_MODE")) type ExecutionRequestWrapper struct { Data []ExecutionRequest `json:"data"` @@ -54,6 +54,7 @@ type ExecutionRequest struct { } var dockercli *dockerclient.Client +var containerId string func init() { var err error @@ -63,48 +64,40 @@ func init() { panic(fmt.Sprintf("Unable to create docker client: %s", err)) } - // FIXME: Move this to global variables? - containerIdentifier := "orborus" - networkIdentifier := "shuffle" + getThisContainerId() +} - ctx := context.Background() - containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{ - All: true, - }) - if err != nil { - log.Printf("Failed getting containers during init - running without network check: %s", err) +// form id of current running container +func getThisContainerId() { + fCol := "" + + // some adjusting based on current running mode + switch runningMode { + case "kubernetes": + // cgroup will be like: + // 11:net_cls,net_prio:/kubepods/besteffort/podf132b44d-cfcf-43f7-9906-79f58e268333/851466f8b5ed5aa0f265b1c95c6d2bafbc51a38dd5c5a1621b6e586572150009 + fCol = "5" + log.Printf("[INFO] Running containerized in Kubernetes!") + + case "docker": + // cgroup will be like: + // 12:perf_event:/docker/0f06810364f52a2cd6e80bfba27419cb8a29758a204cd676388f4913bb366f2b + fCol = "3" + log.Printf("[INFO] Running containerized in Docker!") + + default: + fCol = "3" // for backward-compatibility with production + log.Printf("[WARNING] RUNNING_MODE not set, so I can't figure out the current container ID! Defaulting to Docker (not Kubernetes).") } - // Skip random containers. Only handle things related to Shuffle. - for _, container := range containers { - found := false - - // Bad states - it might just be created sometimes, leading to now netowkr - //if container.State == "restarting" || container.State == "paused" || container.State == "exited" || container.State == "dead" { - // continue - //} - - for _, name := range container.Names { - if !strings.Contains(strings.ToLower(name), containerIdentifier) { - found = true - continue - } + if fCol != "" { + cmd := fmt.Sprintf("cat /proc/self/cgroup | grep memory | tail -1 | cut -d/ -f%s", fCol) + out, err := exec.Command("bash", "-c", cmd).Output() + if err == nil { + containerId = strings.TrimSpace(string(out)) + } else { + log.Printf("Failed getting container ID: %s", err) } - - if found { - for key, _ := range container.NetworkSettings.Networks { - if strings.Contains(strings.ToLower(key), networkIdentifier) { - shuffleNetwork = key - break - } - } - } - } - - if len(shuffleNetwork) > 0 { - log.Printf("Found shuffle network \"%s\" for container %s", shuffleNetwork, containerIdentifier) - } else { - log.Printf("Running Shuffle without a docker network") } } @@ -121,40 +114,24 @@ func deployWorker(image string, identifier string, env []string) { }, } - // Look for Shuffle network and set it - networkConfig := &network.NetworkingConfig{} - if len(shuffleNetwork) > 0 { - log.Printf("Starting worker with network %s", shuffleNetwork) - networkConfig = &network.NetworkingConfig{ - EndpointsConfig: map[string]*network.EndpointSettings{ - shuffleNetwork: { - NetworkID: shuffleNetwork, - }, - }, - } - - env = append(env, fmt.Sprintf("DOCKER_NETWORK=%s", shuffleNetwork)) + // form container id and use it as network source if it's not empty + if containerId != "" { + log.Printf("[INFO] Found container ID %s", containerId) + hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) } else { - log.Printf("Starting worker WITHOUT any specified network: %s", shuffleNetwork) + log.Printf("[WARNING] Empty self container id, continue without NetworkMode") } - // ROFL: https://docker-py.readthedocs.io/en/1.4.0/volumes/ config := &container.Config{ Image: image, Env: env, } - //test := &network.EndpointSettings{ - // Gateway: "helo", - //} - //NetworkID - //if connect.EndpointConfig.NetworkID != "NetworkID" { - cont, err := dockercli.ContainerCreate( context.Background(), config, hostConfig, - networkConfig, + nil, nil, identifier, ) @@ -224,19 +201,18 @@ func initializeImages() { ctx := context.Background() // check whether theyre the same first - //version := "0.1.0" - // fmt.Sprintf("docker.pkg.github.com/frikky/shuffle/orborus:%s", version), - // fmt.Sprintf("docker.pkg.github.com/frikky/shuffle/worker:%s", version), images := []string{ - fmt.Sprintf("docker.io/%s:app_sdk", baseimagename), - fmt.Sprintf("docker.io/%s:worker", baseimagename), + // fmt.Sprintf("docker.io/%s:app_sdk", baseimagename), + // fmt.Sprintf("docker.io/%s:worker", baseimagename), + fmt.Sprintf("%s/worker:%s", baseimagename, workerVersion), + fmt.Sprintf("%s/app_sdk:%s", baseimagename, appSdkVersion), } pullOptions := types.ImagePullOptions{} for _, image := range images { reader, err := dockercli.ImagePull(ctx, image, pullOptions) if err != nil { - log.Printf("Failed getting %s: %s", image, err) + log.Printf("Failed getting image %s: %s", image, err) continue } diff --git a/functions/onprem/orborus/run.sh b/functions/onprem/orborus/run.sh index 9c6d8db5..48e5b701 100644 --- a/functions/onprem/orborus/run.sh +++ b/functions/onprem/orborus/run.sh @@ -3,6 +3,7 @@ docker run \ --env ENVIRONMENT_NAME="Shuffle" \ --env BASE_URL=http://shuffle-backend:5001 \ --env DOCKER_API_VERSION=1.42 \ + --env RUNNING_MODE="Docker" \ --network "shuffle_shuffle" \ -v /var/run/docker.sock:/var/run/docker.sock \ frikky/shuffle:orborus diff --git a/functions/onprem/worker/Dockerfile b/functions/onprem/worker/Dockerfile index 2f8971f8..685b426b 100644 --- a/functions/onprem/worker/Dockerfile +++ b/functions/onprem/worker/Dockerfile @@ -1,21 +1,16 @@ -#from golang as builder -# -#RUN mkdir /app -#WORKDIR /app -#COPY worker.go /app/worker.go -# -#RUN go get github.com/docker/docker/api/types -#RUN go get github.com/docker/docker/api/types/container -#RUN go get -u github.com/docker/docker/client -# -#RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker . -# +from golang as builder -# THis is a workaround until I get docker/docker to build in a dockerfile -# PS: This is tricky to google. -# Might not work on some machines. -from scratch -#COPY --from=builder /app/ / -COPY worker.bin /worker.bin +WORKDIR /app -CMD ["./worker.bin"] +RUN go get -u github.com/docker/docker/api/types +RUN go get -u github.com/docker/docker/api/types/container +RUN go get -u github.com/docker/docker/client + +COPY worker.go /app/worker.go +RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker . + +FROM alpine:3.12 +RUN apk add --no-cache bash +COPY --from=builder /app/ / + +CMD ["./worker"] diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 171308aa..786e92c2 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,11 +1,11 @@ NAME=worker -VERSION=0.1.0 +VERSION=0.6.0 echo "Running docker build with $NAME:$VERSION" CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t frikky/$NAME:$VERSION # Push both for now.. -docker push frikky/$NAME:$VERSION -docker push frikky/shuffle:$NAME -#docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION +#docker push frikky/$NAME:$VERSION +#docker push frikky/shuffle:$NAME +docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 48b11ebc..4a4492a6 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -11,12 +11,12 @@ import ( "log" "net/http" "os" + "os/exec" "strings" "time" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" - network "github.com/docker/docker/api/types/network" dockerclient "github.com/docker/docker/client" ) @@ -25,6 +25,29 @@ var baseUrl = os.Getenv("BASE_URL") var baseimagename = "frikky/shuffle" var sleepTime = 2 +var containerId string + +// form container id of current running container +func getThisContainerId() string { + id := "" + cmd := fmt.Sprintf("cat /proc/self/cgroup | grep memory | tail -1 | cut -d/ -f3") + out, err := exec.Command("bash", "-c", cmd).Output() + if err == nil { + id = strings.TrimSpace(string(out)) + } + + return id +} + +func init() { + containerId = getThisContainerId() + if len(containerId) == 0 { + log.Printf("[ERROR] No container ID found.") + } else { + log.Printf("Found container ID: %s", containerId) + } +} + type User struct { Username string `datastore:"Username" json:"username"` Password string `datastore:"password,noindex" password:"password,omitempty"` @@ -380,11 +403,11 @@ func shutdown(executionId, workflowId string) { authorization := os.Getenv("AUTHORIZATION") if len(authorization) > 0 { req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", authorization)) + } else { + log.Printf("No authorization specified for abort") } req.Header.Add("Content-Type", "application/json") - //req.Header.Add("Authorization", authorization) - client := &http.Client{ Transport: &http.Transport{ Proxy: nil, @@ -414,6 +437,7 @@ func shutdown(executionId, workflowId string) { // Deploys the internal worker whenever something happens func deployApp(cli *dockerclient.Client, image string, identifier string, env []string) error { + // form basic hostConfig hostConfig := &container.HostConfig{ LogConfig: container.LogConfig{ Type: "json-file", @@ -421,28 +445,23 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] }, } + // form container id and use it as network source if it's not empty + if containerId != "" { + hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) + } else { + log.Printf("[WARNING] Empty self container id, continue without NetworkMode") + } + config := &container.Config{ Image: image, Env: env, } - networkConfig := &network.NetworkingConfig{} - shuffleNetwork := os.Getenv("DOCKER_NETWORK") - if len(shuffleNetwork) > 0 { - networkConfig = &network.NetworkingConfig{ - EndpointsConfig: map[string]*network.EndpointSettings{ - shuffleNetwork: { - NetworkID: shuffleNetwork, - }, - }, - } - } - cont, err := cli.ContainerCreate( context.Background(), config, hostConfig, - networkConfig, + nil, nil, identifier, ) @@ -1107,13 +1126,6 @@ func main() { } } - shuffleNetwork := os.Getenv("DOCKER_NETWORK") - if len(shuffleNetwork) > 0 { - log.Printf("Running with Docker network %s", shuffleNetwork) - } else { - log.Printf("No docker network specified for Worker.") - } - // WORKER_TESTING_WORKFLOW should be a workflow ID authorization := "" executionId := "" diff --git a/functions/stitcher.go b/functions/stitcher.go index 1b752f77..aae060f3 100644 --- a/functions/stitcher.go +++ b/functions/stitcher.go @@ -101,15 +101,15 @@ func getRunner(classname string) string { return fmt.Sprintf(` # Run the actual thing after we've checked params def run(request): - action = request.get_json() + action = request.get_json() print(action) print(type(action)) authorization_key = action.get("authorization") current_execution_id = action.get("execution_id") - + if action and "name" in action and "app_name" in action: asyncio.run(%s.run(action), debug=True) - return f'Attempting to execute function {action["name"]} in app {action["app_name"]}' + return f'Attempting to execute function {action["name"]} in app {action["app_name"]}' else: return f'Invalid action' @@ -610,6 +610,7 @@ func buildImage(client *client.Client, tags []string, dockerBuildCtxDir string) PullParent: true, Remove: true, Tags: tags, + NetworkMode: "host", }, )