marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more k8s stuff

Browse Source
This commit is contained in:
dhaval055
2023-09-29 18:14:06 +00:00
parent 288b8afab9
commit 7e0a73d007
19 changed files with 1199 additions and 320 deletions
Download Patch File Download Diff File Expand all files Collapse all files
functions/k8s-confs/backend-deployment.yaml
+79 -36
View File
@@ -3,33 +3,58 @@
apiVersion: v1
kind: PersistentVolume
metadata:
name: shuffle-pv
name: shuffle-apps-pv
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Retain
storageClassName: shuffle-storage
hostPath:
path: /mnt/shuffle-data/backend
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
creationTimestamp: null
labels:
io.kompose.service: backend-files-claim
name: backend-files-claim
spec:
storage: 10Gi
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
status: {}
persistentVolumeReclaimPolicy: Retain
storageClassName: fast
local:
path: /mnt/shuffle-data/backend
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: node
operator: In
values:
- master
# ---
# apiVersion: v1
# kind: PersistentVolume
# metadata:
# name: shuffle-files-pv
# spec:
# capacity:
# storage: 5Gi
# accessModes:
# - ReadWriteOnce
# persistentVolumeReclaimPolicy: Retain
# storageClassName: fast
# hostPath:
# path: /mnt/shuffle-data/backend
# ---
# apiVersion: v1
# kind: PersistentVolumeClaim
# metadata:
# creationTimestamp: null
# labels:
# io.kompose.service: backend-files-claim
# name: backend-files-claim
# spec:
# accessModes:
# - ReadWriteOnce
# storageClassName: fast
# resources:
# requests:
# storage: 1Gi
# status: {}
---
@@ -43,10 +68,11 @@ metadata:
spec:
accessModes:
- ReadWriteOnce
storageClassName: fast
resources:
requests:
storage: 1Gi
status: {}
# status: {}
---
apiVersion: apps/v1
@@ -79,15 +105,21 @@ spec:
name: shuffle-backend
spec:
volumes:
- name: shuffle-files
persistentVolumeClaim:
claimName: backend-files-claim
# - name: shuffle-files
# persistentVolumeClaim:
# claimName: backend-files-claim
# - name: shuffle-apps
# hostPath:
# path: /mnt/shuffle-data/backend
# type: DirectoryOrCreate
- name: shuffle-apps
persistentVolumeClaim:
claimName: backend-apps-claim
- name: docker-socket
hostPath:
path: /var/run/docker.sock
# - name: docker-socket
# hostPath:
# path: /var/run/docker.sock
nodeSelector:
node: master
containers:
- env:
- name: BACKEND_HOSTNAME
@@ -350,18 +382,29 @@ spec:
configMapKeyRef:
key: TZ
name: env
image: ghcr.io/shuffle/shuffle-backend:latest
- name: IS_KUBERNETES
valueFrom:
configMapKeyRef:
key: IS_KUBERNETES
name: env
- name: REGISTRY_URL
valueFrom:
configMapKeyRef:
key: REGISTRY_URL
name: env
image: shuffle-backend:v1
imagePullPolicy: Never
name: shuffle-backend
ports:
- containerPort: 5001
resources: {}
volumeMounts:
- name: docker-socket
mountPath: /var/run/docker.sock
# - name: docker-socket
# mountPath: /var/run/docker.sock
- name: shuffle-apps
mountPath: /shuffle-apps
- name: shuffle-files
mountPath: /shuffle-files
mountPath: /app/generated
# - name: shuffle-files
# mountPath: /shuffle-files
restartPolicy: Always
status: {}
functions/k8s-confs/env-configmap.yaml
+5 -2
View File
@@ -2,7 +2,7 @@ apiVersion: v1
data:
BACKEND_HOSTNAME: shuffle-backend
BACKEND_PORT: "5001"
BASE_URL: http://10.0.244.224:5001
BASE_URL: http://shuffle-backend:5001
DATASTORE_EMULATOR_HOST: shuffle-database:8000
DB_LOCATION: /mnt/shuffle-data/open-search
DOCKER_API_VERSION: "1.40"
@@ -44,7 +44,7 @@ data:
SHUFFLE_OPENSEARCH_PASSWORD: admin
SHUFFLE_OPENSEARCH_PROXY: ""
SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY: "true"
SHUFFLE_OPENSEARCH_URL: https://192.168.49.2:31001
SHUFFLE_OPENSEARCH_URL: https://opensearch:9200
SHUFFLE_OPENSEARCH_USERNAME: admin
SHUFFLE_ORBORUS_STARTUP_DELAY: "\t\t"
SHUFFLE_PASS_APP_PROXY: "FALSE"
@@ -52,6 +52,9 @@ data:
SHUFFLE_RERUN_SCHEDULE: "300"
SSO_REDIRECT_URL: ""
TZ: "Europe/Amsterdam \t\t\t\t\t"
IS_KUBERNETES: "true"
REGISTRY_URL: "172.17.14.71:5000"
#REGISTRY_AUTH: ""
kind: ConfigMap
metadata:
creationTimestamp: null
functions/k8s-confs/kaniko.yaml
+20 -10
View File
@@ -1,3 +1,4 @@
apiVersion: batch/v1
kind: Job
metadata:
@@ -5,20 +6,29 @@ metadata:
spec:
template:
spec:
nodeSelector:
node: master
containers:
- name: kaniko
image: gcr.io/kaniko-project/executor:latest
args: ["--verbosity=debug",
"--dockerfile=/workspace/Dockerfile",
"--context=dir:///workspace",
"--insecure",
"--destination=shuffle-registry:5000/shuffle/dhaval-repo:1.0"
]
image: gcr.io/kaniko-project/executor:debug
# command: ["/busybox/sh", "-c"]
# args: ["-c","while true; do sleep 3600; done"]
args: [
"--verbosity=debug",
"--context=dir:///app/generated/ok-349ec164d4ad2409152dcb901257f718",
"--dockerfile=./Dockerfile",
#"--no-push",
"--skip-tls-verify",
# "--registry-certificate=172.17.14.71:5000=/workspace/certs/cert.pem",
"--destination=172.17.14.71:5000/shuffle/dhaval-kaniko:1.0"
]
volumeMounts:
- name: kaniko-workspace
mountPath: /workspace
mountPath: /app/generated
nodeSelector:
node: master
restartPolicy: Never
volumes:
- name: kaniko-workspace
hostPath:
path: /home/docker/kaniko
persistentVolumeClaim:
claimName: backend-apps-claim
functions/k8s-confs/opensearch-deployment.yaml
+7 -1
View File
@@ -10,7 +10,7 @@ spec:
accessModes:
- ReadWriteOnce # This allows read-write access to a single node
persistentVolumeReclaimPolicy: Retain # Adjust the reclaim policy as per your needs
storageClassName: shuffle-storage # Set the desired storage class
storageClassName: fast # Set the desired storage class
hostPath:
path: /mnt/shuffle-data/open-search
@@ -26,6 +26,7 @@ metadata:
spec:
accessModes:
- ReadWriteOnce
storageClassName: fast
resources:
requests:
storage: 500Mi
@@ -58,6 +59,11 @@ spec:
io.kompose.network/shuffle: "true"
io.kompose.service: opensearch
spec:
# securityContext:
# runAsUser: 1000 # UID
# fsGroup: 1000 # GID
nodeSelector:
node: worker1
containers:
- env:
- name: OPENSEARCH_JAVA_OPTS
functions/k8s-confs/orborus-deployment.yaml
+12 -2
View File
@@ -31,7 +31,7 @@ spec:
containers:
- env:
- name: BASE_URL
value: "http://192.168.49.2:30009"
value: "http://shuffle-backend:5001"
- name: DOCKER_API_VERSION
value: "1.40"
- name: ENVIRONMENT_NAME
@@ -46,8 +46,18 @@ spec:
#value: run
- name: SHUFFLE_WORKER_VERSION
value: nightly
- name: IS_KUBERNETES
valueFrom:
configMapKeyRef:
key: IS_KUBERNETES
name: env
- name: REGISTRY_URL
valueFrom:
configMapKeyRef:
key: REGISTRY_URL
name: env
image: orborus:v2
image: shuffle-orborus:v1
imagePullPolicy: Never
name: shuffle-orborus
resources: {}
functions/k8s-confs/shuffle-role.yaml
+3 -1
View File
@@ -7,4 +7,6 @@ rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "create", "update", "delete"]
- apiGroups: ["batch"]
resources: ["jobs"]
verbs: ["create", "get", "list", "watch", "delete"]
functions/onprem/orborus/Dockerfile
+7
View File
@@ -23,4 +23,11 @@ FROM alpine:3.15.0
RUN apk add --no-cache bash tzdata
COPY --from=builder /app/ /
ENV ENVIRONMENT_NAME=Shuffle
ENV BASE_URL=http://shuffle-backend:5001
ENV DOCKER_API_VERSION=1.39
ENV SHUFFLE_OPENSEARCH_URL=https://opensearch:9200
CMD ["./orborus"]
functions/onprem/orborus/orborus.go
+43 -185
View File
@@ -44,13 +44,13 @@ import (
//k8s deps
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/homedir"
"k8s.io/client-go/rest"
"path/filepath"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/homedir"
"path/filepath"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// Starts jobs in bulk, so this could be increased
@@ -243,11 +243,11 @@ func deployServiceWorkers(image string) {
// this assumes that the machine should have at least 2 network
// interfaces. If not, we will use the default MTU.
// interface 1 is the loopback interface
// interface 2 is eth0, The eth0 interface inside a
// interface 2 is eth0, The eth0 interface inside a
// Docker container corresponds to the virtual Ethernet
// interface that connects the container to the docker0
log.Printf("[ERROR] Failed to get enough network interfaces")
} else {
} else {
// Get the preferred interface
for _, iface := range interfaces {
if strings.Contains(iface.Name, bridgeName) {
@@ -257,7 +257,7 @@ func deployServiceWorkers(image string) {
break
}
}
}
}
// Create the network options with the specified MTU
options := make(map[string]string)
@@ -579,205 +579,63 @@ func buildEnvVars(envMap map[string]string) []corev1.EnvVar {
}
func deployWorker(image string, identifier string, env []string, executionRequest shuffle.ExecutionRequest) error {
// Binds is the actual "-v" volume.
// Max 20% CPU every second
//CPUQuota: 25000,
//CPUPeriod: 100000,
//CPUShares: 256,
if os.Getenv("IS_KUBERNETES") == "true" {
log.Printf("IS_KUBERNETS", os.Getenv("IS_KUBERNETES"))
log.Printf("REGISTRY_URL", os.Getenv("REGISTRY_URL"))
fmt.Printf("ENV: %+v", env)
image = "shuffle-worker:v1" //hard coded image name to test locally
envMap := make(map[string]string)
for _, envStr := range env {
parts := strings.SplitN(envStr, "=", 2)
if len(parts) == 2 {
envMap[parts[0]] = parts[1]
if len(os.Getenv("REGISTRY_URL")) > 0 && os.Getenv("REGISTRY_URL") != "" {
env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL")))
env = append(env, fmt.Sprintf("IS_KUBERNETES=%s", os.Getenv("IS_KUBERNETES")))
}
}
clientset, err := getKubernetesClient()
if err != nil {
fmt.Println("[ERROR]Error getting kubernetes client:", err)
os.Exit(1)
}
image = "shuffle-worker:v1" //hard coded image name to test locally
envMap := make(map[string]string)
for _, envStr := range env {
parts := strings.SplitN(envStr, "=", 2)
if len(parts) == 2 {
envMap[parts[0]] = parts[1]
}
}
log.Printf("envMap", envMap)
clientset, err := getKubernetesClient()
if err != nil {
fmt.Println("[ERROR]Error getting kubernetes client:", err)
os.Exit(1)
}
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: identifier,
Name: identifier,
Labels: map[string]string{"app": "shuffle-worker"},
},
Spec: corev1.PodSpec{
NodeSelector: map[string]string{
"node": "master",
},
Containers: []corev1.Container{
{
Name: identifier,
Image: image,
Env: buildEnvVars(envMap),
Env: buildEnvVars(envMap),
},
},
},
}
createdPod, err := clientset.CoreV1().Pods("shuffle").Create(context.Background(), pod, metav1.CreateOptions{})
if err != nil {
fmt.Fprintf(os.Stderr, "Error creating pod: %v\n", err)
os.Exit(1)
}
fmt.Printf("Created pod %q in namespace %q\n", createdPod.Name, createdPod.Namespace)
/////////////////////////////////////////////
} else {
// hostConfig := &container.HostConfig{
// LogConfig: container.LogConfig{
// Type: "json-file",
// Config: map[string]string{
// "max-size": "10m",
// },
// },
// Resources: container.Resources{},
// }
// if len(os.Getenv("DOCKER_HOST")) == 0 {
// if runtime.GOOS == "windows" {
// hostConfig.Binds = []string{`\\.\pipe\docker_engine:\\.\pipe\docker_engine`}
// } else {
// hostConfig.Binds = []string{"/var/run/docker.sock:/var/run/docker.sock:rw"}
// }
// }
// hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId))
// if strings.ToLower(cleanupEnv) == "true" {
// hostConfig.AutoRemove = true
// }
// config := &container.Config{
// Image: image,
// Env: env,
// }
// //var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG")
// parsedUuid := uuid.NewV4()
// if swarmConfig == "run" || swarmConfig == "swarm" {
// // FIXME: Should we handle replies properly?
// // In certain cases, a workflow may e.g. be aborted already. If it's aborted, that returns
// // a 401 from the worker, which returns an error here
// go sendWorkerRequest(executionRequest)
// //sendWorkerRequest(executionRequest)
// //err := sendWorkerRequest(executionRequest)
// //if err != nil {
// // log.Printf("[ERROR] Failed worker request for %s: %s", executionRequest.ExecutionId, err)
// // if strings.Contains(fmt.Sprintf("%s", err), "connection refused") || strings.Contains(fmt.Sprintf("%s", err), "EOF") {
// // workerImage := fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion)
// // deployServiceWorkers(workerImage)
// // time.Sleep(time.Duration(10) * time.Second)
// // err = sendWorkerRequest(executionRequest)
// // }
// //}
// //if err == nil {
// // // FIXME: Readd this? Removed for rerun reasons
// // // executionIds = append(executionIds, executionRequest.ExecutionId)
// //}
// //}()
// return nil
// }
// //log.Printf("[INFO] Identifier: %s", identifier)
// cont, err := dockercli.ContainerCreate(
// context.Background(),
// config,
// hostConfig,
// nil,
// nil,
// identifier,
// )
// if err != nil {
// if strings.Contains(fmt.Sprintf("%s", err), "Conflict. The container name ") {
// identifier = fmt.Sprintf("%s-%s", identifier, parsedUuid)
// //log.Printf("[INFO] 2 - Identifier: %s", identifier)
// cont, err = dockercli.ContainerCreate(
// context.Background(),
// config,
// hostConfig,
// nil,
// nil,
// identifier,
// )
// if err != nil {
// log.Printf("[ERROR] Container create error(2): %s", err)
// return err
// }
// } else {
// log.Printf("[ERROR] Container create error: %s", err)
// return err
// }
// }
// containerStartOptions := types.ContainerStartOptions{}
// err = dockercli.ContainerStart(context.Background(), cont.ID, containerStartOptions)
// if err != nil {
// // Trying to recreate and start WITHOUT network if it's possible. No extended checks. Old execution system (<0.9.30)
// if strings.Contains(fmt.Sprintf("%s", err), "cannot join network") || strings.Contains(fmt.Sprintf("%s", err), "No such container") {
// hostConfig.NetworkMode = ""
// //container.NetworkMode(fmt.Sprintf("container:%s", containerId))
// cont, err = dockercli.ContainerCreate(
// context.Background(),
// config,
// hostConfig,
// nil,
// nil,
// identifier+"-2",
// )
// if err != nil {
// log.Printf("[ERROR] Failed to CREATE container (2): %s", err)
// }
// err = dockercli.ContainerStart(context.Background(), cont.ID, containerStartOptions)
// if err != nil {
// log.Printf("[ERROR] Failed to start container (2): %s", err)
// }
// } else {
// log.Printf("[ERROR] Failed initial container start. Quitting as this is NOT a simple network issue. Err: %s", err)
// }
// if err != nil {
// log.Printf("[ERROR] Failed to start worker container in environment %s: %s", environment, err)
// return err
// } else {
// log.Printf("[INFO] Worker Container %s was created under environment %s for execution %s: docker logs %s", cont.ID, environment, executionRequest.ExecutionId, cont.ID)
// }
// //stats, err := cli.ContainerInspect(context.Background(), containerName)
// //if err != nil {
// // log.Printf("Failed checking worker %s", containerName)
// // return
// //}
// //containerStatus := stats.ContainerJSONBase.State.Status
// //if containerStatus != "running" {
// // log.Printf("Status of %s is %s. Should be running. Will reset", containerName, containerStatus)
// // err = stopWorker(containerName)
// // if err != nil {
// // log.Printf("Failed stopping worker %s", execution.ExecutionId)
// // return
// // }
// // err = deployWorke(cli, workerImage, containerName, env)
// // if err != nil {
// // log.Printf("Failed executing worker %s in state %s", execution.ExecutionId, containerStatus)
// // return
// // }
// //}
// } else {
// log.Printf("[INFO] Worker Container %s was created under environment %s: docker logs %s", cont.ID, environment, cont.ID)
// }
// docker part here
}
return nil
}
@@ -1026,7 +884,7 @@ func getKubernetesClient() (*kubernetes.Clientset, error) {
// Initial loop etc
func main() {
if isRunningInCluster(){
if isRunningInCluster() {
log.Printf("[INFO] Running inside k8s cluster")
}
@@ -1116,7 +974,7 @@ func main() {
ctx := context.Background()
// Run by default from now
//commenting for now as its stoppoing minikube
// zombiecheck(ctx, workerTimeout)
// zombiecheck(ctx, workerTimeout)
log.Printf("[INFO] Running towards %s (BASE_URL) with environment name %s", baseUrl, environment)
functions/onprem/worker/Dockerfile
+1 -1
View File
@@ -32,7 +32,7 @@ FROM alpine:3.15.0
ENV SHUFFLE_BASE_IMAGE_REGISTRY=docker.io
ENV SHUFFLE_BASE_IMAGE_NAME=frikky/shuffle
ENV SHUFFLE_BASE_IMAGE_TAG_SUFFIX=0.8.70
ENV SHUFFLE_OPENSEARCH_URL=https://192.168.49.2:31001
ENV SHUFFLE_OPENSEARCH_URL=https://opensearch:9200
ENV SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY=true
RUN apk add --no-cache bash tzdata
functions/onprem/worker/worker.go
+25 -24
View File
@@ -38,13 +38,13 @@ import (
"cloud.google.com/go/storage"
//k8s deps
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/homedir"
"k8s.io/client-go/rest"
"path/filepath"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/homedir"
"path/filepath"
// "k8s.io/client-go/util/retry"
)
@@ -223,12 +223,13 @@ func getKubernetesClient() (*kubernetes.Clientset, error) {
// Deploys the internal worker whenever something happens
func deployApp(cli *dockerclient.Client, image string, identifier string, env []string, workflowExecution shuffle.WorkflowExecution, action shuffle.Action) error {
// form basic hostConfig
log.Printf("HELLO FROM DEPLOYAPP")
// log.Printf("workflow execution: %+v", workflowExecution)
log.Printf("image: %s", image)
log.Printf("IS_KUBERNETES: %s", os.Getenv("IS_KUBERNETES"))
log.Printf("REGISTRY_NAME: %s", os.Getenv("REGISTRY_NAME"))
namespace := "shuffle"
// ctx := context.Background()
envMap := make(map[string]string)
for _, envStr := range env {
@@ -243,16 +244,16 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
fmt.Println("[ERROR]Error getting kubernetes client:", err)
os.Exit(1)
}
log.Printf("[DEBUG] Got kubernetes client")
str := strings.ToLower(identifier)
strSplit := strings.Split(str, "_")
value := strSplit[0]
strSplit := strings.Split(str, "_")
value := strSplit[0]
value = strings.ReplaceAll(value, "_", "-")
//fix naming convention
podUuid := uuid.NewV4().String()
podName := fmt.Sprintf("%s-%s", value, podUuid)
podUuid := uuid.NewV4().String()
podName := fmt.Sprintf("%s-%s", value, podUuid)
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
@@ -266,7 +267,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
{
Name: value,
Image: image,
Env: buildEnvVars(envMap),
Env: buildEnvVars(envMap),
},
},
},
@@ -291,22 +292,22 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
// container := corev1.EphemeralContainer{
// EphemeralContainerCommon: corev1.EphemeralContainerCommon{
// Image: image,
// Env: buildEnvVars(envMap),
// ImagePullPolicy: corev1.PullIfNotPresent,
// Env: buildEnvVars(envMap),
// ImagePullPolicy: corev1.PullIfNotPresent,
// },
// }
// pod.Spec.EphemeralContainers = append(pod.Spec.EphemeralContainers, container)
// err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
// _, err := clientset.CoreV1().Pods(namespace).Update(context.Background(), pod, metav1.UpdateOptions{})
// return err
// })
// if err != nil {
// log.Fatalf("Failed to update Pod %v", err)
// }
// _, err := clientset.CoreV1().Pods(namespace).Update(context.Background(), pod, metav1.UpdateOptions{})
// return err
// })
// if err != nil {
// log.Fatalf("Failed to update Pod %v", err)
// }
// fmt.Printf("Ephemeral container added to Pod \n")
// fmt.Printf("Ephemeral container added to Pod \n")
// ephemeralContainer := corev1.EphemeralContainer{
// EphemeralContainerCommon: corev1.EphemeralContainerCommon{
@@ -350,7 +351,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
//CPUShares: 128,
//CPUQuota: 10000,
//CPUPeriod: 100000,
// hostConfig := &container.HostConfig{
// LogConfig: container.LogConfig{
// Type: "json-file",