marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

#135: Started building app exist validation

Browse Source
This commit is contained in:
frikky
2020-09-05 08:56:29 +02:00
parent b635b6f3e6
commit 2633ff4b4d
8 changed files with 152 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
functions/onprem/orborus/build.sh
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=orborus
VERSION=0.6.0
VERSION=0.6.1
echo "Running docker build with $NAME:$VERSION"
#docker rmi frikky/shuffle:$NAME --force
functions/onprem/orborus/orborus.go
+50 -39
View File
@@ -30,11 +30,11 @@ var sleepTime = 3
// Timeout if something rashes
var workerTimeout = 300
var appSdkVersion = "0.6.0"
var workerVersion = "0.6.0"
var appSdkVersion = os.Getenv("SHUFFLE_APP_SDK_VERSION")
var workerVersion = os.Getenv("SHUFFLE_WORKER_VERSION")
//var baseimagename = "docker.pkg.github.com/frikky/shuffle"
var baseimagename = "gchr.io/frikky"
var baseimagename = "ghcr.io/frikky"
var orgId = os.Getenv("ORG_ID")
var baseUrl = os.Getenv("BASE_URL")
@@ -121,7 +121,7 @@ func deployWorker(image string, identifier string, env []string) {
log.Printf("[INFO] Found container ID %s", containerId)
hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId))
} else {
log.Printf("[WARNING] Empty self container id, continue without NetworkMode")
log.Printf("[INFO] Empty self container id, continue without NetworkMode")
}
config := &container.Config{
@@ -145,7 +145,7 @@ func deployWorker(image string, identifier string, env []string) {
err = dockercli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{})
if err != nil {
log.Printf("Failed to start container in environment %s: %s", environment, err)
log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err)
return
//stats, err := cli.ContainerInspect(context.Background(), containerName)
@@ -170,7 +170,7 @@ func deployWorker(image string, identifier string, env []string) {
// }
//}
} else {
log.Printf("Container %s was created under environment %s", cont.ID, environment)
log.Printf("[INFO] Container %s was created under environment %s", cont.ID, environment)
}
return
@@ -184,7 +184,7 @@ func stopWorker(containername string) error {
// })
if err := dockercli.ContainerStop(ctx, containername, nil); err != nil {
log.Printf("Unable to stop container %s - running removal anyway, just in case: %s", containername, err)
log.Printf("[ERROR] Unable to stop container %s - running removal anyway, just in case: %s", containername, err)
}
removeOptions := types.ContainerRemoveOptions{
@@ -193,7 +193,7 @@ func stopWorker(containername string) error {
}
if err := dockercli.ContainerRemove(ctx, containername, removeOptions); err != nil {
log.Printf("Unable to remove container: %s", err)
log.Printf("[ERROR] Unable to remove container: %s", err)
}
return nil
@@ -202,31 +202,42 @@ func stopWorker(containername string) error {
func initializeImages() {
ctx := context.Background()
if appSdkVersion == "" {
appSdkVersion = "0.6.0"
log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion)
}
if workerVersion == "" {
workerVersion = "0.6.0"
log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion)
}
// check whether theyre the same first
images := []string{
// fmt.Sprintf("docker.io/%s:app_sdk", baseimagename),
// fmt.Sprintf("docker.io/%s:worker", baseimagename),
fmt.Sprintf("%s/worker:%s", baseimagename, workerVersion),
fmt.Sprintf("%s/app_sdk:%s", baseimagename, appSdkVersion),
}
pullOptions := types.ImagePullOptions{}
for _, image := range images {
log.Printf("[INFO] Pulling image %s", image)
reader, err := dockercli.ImagePull(ctx, image, pullOptions)
if err != nil {
log.Printf("Failed getting image %s: %s", image, err)
log.Printf("[ERROR] Failed getting image %s: %s", image, err)
continue
}
io.Copy(os.Stdout, reader)
log.Printf("Successfully downloaded and built %s", image)
log.Printf("[INFO] Successfully downloaded and built %s", image)
}
}
// Initial loop etc
func main() {
go zombiecheck()
log.Println("Setting up execution environment")
log.Println("[INFO] Setting up execution environment")
//FIXME
if baseUrl == "" {
@@ -235,30 +246,30 @@ func main() {
}
if orgId == "" {
log.Printf("Org not defined. Set variable ORG_ID based on your org")
log.Printf("[ERROR] Org not defined. Set variable ORG_ID based on your org")
os.Exit(3)
}
log.Printf("Running towards %s with Org %s", baseUrl, orgId)
log.Printf("[INFO] Running towards %s with Org %s", baseUrl, orgId)
httpProxy := os.Getenv("HTTP_PROXY")
httpsProxy := os.Getenv("HTTPS_PROXY")
if environment == "" {
environment = "onprem"
log.Printf("Defaulting to environment name %s. Set environment variable ENVIRONMENT_NAME to change. This should be the same as in the frontend action.", environment)
log.Printf("[INFO] Defaulting to environment name %s. Set environment variable ENVIRONMENT_NAME to change. This should be the same as in the frontend action.", environment)
}
// FIXME - during init, BUILD and/or LOAD worker and app_sdk
// Build/load app_sdk so it can be loaded as 127.0.0.1:5000/walkoff_app_sdk
log.Printf("--- Setting up Docker environment. Downloading worker and App SDK! ---")
log.Printf("[INFO] Setting up Docker environment. Downloading worker and App SDK!")
initializeImages()
//workerName := "worker"
//workerVersion := "0.1.0"
//workerImage := fmt.Sprintf("docker.pkg.github.com/frikky/shuffle/%s:%s", workerName, workerVersion)
workerImage := fmt.Sprintf("%s:worker", baseimagename)
workerImage := fmt.Sprintf("%s/worker:%s", baseimagename, workerVersion)
log.Printf("--- Finished configuring docker environment ---\n")
log.Printf("[INFO] Finished configuring docker environment")
// FIXME - time limit
client := &http.Client{
@@ -271,10 +282,10 @@ func main() {
client = &http.Client{}
} else {
if len(httpProxy) > 0 {
log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
log.Printf("[INFO] Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
}
if len(httpsProxy) > 0 {
log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
log.Printf("[INFO] Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
}
}
@@ -286,21 +297,21 @@ func main() {
)
if err != nil {
log.Printf("Failed making request builder: %s", err)
log.Printf("[ERROR] Failed making request builder: %s", err)
os.Exit(3)
}
zombiecounter := 0
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Org-Id", orgId)
log.Printf("Getting data from %s", fullUrl)
log.Printf("[INFO] Waiting for executions at %s", fullUrl)
hasStarted := false
for {
//log.Printf("Prerequest")
newresp, err := client.Do(req)
//log.Printf("Postrequest")
if err != nil {
log.Printf("Failed making request: %s", err)
log.Printf("[WARNING] Failed making request: %s", err)
zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout {
go zombiecheck()
@@ -313,7 +324,7 @@ func main() {
// FIXME - add check for StatusCode
if newresp.StatusCode != 200 {
if hasStarted {
log.Printf("Bad statuscode: %d", newresp.StatusCode)
log.Printf("[WARNING] Bad statuscode: %d", newresp.StatusCode)
}
} else {
hasStarted = true
@@ -321,7 +332,7 @@ func main() {
body, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("Failed reading body: %s", err)
log.Printf("[ERROR] Failed reading body: %s", err)
zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout {
go zombiecheck()
@@ -334,7 +345,7 @@ func main() {
var executionRequests ExecutionRequestWrapper
err = json.Unmarshal(body, &executionRequests)
if err != nil {
log.Printf("Failed executionrequest in queue unmarshaling: %s", err)
log.Printf("[WARNING] Failed executionrequest in queue unmarshaling: %s", err)
sleepTime = 10
zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout {
@@ -346,7 +357,7 @@ func main() {
}
if hasStarted && len(executionRequests.Data) > 0 {
log.Printf("Body: %s", string(body))
log.Printf("[INFO] Body: %s", string(body))
// Type string `json:"type"`
}
@@ -364,16 +375,16 @@ func main() {
var toBeRemoved ExecutionRequestWrapper
for _, execution := range executionRequests.Data {
if len(execution.ExecutionArgument) > 0 {
log.Printf("Argument: %#v", execution.ExecutionArgument)
log.Printf("[INFO] Argument: %#v", execution.ExecutionArgument)
}
if execution.Type == "schedule" {
log.Printf("SOMETHING ELSE :O: %s", execution.Type)
log.Printf("[INFO] SOMETHING ELSE :O: %s", execution.Type)
continue
}
if execution.Status == "ABORT" || execution.Status == "FAILED" {
log.Printf("Executionstatus issue: ", execution.Status)
log.Printf("[INFO] Executionstatus issue: ", execution.Status)
}
// Now, how do I execute this one?
// FIXME - if error, check the status of the running one. If it's bad, send data back.
@@ -396,7 +407,7 @@ func main() {
go deployWorker(workerImage, containerName, env)
log.Printf("%s is deployed and to be removed from queue.", execution.ExecutionId)
log.Printf("[INFO] %s is deployed and to be removed from queue.", execution.ExecutionId)
zombiecounter += 1
toBeRemoved.Data = append(toBeRemoved.Data, execution)
}
@@ -407,7 +418,7 @@ func main() {
data, err := json.Marshal(toBeRemoved)
if err != nil {
log.Printf("Failed removal marshalling: %s", err)
log.Printf("[WARNING] Failed removal marshalling: %s", err)
time.Sleep(time.Duration(sleepTime) * time.Second)
continue
}
@@ -419,7 +430,7 @@ func main() {
)
if err != nil {
log.Printf("Failed building confirm request: %s", err)
log.Printf("[ERROR] Failed building confirm request: %s", err)
time.Sleep(time.Duration(sleepTime) * time.Second)
continue
}
@@ -429,14 +440,14 @@ func main() {
resultResp, err := client.Do(result)
if err != nil {
log.Printf("Failed making confirm request: %s", err)
log.Printf("[ERROR] Failed making confirm request: %s", err)
time.Sleep(time.Duration(sleepTime) * time.Second)
continue
}
body, err := ioutil.ReadAll(resultResp.Body)
if err != nil {
log.Printf("Failed reading confirm body: %s", err)
log.Printf("[ERROR] Failed reading confirm body: %s", err)
time.Sleep(time.Duration(sleepTime) * time.Second)
continue
}
@@ -450,7 +461,7 @@ func main() {
if len(toBeRemoved.Data) == len(executionRequests.Data) {
//log.Println("Should remove ALL!")
} else {
log.Printf("NOT IMPLEMENTED: Should remove %d workflows from backend because they're executed!", len(toBeRemoved.Data))
log.Printf("[INFO] NOT IMPLEMENTED: Should remove %d workflows from backend because they're executed!", len(toBeRemoved.Data))
}
}
@@ -461,7 +472,7 @@ func main() {
// FIXME - add this to remove exited workers
// Should it check what happened to the execution? idk
func zombiecheck() error {
log.Println("Looking for old containers")
log.Println("[INFO] Looking for old containers")
ctx := context.Background()
containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
@@ -469,7 +480,7 @@ func zombiecheck() error {
})
if err != nil {
log.Printf("Failed creating Containerlist: %s", err)
log.Printf("[ERROR] Failed creating Containerlist: %s", err)
return err
}
@@ -502,7 +513,7 @@ func zombiecheck() error {
continue
}
log.Printf("NAME: %s", name)
log.Printf("[INFO] NAME: %s", name)
// Need to check time here too because a container can be removed the same instant as its created
currenttime := time.Now().Unix()
@@ -522,7 +533,7 @@ func zombiecheck() error {
// FIXME - add killing of apps with same execution ID too
for _, containername := range stopContainers {
log.Printf("Stopping and removing container %s", containerNames[containername])
log.Printf("[INFO] Stopping and removing container %s", containerNames[containername])
go dockercli.ContainerStop(ctx, containername, nil)
removeContainers = append(removeContainers, containername)
}