marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

#415: Added app rebuilds if the container doesn't exist

Browse Source
This commit is contained in:
frikky
2021-06-15 17:41:53 +02:00
parent f970bb351e
commit 642b589953
6 changed files with 280 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/go-app/docker.go
+103 -70
View File
@@ -5,9 +5,9 @@ import (
"github.com/frikky/shuffle-shared"
"archive/tar"
"bufio"
//"bufio"
"path/filepath"
"strconv"
//"strconv"
"bytes"
"context"
@@ -20,7 +20,6 @@ import (
"github.com/docker/docker/client"
newdockerclient "github.com/fsouza/go-dockerclient"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/memfs"
//network "github.com/docker/docker/api/types/network"
//natting "github.com/docker/go-connections/nat"
@@ -624,13 +623,13 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
}
// Just here to verify that the user is logged in
_, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in DOWNLOAD IMAGE: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
//_, err := shuffle.HandleApiAuthentication(resp, request)
//if err != nil {
// log.Printf("[WARNING] Api authentication failed in DOWNLOAD IMAGE: %s", err)
// resp.WriteHeader(401)
// resp.Write([]byte(`{"success": false}`))
// return
//}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
@@ -643,16 +642,6 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
Name string `datastore:"name" json:"name" yaml:"name"`
}
//body = []byte(`swagger: "2.0"`)
//body = []byte(`swagger: '1.0'`)
//newbody := string(body)
//newbody = strings.TrimSpace(newbody)
//body = []byte(newbody)
//log.Println(string(body))
//tmpbody, err := yaml.YAMLToJSON(body)
//log.Println(err)
//log.Println(string(tmpbody))
// This has to be done in a weird way because Datastore doesn't
// support map[string]interface and similar (openapi3.Swagger)
var version requestCheck
@@ -665,15 +654,9 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
}
log.Printf("[DEBUG] Image to load: %s", version.Name)
//cli, err := client.NewEnvClient()
//if err != nil {
// log.Println("Unable to create docker client")
// return err
//}
dockercli, err := client.NewEnvClient()
if err != nil {
log.Printf("Unable to create docker client: %s", err)
log.Printf("[WARNING] Unable to create docker client: %s", err)
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed JSON marshalling: %s"}`, err)))
return
@@ -686,75 +669,125 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
img := types.ImageSummary{}
tagFound := ""
img2 := types.ImageSummary{}
tagFound2 := ""
alternativeNameSplit := strings.Split(version.Name, "/")
alternativeName := version.Name
if len(alternativeNameSplit) == 3 {
alternativeName = strings.Join(alternativeNameSplit[1:3], "/")
}
for _, image := range images {
for _, tag := range image.RepoTags {
//log.Printf("[INFO] Docker Image: %s", tag)
if strings.ToLower(tag) == strings.ToLower(version.Name) {
img = image
tagFound = tag
break
}
if strings.ToLower(tag) == strings.ToLower(alternativeName) {
img2 = image
tagFound2 = tag
}
}
}
// REBUILDS THE APP
if len(img.ID) == 0 {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't find image %s"}`, version.Name)))
return
if len(img2.ID) == 0 {
workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500)
log.Printf("[INFO] Getting workflowapps for a rebuild. Got %d with err %#v", len(workflowapps), err)
if err == nil {
imageName := ""
imageVersion := ""
newNameSplit := strings.Split(version.Name, ":")
if len(newNameSplit) == 2 {
log.Printf("[DEBUG] Found name %#v", newNameSplit)
findVersionSplit := strings.Split(newNameSplit[1], "_")
log.Printf("[DEBUG] Found another split %#v", findVersionSplit)
if len(findVersionSplit) == 2 {
imageVersion = findVersionSplit[len(findVersionSplit)-1]
imageName = findVersionSplit[0]
} else if len(findVersionSplit) >= 2 {
imageVersion = findVersionSplit[len(findVersionSplit)-1]
imageName = strings.Join(findVersionSplit[0:len(findVersionSplit)-1], "_")
} else {
log.Printf("[DEBUG] Couldn't parse appname & version for %#v", findVersionSplit)
}
}
if len(imageName) > 0 && len(imageVersion) > 0 {
log.Printf("Looking for appname %s with version %s", imageName, imageVersion)
foundApp := shuffle.WorkflowApp{}
for _, app := range workflowapps {
if strings.ToLower(strings.Replace(app.Name, " ", "_", -1)) == imageName && app.AppVersion == imageVersion {
if app.Generated {
foundApp = app
break
}
break
}
}
if len(foundApp.ID) > 0 {
openApiApp, err := shuffle.GetOpenApiDatastore(ctx, foundApp.ID)
if err != nil {
log.Printf("[ERROR] Failed getting OpenAPI app %s to database: %s", foundApp.ID, err)
} else {
log.Printf("[DEBUG] Found OpenAPI app for %s as generated - now building!", version.Name)
user := shuffle.User{}
//img = version.Name
if len(alternativeName) > 0 {
tagFound = alternativeName
} else {
tagFound = version.Name
}
buildSwaggerApp(resp, []byte(openApiApp.Body), user)
}
}
}
} else {
log.Printf("[WARNING] Couldn't find an image with registry name %s and %s", version.Name, alternativeName)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't find image %s"}`, version.Name)))
return
}
}
if len(tagFound) == 0 && len(tagFound2) > 0 {
img = img2
tagFound = tagFound2
}
}
log.Printf("[INFO] Img found (%s): %#v", tagFound, img)
basepath := "base"
location := fmt.Sprintf("%s.tar.gz", tagFound)
fs := memfs.New()
//Close after function return
f, err := fs.Create(fmt.Sprintf("%s/%s", basepath, location))
if err != nil {
log.Printf("[WARNING] Failed making file: %s", err)
return
}
//log.Printf("[INFO] Img found (%s): %#v", tagFound, img)
log.Printf("[INFO] Img found to be downloaded: %s", tagFound)
newClient, err := newdockerclient.NewClientFromEnv()
if err != nil {
log.Printf("[WARNING] Failed setting up docker env: %s", newClient)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't make docker client"}`)))
return
}
//https://github.com/fsouza/go-dockerclient/issues/600
defer f.Close()
w := bufio.NewWriter(f)
////https://github.com/fsouza/go-dockerclient/issues/600
//defer fileReader.Close()
opts := newdockerclient.ExportImageOptions{
Name: tagFound,
OutputStream: w,
OutputStream: resp,
}
if err := newClient.ExportImage(opts); err != nil {
log.Printf("[WARNING] FAILED to save image to file: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't export image"}`)))
return
}
w.Flush()
FileHeader := make([]byte, 512)
f.Read(FileHeader)
FileContentType := http.DetectContentType(FileHeader)
//Get the file size
//FileStat, _ := f.Stat() //Get info from file
//FileSize := strconv.FormatInt(f.Size(), 10) //Get file size as a string
//Send the headers
resp.Header().Set("Content-Disposition", "attachment; filename="+location)
resp.Header().Set("Content-Type", FileContentType)
resp.Header().Set("Content-Length", strconv.FormatInt(img.Size, 10))
//Send the file
//We read 512 bytes from the file already, so we reset the offset back to 0
f.Seek(0, 0)
io.Copy(resp, f) //'Copy' the file to the client
//resp.WriteHeader(200)
//resp.Write([]byte(fmt.Sprintf(`{"success": true, "message": "Downloading image %s"}`, version.Name)))
}
backend/go-app/go.sum
+2
View File
@@ -177,6 +177,8 @@ github.com/frikky/shuffle-shared v0.0.60 h1:o6/QLsu3Rbjr4+BQWs9DF4B5qZCg0gPpZWPY
github.com/frikky/shuffle-shared v0.0.60/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc=
github.com/frikky/shuffle-shared v0.0.62 h1:1M8y7rX8nQW7072+bUD4vgHcf65AG0kJ8m3ihmY2bPQ=
github.com/frikky/shuffle-shared v0.0.62/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc=
github.com/frikky/shuffle-shared v0.0.63 h1:btn7V7s98eZmx/9qyapGE3V1TyxftvjMmrzkEcUvu+c=
github.com/frikky/shuffle-shared v0.0.63/go.mod h1:oPDyGFBuurPtE6UQLUrHU/JtbOJftqDljRfEdLlCTI4=
github.com/fsouza/go-dockerclient v1.7.2 h1:bBEAcqLTkpq205jooP5RVroUKiVEWgGecHyeZc4OFjo=
github.com/fsouza/go-dockerclient v1.7.2/go.mod h1:+ugtMCVRwnPfY7d8/baCzZ3uwB0BrG5DB8OzbtxaRz8=
github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE=
backend/go-app/main.go
+42 -66
View File
@@ -3081,29 +3081,7 @@ func handleSwaggerValidation(body []byte) (shuffle.ParsedOpenApi, error) {
return parsed, err
}
// Creates an app from the app builder
func verifySwagger(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
//log.Printf("[INFO] TRY TO SET APP TO LIVE!!!")
user, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in verify swagger: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
return
}
func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) {
type Test struct {
Editing bool `datastore:"editing"`
Id string `datastore:"id"`
@@ -3111,7 +3089,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) {
}
var test Test
err = json.Unmarshal(body, &test)
err := json.Unmarshal(body, &test)
if err != nil {
log.Printf("Failed unmarshalling test: %s", err)
resp.WriteHeader(401)
@@ -3123,7 +3101,8 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) {
hasher := md5.New()
hasher.Write(body)
newmd5 := hex.EncodeToString(hasher.Sum(nil))
if test.Editing {
if test.Editing && len(user.Id) > 0 {
// Quick verification test
ctx := context.Background()
app, err := shuffle.GetApp(ctx, test.Id, user)
@@ -3150,25 +3129,6 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) {
// Test = client side with fetch?
ctx := context.Background()
//s := string(body)
//if !utf8.ValidString(s) {
// v := make([]rune, 0, len(s))
// for i, r := range s {
// if r == utf8.RuneError {
// _, size := utf8.DecodeRuneInString(s[i:])
// if size == 1 {
// continue
// }
// }
// v = append(v, r)
// }
// s = string(v)
//}
//fmt.Printf("%q\n", s)
//body = []byte(strings.Replace(string(body), '<80>', '', -1))
//log.Println(string(body))
swaggerLoader := openapi3.NewSwaggerLoader()
swaggerLoader.IsExternalRefsAllowed = true
swagger, err := swaggerLoader.LoadSwaggerFromData(body)
@@ -3300,20 +3260,6 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) {
log.Printf("[INFO] Successfully stitched ZIPFILE for %s", identifier)
// 4. Upload as cloud function - this apikey is specifically for cloud functions rofl
//environmentVariables := map[string]string{
// "FUNCTION_APIKEY": apikey,
//}
//fullLocation := fmt.Sprintf("gs://%s/%s", bucketName, applocation)
//err = deployCloudFunctionPython(ctx, identifier, defaultLocation, fullLocation, environmentVariables)
//if err != nil {
// log.Printf("Error uploading cloud function: %s", err)
// resp.WriteHeader(500)
// resp.Write([]byte(`{"success": false, "reason": "Failed to upload function"}`))
// return
//}
// 4. Build the image locally.
// FIXME: Should be moved to a local docker registry
dockerLocation := fmt.Sprintf("%s/Dockerfile", basePath)
@@ -3359,12 +3305,14 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) {
user.PrivateApps[foundNumber] = api
}
err = shuffle.SetUser(ctx, &user, true)
if err != nil {
log.Printf("[ERROR] Failed adding verification for user %s: %s", user.Username, err)
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Failed updating user"}`)))
return
if len(user.Id) > 0 {
err = shuffle.SetUser(ctx, &user, true)
if err != nil {
log.Printf("[ERROR] Failed adding verification for user %s: %s", user.Username, err)
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Failed updating user"}`)))
return
}
}
//log.Printf("DO I REACH HERE WHEN SAVING?")
@@ -3405,8 +3353,36 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) {
shuffle.DeleteCache(ctx, cacheKey)
shuffle.DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, api.ID)))
if len(user.Id) > 0 {
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, api.ID)))
}
}
// Creates an app from the app builder
func verifySwagger(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
//log.Printf("[INFO] TRY TO SET APP TO LIVE!!!")
user, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in verify swagger: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
return
}
buildSwaggerApp(resp, body, user)
}
func healthCheckHandler(resp http.ResponseWriter, request *http.Request) {
backend/tests/dockerpull.sh
+4 -1
View File
@@ -1,2 +1,5 @@
#!/bin/sh
curl -XPOST http://localhost:5001/api/v1/get_docker_image -d '{"name":"frikky/shuffle:Testing_1.0.0"}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" --output tarball.tgz
#curl -XPOST http://localhost:5001/api/v1/get_docker_image -d '{"name":"frikky/shuffle:testing_1.0.0"}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" --output tarball.tgz
#curl -XPOST http://localhost:5001/api/v1/get_docker_image -d '{"name":"frikky/shuffle:testing_1.0.0"}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -O -J
#curl -XPOST http://localhost:5001/api/v1/get_docker_image -d '{"name":"frikky/shuffle:testing_1.0.0"}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" --output tarball.tgz
curl -XPOST http://localhost:5001/api/v1/get_docker_image -d '{"name":"frikky/shuffle:shuffle-tools_1.0.0"}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" --output tarball.tgz