diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 9f0c72e2..b7ee7e6d 100755 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -211,7 +211,7 @@ func fixTags(tags []string) []string { func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string, downloadIfFail bool) error { ctx := context.Background() client, err := client.NewEnvClient() - defer client.Close() + defer client.Close() if err != nil { log.Printf("Unable to create docker client: %s", err) return err @@ -473,73 +473,84 @@ func buildImage(tags []string, dockerfileLocation string) error { } } } - } else { - - ctx := context.Background() - client, err := client.NewEnvClient() - defer client.Close() - if err != nil { - log.Printf("Unable to create docker client: %s", err) - return err - } - - log.Printf("[INFO] Docker Tags: %s", tags) - dockerfileSplit := strings.Split(dockerfileLocation, "/") - - // Create a buffer - buf := new(bytes.Buffer) - tw := tar.NewWriter(buf) - defer tw.Close() - baseDir := strings.Join(dockerfileSplit[0:len(dockerfileSplit)-1], "/") - - // Builds the entire folder into buf - err = getParsedTar(tw, baseDir, "") - if err != nil { - log.Printf("Tar issue: %s", err) - } - - dockerFileTarReader := bytes.NewReader(buf.Bytes()) - buildOptions := types.ImageBuildOptions{ - Remove: true, - Tags: tags, - BuildArgs: map[string]*string{}, - } - //NetworkMode: "host", - - httpProxy := os.Getenv("HTTP_PROXY") - if len(httpProxy) > 0 { - buildOptions.BuildArgs["HTTP_PROXY"] = &httpProxy - } - httpsProxy := os.Getenv("HTTPS_PROXY") - if len(httpProxy) > 0 { - buildOptions.BuildArgs["https_proxy"] = &httpsProxy - } - - // Build the actual image - imageBuildResponse, err := client.ImageBuild( - ctx, - dockerFileTarReader, - buildOptions, - ) - - if err != nil { - return err - } - - // Read the STDOUT from the build process - defer imageBuildResponse.Body.Close() - buildBuf := new(strings.Builder) - _, err = io.Copy(buildBuf, imageBuildResponse.Body) - if err != nil { - return err - } else { - if strings.Contains(buildBuf.String(), "errorDetail") { - log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), strings.Join(tags, "\n")) - return errors.New(fmt.Sprintf("Failed building %s. Check backend logs for details. Most likely means you have an old version of Docker.", strings.Join(tags, ","))) - } - } + return nil } + + ctx := context.Background() + client, err := client.NewEnvClient() + defer client.Close() + if err != nil { + log.Printf("Unable to create docker client: %s", err) + return err + } + + log.Printf("[INFO] Docker Tags: %s", tags) + dockerfileSplit := strings.Split(dockerfileLocation, "/") + + // Create a buffer + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + defer tw.Close() + baseDir := strings.Join(dockerfileSplit[0:len(dockerfileSplit)-1], "/") + + // Builds the entire folder into buf + err = getParsedTar(tw, baseDir, "") + if err != nil { + log.Printf("[ERROR] Tar issue during app build: %s", err) + } + + dockerFileTarReader := bytes.NewReader(buf.Bytes()) + buildOptions := types.ImageBuildOptions{ + Remove: true, + Tags: tags, + BuildArgs: map[string]*string{}, + } + //NetworkMode: "host", + + httpProxy := os.Getenv("HTTP_PROXY") + if len(httpProxy) > 0 { + buildOptions.BuildArgs["HTTP_PROXY"] = &httpProxy + } + httpsProxy := os.Getenv("HTTPS_PROXY") + if len(httpProxy) > 0 { + buildOptions.BuildArgs["https_proxy"] = &httpsProxy + } + + // Print the actual file content from dockerFileTarReader + /* + data, err := ioutil.ReadAll(dockerFileTarReader) + if err != nil { + log.Printf("[ERROR] Failed reading Dockerfile TAR reader: %s", err) + } else { + log.Printf("[DEBUG] Dockerfile TAR reader content: %s", string(data)) + } + */ + + // Build the actual image + imageBuildResponse, err := client.ImageBuild( + ctx, + dockerFileTarReader, + buildOptions, + ) + + if err != nil { + return err + } + + // Read the STDOUT from the build process + defer imageBuildResponse.Body.Close() + buildBuf := new(strings.Builder) + _, err = io.Copy(buildBuf, imageBuildResponse.Body) + if err != nil { + return err + } else { + if strings.Contains(buildBuf.String(), "errorDetail") { + log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), strings.Join(tags, "\n")) + return errors.New(fmt.Sprintf("Failed building %s. Check backend logs for details. Most likely means you have an old version of Docker.", strings.Join(tags, ","))) + } + } + return nil } @@ -671,7 +682,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "No image name"}`))) return - + } log.Printf("[INFO] Trying to download image: '%s'. Appname: '%s'. BaseAppname: '%s', Split2: %s", version.Name, appname, baseAppname, appnameSplit2) @@ -870,7 +881,7 @@ func handleRemoteDownloadApp(resp http.ResponseWriter, ctx context.Context, user type tmpapp struct { Success bool `json:"success"` OpenAPI string `json:"openapi"` - App string `json:"app"` + App string `json:"app"` } app := tmpapp{} diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 449a215a..32b30400 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -22,7 +22,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.70 + github.com/shuffle/shuffle-shared v0.8.71 golang.org/x/crypto v0.37.0 google.golang.org/api v0.228.0 google.golang.org/grpc v1.71.1 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 34ccd9f6..d52c6821 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -341,8 +341,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.70 h1:Balbk7kIVUSgM9Lc+VJ09ZpaF18MlFxMPZdkPTD/+gU= -github.com/shuffle/shuffle-shared v0.8.70/go.mod h1:OLAwH/Ym4941Jn5DF1oZaq6iBpmjG2SNrTZ9Xqck5So= +github.com/shuffle/shuffle-shared v0.8.71 h1:OhiBpIEkn+1+uRs4nn9jPtjdT0dVfVl6VQKtaFzUtZY= +github.com/shuffle/shuffle-shared v0.8.71/go.mod h1:OLAwH/Ym4941Jn5DF1oZaq6iBpmjG2SNrTZ9Xqck5So= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 657e4a1f..c78103f5 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -35,9 +35,9 @@ import ( "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/storage/memory" gitProxy "github.com/go-git/go-git/v5/plumbing/transport" http2 "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/storage/memory" // Random xj "github.com/basgys/goxml2json" @@ -2078,7 +2078,6 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Failed to run workflow. Check logs."}`)) - } func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { @@ -3095,7 +3094,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User, s return } - if user.Id == app.Owner || (user.Role == "admin" && user.ActiveOrg.Id == app.ReferenceOrg) || shuffle.ArrayContains(app.Contributors, user.Id) { + if user.Id == app.Owner || (user.Role == "admin" && user.ActiveOrg.Id == app.ReferenceOrg) || shuffle.ArrayContains(app.Contributors, user.Id) { log.Printf("[DEBUG] Editing app %s with user %s (%s) in org %s", test.Id, user.Username, user.Id, user.ActiveOrg.Id) } else { log.Printf("[WARNING] Wrong user (%s) for app %s when verifying swagger", user.Username, app.Name) @@ -3383,7 +3382,6 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User, s } } - log.Printf("[DEBUG] Successfully built app %s (%s)", api.Name, api.ID) if len(user.Id) > 0 { resp.WriteHeader(200) @@ -3826,6 +3824,8 @@ func remoteOrgJobHandler(org shuffle.Org, interval int) error { } } + // Send stats once every 10 times or so..? + // For now, just send every time info, err := shuffle.GetOrgStatistics(ctx, org.Id) if err != nil { log.Printf("[ERROR] Failed getting org statistics backup for org %s: %s", org.Id, err) @@ -4385,7 +4385,7 @@ func runInitEs(ctx context.Context) { } if os.Getenv("SHUFFLE_HEALTHCHECK_DISABLED") != "true" { - healthcheckInterval := 60 + healthcheckInterval := 60 log.Printf("[INFO] Starting healthcheck job every %d minute. Stats available on /api/v1/health/stats, and dashboard on /health. Disable with SHUFFLE_HEALTHCHECK_DISABLED=true", healthcheckInterval) job := func() { // Prepare a fake http.responsewriter