From 68db8aae14fd57f90a79705c2be8c128c48364f1 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Sun, 4 Aug 2024 16:02:25 +0000 Subject: [PATCH 1/4] [fix] git proxy --- backend/go-app/walkoff.go | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 7ff135e3..8f4161e8 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -34,6 +34,7 @@ import ( "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" + gitProxy "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/storage/memory" http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" @@ -106,7 +107,6 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode } log.Printf("[INFO] Starting frequency for execution: %d", newfrequency) - //jobret, err := newscheduler.Every(newfrequency).Seconds().NotImmediately().Run(job) jobret, err := newscheduler.Every(newfrequency).Seconds().Run(job) @@ -309,7 +309,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { if envData.Swarm { env.Licensed = true env.RunType = "docker" - } + } if envData.Kubernetes { env.RunType = "k8s" @@ -741,7 +741,6 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult shuffle.ActionResult, resp http.ResponseWriter) { log.Printf("[DEBUG][%s] Running workflow execution update", workflowExecutionId) - // Should start a tx for the execution here workflowExecution, err := shuffle.GetWorkflowExecution(ctx, workflowExecutionId) if err != nil { @@ -925,7 +924,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { if len(workflow.ParentWorkflowId) > 0 { resp.WriteHeader(403) resp.Write([]byte(`{"success": false, "reason": "Can't delete a workflow distributed from your parent org"}`)) - return + return } if user.Id != workflow.Owner || len(user.Id) == 0 { @@ -984,8 +983,6 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": true}`)) } - - func handleExecution(id string, workflow shuffle.Workflow, request *http.Request, orgId string) (shuffle.WorkflowExecution, string, error) { //go func() { // log.Printf("\n\nPRE TIME: %s\n\n", time.Now().Format("2006-01-02 15:04:05")) @@ -1080,7 +1077,6 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request } } - err := imageCheckBuilder(execInfo.ImageNames) if err != nil { log.Printf("[ERROR] Failed building the required images from %#v: %s", execInfo.ImageNames, err) @@ -1354,7 +1350,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request } } - childNodes := shuffle.FindChildNodes(workflowExecution, workflowExecution.Start, []string{}, []string{}) + childNodes := shuffle.FindChildNodes(workflowExecution.Workflow, workflowExecution.Start, []string{}, []string{}) startFound := false newActions := []shuffle.Action{} @@ -2651,6 +2647,18 @@ func loadGithubWorkflows(url, username, password, userId, branch, orgId string) URL: url, } + if os.Getenv("HTTP_PROXY") != "" { + cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ + URL: os.Getenv("HTTP_PROXY"), + } + } + + if os.Getenv("HTTPS_PROXY") != "" { + cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ + URL: os.Getenv("HTTPS_PROXY"), + } + } + // FIXME: Better auth. if len(username) > 0 && len(password) > 0 { cloneOptions.Auth = &http2.BasicAuth{ @@ -3410,7 +3418,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { // FIXME: Should use environment that is in the source workflow if it exists for i, _ := range workflowExecution.Workflow.Actions { workflowExecution.Workflow.Actions[i].Environment = environment - workflowExecution.Workflow.Actions[i].Label = "TMP" + workflowExecution.Workflow.Actions[i].Label = "TMP" } shuffle.SetWorkflowExecution(ctx, workflowExecution, false) @@ -4193,7 +4201,6 @@ func checkUnfinishedExecution(resp http.ResponseWriter, request *http.Request) { log.Printf("[ERROR] Failed adding execution to db: %s", err) } - resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Reran workflow in %s"}`, parsedEnv))) From 9b37eb34f9f7e0eeeec4e06125e2f23a188720c1 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Tue, 6 Aug 2024 23:58:34 +0530 Subject: [PATCH 2/4] added proxy for git everywhere --- backend/go-app/main.go | 26 ++++++++++++++++++++++++++ backend/go-app/walkoff.go | 13 +++++++++++++ 2 files changed, 39 insertions(+) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 23486930..95a9dadf 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -36,6 +36,7 @@ import ( "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" + gitProxy "github.com/go-git/go-git/v5/plumbing/transport" // Random xj "github.com/basgys/goxml2json" @@ -4238,6 +4239,19 @@ func runInitEs(ctx context.Context) { } } + if os.Getenv("HTTP_PROXY") != "" { + cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ + URL: os.Getenv("HTTP_PROXY"), + } + } + + if os.Getenv("HTTPS_PROXY") != "" { + cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ + URL: os.Getenv("HTTPS_PROXY"), + } + } + + branch := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_BRANCH") if len(branch) > 0 && branch != "master" && branch != "main" { cloneOptions.ReferenceName = plumbing.ReferenceName(branch) @@ -4282,6 +4296,18 @@ func runInitEs(ctx context.Context) { cloneOptions := &git.CloneOptions{ URL: apis, } + + if os.Getenv("HTTP_PROXY") != "" { + cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ + URL: os.Getenv("HTTP_PROXY"), + } + } + + if os.Getenv("HTTPS_PROXY") != "" { + cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ + URL: os.Getenv("HTTPS_PROXY"), + } + } _, err = git.Clone(storer, fs, cloneOptions) if err != nil { log.Printf("[ERROR] Failed loading repo %s into memory: %s", apis, err) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 8f4161e8..ac5dbe33 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -3940,6 +3940,19 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { cloneOptions.ReferenceName = plumbing.ReferenceName(tmpBody.Branch) } + if os.Getenv("HTTP_PROXY") != "" { + cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ + URL: os.Getenv("HTTP_PROXY"), + } + } + + if os.Getenv("HTTPS_PROXY") != "" { + cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ + URL: os.Getenv("HTTPS_PROXY"), + } + } + + // FIXME: Better auth. if len(tmpBody.Field1) > 0 && len(tmpBody.Field2) > 0 { cloneOptions.Auth = &http2.BasicAuth{ From 427bd88bbf529f5e9a5f2067eb6b400f064e0237 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Mon, 12 Aug 2024 19:52:49 +0530 Subject: [PATCH 3/4] added no_proxy --- backend/go-app/main.go | 70 +++++++++++++++++++++++++++------------ backend/go-app/walkoff.go | 30 +++-------------- 2 files changed, 52 insertions(+), 48 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 95a9dadf..f2407a37 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -397,6 +397,52 @@ func checkUsername(Username string) error { return nil } +func isNoProxy(rawURL string) bool { + noProxy := os.Getenv("NO_PROXY") + if noProxy == "" { + return false + } + + if noProxy == "*" { + return true + } + + noProxyList := strings.Split(noProxy, ",") + parsedURL, err := url.Parse(rawURL) + if err != nil { + return false + } + host := parsedURL.Hostname() + + for _,value := range noProxyList { + value = strings.TrimSpace(value) + + if host == value { + return true + } + if strings.HasPrefix(value, "*.") && strings.HasSuffix(host, value[2:]){ + return true + } + } + return false +} + +func checkGitProxy(cloneOptions *git.CloneOptions) *git.CloneOptions { + if os.Getenv("HTTP_PROXY") != "" && !isNoProxy(cloneOptions.URL){ + cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ + URL: os.Getenv("HTTP_PROXY"), + } + } + + if os.Getenv("HTTPS_PROXY") != "" && !isNoProxy(cloneOptions.URL) { + cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ + URL: os.Getenv("HTTPS_PROXY"), + } + } + + return cloneOptions +} + func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) error { // Returns false if there is an issue // Use this for register @@ -4239,18 +4285,7 @@ func runInitEs(ctx context.Context) { } } - if os.Getenv("HTTP_PROXY") != "" { - cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ - URL: os.Getenv("HTTP_PROXY"), - } - } - - if os.Getenv("HTTPS_PROXY") != "" { - cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ - URL: os.Getenv("HTTPS_PROXY"), - } - } - + cloneOptions = checkGitProxy(cloneOptions) branch := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_BRANCH") if len(branch) > 0 && branch != "master" && branch != "main" { @@ -4297,17 +4332,8 @@ func runInitEs(ctx context.Context) { URL: apis, } - if os.Getenv("HTTP_PROXY") != "" { - cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ - URL: os.Getenv("HTTP_PROXY"), - } - } + cloneOptions = checkGitProxy(cloneOptions) - if os.Getenv("HTTPS_PROXY") != "" { - cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ - URL: os.Getenv("HTTPS_PROXY"), - } - } _, err = git.Clone(storer, fs, cloneOptions) if err != nil { log.Printf("[ERROR] Failed loading repo %s into memory: %s", apis, err) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index ac5dbe33..98bb5d09 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -34,7 +34,6 @@ import ( "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" - gitProxy "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/storage/memory" http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" @@ -2647,18 +2646,6 @@ func loadGithubWorkflows(url, username, password, userId, branch, orgId string) URL: url, } - if os.Getenv("HTTP_PROXY") != "" { - cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ - URL: os.Getenv("HTTP_PROXY"), - } - } - - if os.Getenv("HTTPS_PROXY") != "" { - cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ - URL: os.Getenv("HTTPS_PROXY"), - } - } - // FIXME: Better auth. if len(username) > 0 && len(password) > 0 { cloneOptions.Auth = &http2.BasicAuth{ @@ -2673,6 +2660,8 @@ func loadGithubWorkflows(url, username, password, userId, branch, orgId string) cloneOptions.ReferenceName = plumbing.ReferenceName(branch) } + cloneOptions = checkGitProxy(cloneOptions) + storer := memory.NewStorage() r, err := git.Clone(storer, fs, cloneOptions) if err != nil { @@ -3940,19 +3929,6 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { cloneOptions.ReferenceName = plumbing.ReferenceName(tmpBody.Branch) } - if os.Getenv("HTTP_PROXY") != "" { - cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ - URL: os.Getenv("HTTP_PROXY"), - } - } - - if os.Getenv("HTTPS_PROXY") != "" { - cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ - URL: os.Getenv("HTTPS_PROXY"), - } - } - - // FIXME: Better auth. if len(tmpBody.Field1) > 0 && len(tmpBody.Field2) > 0 { cloneOptions.Auth = &http2.BasicAuth{ @@ -3961,6 +3937,8 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { } } + cloneOptions = checkGitProxy(cloneOptions) + storer := memory.NewStorage() r, err := git.Clone(storer, fs, cloneOptions) if err != nil { From 5ec483a21f6574322482530211ea6383b1990ec5 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Thu, 15 Aug 2024 01:39:39 +0530 Subject: [PATCH 4/4] git no-proxy --- backend/go-app/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index f2407a37..7219b351 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -397,7 +397,7 @@ func checkUsername(Username string) error { return nil } -func isNoProxy(rawURL string) bool { +func isGitNoProxy(rawURL string) bool { noProxy := os.Getenv("NO_PROXY") if noProxy == "" { return false @@ -428,13 +428,13 @@ func isNoProxy(rawURL string) bool { } func checkGitProxy(cloneOptions *git.CloneOptions) *git.CloneOptions { - if os.Getenv("HTTP_PROXY") != "" && !isNoProxy(cloneOptions.URL){ + if os.Getenv("HTTP_PROXY") != "" && !isGitNoProxy(cloneOptions.URL){ cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ URL: os.Getenv("HTTP_PROXY"), } } - if os.Getenv("HTTPS_PROXY") != "" && !isNoProxy(cloneOptions.URL) { + if os.Getenv("HTTPS_PROXY") != "" && !isGitNoProxy(cloneOptions.URL) { cloneOptions.ProxyOptions = gitProxy.ProxyOptions{ URL: os.Getenv("HTTPS_PROXY"), }