From 2e40266171f6dfa154b1935337e2b5e9f37b1735 Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 2 Apr 2025 12:20:48 +0200 Subject: [PATCH] Fixed a few minor problems like queue confirms working well for environments with spaces --- backend/go-app/go.mod | 2 +- backend/go-app/go.sum | 4 +-- backend/go-app/walkoff.go | 4 ++- frontend/src/components/Navbar.jsx | 2 +- frontend/src/views/LoginPage.jsx | 2 +- functions/onprem/orborus/orborus.go | 51 ++++++++++++++++------------- 6 files changed, 36 insertions(+), 29 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 2b8f075b..6822373e 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -4,7 +4,7 @@ go 1.23.0 toolchain go1.23.7 -//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared require ( cloud.google.com/go/datastore v1.15.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 5610d4db..01efec32 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -333,8 +333,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.31 h1:APO/BkBxiP9Hn/Fa7SsESRD6ws9E5dBfIzS57pd3MvA= -github.com/shuffle/shuffle-shared v0.8.31/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.32 h1:RizHZ4lZg9wQj3RQvJMgh+1SAMszwhYfcco3aT716RQ= +github.com/shuffle/shuffle-shared v0.8.32/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 5b44d73a..317595c3 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -207,8 +207,10 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque return } + log.Printf("Queue confirm: %#v. Queue: %#v", removeExecutionRequests.Data, id) + // remove items from DB - parsedId := fmt.Sprintf("workflowqueue-%s", id) + parsedId := strings.ReplaceAll(fmt.Sprintf("workflowqueue-%s", id), " ", "-") ids := []string{} for _, execution := range removeExecutionRequests.Data { ids = append(ids, execution.ExecutionId) diff --git a/frontend/src/components/Navbar.jsx b/frontend/src/components/Navbar.jsx index 2c437a8d..fa038a51 100644 --- a/frontend/src/components/Navbar.jsx +++ b/frontend/src/components/Navbar.jsx @@ -1247,7 +1247,7 @@ const Navbar = (props) => { letterSpacing: '0.5px', }} > - Coming Soon + Beta: Coming Soon )} diff --git a/frontend/src/views/LoginPage.jsx b/frontend/src/views/LoginPage.jsx index 9bcc6966..9a04b959 100755 --- a/frontend/src/views/LoginPage.jsx +++ b/frontend/src/views/LoginPage.jsx @@ -375,7 +375,7 @@ const LoginPage = props => { padding: "40px", flex: 1, maxWidth: isMobile ? "100%" : 410, - minWidth: 410, + minWidth: isCloud ? 410 : 475, background: "#212121", borderRadius: "12px", display: "flex", diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 291c861d..b5a3272b 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1,13 +1,9 @@ package main /* - Orborus exists to listen for new jobs which are deployed as workers. + Orborus exists to listen for new jobs from Shuffle. This is to run workflows, pipelines, and other tasks. */ -// Potential issues: -// Default network could be same as on the host -// Ingress network may not exist (default) - import ( "archive/zip" "bytes" @@ -88,7 +84,6 @@ var appServiceAccountName = os.Getenv("SHUFFLE_APP_SERVICE_ACCOUNT_NAME") // var baseimagename = "shuffle/shuffle" var baseimagename = os.Getenv("SHUFFLE_BASE_IMAGE_NAME") var baseimageregistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY") - //var baseimagetagsuffix = os.Getenv("SHUFFLE_BASE_IMAGE_TAG_SUFFIX") // Used for cloud with auth @@ -781,7 +776,7 @@ func handleBackendImageDownload(ctx context.Context, images string) error { log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig) } - err = shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, curimage) + err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, curimage) if err != nil { log.Printf("[ERROR] Failed downloading image: %s", err) } else { @@ -1340,11 +1335,11 @@ func deployWorker(image string, identifier string, env []string, executionReques ) if err != nil { - log.Printf("[ERROR] Container create error(2): %s", err) + log.Printf("[ERROR][%s] Container create error(2): %s", executionRequest.ExecutionId, err) return err } } else { - log.Printf("[ERROR] Container create error: %s", err) + log.Printf("[ERROR][%s] Container create error: %s", executionRequest.ExecutionId, err) return err } } @@ -1369,19 +1364,19 @@ func deployWorker(image string, identifier string, env []string, executionReques identifier+"-2", ) if err != nil { - log.Printf("[ERROR] Failed to CREATE container (2): %s", err) + log.Printf("[ERROR][%s] Failed to CREATE container (2): %s", executionRequest.ExecutionId, err) } err = dockercli.ContainerStart(context.Background(), cont.ID, containerStartOptions) if err != nil { - log.Printf("[ERROR] Failed to start container (2): %s", err) + log.Printf("[ERROR][%s] Failed to start container (2): %s", executionRequest.ExecutionId, err) } } else { - log.Printf("[ERROR] Failed initial container start. Quitting as this is NOT a simple network issue. Err: %s", err) + log.Printf("[ERROR][%s] Failed initial container start. Quitting as this is NOT a simple network issue. Err: %s", executionRequest.ExecutionId, err) } if err != nil { - log.Printf("[ERROR] Failed to start worker container in environment '%s': %s", environment, err) + log.Printf("[ERROR][%s] Failed to start worker container in environment '%s': %s", executionRequest.ExecutionId, environment, err) return err } else { log.Printf("[INFO][%s] Worker Container created (2). Environment %s: docker logs %s", executionRequest.ExecutionId, environment, cont.ID) @@ -1389,13 +1384,13 @@ func deployWorker(image string, identifier string, env []string, executionReques stats, err := dockercli.ContainerInspect(ctx, cont.ID) if err != nil { - log.Printf("[WARNING] Failed checking worker '%s': %s", cont.ID, err) + log.Printf("[WARNING][%s] Failed checking worker '%s': %s", executionRequest.ExecutionId, cont.ID, err) return nil } containerStatus := stats.ContainerJSONBase.State.Status if containerStatus != "running" { - log.Printf("[ERROR] Status of %s is %s. Should be running. Contact support@shuffler.io if this persists.", cont.ID, containerStatus) + log.Printf("[ERROR][%s] Status of %s is %s. Should be running. Contact support@shuffler.io if this persists.", executionRequest.ExecutionId, cont.ID, containerStatus) } /* err = stopWorker(containerName) @@ -1459,17 +1454,30 @@ func initializeImages() { if baseimageregistry == "" { baseimageregistry = "docker.io" // Dockerhub baseimageregistry = "ghcr.io" // Github - log.Printf("[DEBUG] Setting baseimageregistry to %#v", baseimageregistry) + + if len(os.Getenv("REGISTRY_URL")) > 0 { + baseimageregistry = os.Getenv("REGISTRY_URL") + } else { + os.Setenv("REGISTRY_URL", baseimageregistry) + } + + os.Setenv("SHUFFLE_BASE_IMAGE_REGISTRY", baseimageregistry) + + log.Printf("[WARNING] Setting baseimageregistry to %#v", baseimageregistry) } if baseimagename == "" { + // FIXME: This is probably the problem for image names tbh baseimagename = "frikky/shuffle" // Dockerhub baseimagename = "shuffle" // Github (ghcr.io) - log.Printf("[DEBUG] Setting baseimagename to %#v", baseimagename) + + os.Setenv("SHUFFLE_BASE_IMAGE_NAME", baseimagename) + log.Printf("[WARNING] Setting baseimagename to %#v", baseimagename) } log.Printf("[DEBUG] Setting swarm config to %#v. Default is empty.", swarmConfig) + // FIXME: Shuffle Worker vs Apps != same newWorker := fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion) if len(newWorkerImage) > 0 { newWorker = newWorkerImage @@ -2412,11 +2420,11 @@ func main() { } if execution.Status == "ABORT" || execution.Status == "FAILED" { - log.Printf("[INFO] Executionstatus issue: ", execution.Status) + log.Printf("[INFO][%s] Executionstatus issue: ", execution.ExecutionId, execution.Status) } if shuffle.ArrayContains(executionIds, execution.ExecutionId) { - log.Printf("[INFO] Execution already handled (rerun of old executions?): %s", execution.ExecutionId) + log.Printf("[INFO][%s] Execution already handled (rerunning old execution)", execution.ExecutionId) toBeRemoved.Data = append(toBeRemoved.Data, execution) // Should check when last this was ran, and if it's more than 10 minutes ago and it's not finished, we should run it again? @@ -2506,7 +2514,7 @@ func main() { toBeRemoved.Data = append(toBeRemoved.Data, execution) executionIds = append(executionIds, execution.ExecutionId) } else { - log.Printf("[WARNING] Execution ID '%s' failed to deploy: %s", execution.ExecutionId, err) + log.Printf("[WARNING][%s] Failed to deploy: %s", execution.ExecutionId, err) if strings.Contains(err.Error(), "already exists") { toBeRemoved.Data = append(toBeRemoved.Data, execution) executionIds = append(executionIds, execution.ExecutionId) @@ -3638,15 +3646,12 @@ func zombiecheck(ctx context.Context, workerTimeout int) error { All: true, }) - //log.Printf("Len: %d", len(containers)) - if err != nil { log.Printf("[ERROR] Failed creating Containerlist: %s", err) return err } containerNames := map[string]string{} - stopContainers := []string{} removeContainers := []string{} log.Printf("[INFO] Baseimage: %s, Workertimeout: %d", baseimagename, int64(workerTimeout))