From d8b0d5f5b267f0b4aef5044eb9c9071273a6d11d Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Tue, 14 Oct 2025 17:36:10 +0530 Subject: [PATCH 01/42] feat: adding log forwarding features --- functions/onprem/orborus/orborus.go | 69 +++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index eb372e8a..9d7862cf 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -18,12 +18,14 @@ import ( "net/http" "os" "os/exec" + "os/signal" "path/filepath" "regexp" "runtime" "strconv" "strings" "sync" + "syscall" "time" "github.com/shuffle/shuffle-shared" @@ -2092,6 +2094,66 @@ func cleanup() { os.Exit(0) } +func StartAgent() { + log.Printf("[INFO] Starting Orborus agent mode") + + auditLogEnabled := os.Getenv("SHUFFLE_AUDIT_LOG_ENABLED") == "true" + + if auditLogEnabled { + log.Printf("[INFO] Audit log monitoring is enabled") + + // Initialize telemetry configuration + telemetryConfig := shuffle.TelemetryConfig{ + Enabled: true, + Modes: []string{"audit_log"}, + BufferSize: 1000, + FlushInterval: 10 * time.Second, + } + + if excludePatterns := os.Getenv("SHUFFLE_AUDIT_LOG_EXCLUDE"); excludePatterns != "" { + patterns := strings.Split(excludePatterns, ",") + telemetryConfig.Filters = append(telemetryConfig.Filters, shuffle.TelemetryFilter{ + Type: "message", + Exclude: patterns, + }) + } + + if includePatterns := os.Getenv("SHUFFLE_AUDIT_LOG_INCLUDE"); includePatterns != "" { + patterns := strings.Split(includePatterns, ",") + telemetryConfig.Filters = append(telemetryConfig.Filters, shuffle.TelemetryFilter{ + Type: "message", + Include: patterns, + }) + } + + collector, err := shuffle.NewAuditLogCollector(telemetryConfig) + if err != nil { + log.Printf("[ERROR] Failed to create audit log collector: %v", err) + } else { + ctx := context.Background() + if err := collector.LogCollectorStart(ctx); err != nil { + log.Printf("[ERROR] Failed to start audit log collector: %v", err) + } else { + log.Printf("[INFO] Audit log collector started successfully") + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + + go func() { + <-sigChan + log.Printf("[INFO] Received shutdown signal, stopping audit log collector...") + collector.Stop() + + os.Exit(0) + }() + } + } + } else { + log.Printf("[INFO] Audit log monitoring is disabled") + } + select {} +} + // Initial loop etc func main() { // Get arch. amd64 or arm64 @@ -2100,6 +2162,13 @@ func main() { //signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM) //defer cleanup() + agentMode := os.Getenv("SHUFFLE_AGENT_MODE") + if agentMode == "true" { + log.Printf("[INFO] Running in agent mode. Starting the agent.") + StartAgent() + return + } + // Block until a signal is received if shuffle.IsRunningInCluster() { log.Printf("[INFO] Running inside k8s cluster") From 923a7df0e1dab553a73c400bfda488bce49fbb55 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Thu, 16 Oct 2025 10:29:29 +0000 Subject: [PATCH 02/42] fix: backend/Dockerfile to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-ALPINE322-OPENSSL-13174131 - https://snyk.io/vuln/SNYK-ALPINE322-OPENSSL-13174131 - https://snyk.io/vuln/SNYK-ALPINE322-OPENSSL-13174132 - https://snyk.io/vuln/SNYK-ALPINE322-OPENSSL-13174132 - https://snyk.io/vuln/SNYK-ALPINE322-OPENSSL-13174133 --- backend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100755 => 100644 backend/Dockerfile diff --git a/backend/Dockerfile b/backend/Dockerfile old mode 100755 new mode 100644 index 647f74cf..5ccc3de8 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -27,7 +27,7 @@ FROM alpine:latest as certs RUN apk add --update ca-certificates # Sets up the final image -FROM alpine:3.22.1 +FROM alpine:3.22.2 # FIXME: Install cgo because CGO_ENABLED=1 during build RUN apk add --no-cache libc6-compat From 73a8531e4314bf817ed1b6ee2460d2f76fd02cf5 Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 23 Oct 2025 11:27:27 +0200 Subject: [PATCH 03/42] Made 30 second wait time for database mandatory --- backend/go-app/main.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 5b26b726..36a92b51 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -4243,10 +4243,8 @@ func runInitEs(ctx context.Context) { } } - if strings.Contains(os.Getenv("SHUFFLE_OPENSEARCH_URL"), "https") { - log.Printf("[INFO] Waiting 30 seconds during init to make sure the opensearch instance is up and running with security features enabled") - time.Sleep(30 * time.Second) - } + log.Printf("[INFO] Waiting 30 seconds during init to make sure the opensearch instance is up and running with security features enabled") + time.Sleep(30 * time.Second) shuffle.InitOpensearchIndexes() From f638baefcfee24ff08c3a5aebccacafc7e7ac7af Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 24 Oct 2025 16:54:32 +0200 Subject: [PATCH 04/42] Made orborus less verbose --- functions/onprem/orborus/orborus.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 02210982..d1af43a2 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2075,7 +2075,10 @@ func sendRemoveRequest(client *http.Client, toBeRemoved shuffle.ExecutionRequest resultResp, err := client.Do(result) if err != nil { - log.Printf("[ERROR] Failed making confirm request: %s", err) + if !strings.Contains(fmt.Sprintf("%s", err), "timeout") { + log.Printf("[ERROR] Failed making confirm request: %s", err) + } + time.Sleep(time.Duration(sleepTime) * time.Second) return err } @@ -4167,7 +4170,9 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, newresp, err := client.Do(req) if err != nil { // Connection refused? - log.Printf("[ERROR][%s] Error running worker request to %s (1): %s", workflowExecution.ExecutionId, streamUrl, err) + if !strings.Contains(fmt.Sprintf("%s", err), "timeout") { + log.Printf("[ERROR][%s] Error running worker request to %s (1): %s", workflowExecution.ExecutionId, streamUrl, err) + } if strings.Contains(fmt.Sprintf("%s", err), "connection refused") || strings.Contains(fmt.Sprintf("%s", err), "EOF") { workerImage := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) From 97ef960bb0b879189dc632665afe5d33e6fba1e9 Mon Sep 17 00:00:00 2001 From: Frikky Date: Sun, 26 Oct 2025 14:24:38 +0100 Subject: [PATCH 05/42] Refactoring for opensearch control --- backend/go-app/go.mod | 9 ++++----- backend/go-app/go.sum | 44 ++++++++++++++----------------------------- 2 files changed, 18 insertions(+), 35 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 1711153b..07fcc07b 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -4,11 +4,11 @@ go 1.24.0 toolchain go1.24.3 -//replace github.com/shuffle/opensearch-go => ../../../opensearch-go -//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +replace github.com/shuffle/opensearch-go => ../../../opensearch-go + +replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared //replace github.com/frikky/schemaless => ../../../schemaless - //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi require ( @@ -115,8 +115,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/opensearch-project/opensearch-go v1.1.0 // indirect - github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect github.com/osteele/liquid v1.7.0 // indirect github.com/osteele/tuesday v1.0.3 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect @@ -128,6 +126,7 @@ require ( github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/sendgrid/sendgrid-go v3.16.1+incompatible // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/shuffle/opensearch-go/v4 v4.0.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index d7ba4a0e..4f72a37f 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -72,20 +72,6 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/aws/aws-sdk-go v1.42.27/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= -github.com/aws/aws-sdk-go v1.44.263/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4= -github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/basgys/goxml2json v1.1.0 h1:4ln5i4rseYfXNd86lGEB+Vi652IsIXIvggKM/BhUKVw= github.com/basgys/goxml2json v1.1.0/go.mod h1:wH7a5Np/Q4QoECFIU8zTQlZwZkrilY0itPfecMw41Dw= github.com/bitly/go-simplejson v0.5.1 h1:xgwPbetQScXt1gh9BmoJ6j9JMr3TElvuIyjR8pgdoow= @@ -223,7 +209,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -265,8 +250,6 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -325,10 +308,6 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/opensearch-project/opensearch-go v1.1.0 h1:eG5sh3843bbU1itPRjA9QXbxcg8LaZ+DjEzQH9aLN3M= -github.com/opensearch-project/opensearch-go v1.1.0/go.mod h1:+6/XHCuTH+fwsMJikZEWsucZ4eZMma3zNSeLrTtVGbo= -github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ= -github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8= github.com/osteele/liquid v1.7.0 h1:VsbPSchE5D5S5scylAIvERET4dnCxsO6IDri2oSJ5Dk= github.com/osteele/liquid v1.7.0/go.mod h1:xU0Z2dn2hOQIEFEWNmeltOmCtfhtoW/2fCyiNQeNG+U= github.com/osteele/tuesday v1.0.3 h1:SrCmo6sWwSgnvs1bivmXLvD7Ko9+aJvvkmDjB5G4FTU= @@ -363,8 +342,8 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.9.34 h1:4RtGRMoYtBQia2CLbuH7aPFM+EMFXbH0ccIQsBXs8n8= -github.com/shuffle/shuffle-shared v0.9.34/go.mod h1:WbLPLyBrh86ieRBgDyjC43M2qBF8a7E8jO8Pv7t8SOk= +github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU= +github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0= github.com/shuffle/singul v0.0.17 h1:mxaPtj6z85Nf6tl7L2gwDliTfEZtRQqApuu9iKcP75o= github.com/shuffle/singul v0.0.17/go.mod h1:8c42n1NahhCIPxzLxwp9eYbWkvY4+ct0jfbhkRsRRsY= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -393,8 +372,18 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/wI2L/jsondiff v0.7.0 h1:1lH1G37GhBPqCfp/lrs91rf/2j3DktX6qYAKZkLuCQQ= +github.com/wI2L/jsondiff v0.7.0/go.mod h1:KAEIojdQq66oJiHhDyQez2x+sRit0vIzC9KeK0yizxM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= @@ -491,9 +480,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= @@ -536,13 +523,11 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= @@ -554,7 +539,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= From ce1e932a88a62d6c93b5c57731196efff95f7f7c Mon Sep 17 00:00:00 2001 From: Frikky Date: Sun, 26 Oct 2025 14:38:01 +0100 Subject: [PATCH 06/42] Indent fixes gofmt --- backend/go-app/main.go | 42 ++++++++++++------------- backend/go-app/walkoff.go | 64 +++++++++++++++++++-------------------- 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 36a92b51..78dde288 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -14,18 +14,18 @@ import ( "crypto/md5" "strconv" - "os" - "io" - "log" - "fmt" - "errors" - "net/url" - "os/exec" - "net/http" - "io/ioutil" - "math/rand" "encoding/hex" "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "math/rand" + "net/http" + "net/url" + "os" + "os/exec" "net/http/httptest" "strings" @@ -67,6 +67,7 @@ var runningEnvironment = "onprem" var syncUrl = "https://shuffler.io" var debug = false + //var syncUrl = "http://localhost:5002" type retStruct struct { @@ -3416,7 +3417,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User, s // Read and copy the baseline Dockerfile dockerfileContent, err := ioutil.ReadFile(dockerfileSource) if err != nil { - foundDockerfile := shuffle.GetBaseDockerfile() + foundDockerfile := shuffle.GetBaseDockerfile() if len(foundDockerfile) > 0 { dockerfileContent = foundDockerfile } else { @@ -4033,7 +4034,7 @@ func remoteOrgJobHandler(org shuffle.Org, interval int) error { } // Just to prevent it from spamming large outbound requests - if shouldBackupData { + if shouldBackupData { if org.SyncConfig.WorkflowBackup { workflows, err := shuffle.GetAllWorkflowsByQuery(ctx, foundUser, 250, "") if err != nil { @@ -4067,7 +4068,7 @@ func remoteOrgJobHandler(org shuffle.Org, interval int) error { } // Send stats once every 10 times or so..? - // For now, just send every time + // For now, just send every time info, err := shuffle.GetOrgStatistics(ctx, org.Id) if err != nil { log.Printf("[ERROR] Failed getting org statistics backup for org %s: %s", org.Id, err) @@ -4401,7 +4402,7 @@ func runInitEs(ctx context.Context) { } //interval := int(org.SyncConfig.Interval) - interval := 30 + interval := 30 if interval == 0 { log.Printf("[WARNING] Skipping org %s because sync isn't set (0).", org.Id) continue @@ -4503,7 +4504,6 @@ func runInitEs(ctx context.Context) { continue } - respBody, err := ioutil.ReadAll(newresp.Body) if err != nil { log.Printf("[ERROR] Failed setting respbody %s for execution stop. Status: %d", err, newresp.StatusCode) @@ -4574,7 +4574,7 @@ func runInitEs(ctx context.Context) { url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION") if len(url) == 0 { log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/shuffle/python-apps") - + url = "https://github.com/shuffle/python-apps" //url = "" //return @@ -4660,9 +4660,9 @@ func runInitEs(ctx context.Context) { } // Self-cleaning - go func() { + go func() { cursor := "" - cnt := 0 + cnt := 0 newCtx := context.Background() for _, org := range activeOrgs { if len(org.Id) == 0 { @@ -4672,7 +4672,7 @@ func runInitEs(ctx context.Context) { log.Printf("[INFO] Starting self-cleanup of cache keys for org %s", org.Id) - for { + for { keys, newCursor, err := shuffle.GetAllCacheKeys(newCtx, org.Id, "", 1000, cursor) if err != nil { //log.Printf("[ERROR] Failed getting all cache keys for cleanup: %s", err) @@ -5052,7 +5052,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { Interval: responseData.IntervalSeconds, WorkflowBackup: true, - AppBackup: true, + AppBackup: true, } interval := int(responseData.IntervalSeconds) @@ -5675,7 +5675,7 @@ func initHandlers() { r.HandleFunc("/api/v1/dashboards/{key}/widgets", shuffle.HandleNewWidget).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/dashboards/{key}/widgets/{widget_id}", shuffle.HandleGetWidget).Methods("GET", "OPTIONS") - if (strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" || strings.ToLower(os.Getenv("DEBUG_MEMORY")) == "true") { + if strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" || strings.ToLower(os.Getenv("DEBUG_MEMORY")) == "true" { log.Printf("[DEBUG] Memory debugging is enabled on /debug/pprof") r.HandleFunc("/debug/pprof/", pprof.Index) r.HandleFunc("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 54635430..032af73a 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -20,21 +20,21 @@ import ( "strings" "time" - dockerclient "github.com/docker/docker/client" "github.com/docker/docker/api/types/image" + dockerclient "github.com/docker/docker/client" "github.com/h2non/filetype" uuid "github.com/satori/go.uuid" newscheduler "github.com/carlescere/scheduler" - "github.com/go-co-op/gocron" "github.com/frikky/kin-openapi/openapi3" + gyaml "github.com/ghodss/yaml" + "github.com/go-co-op/gocron" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/storage/memory" "github.com/go-git/go-git/v5/plumbing" http2 "github.com/go-git/go-git/v5/plumbing/transport/http" - gyaml "github.com/ghodss/yaml" + "github.com/go-git/go-git/v5/storage/memory" ) var localBase = "http://localhost:5001" @@ -257,7 +257,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { return } - // This is really the environment's name - NOT OrgId + // This is really the environment's name - NOT OrgId environment := request.Header.Get("Org-Id") if len(environment) == 0 { log.Printf("[AUDIT] No org-id header set") @@ -276,7 +276,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { return */ } - + // This section is cloud custom for now auth := request.Header.Get("Authorization") if len(auth) == 0 { @@ -305,7 +305,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } // Only works onprem - shared queues across tenants - // without tenancy + // without tenancy if !found { env, err = shuffle.GetEnvironment(ctx, environment, "") if err != nil { @@ -336,7 +336,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { if len(executionRequests.Data) == 0 { executionRequests.Data = []shuffle.ExecutionRequest{} } else { - // Try again? I don't think this is necessary, and shouldn't really ever occur. + // Try again? I don't think this is necessary, and shouldn't really ever occur. if len(executionRequests.Data) > 50 { executionRequests.Data = executionRequests.Data[0:49] } @@ -457,16 +457,16 @@ func handleGetWorkflowExecutionResult(resp http.ResponseWriter, request *http.Re if workflowExecution.Workflow.Sharing == "form" { newWorkflow := shuffle.Workflow{ - Name: workflowExecution.Workflow.Name, - ID: workflowExecution.Workflow.ID, - Owner: workflowExecution.Workflow.Owner, - OrgId: workflowExecution.Workflow.OrgId, + Name: workflowExecution.Workflow.Name, + ID: workflowExecution.Workflow.ID, + Owner: workflowExecution.Workflow.Owner, + OrgId: workflowExecution.Workflow.OrgId, - Sharing: workflowExecution.Workflow.Sharing, + Sharing: workflowExecution.Workflow.Sharing, Description: workflowExecution.Workflow.Description, InputQuestions: workflowExecution.Workflow.InputQuestions, - FormControl: workflowExecution.Workflow.FormControl, + FormControl: workflowExecution.Workflow.FormControl, } workflowExecution.Results = []shuffle.ActionResult{} @@ -765,7 +765,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { } if user.Id != workflow.Owner || len(user.Id) == 0 { - if workflow.OrgId == user.ActiveOrg.Id { + if workflow.OrgId == user.ActiveOrg.Id { log.Printf("[INFO] User %s is deleting workflow %s as admin. Owner: %s", user.Username, workflow.ID, workflow.Owner) } else { log.Printf("[WARNING] Wrong user (%s) for workflow %s (delete workflow)", user.Username, workflow.ID) @@ -782,7 +782,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { childWorkflows, err := shuffle.ListChildWorkflows(ctx, workflow.ID) if err != nil { log.Printf("[ERROR] Failed to list child workflows: %s", err) - } else { + } else { //log.Printf("\n\n[DEBUG] Found %d child workflows for workflow %s\n\n", len(childWorkflows), workflow.ID) // Find cookies and append them to request.Header to replicate current request as closely as possible @@ -898,10 +898,10 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request } /* - if !workflow.IsValid { - log.Printf("[ERROR] Stopped execution as workflow %s is not valid.", workflow.ID) - return shuffle.WorkflowExecution{}, fmt.Sprintf(`workflow %s is invalid`, workflow.ID), errors.New("Failed getting workflow") - } + if !workflow.IsValid { + log.Printf("[ERROR] Stopped execution as workflow %s is not valid.", workflow.ID) + return shuffle.WorkflowExecution{}, fmt.Sprintf(`workflow %s is invalid`, workflow.ID), errors.New("Failed getting workflow") + } */ maxExecutionDepth := 10 @@ -914,7 +914,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request workflowExecution, execInfo, _, workflowExecErr := shuffle.PrepareWorkflowExecution(ctx, workflow, request, int64(maxExecutionDepth)) if workflowExecErr != nil { - if len(workflowExecution.Workflow.Actions) > 0 && len(workflowExecution.Results) > 0 && len(workflowExecution.ExecutionId) > 0 { + if len(workflowExecution.Workflow.Actions) > 0 && len(workflowExecution.Results) > 0 && len(workflowExecution.ExecutionId) > 0 { err := shuffle.SetWorkflowExecution(ctx, workflowExecution, true) if err != nil { log.Printf("[ERROR] Failed setting workflow execution during init (2): %s", err) @@ -928,7 +928,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request return shuffle.WorkflowExecution{}, "", nil } else { log.Printf("[ERROR] Failed in prepareExecution: '%s'", workflowExecErr) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed running: %s", workflowExecErr), workflowExecErr + return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed running: %s", workflowExecErr), workflowExecErr } } @@ -1337,7 +1337,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { } if user.Id != workflow.Owner || len(user.Id) == 0 { - if workflow.OrgId == user.ActiveOrg.Id { + if workflow.OrgId == user.ActiveOrg.Id { log.Printf("[AUDIT] User %s is accessing workflow %s as admin (stop schedule)", user.Username, workflow.ID) } else { log.Printf("[WARNING] Wrong user (%s) for workflow %s (stop schedule)", user.Username, workflow.ID) @@ -1582,7 +1582,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { } if user.Id != workflow.Owner || len(user.Id) == 0 { - if workflow.OrgId == user.ActiveOrg.Id { + if workflow.OrgId == user.ActiveOrg.Id { log.Printf("[INFO] User %s is deleting workflow %s as admin. Owner: %s", user.Username, workflow.ID, workflow.Owner) } else { log.Printf("[WARNING] Wrong user (%s) for workflow %s (schedule start). Owner: %s", user.Username, workflow.ID, workflow.Owner) @@ -2022,7 +2022,7 @@ func loadGithubWorkflows(url, username, password, userId, branch, orgId string) cloneOptions.ReferenceName = plumbing.ReferenceName(branch) } - cloneOptions = checkGitProxy(cloneOptions) + cloneOptions = checkGitProxy(cloneOptions) storer := memory.NewStorage() r, err := git.Clone(storer, fs, cloneOptions) @@ -2877,7 +2877,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { log.Printf("\n\nACTION TO RUN: %s. Body: %s. Source URL: %s\n\n", appId, string(body), request.URL.String()) workflowExecution, err := shuffle.PrepareSingleAction(ctx, user, appId, body, runValidationAction, decisionId) - if appId == "agent_starter" { + if appId == "agent_starter" { log.Printf("[INFO] Returning early for agent_starter single action execution: %s", workflowExecution.ExecutionId) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization))) @@ -2905,7 +2905,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { return } - workflowExecution.ProjectId = "" + workflowExecution.ProjectId = "" workflowExecution.Locations = []string{""} foundEnv := "" @@ -2923,7 +2923,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { go shuffle.IncrementCache(ctx, workflowExecution.OrgId, "workflow_executions") executionRequest := shuffle.ExecutionRequest{ - Priority: 15, + Priority: 15, ExecutionId: workflowExecution.ExecutionId, WorkflowId: workflowExecution.Workflow.ID, Authorization: workflowExecution.Authorization, @@ -2998,7 +2998,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. continue } - //duringStartup + //duringStartup if duringStartup { // Look for names: shuffle tools, http, email, shuffle ai if shuffle.ArrayContains(startupNames, strings.ToLower(file.Name())) { @@ -3366,7 +3366,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. if len(item.Tags) > 0 && shuffle.ArrayContains(handledImages, item.Tags[0]) { continue - } + } handledImages = append(handledImages, item.Tags[0]) err = buildImageMemory(fs, item.Tags, item.Extra, true) @@ -3402,7 +3402,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. for _, item := range buildLaterList { if len(item.Tags) > 0 && shuffle.ArrayContains(handledImages, item.Tags[0]) { continue - } + } handledImages = append(handledImages, item.Tags[0]) @@ -3493,7 +3493,7 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { } } - cloneOptions = checkGitProxy(cloneOptions) + cloneOptions = checkGitProxy(cloneOptions) storer := memory.NewStorage() r, err := git.Clone(storer, fs, cloneOptions) From 9a8f3bd3bd52fbaecfd82f2fc08c3fbf9f919289 Mon Sep 17 00:00:00 2001 From: Frikky Date: Sun, 26 Oct 2025 16:01:39 +0100 Subject: [PATCH 07/42] Opensearch & Elasticsearch compatibility fixes MOSTLY done. Just a bit more missing. --- backend/go-app/go.mod | 9 ++++----- backend/go-app/go.sum | 2 ++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 07fcc07b..fabd46cd 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -4,12 +4,11 @@ go 1.24.0 toolchain go1.24.3 -replace github.com/shuffle/opensearch-go => ../../../opensearch-go - -replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared - +//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared //replace github.com/frikky/schemaless => ../../../schemaless + //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi +//replace github.com/shuffle/opensearch-go => ../../../opensearch-go require ( cloud.google.com/go/datastore v1.20.0 @@ -26,7 +25,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.34 + github.com/shuffle/shuffle-shared v0.9.35 github.com/shuffle/singul v0.0.17 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 4f72a37f..f1ec1c07 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -344,6 +344,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU= github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0= +github.com/shuffle/shuffle-shared v0.9.35 h1:A6Mrv6zBlChbcu2YJTgEqnfgVOsZfIoaGXjDh9468gU= +github.com/shuffle/shuffle-shared v0.9.35/go.mod h1:czsJf2wOcGG4NhED3lbRwLOHTV5AP868mJCf2HLRVss= github.com/shuffle/singul v0.0.17 h1:mxaPtj6z85Nf6tl7L2gwDliTfEZtRQqApuu9iKcP75o= github.com/shuffle/singul v0.0.17/go.mod h1:8c42n1NahhCIPxzLxwp9eYbWkvY4+ct0jfbhkRsRRsY= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= From f50ba78a56dc3c74e9d43dd9c1013033b9613dc1 Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 27 Oct 2025 15:29:12 +0100 Subject: [PATCH 08/42] Bumped singul version --- backend/go-app/go.mod | 6 +++--- backend/go-app/go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index fabd46cd..5c39faf8 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -25,8 +25,8 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.35 - github.com/shuffle/singul v0.0.17 + github.com/shuffle/shuffle-shared v0.9.37 + github.com/shuffle/singul v0.0.18 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 google.golang.org/grpc v1.72.2 @@ -74,7 +74,7 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frikky/schemaless v0.0.22 // indirect + github.com/frikky/schemaless v0.0.23 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index f1ec1c07..39338b02 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -138,8 +138,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.22 h1:aMc7cc/lr1zpogjGWbY0j6J2f6QqyfPbbW6Y9JgTAqE= -github.com/frikky/schemaless v0.0.22/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= +github.com/frikky/schemaless v0.0.23 h1:fa1JR8HJat2obbAKB5+Rnwmr09LNUXlyjTMgKG/G7BA= +github.com/frikky/schemaless v0.0.23/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= github.com/fsouza/go-dockerclient v1.12.1 h1:FMoLq+Zhv9Oz/rFmu6JWkImfr6CBgZOPcL+bHW4gS0o= github.com/fsouza/go-dockerclient v1.12.1/go.mod h1:OqsgJJcpCwqyM3JED7TdfM9QVWS5O7jSYwXxYKmOooY= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= @@ -344,10 +344,10 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU= github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0= -github.com/shuffle/shuffle-shared v0.9.35 h1:A6Mrv6zBlChbcu2YJTgEqnfgVOsZfIoaGXjDh9468gU= -github.com/shuffle/shuffle-shared v0.9.35/go.mod h1:czsJf2wOcGG4NhED3lbRwLOHTV5AP868mJCf2HLRVss= -github.com/shuffle/singul v0.0.17 h1:mxaPtj6z85Nf6tl7L2gwDliTfEZtRQqApuu9iKcP75o= -github.com/shuffle/singul v0.0.17/go.mod h1:8c42n1NahhCIPxzLxwp9eYbWkvY4+ct0jfbhkRsRRsY= +github.com/shuffle/shuffle-shared v0.9.37 h1:rk/TKhcqXFxRdlMiUjPfm2dCRjoGI360i12znF/Rq+A= +github.com/shuffle/shuffle-shared v0.9.37/go.mod h1:sYRBTvGlYRsh9bFJeSbA1Rq9DA274+rGzvIH+Q1j3GU= +github.com/shuffle/singul v0.0.18 h1:SMM18+MHC41UexDToNdVkBTkUCxI9kRf8y30TPzvTnA= +github.com/shuffle/singul v0.0.18/go.mod h1:8zZevcJhlQ9hW8UfDS/X4e6UYS8h03gy5qkFtno+lJM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= From f027e6196c97ce90460e16ae9e17b09ee1b5bd62 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Tue, 28 Oct 2025 15:50:49 +0530 Subject: [PATCH 09/42] fix: subflow stream result returning 400 --- functions/onprem/worker/worker.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 13c507d3..5a858888 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -2116,6 +2116,11 @@ func handleSubflowPoller(ctx context.Context, workflowExecution shuffle.Workflow } } + if len(data) == 0 { + log.Printf("[WARNING] Stream result missing execution ID and authorization; injecting them from workflow execution") + data = fmt.Sprintf(`{"execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization) + } + req, err := http.NewRequest( "POST", streamResultUrl, From b99565dc90fbfda53f7f394f575e98a443d0fdfe Mon Sep 17 00:00:00 2001 From: "lalitdeore12@gmail.com" Date: Tue, 28 Oct 2025 19:45:35 +0530 Subject: [PATCH 10/42] fix minor ui issue --- backend/go-app/main.go | 28 ++++++++++++++++++++++++ frontend/src/components/CloudSyncTab.jsx | 4 +++- frontend/src/components/LeftSideBar.jsx | 5 +++-- frontend/src/context/ContextApi.jsx | 3 +++ frontend/src/views/Admin2.jsx | 13 +++++++++-- 5 files changed, 48 insertions(+), 5 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 78dde288..5f54465a 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -4956,6 +4956,17 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { resp.Write(b) } else { + + // Delete cache for the sync features + cacheKey := fmt.Sprintf("org_sync_features_%s", org.Id) + shuffle.DeleteCache(ctx, cacheKey) + + subscriptionCacheKey := fmt.Sprintf("org_subscriptions_%s", org.Id) + shuffle.DeleteCache(ctx, subscriptionCacheKey) + + licenseCacheKey := fmt.Sprintf("org_licensed_%s", org.Id) + shuffle.DeleteCache(ctx, licenseCacheKey) + resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Successfully disabled cloud sync for org."}`))) } @@ -5047,6 +5058,23 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { shuffle.SetCache(ctx, cacheKey, featuresBytes, 1800) } + subscriptionCacheKey := fmt.Sprintf("org_subscriptions_%s", org.Id) + subscriptionsBytes, err := json.Marshal(responseData.Subscriptions) + if err != nil { + log.Printf("[ERROR] Failed to marshal Subscriptions for cache: %s", err) + } else { + shuffle.SetCache(ctx, subscriptionCacheKey, subscriptionsBytes, 1800) + } + + licenseCacheKey := fmt.Sprintf("org_licensed_%s", org.Id) + licensedBytes, err := json.Marshal(responseData.Licensed) + if err != nil { + log.Printf("[ERROR] Failed to marshal Licensed for cache: %s", err) + } else { + + shuffle.SetCache(ctx, licenseCacheKey, licensedBytes, 1800) + } + org.SyncConfig = shuffle.SyncConfig{ Apikey: responseData.SessionKey, Interval: responseData.IntervalSeconds, diff --git a/frontend/src/components/CloudSyncTab.jsx b/frontend/src/components/CloudSyncTab.jsx index 88df2610..75011dbf 100644 --- a/frontend/src/components/CloudSyncTab.jsx +++ b/frontend/src/components/CloudSyncTab.jsx @@ -49,7 +49,7 @@ const CloudSyncTab = (props) => { const itemColor = "white"; const isCloud = window?.location?.host === "localhost:3002" || window?.location?.host === "shuffler.io"; - const { themeMode, brandColor } = useContext(Context); + const { themeMode, brandColor, setUpdateOrg } = useContext(Context); const theme = getTheme(themeMode, brandColor); useEffect(() => { getSettings(); }, []); @@ -457,6 +457,8 @@ const CloudSyncTab = (props) => { setLoading(false); if (response.status === 200) { console.log("Cloud sync success?"); + handleGetOrg(userdata.active_org.id); + setUpdateOrg(true); } else { console.log("Cloud sync fail?"); } diff --git a/frontend/src/components/LeftSideBar.jsx b/frontend/src/components/LeftSideBar.jsx index ab256f75..cacef366 100644 --- a/frontend/src/components/LeftSideBar.jsx +++ b/frontend/src/components/LeftSideBar.jsx @@ -64,7 +64,7 @@ const ExpandMoreAndLessIcon = "/icons/expandMoreIcon.svg"; const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => { const navigate = useNavigate(); - const {setLeftSideBarOpenByClick, leftSideBarOpenByClick, setSearchBarModalOpen, searchBarModalOpen, logoutUrl, themeMode, handleThemeChange, isDocSearchModalOpen, setIsDocSearchModalOpen, supportEmail} = useContext(Context); + const {setLeftSideBarOpenByClick, leftSideBarOpenByClick, updateOrg, setUpdateOrg, setSearchBarModalOpen, searchBarModalOpen, logoutUrl, themeMode, handleThemeChange, isDocSearchModalOpen, setIsDocSearchModalOpen, supportEmail} = useContext(Context); const [expandLeftNav, setExpandLeftNav] = useState(false); const [activeOrgName, setActiveOrgName] = useState( userdata?.active_org?.name || "Select Organziation" @@ -956,6 +956,7 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => { .then((org) => { if (!fetched && org) { setActiveOrgData(org); + setUpdateOrg(false); if (!isCloud) { if (org?.cloud_sync && org?.subscriptions[0]?.name?.toLowerCase().includes("enterprise") && org?.subscriptions[0]?.active) { setIsProdStatusOn(true); @@ -972,7 +973,7 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => { return () => { fetched = true; }; - }, [userdata?.active_org?.id, globalUrl]); + }, [userdata?.active_org?.id, globalUrl, updateOrg]); return (
{ const [windowWidth, setWindowWidth] = useState(serverside === true ? 100 : window.innerWidth); const [brandColor, setBrandColor] = useState(() => localStorage.getItem("brandColor") || "#ff8544"); const [brandName, setBrandName] = useState(()=> localStorage.getItem("brandName") || "Shuffle"); + const [updateOrg, setUpdateOrg] = useState(false); const [themeMode, setThemeMode] = useState( () => localStorage.getItem("theme") || "dark" @@ -117,6 +118,8 @@ export const AppContext = (props) => { setBrandColor, brandName, setBrandName, + updateOrg, + setUpdateOrg, }}> {props.children} diff --git a/frontend/src/views/Admin2.jsx b/frontend/src/views/Admin2.jsx index fcb675bc..fc2fe2e8 100644 --- a/frontend/src/views/Admin2.jsx +++ b/frontend/src/views/Admin2.jsx @@ -5,14 +5,14 @@ import { Context } from '../context/ContextApi.jsx'; const Admin2 = (props) => { // Destructure props if needed - const { userdata, globalUrl, serverside, checkLogin, notifications, setNotifications, stripeKey, isLoaded, isLoggedIn} = props; + const { userdata, globalUrl, serverside, checkLogin, notifications, setNotifications, stripeKey, isLoaded, } = props; const [selectedTab, setSelectedTab] = useState('editdetails'); const [selectedStatus, setSelectedStatus] = React.useState([]); const [selectedOrganization, setSelectedOrganization] = useState({}); const [organizationFeatures, setOrganizationFeatures] = useState({}); const [orgRequest, setOrgRequest] = React.useState(true); const [isOrgLoaded, setIsOrgLoaded] = React.useState(false); - const {brandName} = useContext(Context) + const {brandName, updateOrg, setUpdateOrg} = useContext(Context) const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; if (document !== undefined) { @@ -47,6 +47,8 @@ const Admin2 = (props) => { window.location.href = "/workflows"; }, 3000); } else { + + setUpdateOrg(false); if ( responseJson.sync_features === undefined || responseJson.sync_features === null @@ -163,6 +165,13 @@ const Admin2 = (props) => { }); }; + useEffect(() => { + if (updateOrg && userdata?.active_org?.id !== undefined && userdata?.active_org?.id !== null && userdata?.active_org?.id.length > 0) { + handleGetOrg(userdata.active_org.id); + setUpdateOrg(false); + } + + }, [updateOrg]); useEffect(() => { const urlSearchParams = new URLSearchParams(window.location.search); From 0f737540d797f5a7fc80cfe6c4ba229f468d54d1 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Thu, 30 Oct 2025 21:05:54 +0530 Subject: [PATCH 11/42] fix: no need to compress data in workers --- functions/onprem/worker/go.mod | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index c0cd00f7..b1252739 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -11,7 +11,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.30 + github.com/shuffle/shuffle-shared v0.9.39 github.com/shuffle/singul v0.0.16 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 @@ -60,7 +60,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/kin-openapi v0.42.0 // indirect - github.com/frikky/schemaless v0.0.20 // indirect + github.com/frikky/schemaless v0.0.23 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect @@ -95,8 +95,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/opensearch-project/opensearch-go v1.1.0 // indirect - github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect github.com/osteele/liquid v1.7.0 // indirect github.com/osteele/tuesday v1.0.3 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect @@ -107,6 +105,7 @@ require ( github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/sendgrid/sendgrid-go v3.16.1+incompatible // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/shuffle/opensearch-go/v4 v4.0.0 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/spf13/pflag v1.0.6 // indirect From a7cd4bde29924f10f0bd3c65b16f06c3ef1e737b Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Thu, 30 Oct 2025 21:27:14 +0530 Subject: [PATCH 12/42] fix: no need to compress data in workers --- functions/onprem/worker/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index b1252739..085375c5 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 github.com/shuffle/shuffle-shared v0.9.39 - github.com/shuffle/singul v0.0.16 + github.com/shuffle/singul v0.0.19 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 k8s.io/client-go v0.33.1 From f158979e0b7394b637f63c7550ab46e84262d0f0 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Fri, 31 Oct 2025 19:10:21 +0530 Subject: [PATCH 13/42] fix: disabling compressExecution for onprem --- backend/go-app/go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 5c39faf8..664123ab 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -25,8 +25,8 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.37 - github.com/shuffle/singul v0.0.18 + github.com/shuffle/shuffle-shared v0.9.40 + github.com/shuffle/singul v0.0.19 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 google.golang.org/grpc v1.72.2 From 5cb0acdb715ab566239cca8acdadb6a6b716fc7e Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Fri, 31 Oct 2025 19:10:36 +0530 Subject: [PATCH 14/42] fix: disabling compressExecution for onprem --- functions/onprem/worker/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 085375c5..9f046605 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -11,7 +11,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.39 + github.com/shuffle/shuffle-shared v0.9.40 github.com/shuffle/singul v0.0.19 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 From ee300ece3c59e9660e1e6e75d478179f7981eae6 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Fri, 31 Oct 2025 19:18:40 +0530 Subject: [PATCH 15/42] fix: no need to compress data in workers --- functions/onprem/worker/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 9f046605..085375c5 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -11,7 +11,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.40 + github.com/shuffle/shuffle-shared v0.9.39 github.com/shuffle/singul v0.0.19 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 From 392a05100ec7a2b2c2569d1f141a3cb07d0b5e9c Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Fri, 31 Oct 2025 19:58:38 +0530 Subject: [PATCH 16/42] log info over what we got from backend --- functions/onprem/worker/worker.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 5a858888..11c7b524 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -2555,6 +2555,11 @@ func getWorkerBackendExecution(auth string, executionId string) (*shuffle.Workfl return workflowExecution, err } + if debug { + log.Printf("[INFO] Here is the result we got back from backend: %s", workflowExecution.Results) + } + + setWorkflowExecution(context.Background(), *workflowExecution, false) return workflowExecution, nil } From 70937213a1c69e48c4963c5e7077653178f5fddc Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Fri, 31 Oct 2025 21:39:47 +0530 Subject: [PATCH 17/42] fix: memcache does not consider env --- docker-compose.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8ab7e3b9..461acf9a 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,9 @@ services: #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - SHUFFLE_FILE_LOCATION=/shuffle-files + - HTTP_PROXY=http://192.168.29.215:5000 + - HTTPS_PROXY=http://192.168.29.215:5000 + - NO_PROXY=192.168.29.215:3001, shuffler.io, 192.168.29.215:9200 restart: unless-stopped orborus: image: ghcr.io/shuffle/shuffle-orborus:latest @@ -122,11 +125,10 @@ services: # - MEMCACHED_MAX_CONNECTIONS=2500 # ports: # - 11211:11211 - # deploy: - # resources: - # limits: - # memory: 2048M - + # networks: + # - shuffle + # - swarm_executions + #docker-socket-proxy: # image: tecnativa/docker-socket-proxy # container_name: docker-socket-proxy From 61f499807bba50b4adc53f5c19046d7bd3e713b0 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Fri, 31 Oct 2025 21:40:49 +0530 Subject: [PATCH 18/42] remove test data --- docker-compose.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 461acf9a..0d96bcdf 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,9 +31,6 @@ services: #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - SHUFFLE_FILE_LOCATION=/shuffle-files - - HTTP_PROXY=http://192.168.29.215:5000 - - HTTPS_PROXY=http://192.168.29.215:5000 - - NO_PROXY=192.168.29.215:3001, shuffler.io, 192.168.29.215:9200 restart: unless-stopped orborus: image: ghcr.io/shuffle/shuffle-orborus:latest From 27f70ec2d9025e20725c61e69cb3f449a38a787b Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Sat, 1 Nov 2025 02:06:05 +0530 Subject: [PATCH 19/42] fix: more debug info --- backend/go-app/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 664123ab..a5654a1e 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -25,7 +25,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.40 + github.com/shuffle/shuffle-shared v0.9.41 github.com/shuffle/singul v0.0.19 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 From d87ca7b2e33e2697adfa233704d394027d1ee15c Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Sat, 1 Nov 2025 02:28:27 +0530 Subject: [PATCH 20/42] fix: more logs --- backend/go-app/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index a5654a1e..ab7c2df2 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -25,7 +25,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.41 + github.com/shuffle/shuffle-shared v0.9.43 github.com/shuffle/singul v0.0.19 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 From 305b6f61d8214578ba4696cc1ab2273c8c4e6b0e Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Sat, 1 Nov 2025 03:56:49 +0530 Subject: [PATCH 21/42] fix: more logs --- backend/go-app/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index ab7c2df2..8d0346b4 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -25,7 +25,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.43 + github.com/shuffle/shuffle-shared v0.9.44 github.com/shuffle/singul v0.0.19 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 From 3b9cc6d32462e0afba7c1992731aca3e2afe8448 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Sat, 1 Nov 2025 05:01:42 +0530 Subject: [PATCH 22/42] fix: hotfix for prefixes --- backend/go-app/go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 8d0346b4..f2708784 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -4,7 +4,7 @@ go 1.24.0 toolchain go1.24.3 -//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared //replace github.com/frikky/schemaless => ../../../schemaless //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi @@ -25,7 +25,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.44 + github.com/shuffle/shuffle-shared v0.9.45 github.com/shuffle/singul v0.0.19 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 From 1903f6bb03e87af92f60131f56ec0530b27a8a84 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Sat, 1 Nov 2025 05:08:22 +0530 Subject: [PATCH 23/42] fix: removing local lib reference --- backend/go-app/go.mod | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index f2708784..fef7eeef 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -4,7 +4,8 @@ go 1.24.0 toolchain go1.24.3 -replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared + //replace github.com/frikky/schemaless => ../../../schemaless //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi From 6ee810659c179a4234ad172e2f5d7064fc933a73 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Sat, 1 Nov 2025 05:24:55 +0530 Subject: [PATCH 24/42] fix: lowercase the index name --- backend/go-app/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index fef7eeef..2959b662 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -26,7 +26,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.45 + github.com/shuffle/shuffle-shared v0.9.46 github.com/shuffle/singul v0.0.19 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 From 5564814ada701fd4660d9e37de67cb677706f27f Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Sun, 2 Nov 2025 10:25:58 +0000 Subject: [PATCH 25/42] fix: frontend/Dockerfile to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-DEBIAN12-LIBXML2-10350658 - https://snyk.io/vuln/SNYK-DEBIAN12-LIBXML2-10350668 - https://snyk.io/vuln/SNYK-DEBIAN12-EXPAT-6227597 - https://snyk.io/vuln/SNYK-DEBIAN12-EXPAT-9407503 - https://snyk.io/vuln/SNYK-DEBIAN12-TIFF-13019601 --- frontend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100755 => 100644 frontend/Dockerfile diff --git a/frontend/Dockerfile b/frontend/Dockerfile old mode 100755 new mode 100644 index 20051475..5df910ee --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -27,7 +27,7 @@ COPY ./*.json /usr/src/app/ RUN npm run build --loglevel verbose 2>&1 # Production environment -FROM nginx:1.29.0 +FROM nginx:1.29.3 RUN mkdir -p /usr/share/nginx/html/build RUN mkdir -p /usr/share/nginx/html/css From 222899f1895e6261b37e3f03a90e75762cb4780a Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Tue, 4 Nov 2025 01:14:40 +0530 Subject: [PATCH 26/42] fix: delete API when data has no category --- frontend/src/components/CacheView.jsx | 311 +++++++++++++------------- 1 file changed, 157 insertions(+), 154 deletions(-) diff --git a/frontend/src/components/CacheView.jsx b/frontend/src/components/CacheView.jsx index 3c19da36..564af782 100644 --- a/frontend/src/components/CacheView.jsx +++ b/frontend/src/components/CacheView.jsx @@ -38,8 +38,8 @@ import { Avatar, } from "@mui/material"; -import { - DataGrid, +import { + DataGrid, GridColDef, } from '@mui/x-data-grid'; @@ -70,15 +70,15 @@ import { VisibilityOff as VisibilityOffIcon, Clear as ClearIcon, Add as AddIcon, - Rocket as RocketIcon, - Webhook as WebhookIcon, - Air as AirIcon, - RocketLaunch as RocketLaunchIcon, + Rocket as RocketIcon, + Webhook as WebhookIcon, + Air as AirIcon, + RocketLaunch as RocketLaunchIcon, Send as SendIcon, SmartToy as SmartToyIcon, Settings as SettingsIcon, FilterAlt as FilterAltIcon, - CompareArrows as CompareArrowsIcon, + CompareArrows as CompareArrowsIcon, } from "@mui/icons-material"; import { Context } from "../context/ContextApi.jsx"; @@ -104,7 +104,7 @@ const useStyles = makeStyles({ }, }); -// +// //const CacheView = (props) => { const CacheView = memo((props) => { @@ -130,7 +130,7 @@ const CacheView = memo((props) => { const [cursors, setCursors] = useState({ 0: "", }) - const [_, setUpdate] = useState(Math.random()) + const [_, setUpdate] = useState(Math.random()) const [selectedRows, setSelectedRows] = useState([]); // Direct category migration from ../components/Files.jsx const [selectAllChecked, setSelectAllChecked] = React.useState(false) @@ -170,7 +170,7 @@ const CacheView = memo((props) => { "key": "workflow_id", "value": "", }], - "icon": , + "icon": , "enabled": false, }, { @@ -203,7 +203,7 @@ const CacheView = memo((props) => { "key": "webhook_url", "value": "", }], - "icon": , + "icon": , "enabled": false, }, @@ -237,7 +237,7 @@ const CacheView = memo((props) => { const [categoryAutomations, setCategoryAutomations] = useState(defaultAutomation) const [categoryConfig, setCategoryConfig] = useState(undefined) - + const { themeMode, brandColor } = useContext(Context); const theme = getTheme(themeMode, brandColor); const classes = useStyles(); @@ -312,9 +312,9 @@ const CacheView = memo((props) => { useEffect(() => { getWorkflows() - getApps() + getApps() - var chosenCategory = selectedCategory + var chosenCategory = selectedCategory const urlParams = new URLSearchParams(window.location.search) const categoryParam = urlParams.get("category") if (categoryParam && categoryParam !== undefined && categoryParam !== "default" && categoryParam !== "") { @@ -335,7 +335,7 @@ const CacheView = memo((props) => { if (event.key === 'Escape'){ // not working for some reasons console.log('escape pressed') - setRenderTextBox(false); + setRenderTextBox(false); } } @@ -359,7 +359,7 @@ const CacheView = memo((props) => { if (page !== undefined && page !== null && page >= 0) { if (cursors[page-1] !== undefined && cursors[page-1] !== null && cursors[page-1] !== "") { - url += "&cursor=" + cursors[page-1] + url += "&cursor=" + cursors[page-1] } } @@ -405,8 +405,8 @@ const CacheView = memo((props) => { } // Especially important during first load - if (index < 2 && (category === "default" || category === "" || category === undefined)) { - // If it exists and isn't blank/default, load it + if (index < 2 && (category === "default" || category === "" || category === undefined)) { + // If it exists and isn't blank/default, load it const urlParams = new URLSearchParams(window.location.search); const categoryParam = urlParams.get("category"); if (categoryParam && categoryParam !== undefined && categoryParam !== "default" && categoryParam !== "") { @@ -414,7 +414,7 @@ const CacheView = memo((props) => { if (index === undefined || index === null) { index = 0 } - + listOrgCache(orgId, categoryParam, index+1, amount, page) } else { setSelectedCategory("default"); @@ -478,10 +478,10 @@ const CacheView = memo((props) => { } else { setCategoryAutomations(defaultAutomation) } - } + } } else { //toast.warn("Failed to load keys. Please try again or contact support@shuffler if this persists.") - + if (category !== undefined && category !== null && category !== "" && category !== "default") { toast.info(`No keys to load in category ${category}`) setSelectedCategory(category) @@ -495,6 +495,9 @@ const CacheView = memo((props) => { const deleteEntry = (orgId, key, itemCategory, refreshList) => { + + console.log(`Deleting entry for orgId: ${orgId}, key: ${key}, category: ${itemCategory}`); + const method = "POST" const url = `${globalUrl}/api/v1/orgs/${orgId}/delete_cache` var parsed = { @@ -534,8 +537,8 @@ const CacheView = memo((props) => { }; const editOrgCache = (orgId) => { - var entry = { - key: dataValue.key, + var entry = { + key: dataValue.key, value: value, category: selectedCategory, } @@ -587,10 +590,10 @@ const CacheView = memo((props) => { }; const addOrgCache = (orgId) => { - const cache = { - key: key, + const cache = { + key: key, value: value, - category: selectedCategory, + category: selectedCategory, } setCacheInput([cache]); @@ -709,7 +712,7 @@ const CacheView = memo((props) => { fullWidth={true} autoComplete="Key" placeholder="abc" - id="keyfield" + id="keyfield" margin="normal" variant="outlined" value={editCache ? dataValue.key : key} @@ -728,7 +731,7 @@ const CacheView = memo((props) => { autoFixJson(value) }} > - +
@@ -758,7 +761,7 @@ const CacheView = memo((props) => { /> - {editCache ? + {editCache ?
Created: {timestamp(dataValue?.created)} @@ -841,7 +844,7 @@ const CacheView = memo((props) => { } }; - const changeDistribution = (id, selectedSubOrg) => { + const changeDistribution = (id, selectedSubOrg) => { editFileConfig(id, [...new Set(selectedSubOrg)], selectedCategory) } @@ -855,8 +858,8 @@ const CacheView = memo((props) => { category: category === undefined || category === "" || category === "default" ? "" : category, } - console.log("data: ", data); - + console.log("data: ", data); + const url = `${globalUrl}/api/v1/orgs/${orgId}/cache/config`; fetch(url, { @@ -989,7 +992,7 @@ const CacheView = memo((props) => { const originalIcon = automation.icon if (typeof automation.icon !== "string") { automation.icon = ""; - } + } }) const url = `${globalUrl}/api/v2/datastore/automate` @@ -999,7 +1002,7 @@ const CacheView = memo((props) => { } if (settings !== undefined && settings !== null && Object.keys(settings).length > 0) { - data["settings"] = settings + data["settings"] = settings } fetch(url, { @@ -1049,7 +1052,7 @@ const CacheView = memo((props) => { } } - const runSave = () => { + const runSave = () => { setUpdated(false) const newAutomations = [...categoryAutomations] @@ -1062,11 +1065,11 @@ const CacheView = memo((props) => { } return ( -
setHovered(true)} onMouseLeave={() => setHovered(false)} > @@ -1085,15 +1088,15 @@ const CacheView = memo((props) => { setUpdatedAutomation(updatedAutomation) setUpdated(true) - setUpdate(Math.random()) + setUpdate(Math.random()) }} /> -
{ @@ -1109,30 +1112,30 @@ const CacheView = memo((props) => { } }} > - {typeof automation?.icon === "string" && automation?.icon?.length > 0 ? + {typeof automation?.icon === "string" && automation?.icon?.length > 0 ? {automation.name} : automation.icon } - {automation.name}
- {automation?.disabled !== true ? + {automation?.disabled !== true ? - + : null}
@@ -1957,14 +1960,14 @@ const CacheView = memo((props) => { style={{textTransform: isSelectedDataStore ? 'capitalize':null, width : isSelectedDataStore ? 120 : null, height:isSelectedDataStore?40:null}} variant="contained" color="primary" - onClick={() =>{ + onClick={() =>{ setEditCache(false) setModalOpen(true) setValue("") }} > - Add Key + Add Key - + } {renderTextBox && { const foundValue = event.target.value.trim(); if(event.key === 'Enter' && foundValue?.length > 0){ setUpdateToThisCategory(event.target.value) - + listOrgCache(orgId, event.target.value, 0, pageSize, 0) setPage(0) } - + }} style={{ height: 35, @@ -2151,10 +2154,10 @@ const CacheView = memo((props) => { - + - + - {showAutomationMenu || showSettingsMenu ? + {showAutomationMenu || showSettingsMenu ? { @@ -2227,7 +2230,7 @@ const CacheView = memo((props) => { - {showSettingsMenu === true ? + {showSettingsMenu === true ?
@@ -2237,7 +2240,7 @@ const CacheView = memo((props) => {
- Timeout + Timeout You can set a timeout for the category. This will delete all keys in this category after the specified time. Timeout is in seconds and based on last EDITED time. @@ -2266,7 +2269,7 @@ const CacheView = memo((props) => { toast.info("Timeout must be a number. Setting to 0.") } else { timeoutValue = parseInt(e.target.value, 10) - if (timeoutValue < 60) { + if (timeoutValue < 60) { toast.info("Timeout must be between 60 seconds or more. Setting to 0.") } @@ -2283,7 +2286,7 @@ const CacheView = memo((props) => {
- {categoryConfig?.settings?.public === true ? "" : "NOT"} Public + {categoryConfig?.settings?.public === true ? "" : "NOT"} Public This will make the url for this category public. Metadata will be cleared, except for timestamps. Types: keys,ndjson,csv,values,json,meta @@ -2330,7 +2333,7 @@ const CacheView = memo((props) => { When - A key is edited + A key is edited @@ -2339,16 +2342,16 @@ const CacheView = memo((props) => { {categoryAutomations.map((automation, index) => { return ( - + automation={automation} + index={index} + /> ) })}
} - +
: null} @@ -2374,17 +2377,17 @@ const CacheView = memo((props) => { setSelectedRows(newSelection); }} keepNonExistentRowsSelected={false} - getRowId={(row) => `${row?.key}_${row?.category}`} + getRowId={(row) => row?.category ? `${row.key}_${row.category}` : row.key} autoHeight={true} sx={{ - marginTop: 1, + marginTop: 1, height: listCache.length*52+500, width: "100%", '.MuiTablePagination-selectLabel, .MuiTablePagination-select, .MuiTablePagination-selectIcon': { display: 'none', }, - marginBottom: 20, + marginBottom: 20, }} loading={cachedLoaded === false} @@ -2404,7 +2407,7 @@ const CacheView = memo((props) => { setCursors({ 0: "", - }) + }) }} @@ -2441,20 +2444,20 @@ const CacheView = memo((props) => { }} >
{page * pageSize + 1} - {Math.min((page + 1) * pageSize, totalAmount)} of {totalAmount} - + { var disabled = false if (item?.type === "page") { if (cursors[item.page-1] === undefined) { - disabled = true + disabled = true } } if (item?.type === "previous") { @@ -2470,7 +2473,7 @@ const CacheView = memo((props) => { } if (cachedLoaded === false) { - disabled = true + disabled = true } return ( @@ -2530,10 +2533,10 @@ const CacheView = memo((props) => { variant={"outlined"} color="secondary" > - Delete {selectedRows.length} Key{ selectedRows.length > 1 ? "s" : "" } From 8b924f77b244123f8611ca7fa0eebeef2522a6fd Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 5 Nov 2025 21:25:42 +0530 Subject: [PATCH 27/42] chore: update shuffle-shared to v0.9.49 --- backend/go-app/go.mod | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 2959b662..14a5de26 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -4,7 +4,7 @@ go 1.24.0 toolchain go1.24.3 -//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared //replace github.com/frikky/schemaless => ../../../schemaless @@ -26,8 +26,8 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.46 - github.com/shuffle/singul v0.0.19 + github.com/shuffle/shuffle-shared v0.9.49 + github.com/shuffle/singul v0.0.20 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 google.golang.org/grpc v1.72.2 @@ -75,7 +75,7 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frikky/schemaless v0.0.23 // indirect + github.com/frikky/schemaless v0.0.24 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect From 1ec03ccf72de1a4a6ab76878d953a9ba448c1c69 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 5 Nov 2025 21:37:55 +0530 Subject: [PATCH 28/42] chore: update shuffle-shared to v0.9.50 --- backend/go-app/go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 14a5de26..b58e87e5 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -4,7 +4,7 @@ go 1.24.0 toolchain go1.24.3 -replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared //replace github.com/frikky/schemaless => ../../../schemaless @@ -26,7 +26,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.49 + github.com/shuffle/shuffle-shared v0.9.50 github.com/shuffle/singul v0.0.20 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 From 25fcb245fad54c3f4ea746f68287831b6fbb7bc7 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 5 Nov 2025 21:43:03 +0530 Subject: [PATCH 29/42] chore: update shuffle-shared to v0.9.51 --- backend/go-app/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index b58e87e5..02239000 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -26,7 +26,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.50 + github.com/shuffle/shuffle-shared v0.9.51 github.com/shuffle/singul v0.0.20 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 From 2077e8beef115ae3de05c2c316fe3209663f0c59 Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 5 Nov 2025 23:51:58 +0100 Subject: [PATCH 30/42] Added env SHUFFLE_PIPELINE_STANDALONE=true to allow for use of third-party tenzir nodes instead of local --- functions/onprem/orborus/go.mod | 7 ++-- functions/onprem/orborus/go.sum | 51 ++++++++++------------------- functions/onprem/orborus/orborus.go | 24 ++++++++++++-- 3 files changed, 42 insertions(+), 40 deletions(-) diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index aebad855..36ff423a 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.30 + github.com/shuffle/shuffle-shared v0.9.51 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 ) @@ -58,7 +58,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/kin-openapi v0.42.0 // indirect - github.com/frikky/schemaless v0.0.20 // indirect + github.com/frikky/schemaless v0.0.24 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect @@ -93,8 +93,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/opensearch-project/opensearch-go v1.1.0 // indirect - github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect github.com/osteele/liquid v1.7.0 // indirect github.com/osteele/tuesday v1.0.3 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect @@ -105,6 +103,7 @@ require ( github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/sendgrid/sendgrid-go v3.16.1+incompatible // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/shuffle/opensearch-go/v4 v4.0.0 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 29bf0bc2..8f77fe56 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -70,20 +70,6 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/aws/aws-sdk-go v1.42.27/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= -github.com/aws/aws-sdk-go v1.44.263/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4= -github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf h1:TqhNAT4zKbTdLa62d2HDBFdvgSbIGB3eJE8HqhgiL9I= github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf/go.mod h1:r5xuitiExdLAJ09PR7vBVENGvp4ZuTBeWTGtxuX3K+c= github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 h1:/P9/RL0xgWE+ehnCUUN5h3RpG3dmoMCOONO1CCvq23Y= @@ -142,8 +128,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.20 h1:S/A2pQcRN9qa2RnufvxwCeM06trjG0JLTF3urt1tFQI= -github.com/frikky/schemaless v0.0.20/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= +github.com/frikky/schemaless v0.0.24 h1:1bNg96gm82yLbRlDS0haQH/96X3GlCAEuTCSasXIJZE= +github.com/frikky/schemaless v0.0.24/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -209,7 +195,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -246,8 +231,6 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -295,10 +278,6 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/opensearch-project/opensearch-go v1.1.0 h1:eG5sh3843bbU1itPRjA9QXbxcg8LaZ+DjEzQH9aLN3M= -github.com/opensearch-project/opensearch-go v1.1.0/go.mod h1:+6/XHCuTH+fwsMJikZEWsucZ4eZMma3zNSeLrTtVGbo= -github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ= -github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8= github.com/osteele/liquid v1.7.0 h1:VsbPSchE5D5S5scylAIvERET4dnCxsO6IDri2oSJ5Dk= github.com/osteele/liquid v1.7.0/go.mod h1:xU0Z2dn2hOQIEFEWNmeltOmCtfhtoW/2fCyiNQeNG+U= github.com/osteele/tuesday v1.0.3 h1:SrCmo6sWwSgnvs1bivmXLvD7Ko9+aJvvkmDjB5G4FTU= @@ -328,8 +307,10 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.9.30 h1:3CYvNyD7sTxdxoZjTVrtaDqFvSWQRKAFGaga6rPGf8A= -github.com/shuffle/shuffle-shared v0.9.30/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw= +github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU= +github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0= +github.com/shuffle/shuffle-shared v0.9.51 h1:jOlFr0cdhPkJxRxlFHddBSIucm62T8e2DVM2PGDkahY= +github.com/shuffle/shuffle-shared v0.9.51/go.mod h1:g03P/ZM8pUwBbdxW0l9A1siMMuemEQshgDpO9hF23eE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -354,10 +335,19 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/wI2L/jsondiff v0.7.0 h1:1lH1G37GhBPqCfp/lrs91rf/2j3DktX6qYAKZkLuCQQ= +github.com/wI2L/jsondiff v0.7.0/go.mod h1:KAEIojdQq66oJiHhDyQez2x+sRit0vIzC9KeK0yizxM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= @@ -452,9 +442,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= @@ -497,13 +485,11 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= @@ -515,7 +501,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index e2dfac44..84624a1b 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -115,8 +115,8 @@ var memcached = os.Getenv("SHUFFLE_MEMCACHED") var queuePerMinute = os.Getenv("SHUFFLE_EXECUTION_PER_MINIUTE") var queuePerMinuteInt int -// For it to download from Sigma? -var pipelineApikey = os.Getenv("SHUFFLE_PIPELINE_AUTH") +// Used to download file categories. Not required since 2.1.1 +var pipelineApikey = "" var pipelineUrl = os.Getenv("SHUFFLE_PIPELINE_URL") var executionIds = []string{} @@ -2182,6 +2182,18 @@ func main() { return } + if os.Getenv("SHUFFLE_PIPELINE_STANDALONE") == "true" { + log.Printf("[INFO] Allowing use of standalone pipeline (tenzir). URL: %s", pipelineUrl) + + //if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "false" { + // os.Setenv("SHUFFLE_SKIP_PIPELINES", "true") + //} + + //if os.Getenv("SHUFFLE_PIPELINE_ENABLED") == "true" { + // os.Setenv("SHUFFLE_PIPELINE_ENABLED", "false") + //} + } + // Block until a signal is received if shuffle.IsRunningInCluster() { log.Printf("[INFO] Running inside k8s cluster") @@ -3003,6 +3015,7 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { } func deployTenzirNode() error { + // Disabled all pipeline features if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "false" || os.Getenv("SHUFFLE_PIPELINE_ENABLED") == "true" { // return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") //log.Printf("[INFO] Pipelines are enabled by user") @@ -3010,8 +3023,13 @@ func deployTenzirNode() error { return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") } + // Specifically for standalone tenzir + if os.Getenv("SHUFFLE_PIPELINE_STANDALONE") == "true" { + return nil + } + if isKubernetes == "true" { - return errors.New("Kubernetes not implemented for Tenzir node") + return errors.New("Tenzir not implemented for k8s") } err := checkTenzirNode() From 438b1531f85aebaf1f683419c4c4fc3a19bf2fd6 Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 7 Nov 2025 16:24:45 +0100 Subject: [PATCH 31/42] Removed static ip for tenzir and instead rely on dhcp in predefined network --- functions/onprem/orborus/orborus.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 84624a1b..0c5c81fc 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -3235,12 +3235,21 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri hostConfig.Mounts = []mount.Mount{} } + //networkingConfig := &network.NetworkingConfig{ + // EndpointsConfig: map[string]*network.EndpointSettings{ + // "tenzir-network": { + // IPAMConfig: &network.EndpointIPAMConfig{ + // IPv4Address: "192.168.102.100", + // }, + // }, + // }, + //} + networkingConfig := &network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ "tenzir-network": { - IPAMConfig: &network.EndpointIPAMConfig{ - IPv4Address: "192.168.102.100", - }, + IPAMConfig: nil, + Aliases: []string{"tenzir-node"}, }, }, } From e470dbef7faf48bb6589ed353575e31593e96882 Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 7 Nov 2025 16:28:43 +0100 Subject: [PATCH 32/42] Minor tweak to tcp workflow --- functions/onprem/orborus/orborus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 0c5c81fc..1d9d8fed 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -3304,7 +3304,7 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri log.Printf("[INFO] Successfully deployed Tenzir Node! Setting up default syslog listener on TCP/1514 AND UDP/1514") - command := `from "tcp://0.0.0.0:1514" { read_syslog } | import` + command := `load_tcp "0.0.0.0:1514" { read_syslog } | import` _, err = createPipeline(command, "default-syslog-tcp-514") if err != nil { log.Printf("[ERROR] Failed to create tcp syslog pipeline: %s", err) From 2dc1966d08df1b4aa7c2469ec01df89c5bed9841 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Mon, 10 Nov 2025 13:54:14 +0530 Subject: [PATCH 33/42] change the order of stopSchedule --- backend/go-app/walkoff.go | 41 ++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 032af73a..0c015470 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1505,30 +1505,39 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { return } -func deleteSchedule(ctx context.Context, id string) error { - log.Printf("[DEBUG] Should stop schedule %s!", id) +func deleteKeySchedule(ctx context.Context, id string) error { err := shuffle.DeleteKey(ctx, "schedules", id) if err != nil { - log.Printf("[ERROR] Failed to delete schedule: %s", err) return err + } + return nil +} + +func deleteSchedule(ctx context.Context, id string) error { + log.Printf("[DEBUG] Should stop schedule %s!", id) + if value, exists := scheduledJobs[id]; exists { + // Stops the schedule properly + value.Lock() } else { - if value, exists := scheduledJobs[id]; exists { - // Stops the schedule properly - value.Lock() - } else { - // FIXME - allow it to kind of stop anyway? - if j, ok := cronJobs[id]; ok { - err := CronScheduler.RemoveByID(j) - if err != nil { - log.Printf("[ERROR] Failed to remove the scheduler %s", err) - return err - } - } else { - return errors.New("Can't find the schedule.") + // FIXME - allow it to kind of stop anyway? + if j, ok := cronJobs[id]; ok { + err := CronScheduler.RemoveByID(j) + if err != nil { + log.Printf("[ERROR] Failed to remove the scheduler %s", err) + return err } + } else { + // Just stop and delete anyway if not in memory + deleteKeySchedule(ctx, id) + return errors.New("Can't find the schedule.") } } + err := deleteKeySchedule(ctx, id) + if err != nil { + log.Printf("[ERROR] Failed to stop schedule in db %s: %s", id, err) + return err + } return nil } From ff0f0149c24dd2aec082323d76960a1ff9529088 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Mon, 10 Nov 2025 13:58:59 +0530 Subject: [PATCH 34/42] change order of deleteSchedule --- backend/go-app/walkoff.go | 41 ++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 032af73a..0c015470 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1505,30 +1505,39 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { return } -func deleteSchedule(ctx context.Context, id string) error { - log.Printf("[DEBUG] Should stop schedule %s!", id) +func deleteKeySchedule(ctx context.Context, id string) error { err := shuffle.DeleteKey(ctx, "schedules", id) if err != nil { - log.Printf("[ERROR] Failed to delete schedule: %s", err) return err + } + return nil +} + +func deleteSchedule(ctx context.Context, id string) error { + log.Printf("[DEBUG] Should stop schedule %s!", id) + if value, exists := scheduledJobs[id]; exists { + // Stops the schedule properly + value.Lock() } else { - if value, exists := scheduledJobs[id]; exists { - // Stops the schedule properly - value.Lock() - } else { - // FIXME - allow it to kind of stop anyway? - if j, ok := cronJobs[id]; ok { - err := CronScheduler.RemoveByID(j) - if err != nil { - log.Printf("[ERROR] Failed to remove the scheduler %s", err) - return err - } - } else { - return errors.New("Can't find the schedule.") + // FIXME - allow it to kind of stop anyway? + if j, ok := cronJobs[id]; ok { + err := CronScheduler.RemoveByID(j) + if err != nil { + log.Printf("[ERROR] Failed to remove the scheduler %s", err) + return err } + } else { + // Just stop and delete anyway if not in memory + deleteKeySchedule(ctx, id) + return errors.New("Can't find the schedule.") } } + err := deleteKeySchedule(ctx, id) + if err != nil { + log.Printf("[ERROR] Failed to stop schedule in db %s: %s", id, err) + return err + } return nil } From 52fed7105e42e1d6ae8f15b51210beaef574b668 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Tue, 11 Nov 2025 22:43:57 +0530 Subject: [PATCH 35/42] fix: tenzir disable + networkID focused --- functions/onprem/orborus/orborus.go | 57 +++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 1d9d8fed..a0f78845 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -441,12 +441,32 @@ func deployServiceWorkers(image string) { if err != nil { if strings.Contains(fmt.Sprintf("%s", err), "already exists") { // Try patching for attachable - + if debug { + log.Printf("[DEBUG] Network %s already exists", networkName) + } } else { log.Printf("[DEBUG] Failed to create network %s for workers: %s. This is not critical, and containers will still be added", networkName, err) } } + networkID := "" + + // find network ID + networks, err := dockercli.NetworkList(ctx, network.ListOptions{}) + if err == nil { + for _, net := range networks { + if net.Name == networkName { + if net.Scope == "swarm" { + log.Printf("[DEBUG] Found swarm-scoped network: %s (%s)", networkName, net.ID) + networkID = net.ID + } else { + log.Printf("[WARNING] Network %s exists but is not swarm scoped (scope=%s)", networkName, net.Scope) + } + break + } + } + } + /* isMemcachedRunning, err := checkMemcached(ctx, dockercli) if err != nil { @@ -463,12 +483,17 @@ func deployServiceWorkers(image string) { } */ + if networkID == "" { + log.Printf("[ERROR] Network %s does not exist", networkName) + networkID = networkName + } + defaultNetworkAttach := false if containerId != "" { log.Printf("[DEBUG] Should connect orborus container to worker network as it's running in Docker with name %#v!", containerId) // https://pkg.go.dev/github.com/docker/docker@v20.10.12+incompatible/api/types/network#EndpointSettings networkConfig := &network.EndpointSettings{} - err := dockercli.NetworkConnect(ctx, networkName, containerId, networkConfig) + err := dockercli.NetworkConnect(ctx, networkID, containerId, networkConfig) if err != nil { log.Printf("[ERROR] Failed connecting Orborus to docker network %s: %s", networkName, err) } @@ -491,7 +516,7 @@ func deployServiceWorkers(image string) { for _, container := range containers { if strings.Contains(strings.ToLower(container.Image), "docker-socket-proxy") { networkConfig := &network.EndpointSettings{} - err := dockercli.NetworkConnect(ctx, networkName, container.ID, networkConfig) + err := dockercli.NetworkConnect(ctx, networkID, container.ID, networkConfig) if err != nil { log.Printf("[ERROR] Failed connecting Docker socket proxy to docker network %s: %s", networkName, err) } else { @@ -571,7 +596,7 @@ func deployServiceWorkers(image string) { }, Networks: []swarm.NetworkAttachmentConfig{ swarm.NetworkAttachmentConfig{ - Target: networkName, + Target: networkID, }, swarm.NetworkAttachmentConfig{ Target: "ingress", @@ -2109,12 +2134,12 @@ func cleanup() { func StartAgent() { log.Printf("[INFO] Starting Orborus agent mode") - + auditLogEnabled := os.Getenv("SHUFFLE_AUDIT_LOG_ENABLED") == "true" - + if auditLogEnabled { log.Printf("[INFO] Audit log monitoring is enabled") - + // Initialize telemetry configuration telemetryConfig := shuffle.TelemetryConfig{ Enabled: true, @@ -2130,7 +2155,7 @@ func StartAgent() { Exclude: patterns, }) } - + if includePatterns := os.Getenv("SHUFFLE_AUDIT_LOG_INCLUDE"); includePatterns != "" { patterns := strings.Split(includePatterns, ",") telemetryConfig.Filters = append(telemetryConfig.Filters, shuffle.TelemetryFilter{ @@ -2151,7 +2176,7 @@ func StartAgent() { sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - + go func() { <-sigChan log.Printf("[INFO] Received shutdown signal, stopping audit log collector...") @@ -2184,7 +2209,7 @@ func main() { if os.Getenv("SHUFFLE_PIPELINE_STANDALONE") == "true" { log.Printf("[INFO] Allowing use of standalone pipeline (tenzir). URL: %s", pipelineUrl) - + //if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "false" { // os.Setenv("SHUFFLE_SKIP_PIPELINES", "true") //} @@ -3016,10 +3041,7 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { func deployTenzirNode() error { // Disabled all pipeline features - if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "false" || os.Getenv("SHUFFLE_PIPELINE_ENABLED") == "true" { - // return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") - //log.Printf("[INFO] Pipelines are enabled by user") - } else { + if os.Getenv("SHUFFLE_SKIP_PIPELINES") != "true" { return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") } @@ -3249,7 +3271,7 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri EndpointsConfig: map[string]*network.EndpointSettings{ "tenzir-network": { IPAMConfig: nil, - Aliases: []string{"tenzir-node"}, + Aliases: []string{"tenzir-node"}, }, }, } @@ -3909,7 +3931,10 @@ func sendPipelineHealthStatus() (shuffle.LakeConfig, error) { } else { //tenzirDisabled = true - log.Printf("[WARNING] Disabling pipelines: %s. You will need to restart the Orborus to fix this.", err) + if debug { + log.Printf("[WARNING] Disabling pipelines: %s. You will need to restart the Orborus to fix this.", err) + } + } return pipelinePayload, err From 697632c916efea079ee758d00a1f765ddc272e6e Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 12 Nov 2025 02:13:16 +0530 Subject: [PATCH 36/42] fix: patch network on the fly for workers --- functions/onprem/orborus/orborus.go | 30 ++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index a0f78845..b5bcf486 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -765,7 +765,36 @@ func deployServiceWorkers(image string) { if err == nil { log.Printf("[DEBUG] Successfully deployed workers with %d replica(s) on %d node(s)", replicas, cnt) + // wait for service to be ready + time.Sleep(time.Duration(rand.Intn(4)+1) * time.Second) + //log.Printf("[DEBUG] Servicecreate request: %#v %#v", service, err) + // patch service network + // this is an edgecase that we noticed on docker version 29 + // and API version 1.44 + services, serr := dockercli.ServiceList(ctx, types.ServiceListOptions{}) + if serr == nil { + for _, svc := range services { + if svc.Spec.Annotations.Name == innerContainerName { + log.Printf("[DEBUG] Found service %s (%s) — patching network attach", innerContainerName, svc.ID) + + spec := svc.Spec + spec.TaskTemplate.Networks = append(spec.TaskTemplate.Networks, swarm.NetworkAttachmentConfig{ + Target: networkID, + }) + + _, uerr := dockercli.ServiceUpdate(ctx, svc.ID, svc.Version, spec, types.ServiceUpdateOptions{}) + if uerr != nil { + log.Printf("[WARNING] Failed to patch service %s with network %s: %v", innerContainerName, networkID, uerr) + } else { + log.Printf("[INFO] Successfully attached network %s to service %s", networkID, innerContainerName) + } + break + } + } + } else { + log.Printf("[WARNING] Failed to list services for patching network attach: %v", serr) + } } else { if !strings.Contains(fmt.Sprintf("%s", err), "Already Exists") && !strings.Contains(fmt.Sprintf("%s", err), "is already in use by service") { log.Printf("[ERROR] Failed making service: %s", err) @@ -3926,7 +3955,6 @@ func sendPipelineHealthStatus() (shuffle.LakeConfig, error) { err := deployTenzirNode() if err != nil { if (!strings.Contains(err.Error(), "SHUFFLE_SKIP_PIPELINES") && !strings.Contains(err.Error(), "Kubernetes not implemented for Tenzir node")) && !strings.Contains(err.Error(), "Tenzir Node is already running") && !strings.Contains(err.Error(), "docker daemon") { - log.Printf("[ERROR] Tenzir node connection problem: %s", err) } else { From 7be4e6adf5d5dd05efca58114c195ab9a707188d Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 12 Nov 2025 02:48:32 +0530 Subject: [PATCH 37/42] fix: shuffle swarm network bug where it wasn't getting attached in swarm --- functions/onprem/worker/worker.go | 58 ++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 5a858888..c0596a8a 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -3454,7 +3454,6 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl _ = service if err != nil { - if strings.Contains(fmt.Sprintf("%s", err), "network") && strings.Contains(fmt.Sprintf("%s", err), "not found") { log.Printf("[DEBUG] Network %s not found. Trying to initialize it.", networkName) networkErr := initSwarmNetwork() @@ -3481,6 +3480,63 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl log.Printf("[DEBUG] Failed deploying %s with image %s: %s", name, image, err) return err + } else { + // wait for service to be ready + time.Sleep(time.Duration(rand.Intn(4)+1) * time.Second) + //log.Printf("[DEBUG] Servicecreate request: %#v %#v", service, err) + // patch service network + // this is an edgecase that we noticed on docker version 29 + // and API version 1.44 + + // get networkID of swarmNetworkName + networkID := "" + + ctx := context.Background() + + // find network ID + networks, err := dockercli.NetworkList(ctx, network.ListOptions{}) + if err == nil { + for _, net := range networks { + if net.Name == networkName { + if net.Scope == "swarm" { + log.Printf("[DEBUG] Found swarm-scoped network: %s (%s)", networkName, net.ID) + networkID = net.ID + } else { + log.Printf("[WARNING] Network %s exists but is not swarm scoped (scope=%s)", networkName, net.Scope) + } + break + } + } + } + + if networkID == "" { + log.Printf("[ERROR] Network %s not found", networkName) + networkID = networkName + } + + services, serr := dockercli.ServiceList(ctx, types.ServiceListOptions{}) + if serr == nil { + for _, svc := range services { + if svc.ID == service.ID { + log.Printf("[DEBUG] Found service %s (%s) — patching network attach", service.ID, svc.ID) + + spec := svc.Spec + spec.TaskTemplate.Networks = append(spec.TaskTemplate.Networks, swarm.NetworkAttachmentConfig{ + Target: networkID, + }) + + _, uerr := dockercli.ServiceUpdate(ctx, svc.ID, svc.Version, spec, types.ServiceUpdateOptions{}) + if uerr != nil { + log.Printf("[WARNING] Failed to patch service %s with network %s: %v", service.ID, networkID, uerr) + } else { + log.Printf("[INFO] Successfully attached network %s to service %s", networkID, service.ID) + } + break + } + } + } else { + log.Printf("[WARNING] Failed to list services for patching network attach: %v", serr) + } } log.Printf("[DEBUG] Successfully deployed service %s with image %s on port %d", name, image, deployport) From ff6cad8049d10248264750f6e5749e5dad48102a Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Wed, 12 Nov 2025 10:09:33 +0100 Subject: [PATCH 38/42] k8s: remove SHUFFLE_SWARM_CONFIG env variable from orborus --- .../charts/shuffle/templates/orborus/orborus-dpl.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml index 05ca1ac3..a786dcd2 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml +++ b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml @@ -86,8 +86,6 @@ spec: value: kubernetes - name: IS_KUBERNETES value: "true" - - name: SHUFFLE_SWARM_CONFIG - value: run {{- if .Values.orborus.extraEnvVars }} {{- include "common.tplvalues.render" (dict "value" .Values.orborus.extraEnvVars "context" $) | nindent 12 }} {{- end }} From 182a6a363fb3d6c6dd6fbe6248caf75dabd6cf88 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 12 Nov 2025 15:03:15 +0530 Subject: [PATCH 39/42] chore: bump shuffle-shared version --- backend/go-app/go.mod | 2 +- functions/onprem/orborus/go.mod | 2 +- functions/onprem/worker/go.mod | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 02239000..3e8a4c0d 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -26,7 +26,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.51 + github.com/shuffle/shuffle-shared v0.9.52 github.com/shuffle/singul v0.0.20 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 36ff423a..67e428c7 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.51 + github.com/shuffle/shuffle-shared v0.9.52 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 ) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 085375c5..1227cfba 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -11,7 +11,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.39 + github.com/shuffle/shuffle-shared v0.9.52 github.com/shuffle/singul v0.0.19 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 @@ -60,7 +60,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/kin-openapi v0.42.0 // indirect - github.com/frikky/schemaless v0.0.23 // indirect + github.com/frikky/schemaless v0.0.24 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect From 48f19db0a6ab45efaba926ada58f38b6472f6e95 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 12 Nov 2025 15:09:25 +0530 Subject: [PATCH 40/42] singul version bump --- functions/onprem/worker/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 1227cfba..f6d2762b 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 github.com/shuffle/shuffle-shared v0.9.52 - github.com/shuffle/singul v0.0.19 + github.com/shuffle/singul v0.0.20 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 k8s.io/client-go v0.33.1 From 822e7c1eb33ecf3b6e89271bc336e01d91b64070 Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 12 Nov 2025 11:39:18 +0100 Subject: [PATCH 41/42] Tons of minor fixes to the based on cloud tests --- frontend/src/components/AdminNavBar.jsx | 59 +- frontend/src/components/ApiExplorer.jsx | 2 +- frontend/src/components/AppGrid.jsx | 2 +- frontend/src/components/Appsearch.jsx | 8 +- frontend/src/components/Billing.jsx | 2 +- frontend/src/components/CacheView.jsx | 1155 +++++++++-------- frontend/src/components/ChatBot.jsx | 653 ++++++++-- frontend/src/components/CloudSyncTab.jsx | 12 +- frontend/src/components/CreatorGrid.jsx | 2 +- frontend/src/components/DocsGrid.jsx | 2 +- frontend/src/components/EnvironmentTab.jsx | 24 +- frontend/src/components/ExecutionPanel.jsx | 2 +- frontend/src/components/LeftSideBar.jsx | 12 +- .../src/components/PartnersUsecasesTab.jsx | 24 +- frontend/src/components/RunDetectionTest.jsx | 33 +- .../src/components/RunsOverTimeWidget.jsx | 305 ++++- frontend/src/components/RuntimeDebugger.jsx | 5 +- frontend/src/components/SchedulesTab.jsx | 96 +- .../src/components/ShuffleCodeEditor1.jsx | 6 +- .../components/SuccessFailedRunsWidget.jsx | 483 +++++-- frontend/src/components/WorkflowGrid.jsx | 2 +- .../components/WorkflowValidationTimeline.jsx | 2 +- frontend/src/defaultCytoscapeStyle.jsx | 93 +- frontend/src/views/Admin.jsx | 12 +- frontend/src/views/Admin2.jsx | 54 +- frontend/src/views/AgentUI.jsx | 412 +++++- frontend/src/views/AngularWorkflow.jsx | 371 +++--- frontend/src/views/ApiExplorerWrapper.jsx | 6 +- frontend/src/views/AppCreator.jsx | 6 +- frontend/src/views/AppExplorer.jsx | 4 +- frontend/src/views/Docs.jsx | 172 ++- frontend/src/views/LoginPage.jsx | 2 +- frontend/src/views/NewDashboard.jsx | 213 ++- frontend/src/views/RunWorkflow.jsx | 173 ++- frontend/src/views/Workflows.jsx | 576 -------- frontend/src/views/Workflows2.jsx | 250 +++- 36 files changed, 3353 insertions(+), 1882 deletions(-) diff --git a/frontend/src/components/AdminNavBar.jsx b/frontend/src/components/AdminNavBar.jsx index 9ebea75b..f32a284f 100644 --- a/frontend/src/components/AdminNavBar.jsx +++ b/frontend/src/components/AdminNavBar.jsx @@ -23,7 +23,7 @@ import { } from '@mui/icons-material'; import theme, { getTheme } from '../theme.jsx'; -import { Button, Skeleton, Tooltip } from '@mui/material'; +import { Box, Button, Skeleton, Tooltip, Typography } from '@mui/material'; import { Index } from 'react-instantsearch-dom'; import { Context } from '../context/ContextApi.jsx'; import { toast } from 'react-toastify'; @@ -265,7 +265,7 @@ const AdminNavBar = (props) => { : selectedOrganization?.image; return ( - !isOrgLoaded && !isUserDataLoaded ? : + !isOrgLoaded || !isUserDataLoaded ? :