diff --git a/.github/workflows/helm-release.yml b/.github/workflows/helm-release.yml index 4c3c6b6f..153bb9ee 100644 --- a/.github/workflows/helm-release.yml +++ b/.github/workflows/helm-release.yml @@ -24,13 +24,13 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Install apt dependencies - run: | - curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null - sudo apt-get install apt-transport-https -y --no-install-recommends - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list - sudo apt-get update - sudo apt-get install helm -y --no-install-recommends + - name: Setup Helm + uses: azure/setup-helm@v4 + with: + version: v3.18.4 # I run it locally so I know it works :b + + - name: Verify Helm + run: helm version - name: Set versions run: | diff --git a/.github/workflows/helm-test.yml b/.github/workflows/helm-test.yml index fb0e34aa..49c20240 100644 --- a/.github/workflows/helm-test.yml +++ b/.github/workflows/helm-test.yml @@ -18,9 +18,9 @@ jobs: - name: Install apt dependencies run: | - curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null sudo apt-get install apt-transport-https -y --no-install-recommends - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list + curl -fsSL https://packages.buildkite.com/helm-linux/helm-debian/gpgkey | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null + echo "deb [signed-by=/usr/share/keyrings/helm.gpg] https://packages.buildkite.com/helm-linux/helm-debian/any/ any main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list sudo apt-get update sudo apt-get install helm -y --no-install-recommends diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 4896209c..1711153b 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -4,7 +4,9 @@ go 1.24.0 toolchain go1.24.3 +//replace github.com/shuffle/opensearch-go => ../../../opensearch-go //replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared + //replace github.com/frikky/schemaless => ../../../schemaless //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi @@ -24,8 +26,8 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.25 - github.com/shuffle/singul v0.0.16 + github.com/shuffle/shuffle-shared v0.9.34 + github.com/shuffle/singul v0.0.17 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 google.golang.org/grpc v1.72.2 @@ -73,7 +75,7 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frikky/schemaless v0.0.20 // indirect + github.com/frikky/schemaless v0.0.22 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index dc8366b0..d7ba4a0e 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -152,8 +152,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.20 h1:S/A2pQcRN9qa2RnufvxwCeM06trjG0JLTF3urt1tFQI= -github.com/frikky/schemaless v0.0.20/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= +github.com/frikky/schemaless v0.0.22 h1:aMc7cc/lr1zpogjGWbY0j6J2f6QqyfPbbW6Y9JgTAqE= +github.com/frikky/schemaless v0.0.22/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= github.com/fsouza/go-dockerclient v1.12.1 h1:FMoLq+Zhv9Oz/rFmu6JWkImfr6CBgZOPcL+bHW4gS0o= github.com/fsouza/go-dockerclient v1.12.1/go.mod h1:OqsgJJcpCwqyM3JED7TdfM9QVWS5O7jSYwXxYKmOooY= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= @@ -363,10 +363,10 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.9.21 h1:ZxA/g0Lk93miHJpVYAhtTQmtWLJWcCtm6kFNF772Ynk= -github.com/shuffle/shuffle-shared v0.9.21/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw= -github.com/shuffle/singul v0.0.16 h1:dW+0Mln9R1aUJ0fjikpWcxbjoQWqJHxe4kSxh2tQN5E= -github.com/shuffle/singul v0.0.16/go.mod h1:LYkp320A6gsoPlYbXUM+WvEPUVAuutlSsqnVKyRy4gs= +github.com/shuffle/shuffle-shared v0.9.34 h1:4RtGRMoYtBQia2CLbuH7aPFM+EMFXbH0ccIQsBXs8n8= +github.com/shuffle/shuffle-shared v0.9.34/go.mod h1:WbLPLyBrh86ieRBgDyjC43M2qBF8a7E8jO8Pv7t8SOk= +github.com/shuffle/singul v0.0.17 h1:mxaPtj6z85Nf6tl7L2gwDliTfEZtRQqApuu9iKcP75o= +github.com/shuffle/singul v0.0.17/go.mod h1:8c42n1NahhCIPxzLxwp9eYbWkvY4+ct0jfbhkRsRRsY= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 38752409..633cd6f4 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5,6 +5,8 @@ import ( "github.com/shuffle/shuffle-shared" "github.com/shuffle/singul/pkg" + "net/http/pprof" + "archive/zip" "bufio" "bytes" @@ -46,6 +48,7 @@ import ( newscheduler "github.com/carlescere/scheduler" "golang.org/x/crypto/bcrypt" "gopkg.in/yaml.v3" + "sort" // Web "github.com/gorilla/mux" @@ -67,11 +70,13 @@ var debug = false //var syncUrl = "http://localhost:5002" type retStruct struct { - Success bool `json:"success"` - SyncFeatures shuffle.SyncFeatures `json:"sync_features"` - SessionKey string `json:"session_key"` - IntervalSeconds int64 `json:"interval_seconds"` - Reason string `json:"reason"` + Success bool `json:"success"` + SyncFeatures shuffle.SyncFeatures `json:"sync_features"` + SessionKey string `json:"session_key"` + IntervalSeconds int64 `json:"interval_seconds"` + Reason string `json:"reason"` + Subscriptions []shuffle.PaymentSubscription `json:"subscriptions"` + Licensed bool `json:"licensed"` } type Contact struct { @@ -933,6 +938,78 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { log.Printf("[DEBUG] Failed to get org during getinfo: %s", err) } + childOrgs := []shuffle.Org{} + if len(org.CreatorOrg) > 0 { + childOrgs, err = shuffle.GetAllChildOrgs(ctx, org.CreatorOrg) + if err != nil { + log.Printf("[ERROR] Failed to get child orgs during getinfo: %s", err) + childOrgs = []shuffle.Org{} + } + } + + failToLoadOrgs := []string{} + sort.Slice(childOrgs, func(i, j int) bool { + return childOrgs[i].Created < childOrgs[j].Created + }) + + parentOrg := &shuffle.Org{} + if len(org.CreatorOrg) > 0 { + parentOrg, err = shuffle.GetOrg(ctx, org.CreatorOrg) + if err != nil { + log.Printf("[ERROR] Failed to get parent org during getinfo: %s", err) + parentOrg = &shuffle.Org{} + } + licenseOrg := shuffle.HandleCheckLicense(ctx, *parentOrg) + parentOrg = &licenseOrg + } + + limit := 3 + + if parentOrg.Licensed && parentOrg.SyncFeatures.MultiTenant.Active { + limit = int(parentOrg.SyncFeatures.MultiTenant.Limit) + } + + for index, org := range childOrgs { + + if index < limit { + continue + } + + failToLoadOrgs = append(failToLoadOrgs, org.Id) + } + + if len(org.CreatorOrg) > 0 && len(childOrgs) > limit && shuffle.ArrayContains(failToLoadOrgs, userInfo.ActiveOrg.Id) { + + userInfo.ActiveOrg = shuffle.OrgMini{ + Id: parentOrg.Id, + Name: parentOrg.Name, + Role: userInfo.Role, + Branding: parentOrg.Branding, + Image: parentOrg.Image, + } + + if parentOrg.Licensed { + log.Printf("[INFO] Parent org %s is licensed. But Multi-tenant feature is not active. Moving user %s to parent org %s", parentOrg.Name, userInfo.Username) + } else { + log.Printf("[INFO] Parent org %s has more than 3 child orgs and is not licensed. Moving user %s to parent org %s", parentOrg.Name, userInfo.Username) + } + + err = shuffle.SetUser(ctx, &userInfo, false) + if err != nil { + log.Printf("[WARNING] Failed setting user to parent org: %s", err) + } + + reason := "Parent org has more than 3 child orgs and is not licensed. Moving user to parent org. Contact support@shuffler.io for more information" + + if parentOrg.Licensed { + reason = fmt.Sprintf("Parent organization is licensed, but the maximum number of sub-organizations (%d) has been reached. You have been moved to the parent organization. Please contact support@shuffler.io for further assistance.", parentOrg.SyncFeatures.MultiTenant.Limit) + } + + resp.WriteHeader(200) + resp.Write([]byte(`{"success": true, "reason": "` + reason + `", "switch_parent": true}`)) + return + } + //if err == nil { if len(org.Id) > 0 { if userInfo.Role == "" { @@ -1082,7 +1159,8 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { // Check for licensing/branding of parent and override parentOrg, err := shuffle.GetOrg(ctx, parentOrgId) if err == nil { - if parentOrg.LeadInfo.IntegrationPartner { + parent := shuffle.HandleCheckLicense(ctx, *parentOrg) + if parent.SyncFeatures.Branding.Active { parsedStatus = append(parsedStatus, "integration_partner") // except theme take from parent org @@ -1120,11 +1198,15 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { break } } + } else { + userInfo.ActiveOrg.Branding = shuffle.OrgBranding{} } } } else { // for parent org branding - if org.LeadInfo.IntegrationPartner { + licenseOrg := shuffle.HandleCheckLicense(ctx, *org) + org = &licenseOrg + if org.SyncFeatures.Branding.Active { userInfo.ActiveOrg.Branding.Theme = org.Branding.Theme userInfo.ActiveOrg.Branding.DocumentationLink = org.Defaults.DocumentationReference userInfo.ActiveOrg.Branding.SupportEmail = org.Branding.SupportEmail @@ -1133,6 +1215,8 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { userInfo.ActiveOrg.Branding.BrandName = org.Branding.BrandName parsedStatus = append(parsedStatus, "integration_partner") + } else { + userInfo.ActiveOrg.Branding = shuffle.OrgBranding{} } } @@ -1198,7 +1282,6 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { } count := len(users) - if count == 0 { log.Printf("[WARNING] No users - redirecting for management user") resp.WriteHeader(200) @@ -2061,7 +2144,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { } if len(hook.Workflows) == 1 { - workflow, err := shuffle.GetWorkflow(ctx, hook.Workflows[0]) + workflow, err := shuffle.GetWorkflow(ctx, hook.Workflows[0], true) if err == nil { for _, branch := range workflow.Branches { if branch.SourceID == hook.Id { @@ -3816,9 +3899,12 @@ func handleCloudJob(job shuffle.CloudSyncJob) error { // Handles jobs from remote (cloud) func remoteOrgJobController(org shuffle.Org, body []byte) error { type retStruct struct { - Success bool `json:"success"` - Reason string `json:"reason"` - Jobs []shuffle.CloudSyncJob `json:"jobs"` + Success bool `json:"success"` + Reason string `json:"reason"` + Jobs []shuffle.CloudSyncJob `json:"jobs"` + SyncFeatures shuffle.SyncFeatures `json:"sync_features"` + Subscriptions []shuffle.PaymentSubscription `json:"subscriptions"` + Licensed bool `json:"licensed"` } responseData := retStruct{} @@ -3885,6 +3971,31 @@ func remoteOrgJobController(org shuffle.Org, body []byte) error { log.Printf("Got job with reason %s and %d job(s)", responseData.Reason, len(responseData.Jobs)) } + cacheKey := fmt.Sprintf("org_sync_features_%s", org.Id) + featuresBytes, err := json.Marshal(responseData.SyncFeatures) + if err != nil { + log.Printf("[ERROR] Failed to marshal SyncFeatures for cache: %s", err) + } else { + shuffle.SetCache(ctx, cacheKey, featuresBytes, 1800) + } + + subscriptionCacheKey := fmt.Sprintf("org_subscriptions_%s", org.Id) + subscriptionsBytes, err := json.Marshal(responseData.Subscriptions) + if err != nil { + log.Printf("[ERROR] Failed to marshal Subscriptions for cache: %s", err) + } else { + shuffle.SetCache(ctx, subscriptionCacheKey, subscriptionsBytes, 1800) + } + + licenseCacheKey := fmt.Sprintf("org_licensed_%s", org.Id) + licensedBytes, err := json.Marshal(responseData.Licensed) + if err != nil { + log.Printf("[ERROR] Failed to marshal Licensed for cache: %s", err) + } else { + + shuffle.SetCache(ctx, licenseCacheKey, licensedBytes, 1800) + } + for _, job := range responseData.Jobs { err = handleCloudJob(job) if err != nil { @@ -4550,6 +4661,45 @@ func runInitEs(ctx context.Context) { } } + // Self-cleaning + go func() { + cursor := "" + cnt := 0 + newCtx := context.Background() + for _, org := range activeOrgs { + if len(org.Id) == 0 { + log.Printf("[DEBUG] No ID found for org with name '%s'. Why was it made?", org.Name) + continue + } + + log.Printf("[INFO] Starting self-cleanup of cache keys for org %s", org.Id) + + for { + keys, newCursor, err := shuffle.GetAllCacheKeys(newCtx, org.Id, "", 1000, cursor) + if err != nil { + //log.Printf("[ERROR] Failed getting all cache keys for cleanup: %s", err) + break + } + + if newCursor == cursor || len(newCursor) == 0 { + break + } + + if len(keys) == 0 { + break + } + + cursor = newCursor + cnt += 1 + if cnt > 10 { + break + } + } + + log.Printf("[INFO] Finished self-cleanup of cache keys for org %s", org.Id) + } + }() + log.Printf("[INFO] Finished INIT (ES)") } @@ -4648,6 +4798,7 @@ func handleStopCloudSync(syncUrl string, org shuffle.Org) (*shuffle.Org, error) org.CloudSync = false org.SyncFeatures = shuffle.SyncFeatures{} org.SyncConfig = shuffle.SyncConfig{} + org.Subscriptions = []shuffle.PaymentSubscription{} err = shuffle.SetOrg(ctx, org, org.Id) if err != nil { @@ -4863,8 +5014,6 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("[DEBUG] Respbody from sync: %s", string(respBody)) - responseData := retStruct{} err = json.Unmarshal(respBody, &responseData) if err != nil { @@ -4890,7 +5039,15 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { // 2. Add iterative sync schedule for interval seconds // 3. Add another environment for the org's users org.CloudSync = true - org.SyncFeatures = responseData.SyncFeatures + + // set cache here for 30 min + cacheKey := fmt.Sprintf("org_sync_features_%s", org.Id) + featuresBytes, err := json.Marshal(responseData.SyncFeatures) + if err != nil { + log.Printf("[ERROR] Failed to marshal SyncFeatures for cache: %s", err) + } else { + shuffle.SetCache(ctx, cacheKey, featuresBytes, 1800) + } org.SyncConfig = shuffle.SyncConfig{ Apikey: responseData.SessionKey, @@ -5362,6 +5519,11 @@ func initHandlers() { r.HandleFunc("/api/v2/workflows/{key}/executions", shuffle.GetWorkflowExecutionsV2).Methods("GET", "OPTIONS") r.HandleFunc("/api/v2/workflows/generate/llm", shuffle.HandleWorkflowGenerationResponse).Methods("POST", "OPTIONS") r.HandleFunc("/api/v2/workflows/edit/llm", shuffle.HandleEditWorkflowWithLLM).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v2/workflows/generate", shuffle.GenerateSingulWorkflows).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v2/datastore", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v2/datastore", shuffle.HandleSetDatastoreKey).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v2/datastore/category/{category_key}", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v2/datastore/automate", shuffle.HandleDatastoreCategoryConfig).Methods("POST", "OPTIONS") // New for recommendations in Shuffle r.HandleFunc("/api/v1/recommendations/get_actions", shuffle.HandleActionRecommendation).Methods("POST", "OPTIONS") @@ -5456,6 +5618,10 @@ func initHandlers() { r.HandleFunc("/api/v1/orgs/{orgId}/stats/{key}", shuffle.GetSpecificStats).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/statistics", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/stats", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/stats", shuffle.HandleAppendStatistics).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/stats/{key}", shuffle.GetSpecificStats).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/orgs/{orgId}/cache", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/cache", shuffle.HandleSetCacheKey).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/cache/{cache_key}", shuffle.HandleDeleteCacheKey).Methods("DELETE", "OPTIONS") @@ -5511,6 +5677,17 @@ func initHandlers() { r.HandleFunc("/api/v1/dashboards/{key}/widgets", shuffle.HandleNewWidget).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/dashboards/{key}/widgets/{widget_id}", shuffle.HandleGetWidget).Methods("GET", "OPTIONS") + if (strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" || strings.ToLower(os.Getenv("DEBUG_MEMORY")) == "true") { + log.Printf("[DEBUG] Memory debugging is enabled on /debug/pprof") + r.HandleFunc("/debug/pprof/", pprof.Index) + r.HandleFunc("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP) + r.HandleFunc("/debug/pprof/profile", pprof.Profile) + r.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + r.HandleFunc("/debug/pprof/trace", pprof.Trace) + } else { + log.Printf("[DEBUG] Memory debugging is disabled. To enable, set SHUFFLE_DEBUG_MEMORY or DEBUG_MEMORY to true") + } + r.Use(shuffle.RequestMiddleware) http.Handle("/", r) } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 3cf355ab..54635430 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -249,7 +249,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque resp.Write([]byte(`{"success": true}`)) } -// FIXME: Authenticate this one? Can org ID be auth enough? +// FIXME: Authenticate this one. Can org ID be auth enough? // (especially since we have a default: shuffle) func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { cors := shuffle.HandleCors(resp, request) @@ -325,7 +325,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { orgId = env.OrgId } - executionRequests, err := shuffle.GetWorkflowQueue(ctx, environment, 100) + executionRequests, err := shuffle.GetWorkflowQueue(ctx, environment, 100, *env) if err != nil { resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) @@ -336,47 +336,11 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { if len(executionRequests.Data) == 0 { executionRequests.Data = []shuffle.ExecutionRequest{} } else { - // Try again? I don't think this is necessary, and shouldn't really ever occur. - /* - if len(env.Id) == 0 && len(env.Name) == 0 { - timeNow := int64(time.Now().Unix()) - foundId := "" - for _, requestData := range executionRequests.Data { - execution, err := shuffle.GetWorkflowExecution(ctx, requestData.ExecutionId) - if err == nil { - if len(execution.ExecutionOrg) > 0 { - foundId = execution.ExecutionOrg - break - } - } - } - - if len(environment) > 0 { - - env, err := shuffle.GetEnvironment(ctx, environment, foundId) - if err != nil { - log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", environment, err) - //resp.WriteHeader(401) - //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No env found matching %s"}`, id))) - //return - } else { - if timeNow > env.Edited+60 { - env.RunningIp = request.RemoteAddr - env.Checkin = timeNow - err = shuffle.SetEnvironment(ctx, env) - if err != nil { - log.Printf("[WARNING] Failed updating environment: %s", err) - } - } - } - } - } - */ - if len(executionRequests.Data) > 50 { executionRequests.Data = executionRequests.Data[0:49] } + } newjson, err := json.Marshal(executionRequests) @@ -1847,13 +1811,12 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000, 0) if err != nil { log.Printf("{WARNING] Failed getting apps (getworkflowapps): %s", err) - resp.WriteHeader(401) + resp.WriteHeader(400) resp.Write([]byte(`{"success": false}`)) return } newapps := workflowapps - if len(user.PrivateApps) > 0 { found := false for _, item := range user.PrivateApps { diff --git a/docker-compose.yml b/docker-compose.yml index c1c27efb..8ab7e3b9 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -91,6 +91,27 @@ services: - shuffle restart: unless-stopped + # OPTIONAL: advanced monitoring with cAdvisor + # If you have a seperate orborus config YAML on another server, + # Please add it there as well if you want to monitor that server too. + # FYI: both servers will show up on different dashboards. + # cadvisor: + # image: gcr.io/cadvisor/cadvisor:latest + # volumes: + # - /:/rootfs:ro + # - /var/run:/var/run:ro + # - /sys:/sys:ro + # - /var/lib/docker/:/var/lib/docker:ro + # - /dev/disk/:/dev/disk:ro + # - /var/run/docker.sock:/var/run/docker.sock:ro + # ports: + # - "8080:8080" + # privileged: true + # devices: + # - /dev/kmsg:/dev/kmsg + # networks: + # - shuffle + #memcached: # image: memcached:latest # container_name: shuffle-cache diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 9425a1c6..18083aea 100755 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -25,6 +25,7 @@ import AgentUI from "./views/AgentUI.jsx"; import Welcome from "./views/Welcome.jsx"; import Dashboard from "./views/Dashboard.jsx"; import DashboardView from "./views/DashboardViews.jsx"; +import NewDashboard from "./views/NewDashboard.jsx"; import AdminSetup from "./views/AdminSetup.jsx"; import Admin from "./views/Admin.jsx"; import Docs from "./views/Docs.jsx"; @@ -167,6 +168,47 @@ const App = (message, props) => { }); }; + useEffect(() => { + if (userdata && + userdata?.org_status?.includes("integration_partner") && + userdata?.active_org?.image) { + + const existingLinks = document.querySelectorAll("link[rel*='icon']"); + existingLinks.forEach(link => link.parentNode.removeChild(link)); + const newLink = document.createElement('link'); + newLink.rel = 'icon'; + newLink.type = 'image/x-icon'; + try { + if (typeof userdata.active_org.image === 'string') { + if (userdata.active_org.image.startsWith('data:')) { + newLink.href = userdata.active_org.image; + } else if (userdata.active_org.image.startsWith('/')) { + newLink.href = userdata.active_org.image; + } else { + let mimeType = 'image/png'; + newLink.href = `data:${mimeType};base64,${userdata.active_org.image}`; + } + } else { + console.error("Image data is not in expected format:", userdata.active_org.image); + return; + } + + // Add the new favicon link to head + document.head.appendChild(newLink); + const iframe = document.createElement('iframe'); + iframe.style.display = 'none'; + document.body.appendChild(iframe); + iframe.contentDocument.write(''); + setTimeout(() => { + document.body.removeChild(iframe); + }, 100); + + } catch (error) { + console.error("Error updating favicon:", error); + } + } + }, [userdata]); + const checkLogin = () => { var baseurl = globalUrl; fetch(`${globalUrl}/api/v1/getinfo`, { @@ -180,6 +222,13 @@ const App = (message, props) => { var userInfo = {}; if (responseJson.success === true) { //console.log("USER: ", responseJson); + if (responseJson?.switch_parent === true) { + toast.info(responseJson.reason) + setTimeout(() => { + window.location.reload(); + }, 3000); + return + } userInfo = responseJson; setIsLoggedIn(true); @@ -824,6 +873,21 @@ const App = (message, props) => { /> } /> + + + } + /> + { App Authentication
- Control the authentication options for individual apps. + Control the authentication options for individual apps. These keys are write-only, and cannot be viewed after creation. If you want editable secrets (e.g. for use in code), use Protected Keys.   - Learn more about App Authentication + Learn more
@@ -1787,7 +1787,7 @@ const Hits = ({ if (selectedAppData.authentication === undefined || selectedAppData.authentication === null) { setAuthenticationType({ - type: "", + type: "", }) selectedAppData.authentication = { @@ -1955,6 +1955,7 @@ const Hits = ({ if (data === undefined || data === null) { return; } + const filteredData = data.filter((appAuth) => appAuth?.app?.id === appid); if (filteredData.length === 0) { setAppAuthentication([]); @@ -1965,7 +1966,7 @@ const Hits = ({ } }; - const HandleAppAuthentication = ()=>{ + const HandleAppAuthentication = () => { const url = `${globalUrl}/api/v1/apps/authentication`; diff --git a/frontend/src/components/AppCreationModal.jsx b/frontend/src/components/AppCreationModal.jsx index 28dea461..6a66f0e0 100644 --- a/frontend/src/components/AppCreationModal.jsx +++ b/frontend/src/components/AppCreationModal.jsx @@ -21,8 +21,9 @@ import AutoFixHighIcon from '@mui/icons-material/AutoFixHigh' import CreateIcon from '@mui/icons-material/Create' import { toast } from 'react-toastify' import YAML from "yaml"; +import Dropzone from "./Dropzone.jsx"; -const AppCreationModal = ({ open, onClose, theme, globalUrl, isCloud }) => { +const AppCreationModal = ({ open, onClose, theme, globalUrl, isCloud, startOpenApi = false, prefillOpenApiData = "" }) => { const [openApiModal, setOpenApiModal] = useState(false) const [generateAppModal, setGenerateAppModal] = useState(false) const [openApi, setOpenApi] = useState("") @@ -35,6 +36,16 @@ const AppCreationModal = ({ open, onClose, theme, globalUrl, isCloud }) => { const navigate = useNavigate() const upload = useRef() + useEffect(() => { + if (open && (startOpenApi || (prefillOpenApiData && prefillOpenApiData.length > 0))) { + if (prefillOpenApiData && prefillOpenApiData.length > 0) { + setOpenApiData(prefillOpenApiData) + setIsDropzone(true) + } + setOpenApiModal(true) + } + }, [open, startOpenApi, prefillOpenApiData]) + // Style for the create options const AppCreateButton = ({ text, func, icon }) => { const [hover, setHover] = React.useState(false) @@ -467,6 +478,7 @@ const AppCreationModal = ({ open, onClose, theme, globalUrl, isCloud }) => { } }} > + { Paste in the URI for the OpenAPI or find out - How to find URI for openAPI? @@ -568,31 +583,54 @@ const AppCreationModal = ({ open, onClose, theme, globalUrl, isCloud }) => { Must point to a version 2 or 3 OpenAPI specification. - + Or upload a YAML or JSON specification - - +
+ + Drag & drop your OpenAPI (YAML/JSON) anywhere + + + or click to browse files + +
+ + { Continue + {/* Generate App Modal */} diff --git a/frontend/src/components/AppGrid.jsx b/frontend/src/components/AppGrid.jsx index e647085b..33998bfb 100644 --- a/frontend/src/components/AppGrid.jsx +++ b/frontend/src/components/AppGrid.jsx @@ -33,6 +33,7 @@ import { ClearRefinements, connectStateResults } from "react-instantsearch-dom"; +import { useDebouncedCallback } from "../utils/useDebouncedCallback"; import aa from "search-insights"; import { useLocation } from 'react-router-dom'; @@ -160,6 +161,8 @@ const AppGrid = (props) => { refine(searchQuery.trim()); }; + const debouncedRefine = useDebouncedCallback((value) => refine(value), 300); + return (
{ placeholder="Search more than 2500 Apps" id="shuffle_search_field" onChange={(event) => { - setSearchQuery(event.currentTarget.value); + const value = event.currentTarget.value; + setSearchQuery(value); removeQuery("q"); - refine(event.currentTarget.value); + debouncedRefine(value); }} onKeyDown={(event) => { if(event.key === "Enter") { diff --git a/frontend/src/components/AuthenticationModal.jsx b/frontend/src/components/AuthenticationModal.jsx new file mode 100644 index 00000000..62f15e7f --- /dev/null +++ b/frontend/src/components/AuthenticationModal.jsx @@ -0,0 +1,891 @@ +import React, { useState, useEffect, useContext, memo } from "react"; +import { CodeHandler, Img, OuterLink, } from '../views/Docs.jsx' +import { getTheme } from "../theme.jsx"; +import { isMobile } from "react-device-detect" +import Markdown from "react-markdown"; +import { Context } from '../context/ContextApi.jsx'; +import PaperComponent from "../components/PaperComponent.jsx"; +import { toast } from "react-toastify"; +import { v4 as uuidv4} from "uuid"; +import AuthenticationOauth2 from "../components/Oauth2Auth.jsx"; + +import { + Edit as EditIcon, + DragIndicator as DragIndicatorIcon, + Close as CloseIcon, + LockOpen as LockOpenIcon, +} from "@mui/icons-material"; + +import { + Button, + Typography, + Dialog, + DialogContent, + DialogTitle, + DialogActions, + MenuItem, + Select, + TextField, + IconButton, + Tooltip, + Divider, +} from "@mui/material"; + +const AuthenticationModal = (props) => { + const { + globalUrl, + userdata, + + selectedAppData, + getAppAuthentication, + appAuthentication, + setSelectedAction, + + selectedMeta, + setSelectedMeta, + + setAppAuthentication, + } = props; + + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" || window.location.host === "migration.shuffler.io"; + const [selectedAuthentication, setSelectedAuthentication] = React.useState({}); + const [authenticationModalOpen, setAuthenticationModalOpen] = React.useState(false); + const [authenticationType, setAuthenticationType] = React.useState({}) + + const [appid, setAppId] = useState("") + + //const [appAuthentication, setAppAuthentication] = useState([]); + + const { themeMode, supportEmail, brandColor } = useContext(Context) + const theme = getTheme(themeMode, brandColor) + + useEffect(() => { + if (selectedAppData === undefined || selectedAppData === null || Object.getOwnPropertyNames(selectedAppData).length === 0) { + return + } + + if (selectedAppData.authentication === undefined || selectedAppData.authentication === null) { + setAuthenticationType({ + type: "", + }) + + selectedAppData.authentication = { + type: "", + required: false, + } + } else { + setAuthenticationType( + selectedAppData.authentication.type === "oauth2-app" || (selectedAppData.authentication.type === "oauth2" && selectedAppData.authentication.redirect_uri !== undefined && selectedAppData.authentication.redirect_uri !== null) ? { + type: selectedAppData.authentication.type, + redirect_uri: selectedAppData.authentication.redirect_uri, + refresh_uri: selectedAppData.authentication.refresh_uri, + token_uri: selectedAppData.authentication.token_uri, + scope: selectedAppData.authentication.scope, + client_id: selectedAppData.authentication.client_id, + client_secret: selectedAppData.authentication.client_secret, + grant_type: selectedAppData.authentication.grant_type, + } : { + type: "", + } + ) + } + }, [selectedAppData]) + + if (selectedAppData === undefined || selectedAppData === null || Object.getOwnPropertyNames(selectedAppData).length === 0) { + console.log("No app data for authentication modal"); + return null + } + + if (authenticationModalOpen === false) { + return ( + + ) + } + + function Heading(props) { + const element = React.createElement( + `h${props.level}`, + { style: { marginTop: 40 } }, + props.children + ); + return ( + + {props.level !== 1 ? ( + + ) : null} + {element} + + ); + } + + const UpdateAppAuthentication = (data) => { + if (data === undefined || data === null) { + return; + } + + const filteredData = data.filter((appAuth) => appAuth?.app?.id === appid); + if (filteredData.length === 0) { + setAppAuthentication([]); + setSelectedAuthentication({}); + } else { + setAppAuthentication(filteredData); + setSelectedAuthentication(filteredData[0]); + } + }; + + const HandleAppAuthentication = () => { + + const url = `${globalUrl}/api/v1/apps/authentication`; + + fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + }, + credentials: "include", + }).then((response) => { + if (response.status !== 200) { + return; + } + return response.json(); + }).then((responseJson) => { + if (responseJson.success === true) { + UpdateAppAuthentication(responseJson.data); + } else { + toast.error("Failed to get app authentication data"); + } + }).catch((error) => { + console.error("error for app is :", error); + }); + } + + const AuthenticationData = (props) => { + const selectedApp = props.app; + + const [authenticationOption, setAuthenticationOptions] = React.useState({ + app: JSON.parse(JSON.stringify(selectedApp)), + fields: {}, + label: "", + usage: [ + { + // workflow_id: workflow.id, + }, + ], + id: uuidv4(), + active: true, + }); + + if ( + selectedApp.authentication === undefined || + selectedApp.authentication.parameters === null || + selectedApp.authentication.parameters === undefined || + selectedApp.authentication.parameters.length === 0 + ) { + return ( + + + {selectedApp.name} does not require authentication + + + ); + } + + authenticationOption.app.actions = []; + + for (let paramkey in selectedApp.authentication.parameters) { + if ( + authenticationOption.fields[ + selectedApp.authentication.parameters[paramkey].name + ] === undefined + ) { + authenticationOption.fields[ + selectedApp.authentication.parameters[paramkey].name + ] = ""; + } + } + + const setNewAppAuth = (appAuthData, refresh) => { + setSelectedAuthentication(appAuthData); + var headers = { + "Content-Type": "application/json", + "Accept": "application/json", + } + + headers["Org-Id"] = userdata?.active_org?.id + + fetch(globalUrl + "/api/v1/apps/authentication", { + method: "PUT", + headers: headers, + body: JSON.stringify(appAuthData), + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for setting app auth :O!"); + + if (response.status === 400) { + toast.error("Failed setting new auth. Please try again", { + "autoClose": true, + }) + } + } + + return response.json(); + }) + .then((responseJson) => { + if (!responseJson.success) { + toast.error("Error: " + responseJson.reason, { + "autoClose": false, + }) + + } else { + HandleAppAuthentication() + setAuthenticationModalOpen(false) + getAppAuthentication() + } + }) + .catch((error) => { + console.log("New auth error: ", error.toString()); + }); + }; + + const handleSubmitCheck = () => { + if (authenticationOption.label.length === 0) { + authenticationOption.label = `Auth for ${selectedApp.name}`; + } + for (let paramkey in selectedApp.authentication.parameters) { + if ( + authenticationOption.fields[ + selectedApp.authentication.parameters[paramkey].name + ].length === 0 + ) { + if ( + selectedApp.authentication.parameters[paramkey].value !== undefined && + selectedApp.authentication.parameters[paramkey].value !== null && + selectedApp.authentication.parameters[paramkey].value.length > 0 + ) { + authenticationOption.fields[ + selectedApp.authentication.parameters[paramkey].name + ] = selectedApp.authentication.parameters[paramkey].value; + } else { + if ( + selectedApp.authentication.parameters[paramkey].schema.type === "bool" + ) { + authenticationOption.fields[ + selectedApp.authentication.parameters[paramkey].name + ] = "false"; + } else { + toast( + "Field " + + selectedApp.authentication.parameters[paramkey].name + + " can't be empty" + ); + return; + } + } + } + } + + var newAuthOption = JSON.parse(JSON.stringify(authenticationOption)); + var newFields = []; + for (let authkey in newAuthOption.fields) { + const value = newAuthOption.fields[authkey]; + newFields.push({ + "key": authkey, + "value": value, + }); + } + + newAuthOption.fields = newFields + setNewAppAuth(newAuthOption) + } + + if (authenticationOption.label === null || authenticationOption.label === undefined) { + authenticationOption.label = selectedApp.name + " authentication"; + } + + return ( +
+ +
+ Authentication for {selectedApp.name.replaceAll("_", " ", -1)} +
+ + + + What is app authentication? + +
+ These are required fields for authenticating with {selectedApp.name} +
+ Label for you to remember + { + authenticationOption.label = event.target.value; + }} + /> + +
+ {selectedApp.authentication.parameters.map((data, index) => { + if (data.value === "" || data.value === null || data.value === undefined || data.name === "url") { + } + + + return ( +
+ + {data.name} + + {data.schema !== undefined && + data.schema !== null && + data.schema.type === "bool" ? ( + + ) : ( + { + authenticationOption.fields[data.name] = + event.target.value; + }} + /> + )} +
+ ); + })} + + + + + +
+ ); + }; + + const authenticationModal = authenticationModalOpen ? ( + {setSelectedMeta(undefined)}} + PaperProps={{ + style: { + pointerEvents: "auto", + color: theme.palette.textColor, + minWidth: 1100, + minHeight: 800, + maxHeight: 800, + padding: 15, + overflow: "hidden", + zIndex: 10012, + border: theme.palette.defaultBorder, + }, + }} + > +
+ { selectedAppData.reference_info === undefined || + selectedAppData.reference_info === null || + selectedAppData.reference_info.github_url === undefined || + selectedAppData.reference_info.github_url === null || + selectedAppData.reference_info.github_url.length === 0 ? ( + + + {`Documentation + + ) : ( + + {`Documentation + + )} +
+ + + + + + { + setAuthenticationModalOpen(false); + }} + > + + +
+
+ {authenticationType?.type === "oauth2" || authenticationType?.type === "oauth2-app" ? + + : + + } +
+
+ {selectedAppData?.documentation === undefined || + selectedAppData?.documentation === null || + selectedAppData?.documentation.length === 0 ? ( + +
+ + {selectedAppData?.description} + +
+ + +
+ + There is no Shuffle-specific documentation for this app yet outside of the general description above. Documentation is written for each api, and is a community effort. We hope to see your contribution! + + +
+ + + Want to help the making of, or improve this app?{" "} +
+ + Join the community on Discord! + + + + + Want to help change this app directly? + + {selectedAppData.reference_info === undefined || + selectedAppData.reference_info === null || + selectedAppData.reference_info.github_url === undefined || + selectedAppData.reference_info.github_url === null || + selectedAppData.reference_info.github_url.length === 0 ? ( + + + + Check it out on Github! + + + + ) : ( + + + + Check it out on Github! + + + + )} + + ) : ( +
+ {selectedMeta !== undefined && selectedMeta !== null && Object.getOwnPropertyNames(selectedMeta).length > 0 && selectedMeta.name !== undefined && selectedMeta.name !== null ? +
+
+ {isMobile ? null : ( + + + + + + )} + {isMobile ? null : ( +
+ )} + + {selectedMeta.read_time} minute + {selectedMeta.read_time === 1 ? "" : "s"} to read + +
+
+ {isMobile || + selectedMeta.contributors === undefined || + selectedMeta.contributors === null ? ( + "" + ) : ( +
+ {selectedMeta.contributors.slice(0, 7).map((data, index) => { + return ( + + + {data.url} + + + ); + })} +
+ )} +
+
+ : null} + + + {selectedAppData.documentation} + +
+ )} +
+
+
+ ) : null; + + return authenticationModal +} + +export default AuthenticationModal; diff --git a/frontend/src/components/Billing.jsx b/frontend/src/components/Billing.jsx index 8b14eaf5..2bc8290b 100644 --- a/frontend/src/components/Billing.jsx +++ b/frontend/src/components/Billing.jsx @@ -47,7 +47,9 @@ import { CheckCircle, Padding, Edit, - Search as SearchIcon + Search as SearchIcon, + CheckCircle as CheckCircleIcon, + Cancel as CancelIcon, } from "@mui/icons-material"; //import { useAlert @@ -61,6 +63,62 @@ import { Context } from "../context/ContextApi.jsx"; import DeleteIcon from '@mui/icons-material/Delete'; import { DataGrid } from "@mui/x-data-grid"; +const ProductionStatus = ({ selectedOrganization, userdata, isCloud, theme }) => { + var isProdStatusOn; + if (selectedOrganization !== undefined && selectedOrganization?.subscriptions !== undefined && selectedOrganization?.subscriptions[0] !== undefined) { + isProdStatusOn = selectedOrganization?.subscriptions[0]?.name?.toLowerCase()?.includes("enterprise") && selectedOrganization?.subscriptions[0]?.active; + } else { + isProdStatusOn = false; + } + const rows = [ + { label: 'Licensed', ok: isProdStatusOn }, + { label: 'Multi-Tenant', ok: isProdStatusOn }, + { label: 'High Availability', ok: isProdStatusOn }, + { label: 'Robust Infrastructure', ok: isProdStatusOn }, + ]; + + return ( +
+
+ Production Status +
+ + {isProdStatusOn ? "ON" : "OFF"} +
+
+ + Monitor your production status to stay informed about available features. + +
+ {rows.map((row) => ( +
+ {row.ok ? ( + + ) : ( + + )} + {row.label} +
+ ))} +
+ + + + Shuffle Enterprise is designed for organizations that require scalability, high availability, dedicated support and more to run mission-critical workflows in production environments. + + + More about upgrading below. If you want to know more, please contact support@shuffler.io directly. + +
+ ); +}; + const Billing = memo((props) => { const { globalUrl, userdata, serverside, billingInfo, stripeKey,isLoaded, selectedOrganization, handleGetOrg, clickedFromOrgTab, removeCookie} = props; //const alert = useAlert(); @@ -87,6 +145,8 @@ const Billing = memo((props) => { const [currentIndex, setCurrentIndex] = useState(0); const [deleteAlertIndex, setDeleteAlertIndex] = useState(-1); const [deleteAlertVerification, setDeleteAlertVerification] = useState(false); + const [supportAppRunLimit, setSupportAppRunLimit] = useState(selectedOrganization?.billing?.internal_app_runs_hard_limit || ''); + const [supportLimitDialogOpen, setSupportLimitDialogOpen] = useState(false); const [isScale, setIsScale] = useState(false); const [currentTab, setCurrentTab] = useState(0) const [allChildOrgs, setAllChildOrgs] = useState([]) @@ -145,6 +205,9 @@ const Billing = memo((props) => { const findCurrentIndex = sortedAlertThresholds.some(threshold => threshold.Email_send === false); setCurrentIndex(findCurrentIndex ? sortedAlertThresholds.findIndex(threshold => threshold.Email_send === false) : - 1); + if (selectedOrganization?.Billing?.internal_app_runs_hard_limit !== undefined && selectedOrganization?.Billing?.internal_app_runs_hard_limit !== null && selectedOrganization?.Billing?.internal_app_runs_hard_limit > 0) { + setSupportAppRunLimit(selectedOrganization?.Billing?.internal_app_runs_hard_limit) + } }, [selectedOrganization]); @@ -1903,6 +1966,52 @@ const Billing = memo((props) => { setAlertThresholds([...alertThresholds, { percentage: '', count: '', Email_send: false }]); }; + + const handleUpdateSupportAppRunLimit = () => { + if (!supportAppRunLimit || isNaN(supportAppRunLimit) || supportAppRunLimit < 0) { + toast.error("Please enter a valid app run limit"); + return; + } + + toast("Updating app run limit. Please wait..."); + + const data = { + org_id: selectedOrganization.id, + editing: "internal_appruns_hard_limit", + billing: { + internal_app_runs_hard_limit: parseInt(supportAppRunLimit) || 0, + } + }; + + const url = globalUrl + "/api/v1/orgs/" + selectedOrganization.id; + fetch(url, { + mode: "cors", + method: "POST", + body: JSON.stringify(data), + credentials: "include", + crossDomain: true, + withCredentials: true, + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + }) + .then((response) => { + if (response.status === 200) { + toast.success("Successfully updated app run limit"); + setSupportLimitDialogOpen(false); + if (handleGetOrg !== undefined) { + handleGetOrg(selectedOrganization.id); + } + } else { + toast.error("Failed to update app run limit. Please try again."); + } + }) + .catch((error) => { + console.log("Error updating app run limit:", error); + toast.error("Failed to update app run limit. Please try again."); + }); + }; + const updateAlertThreshold = (index, field, value) => { const totalValue = userdata.app_execution_limit; @@ -2025,29 +2134,41 @@ const Billing = memo((props) => { } }, [isChildOrg, currentTab]); + // Update supportAppRunLimit when selectedOrganization changes + useEffect(() => { + if (selectedOrganization?.billing?.internal_app_runs_hard_limit !== undefined) { + setSupportAppRunLimit(selectedOrganization.billing.internal_app_runs_hard_limit); + } + }, [selectedOrganization?.billing?.internal_app_runs_hard_limit]); + return ( -
-
+
+
+ + {isCloud ? null : } + {addDealModal} - {clickedFromOrgTab ? - Billing & Licensing : - - Billing & Licensing - } + {clickedFromOrgTab ? + Billing & Licensing + : + + Billing & Licensing + + } {userdata?.org_status?.includes("integration_partner") && userdata?.org_status?.includes("sub_org") ? null : <> {clickedFromOrgTab ? {isCloud ? "Get more out of Shuffle by adding your credit card, such as no App Run limitations, and priority support from our team. We use Stripe to manage subscriptions and do not store any of your billing information. You can manage your subscription and billing information below." : - "Shuffle is an Open Source automation platform, and no license is required. We do however offer a Scale license with HA guarantees, along with support hours. By buying a license on https://shuffler.io, you can get access to the license immediately, and if Cloud Syncronisation is enabled, the UI in your local instance will also update." + !(selectedOrganization?.subscriptions !== undefined && selectedOrganization?.subscriptions.length > 0 && selectedOrganization?.subscriptions[0]?.name?.toLowerCase().includes("enterprise") && selectedOrganization?.subscriptions[0]?.active) ? "Shuffle is an Enterprise automation platform, and a license is required at scale. We offer a license with HA guarantees, higher limits, along along with support hours. By buying a license on https://shuffler.io, you can get access to the license immediately, and if Cloud Syncronisation is enabled, the UI in your local instance will also update." : "Here you can check your license and billing information." } : {isCloud ? "Get more out of Shuffle by adding your credit card, such as no App Run limitations, and priority support from our team. We use Stripe to manage subscriptions and do not store any of your billing information. You can manage your subscription and billing information below." : - "Shuffle is an Open Source automation platform, and no license is required. We do however offer a Scale license with HA guarantees, along with support hours. By buying a license on https://shuffler.io, you can get access to the license immediately, and if Cloud Syncronisation is enabled, the UI in your local instance will also update." + !(selectedOrganization?.subscriptions !== undefined && selectedOrganization?.subscriptions.length > 0 && selectedOrganization?.subscriptions[0]?.name?.toLowerCase().includes("enterprise") && selectedOrganization?.subscriptions[0]?.active) ? "Shuffle is an Enterprise automation platform, and a license is required at scale. We offer a Scale license with HA guarantees, along with support hours. By buying a license on https://shuffler.io, you can get access to the license immediately, and if Cloud Syncronisation is enabled, the UI in your local instance will also update." : "Here you can check your license and billing information." } } } @@ -2118,6 +2239,8 @@ const Billing = memo((props) => { globalUrl={globalUrl} selectedOrganization={selectedOrganization} billingInfo={billingInfo} + monthlyAppRunsParent={monthlyAppRunsParent} + monthlyAllSuborgExecutions={monthlyAllSuborgExecutions} isCloud={isCloud} userdata={userdata} stripeKey={stripeKey} @@ -2154,6 +2277,8 @@ const Billing = memo((props) => { isLoggedIn={isLoggedIn} globalUrl={globalUrl} selectedOrganization={selectedOrganization} + monthlyAppRunsParent={monthlyAppRunsParent} + monthlyAllSuborgExecutions={monthlyAllSuborgExecutions} billingInfo={billingInfo} isCloud={isCloud} userdata={userdata} @@ -2639,6 +2764,122 @@ const Billing = memo((props) => { Save + {userdata.support === true && ( +
+ + ⚠️ Support Only - App Run Limit Control + + + Note: Setting an app run hard limit below current usage will immediately stop all workflow executions for this organization. + + + Current app runs this month: {Number(monthlyAppRunsParent ?? 0) + Number(monthlyAllSuborgExecutions ?? 0)} / {userdata.app_execution_limit} + + + + + {/* Support Limit Dialog */} + setSupportLimitDialogOpen(false)} + maxWidth="sm" + fullWidth + PaperProps={{ + sx: { + borderRadius: theme?.palette?.DialogStyle?.borderRadius, + border: theme?.palette?.DialogStyle?.border, + minWidth: '440px', + fontFamily: theme?.typography?.fontFamily, + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + zIndex: 1000, + '& .MuiDialogContent-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + }, + '& .MuiDialogTitle-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + }, + '& .MuiDialogActions-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + } + } + }} + > + + ⚠️ Set App Run Limit + + + + Note: This will set a hard limit on app executions. If the organization reaches this limit, all workflow executions will be stopped. + + + Current usage: {Number(monthlyAppRunsParent ?? 0) + Number(monthlyAllSuborgExecutions ?? 0)} app runs this month + + + Current hard limit: {selectedOrganization?.billing?.internal_app_runs_hard_limit || 'Not set'} + + setSupportAppRunLimit(e.target.value)} + inputProps={{ min: 0 }} + style={{ marginTop: 10 }} + helperText="Set to 0 to completely disable app run hard limit" + /> + + + + + + +
+ )} +
): null}
@@ -2746,6 +2987,7 @@ const Billing = memo((props) => { userdata={userdata} currentTab={currentTab} syncStats={true} + statistics={statistics} /> }
@@ -3275,7 +3517,7 @@ const PaddingWrapper = memo(({ clickedFromOrgTab, children }) => { height: '100%', boxSizing: 'border-box', overflow: 'hidden', - maxHeight: "1700px", + maxHeight: 3000, overflowY: "auto", scrollbarColor: theme.palette.scrollbarColorTransparent, scrollbarWidth: 'thin' diff --git a/frontend/src/components/BillingStats.jsx b/frontend/src/components/BillingStats.jsx index 8e011d42..19c0c176 100644 --- a/frontend/src/components/BillingStats.jsx +++ b/frontend/src/components/BillingStats.jsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect, useContext, memo, useMemo } from 'react'; +import React, { useState, useEffect, useContext, useCallback } from 'react'; import {getTheme} from '../theme.jsx'; import classNames from "classnames"; @@ -73,6 +73,7 @@ const AppStats = (defaultprops) => { const [resultRows, setResultRows] = useState([]) const [resultLoading, setResultLoading] = useState(true) const { themeMode, brandColor } = useContext(Context); + const [onpremAppRuns, setOnpremAppRuns] = useState(0) const theme = getTheme(themeMode, brandColor) const includedExecutions = selectedOrganization?.sync_features?.app_executions !== undefined ? selectedOrganization?.sync_features?.app_executions?.limit : 0 @@ -83,12 +84,156 @@ const AppStats = (defaultprops) => { } }, []) + const handleDataSetting = useCallback((inputdata, grouping) => { + if (inputdata === undefined || inputdata === null) { + return + } + + const statKey = syncStats === true ? "onprem_stats" : "daily_statistics" + const dailyStats = inputdata[statKey] + if (dailyStats === undefined || dailyStats === null) { + return + } + + var appRuns = { + "key": "App Runs", + "data": [] + } + + var childorgappRuns = { + "key": "Child Org App Runs", + "data": [] + } + + var workflowRuns = { + "key": "Workflow Runs (includes subflows)", + "data": [] + } + + var subflowRuns = { + "key": "Subflow Runs", + "data": [] + } + + var appcostRuns = { + "key": "Cost of App Runs", + "data": [] + } + + for (let key in dailyStats) { + // Always skips first one as it has accumulated data in it + if (key === 0) { + continue + } + + const item = dailyStats[key] + if (item["date"] === undefined) { + console.log("No date: ", item) + continue + } + + // Check if app_executions key in item + if (item["app_executions"] !== undefined && item["app_executions"] !== null) { + appRuns["data"].push({ + key: new Date(item["date"]).toISOString(), + data: item["app_executions"] + }) + + // Add number + appcostRuns["data"].push({ + key: new Date(item["date"]).toISOString(), + data: (item["app_executions"] * invocationCost).toFixed(2) + }) + } + + if (item["child_app_executions"] !== undefined && item["child_app_executions"] !== null) { + childorgappRuns["data"].push({ + key: new Date(item["date"]).toISOString(), + data: item["child_app_executions"] + }) + } + + // Check if workflow_executions key in item + if (item["workflow_executions"] !== undefined && item["workflow_executions"] !== null) { + workflowRuns["data"].push({ + key: new Date(item["date"]).toISOString(), + data: item["workflow_executions"] + }) + } + + if (item["subflow_executions"] !== undefined && item["subflow_executions"] !== null) { + subflowRuns["data"].push({ + key: new Date(item["date"]).toISOString(), + data: item["subflow_executions"] + }) + } + } + + // Only add today's data if endTime is not set or if today falls within the selected date range + const today = new Date() + const todayStartOfDay = new Date(today) + todayStartOfDay.setHours(0, 0, 0, 0) + const shouldAddTodayData = endTime === "" || endTime === undefined || endTime === null || + (new Date(endTime) >= todayStartOfDay) + + if (!syncStats && shouldAddTodayData) { + // Adds data for today + if (inputdata["daily_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) { + appRuns["data"].push({ + key: new Date().toISOString(), + data: inputdata["daily_app_executions"] + }) + + appcostRuns["data"].push({ + key: new Date().toISOString(), + data: (inputdata["daily_app_executions"] * invocationCost).toFixed(2) + }) + } + + if (inputdata["daily_child_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) { + childorgappRuns["data"].push({ + key: new Date().toISOString(), + data: inputdata["daily_child_app_executions"] + }) + } + + if (inputdata["daily_workflow_executions"] !== undefined && inputdata["daily_workflow_executions"] !== null) { + workflowRuns["data"].push({ + key: new Date().toISOString(), + data: inputdata["daily_workflow_executions"] + }) + } + + if (inputdata["daily_subflow_executions"] !== undefined && inputdata["daily_subflow_executions"] !== null) { + subflowRuns["data"].push({ + key: new Date().toISOString(), + data: inputdata["daily_subflow_executions"] + }) + } + } + + // Only for parent orgs + if (childorgappRuns["data"].length > 0) { + setChildOrgsAppRuns(childorgappRuns) + } + + setSubflowRuns(subflowRuns) + setWorkflowRuns(workflowRuns) + setAppruns(appRuns) + setApprunCosts(appcostRuns) + }, [syncStats, endTime, startTime]) + useEffect(() => { if (statistics && statistics?.org_id?.length > 0) { handleDataSetting(statistics, "day") } }, [statistics]) + useEffect(() => { + setStartTime("") + setEndTime("") + }, [currentTab]) + const getWorkflowStats = async (workflow, startTime, endTime) => { if (workflow.id === undefined || workflow.id === null || workflow.id === "") { @@ -227,7 +372,7 @@ const AppStats = (defaultprops) => { } const statKey = syncStats === true ? "onprem_stats" : "daily_statistics" - if (statistics[statKey] === undefined || statistics[statKey] === null) { + if (!syncStats && (statistics[statKey] === undefined || statistics[statKey] === null)) { setFilteredStatistics(statistics) setMonthlyAppRunsParent(statistics["monthly_app_executions"] ?? 0) return @@ -356,17 +501,55 @@ const AppStats = (defaultprops) => { workflowexecutions += item["workflow_executions"] appexecutions += item["app_executions"] - if (currentTab === 0) { + if (currentTab === 0 || currentTab === 3) { appexecutions += (item["child_app_executions"] ?? 0) } estimatedcost += (item["app_executions"] * invocationCost) } + const today = new Date(); + const isCurrentMonthSelected = + (startTime === "" && endTime === "") || + ( + new Date(foundstarttime).getMonth() === today.getMonth() && + new Date(foundstarttime).getFullYear() === today.getFullYear() && + new Date(foundendtime).getMonth() === today.getMonth() && + new Date(foundendtime).getFullYear() === today.getFullYear() + ); + + if (!syncStats && isCurrentMonthSelected) { + if (statistics["daily_app_executions"] !== undefined && statistics["daily_app_executions"] !== null) { + appexecutions += statistics["daily_app_executions"] + (statistics["daily_child_app_executions"] ?? 0) + } + } + tmpstats["monthly_workflow_executions"] = workflowexecutions tmpstats["monthly_app_executions"] = appexecutions + if (syncStats) { + setOnpremAppRuns(appexecutions) + } + } else { + const today = new Date(); + const isCurrentMonthSelected = + (startTime === "" && endTime === "") || + ( + new Date(foundstarttime).getMonth() === today.getMonth() && + new Date(foundstarttime).getFullYear() === today.getFullYear() && + new Date(foundendtime).getMonth() === today.getMonth() && + new Date(foundendtime).getFullYear() === today.getFullYear() + ); + + if (!syncStats && isCurrentMonthSelected) { + if (statistics["daily_app_executions"] !== undefined && statistics["daily_app_executions"] !== null) { + appexecutions += statistics["daily_app_executions"] + (statistics["daily_child_app_executions"] ?? 0) + } + } + + tmpstats["monthly_app_executions"] = appexecutions } + // Make estimatedcost have max 2 decimals if (isCloud) { // Exclude includedExecutions*month @@ -380,11 +563,11 @@ const AppStats = (defaultprops) => { handleDataSetting(tmpstats, "day") // if we have done monthly reset than only show monthly app runs as current month app run const currentMonth = new Date().getMonth() + 1 - if (!monthlyAppRunsParent && statistics["monthly_app_executions"] > 0 && currentMonth === statistics["last_monthly_reset_month"]) { + if (!syncStats && !monthlyAppRunsParent && statistics["monthly_app_executions"] > 0 && currentMonth === statistics["last_monthly_reset_month"]) { setMonthlyAppRunsParent(statistics["monthly_app_executions"]) } - if (!monthlyAllSuborgExecutions && statistics["monthly_child_app_executions"]> 0 && currentMonth === statistics["last_monthly_reset_month"]) { + if (!syncStats && !monthlyAllSuborgExecutions && statistics["monthly_child_app_executions"]> 0 && currentMonth === statistics["last_monthly_reset_month"]) { setMonthlyAllSuborgExecutions(statistics["monthly_child_app_executions"]) } @@ -397,7 +580,7 @@ const AppStats = (defaultprops) => { loadWorkflowStats(foundWorkflows, startTime, endTime) } - }, [statistics, startTime, endTime]) + }, [statistics, startTime, endTime, syncStats, currentTab, handleDataSetting]) const handleStartTimeChange = (date) => { setStartTime(date) @@ -406,143 +589,6 @@ const AppStats = (defaultprops) => { const handleEndTimeChange = (date) => { setEndTime(date) } - - const handleDataSetting = (inputdata, grouping) => { - if (inputdata === undefined || inputdata === null) { - return - } - - const statKey = syncStats === true ? "onprem_stats" : "daily_statistics" - const dailyStats = inputdata[statKey] - if (dailyStats === undefined || dailyStats === null) { - return - } - - var appRuns = { - "key": "App Runs", - "data": [] - } - - var childorgappRuns = { - "key": "Child Org App Runs", - "data": [] - } - - var workflowRuns = { - "key": "Workflow Runs (includes subflows)", - "data": [] - } - - var subflowRuns = { - "key": "Subflow Runs", - "data": [] - } - - var appcostRuns = { - "key": "Cost of App Runs", - "data": [] - } - - for (let key in dailyStats) { - // Always skips first one as it has accumulated data in it - if (key === 0) { - continue - } - - const item = dailyStats[key] - if (item["date"] === undefined) { - console.log("No date: ", item) - continue - } - - // Check if app_executions key in item - if (item["app_executions"] !== undefined && item["app_executions"] !== null) { - appRuns["data"].push({ - key: new Date(item["date"]), - data: item["app_executions"] - }) - - // Add number - appcostRuns["data"].push({ - key: new Date(item["date"]), - data: (item["app_executions"] * invocationCost).toFixed(2) - }) - } - - if (item["child_app_executions"] !== undefined && item["child_app_executions"] !== null) { - childorgappRuns["data"].push({ - key: new Date(item["date"]), - data: item["child_app_executions"] - }) - } - - // Check if workflow_executions key in item - if (item["workflow_executions"] !== undefined && item["workflow_executions"] !== null) { - workflowRuns["data"].push({ - key: new Date(item["date"]), - data: item["workflow_executions"] - }) - } - - if (item["subflow_executions"] !== undefined && item["subflow_executions"] !== null) { - subflowRuns["data"].push({ - key: new Date(item["date"]), - data: item["subflow_executions"] - }) - } - } - - // Only add today's data if endTime is not set or if today falls within the selected date range - const today = new Date() - const shouldAddTodayData = endTime === "" || endTime === undefined || endTime === null || - (new Date(endTime) >= today.setHours(0, 0, 0, 0)) - - if (shouldAddTodayData) { - // Adds data for today - if (inputdata["daily_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) { - appRuns["data"].push({ - key: new Date(), - data: inputdata["daily_app_executions"] - }) - - appcostRuns["data"].push({ - key: new Date(), - data: (inputdata["daily_app_executions"] * invocationCost).toFixed(2) - }) - } - - if (inputdata["daily_child_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) { - childorgappRuns["data"].push({ - key: new Date(), - data: inputdata["daily_child_app_executions"] - }) - } - - if (inputdata["daily_workflow_executions"] !== undefined && inputdata["daily_workflow_executions"] !== null) { - workflowRuns["data"].push({ - key: new Date(), - data: inputdata["daily_workflow_executions"] - }) - } - - if (inputdata["daily_subflow_executions"] !== undefined && inputdata["daily_subflow_executions"] !== null) { - subflowRuns["data"].push({ - key: new Date(), - data: inputdata["daily_subflow_executions"] - }) - } - } - - // Only for parent orgs - if (childorgappRuns["data"].length > 0) { - setChildOrgsAppRuns(childorgappRuns) - } - - setSubflowRuns(subflowRuns) - setWorkflowRuns(workflowRuns) - setAppruns(appRuns) - setApprunCosts(appcostRuns) - } const paperStyle = { textAlign: "center", @@ -708,22 +754,26 @@ const AppStats = (defaultprops) => { } */} - {syncStats === true ? null : + {/* {syncStats === true ? null : */} App runs in the selected period }> + {syncStats === true ? + + {onpremAppRuns} + : {filteredStatistics.monthly_app_executions === null || filteredStatistics.monthly_app_executions === undefined ? 0 : filteredStatistics.monthly_app_executions} - + } App Runs - } + {/* } */} {syncStats === true || currentTab === 0 ? null : { const [showSettingsMenu, setShowSettingsMenu] = useState(false); const [showCollectIngestMenu, setShowCollectIngestMenu] = useState(false); + useEffect(() => { + if (selectedCategory === "" || selectedCategory === null || selectedCategory === undefined || selectedCategory === "default") { + return + } + + if (datastoreCategories === undefined || datastoreCategories === null || datastoreCategories.length === 0) { + return + } + + if (!datastoreCategories.includes(selectedCategory)) { + setDatastoreCategories([...datastoreCategories, selectedCategory]) + } + }, [datastoreCategories, selectedCategory]) + var to_be_copied = ""; const defaultAutomation = [ { @@ -299,7 +313,16 @@ const CacheView = memo((props) => { useEffect(() => { getWorkflows() getApps() - listOrgCache(orgId, selectedCategory, 0, pageSize, page) + + var chosenCategory = selectedCategory + const urlParams = new URLSearchParams(window.location.search) + const categoryParam = urlParams.get("category") + if (categoryParam && categoryParam !== undefined && categoryParam !== "default" && categoryParam !== "") { + chosenCategory = categoryParam + setSelectedCategory(categoryParam) + } + + listOrgCache(orgId, chosenCategory, 0, pageSize, page) }, []) @@ -423,7 +446,6 @@ const CacheView = memo((props) => { setDatastoreCategories(newcategories) } - if (responseJson?.category_config !== undefined && responseJson?.category_config !== null) { if (responseJson?.category_config?.id !== undefined && responseJson?.category_config?.id !== null && responseJson?.category_config?.id !== "") { @@ -458,7 +480,12 @@ const CacheView = memo((props) => { } } } else { - toast.warn("Failed to load keys. Please try again or contact support@shuffler if this persists.") + //toast.warn("Failed to load keys. Please try again or contact support@shuffler if this persists.") + + if (category !== undefined && category !== null && category !== "" && category !== "default") { + toast.info(`No keys to load in category ${category}`) + setSelectedCategory(category) + } } }) .catch((error) => { @@ -513,8 +540,8 @@ const CacheView = memo((props) => { category: selectedCategory, } - if (dataValue?.category !== "" && dataValue?.category !== "default") { - entry.category = dataValue.category.replaceAll(" ", "_"); + if (dataValue?.category !== undefined && dataValue?.category !== "" && dataValue?.category !== "default") { + entry.category = dataValue?.category?.replaceAll(" ", "_"); } @@ -548,7 +575,10 @@ const CacheView = memo((props) => { .then((responseJson) => { setAddCache(responseJson); toast.success("Edit saved"); - listOrgCache(orgId, selectedCategory, 0, pageSize, page); + setTimeout(() => { + listOrgCache(orgId, selectedCategory, 0, pageSize, page); + }, 7500); + setModalOpen(false); }) .catch((error) => { @@ -586,7 +616,10 @@ const CacheView = memo((props) => { .then((responseJson) => { setAddCache(responseJson); toast.success("New key added!"); - listOrgCache(orgId, selectedCategory, 0, pageSize, page); + + setTimeout(() => { + listOrgCache(orgId, selectedCategory, 0, pageSize, page); + }, 5000); setModalOpen(false); }) .catch((error) => { @@ -1513,7 +1546,7 @@ const CacheView = memo((props) => { name={null} /> : - + {data.value} } @@ -1712,7 +1745,7 @@ const CacheView = memo((props) => { { e.preventDefault() e.stopPropagation() @@ -1911,7 +1944,7 @@ const CacheView = memo((props) => { {selectedCategory === "protected" ?
- Protected keys are encrypted, only available to admins, and will be masked when used in workflows. This is a basic protection, and is NOT bulletproof. + Protected keys are encrypted, only available to admins, and will be masked when used in workflows. If you want unreadable secrets, use App Auth.
: null} @@ -2051,7 +2084,7 @@ const CacheView = memo((props) => { : - +
+ + + + + + + ); +}; + +export default DashboardOnboarding; + + diff --git a/frontend/src/components/Detection.jsx b/frontend/src/components/Detection.jsx index bd75e99d..35c1a994 100644 --- a/frontend/src/components/Detection.jsx +++ b/frontend/src/components/Detection.jsx @@ -143,6 +143,7 @@ const Detection = (props) => { size="small" sx={{ mr: 2 }} value={searchQuery} + disabled onChange={(e) => setSearchQuery(e.target.value)} /> {/*
: */} +
+ +
+ + {/**/} {detectionInfo?.category === "SIGMA" || detectionInfo?.category === "SIEM" ? - + 0 ? green : red}} /> @@ -345,7 +553,7 @@ const DetectionExplorer = (props) => {
- {filteredRules?.length > 0 ? + {ruleInfo?.length > 0 ? { size="small" sx={{ mr: 2 }} value={searchQuery} - onChange={(e) => setSearchQuery(e.target.value)} + onChange={(e) => { + setSearchQuery(e?.target?.value?.replaceAll(" ", "_")?.toLowerCase()) + }} /> @@ -386,7 +596,7 @@ const DetectionExplorer = (props) => { { folderDisabled={folderDisabled} isDetectionActive={isDetectionActive} + ruleDetails={rule} ruleMapping={ruleMapping} setRuleMapping={setRuleMapping} diff --git a/frontend/src/components/DetectionRuleCard.jsx b/frontend/src/components/DetectionRuleCard.jsx index 1a2e9760..a9fcb44b 100644 --- a/frontend/src/components/DetectionRuleCard.jsx +++ b/frontend/src/components/DetectionRuleCard.jsx @@ -12,16 +12,18 @@ import { FormLabel, } from "@mui/material"; -import DashboardBarchart, { LoadStats } from '../components/DashboardBarchart.jsx'; +import LineChartWrapper, { LoadStats } from "../components/LineChartWrapper.jsx"; import { Edit as EditIcon, + Refresh as RefreshIcon, } from "@mui/icons-material"; import { toast } from "react-toastify"; import ShuffleCodeEditor from "../components/ShuffleCodeEditor1.jsx"; import theme from '../theme.jsx'; +const RuleCard = (props) => { + const { ruleName, description, file_id, globalUrl, folderDisabled, isDetectionActive, availableDetection, ruleMapping, setRuleMapping, ruleDetails, key, ...otherProps } = props -const RuleCard = ({ ruleName, description, file_id, globalUrl, folderDisabled, isTenzirActive, availableDetection, ruleMapping, setRuleMapping, ...otherProps }) => { const [openCodeEditor, setOpenCodeEditor] = React.useState(false); const [fileData, setFileData] = React.useState(""); const [isEnabled, setIsEnabled] = React.useState(otherProps.is_enabled); @@ -30,35 +32,33 @@ const RuleCard = ({ ruleName, description, file_id, globalUrl, folderDisabled, i const [responseValue, setResponseValue] = React.useState("No response action") const isCloud = ["localhost:3002", "shuffler.io"].includes(window.location.host); - console.log("Rulemapping: ", ruleMapping) useEffect(() => { - - //const url = `${globalUrl}/api/v1/stats/app_executions_test2` - //const resp = LoadStats(globalUrl, ruleName) - //const resp = LoadStats(globalUrl, "app_executions_test2") - const resp = LoadStats(globalUrl, "app_executions_cloud") - resp.then((data) => { - if (data === undefined) { - setFilteredBarchart([]) - } else { - setFilteredBarchart(data) + if (key < 10) { + console.log("RuleCard Key: ", key, ruleName, file_id, otherProps) } - }) - if (ruleMapping !== undefined && ruleMapping !== null && ruleMapping.value !== undefined && ruleMapping.value !== null) { - console.log("FIX MAPPING FROM ruleMapping.value: ", ruleMapping) - } + if (ruleDetails?.title === undefined || ruleDetails?.title === null || ruleDetails?.title.length === 0) { + //toast.error("Can't load stats for this rule. Contact support@shuffler.io if this persists.") + return + } + + const resp = LoadStats(globalUrl, `detection_rule_${ruleDetails?.title.replaceAll(" ", "_").toLowerCase()}`) + resp.then((data) => { + if (data === undefined) { + setFilteredBarchart([]) + } else { + setFilteredBarchart(data) + } + }) }, []) - console.log("Response Value: ", responseValue) - const handleSwitchChange = (event) => { if (folderDisabled) { toast.warn("Enable the directory to enable individual rules"); return; } - if (!isTenzirActive) { + if (!isDetectionActive) { toast.warn("Connect to the siem first to enable/disable the rule"); return; } @@ -96,6 +96,7 @@ const RuleCard = ({ ruleName, description, file_id, globalUrl, folderDisabled, i }); }; + var parsedRulename = ruleName.charAt(0).toUpperCase() + ruleName.slice(1).replaceAll("_", " ") return ( - {ruleName.replaceAll("_", " ")} ({filteredBarchart === null || filteredBarchart.total === undefined ? 0 : filteredBarchart.total}) + + {parsedRulename} {/*({filteredBarchart === null || filteredBarchart.total === undefined ? 0 : filteredBarchart.total})*/} +
- (upload = ref)} - onChange={(event) => { - //const file = event.target.value - //const fileObject = URL.createObjectURL(actualFile) - //setFile(fileObject) - //const files = event.target.files[0] - uploadFiles(event.target.files); + + + {/* */} + (upload = ref)} + onChange={(event) => { + //const file = event.target.value + //const fileObject = URL.createObjectURL(actualFile) + //setFile(fileObject) + //const files = event.target.files[0] + uploadFiles(event.target.files); - }} - /> - + }} + /> + + + {/*
*/} + {selectedCategory === "sigma" || selectedCategory === "yara" ? + + + + + + : null} + {fileCategories !== undefined && fileCategories !== null && fileCategories.length > 1 ? ( - + + + Category + { /> + {!selectedOrganization || selectedOrganization?.creator_org === undefined || selectedOrganization?.creator_org || null || selectedOrganization?.creator_org?.length > 0 ? null : + />} Workflow Backup Repository diff --git a/frontend/src/components/OrganizationTab.jsx b/frontend/src/components/OrganizationTab.jsx index ce4e5c9e..be65b078 100644 --- a/frontend/src/components/OrganizationTab.jsx +++ b/frontend/src/components/OrganizationTab.jsx @@ -1,5 +1,6 @@ import React, { useEffect, useState, useCallback, useContext } from 'react'; import { Link, useNavigate, useLocation } from "react-router-dom"; + import Billing from "../components/Billing.jsx"; import Priorities from "../components/Priorities.jsx"; import Branding from "../components/Branding.jsx"; @@ -7,9 +8,20 @@ import EditOrgTab from '../components/EditOrgTab.jsx'; import CloudSyncTab from '../components/CloudSyncTab.jsx'; import SSOTab from "../components/ssoTab.jsx" import { ToastContainer, toast } from "react-toastify"; -import { Button, Tooltip } from '@mui/material'; +import { + Button, + Tooltip, + Typography, + Divider, +} from '@mui/material'; + +import { + CheckCircle as CheckCircleIcon, + Cancel as CancelIcon, +} from '@mui/icons-material'; import { getTheme } from '../theme.jsx'; import { Context } from '../context/ContextApi.jsx'; + const OrganizationTab = (props) => { const location = useLocation(); const navigate = useNavigate(); @@ -49,14 +61,18 @@ const OrganizationTab = (props) => { useEffect(() => { if (isIntegrationPartner && isChildOrg && !isGlobalUser) { setVisibleTabs(items.filter((item) => item !== 'Branding' && item !== 'SSO')); - }else { + } else { if (userdata && userdata.active_org && userdata.active_org.role === 'admin') { setVisibleTabs(items); - }else { + } else { setVisibleTabs(items.filter((item) => item !== 'SSO')); } } - },[isIntegrationPartner, isChildOrg, isGlobalUser, userdata]); + + //if (isCloud) { + // setVisibleTabs(items.filter((item) => item !== 'Production Status')); + //} + },[isIntegrationPartner, isChildOrg, isGlobalUser, userdata, isCloud]); useEffect(() => { const queryParams = new URLSearchParams(location.search); @@ -66,30 +82,32 @@ const OrganizationTab = (props) => { setSelectedTab(decodedTabName); if (decodedTabName === 'org_config') { setCurIndex(0); + } else if (decodedTabName === 'prodstatus' || decodedTabName === 'productionstatus' && !isCloud) { + setCurIndex(1); } else if(decodedTabName === 'sso'){ - setCurIndex(1) + setCurIndex(isCloud ? 1 : 2) }else if (decodedTabName === 'notifications' || decodedTabName === 'priorities') { if (isIntegrationPartner && isChildOrg && !isGlobalUser) { - setCurIndex(1); + setCurIndex(isCloud ? 1 : 2); }else { if (userdata && userdata.active_org && userdata.active_org.role === 'admin') { - setCurIndex(2); + setCurIndex(isCloud ? 2 : 3); }else { - setCurIndex(1); + setCurIndex(isCloud ? 1 : 2); } } } else if (decodedTabName === 'billingstats' || decodedTabName === 'billing') { if (isIntegrationPartner && isChildOrg && !isGlobalUser) { - setCurIndex(2); + setCurIndex(isCloud ? 2 : 3); } else { if (userdata && userdata?.active_org && userdata?.active_org?.role === 'admin') { - setCurIndex(3); + setCurIndex(isCloud ? 3 : 4); } else { - setCurIndex(2); + setCurIndex(isCloud ? 2 : 3); } } } else if (decodedTabName === 'branding') { - setCurIndex(4); + setCurIndex(isCloud ? 4 : 5); } // else if (decodedTabName === 'analytics') { // setCurIndex(5); @@ -123,6 +141,7 @@ const OrganizationTab = (props) => { case 'sso': return case `notifications`: + case `errors`: case `priorities`: return ( { key={index} title={ ((tabName === "sso")) && !(userdata?.support || userdata?.active_org?.role === "admin") - ? "Your role is not admin. Please ask the admin to change your role." + ? "Your role is not admin. Please ask an admin to change your role." : "" } placement="right" @@ -248,7 +267,7 @@ const OrganizationTab = (props) => { ))}
-
+
{renderContent()}
@@ -256,3 +275,4 @@ const OrganizationTab = (props) => { }; export default OrganizationTab; + diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index f86bc1d7..6af8691e 100755 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -437,7 +437,8 @@ const ParsedAction = (props) => { ]; const getApp = (appId, setApp) => { - fetch(globalUrl + "/api/v1/apps/" + appId + "/config?openapi=false", { + const url = `${globalUrl}/api/v1/apps/${appId}/config?openapi=false`; + fetch(url, { headers: { Accept: "application/json", }, @@ -447,7 +448,7 @@ const ParsedAction = (props) => { if (response.status === 200) { //toast("Successfully GOT app "+appId) } else { - toast("Failed getting app"); + toast.error("Failed getting app. Please try again or contact support@shuffler.io"); } return response.json(); @@ -1711,6 +1712,7 @@ const ParsedAction = (props) => { } const sortByCategoryLabel = (a, b) => { + const aHasCategoryLabel = a.category_label !== undefined && a.category_label !== null && a.category_label.length > 0 const bHasCategoryLabel = b.category_label !== undefined && b.category_label !== null && b.category_label.length > 0 @@ -1739,11 +1741,12 @@ const ParsedAction = (props) => { }) } + // Gets the most important actions first const renderedActionOptions = deduplicateByName(( - selectedApp.actions === undefined || selectedApp.actions === null ? [] : - selectedApp.actions.filter((a) => - a.category_label !== undefined && a.category_label !== null && a.category_label.length > 0).concat(sortByKey(selectedApp.actions, "label")) + selectedApp.actions === undefined || selectedApp.actions === null ? [] : + isIntegration ? selectedApp.actions : + selectedApp.actions.filter((a) => a.category_label !== undefined && a.category_label !== null && a.category_label.length > 0).concat(sortByKey(selectedApp.actions, "label")) ).sort(sortByCategoryLabel)) @@ -2981,7 +2984,6 @@ const ParsedAction = (props) => { dataLPIgnore="true" autoComplete="off" - id="checkbox-search" style={{ ...theme.palette.textFieldStyle, diff --git a/frontend/src/components/PartnerDetails.jsx b/frontend/src/components/PartnerDetails.jsx index 9ca0aeff..7b7c102c 100644 --- a/frontend/src/components/PartnerDetails.jsx +++ b/frontend/src/components/PartnerDetails.jsx @@ -223,7 +223,7 @@ const PartnerDetails = (props) => {
- Name + Company Name { />
*/}
-
+
Solutions
{ variant="text" style={{ color: theme.palette.text.primary, fontFamily: theme?.typography?.fontFamily }} > - Name + Company Name { cursor: isDisabled ? "not-allowed" : "pointer", }} fullWidth={true} - placeholder="Name" + placeholder="Company Name" type="name" id="standard-required" margin="normal" @@ -544,7 +544,8 @@ const PartnerDetails = (props) => { style={{ marginRight: "12px", color: theme.palette.text.primary, - fontFamily: theme?.typography?.fontFamily + fontFamily: theme?.typography?.fontFamily, + marginTop: 2.5, }} > Solutions @@ -895,7 +896,7 @@ const PartnerDetails = (props) => { cursor: isDisabled ? "not-allowed" : "pointer", }} fullWidth={true} - placeholder="support@shuffler.io" + placeholder="example@company.com" type="name" id="standard-required" margin="normal" diff --git a/frontend/src/components/Priorities.jsx b/frontend/src/components/Priorities.jsx index 2ff15cdf..426c80ea 100644 --- a/frontend/src/components/Priorities.jsx +++ b/frontend/src/components/Priorities.jsx @@ -1362,10 +1362,10 @@ print('"' + encoded + '"')
- Notification Workflow + Error Workflow - The notification workflow triggers when an error occurs in one of your workflows. Each individual one will only start a workflow once every 2 minutes. You can point child org notifications into the parent org notification by choosing it in the list. + The error workflow triggers when an error occurs in one of your workflows. Each individual one will only start a workflow once every 2 minutes. You can point child org errors to a parent org's error workflow by choosing it in the list. {modalView} @@ -1518,6 +1518,7 @@ print('"' + encoded + '"') /> : } - Notifications ({ + Errors ({ notifications?.filter((notification) => showRead === true || notification.read === false).length }) - Notifications help you find potential problems with your workflows and apps.  + Error help you find potential problems with your workflows and apps.  { if (item !== null) { descsplit[1] = item.large_image - console.log("DESCSPLIT name: ", descsplit[0]) - if (descsplit[0].includes(":default")) { realignedSrc = true } diff --git a/frontend/src/components/RunDetectionTest.jsx b/frontend/src/components/RunDetectionTest.jsx new file mode 100644 index 00000000..d6294722 --- /dev/null +++ b/frontend/src/components/RunDetectionTest.jsx @@ -0,0 +1,223 @@ +import React, { useEffect, useState } from 'react'; +import { toast } from 'react-toastify'; + +import { + Button, + ButtonGroup, + CircularProgress, + Tooltip, +} from "@mui/material"; + +import { + Check as CheckIcon, + OpenInNew as OpenInNewIcon, +} from "@mui/icons-material"; +import { green, yellow, red } from '../views/AngularWorkflow.jsx' + +const RunDetectionTest = (props) => { + const { + globalUrl, + pipelines, + workflows, + ticketWebhook, + detectionWorkflowId, + + changePipelineState, + submitPipelineWrapper, + } = props + + const [executions, setExecutions] = React.useState([]); + const [detectionTestRunning, setDetectionTestRunning] = React.useState(false); + const [detectionTestExecutionId, setDetectionTestExecutionId] = React.useState(""); + + useEffect(() => { + if (detectionWorkflowId !== "") { + handleLoadExecutions(detectionWorkflowId) + } + }, [detectionWorkflowId]) + + if (workflows === undefined || workflows === null || workflows.length === 0) { + return null + } + + const handleLoadExecutions = (workflowId, detectionTestRunning) => { + const url = `${globalUrl}/api/v2/workflows/${workflowId}/executions` + + fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }).then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for getting all executions"); + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success !== false && responseJson?.executions?.length > 0) { + if (detectionTestRunning === true) { + console.log("Checking executions in workflow: ", workflowId, responseJson.executions) + for (var executionKey in responseJson.executions) { + const curExec = responseJson.executions[executionKey] + + if (curExec.execution_id === detectionTestExecutionId) { + continue + } + + // started_at = unix timestamp + // check within the last 60 seconds + const datecomparison = (Date.now() / 1000) - 60 + if (curExec.started_at >= datecomparison) { + if (curExec?.execution_argument?.includes("rule") && curExec?.execution_argument?.includes("Test Notepad Event")) { + setDetectionTestRunning(false) + setDetectionTestExecutionId(curExec.execution_id) + } + + break; + } + } + } else { + setExecutions(responseJson.executions || []) + } + } + }) + .catch((error) => { + toast(error.toString()); + }) + } + + const runDetectionTest = () => { + setDetectionTestRunning(true) + if (ticketWebhook === "") { + setDetectionTestRunning(false) + toast.error("No ticketing webhook found. Please enable the ticketing workflow first.") + return + } + + if (detectionWorkflowId === "") { + setDetectionTestRunning(false) + toast.error("No ticketing workflow found. Please enable the ticketing workflow first.") + return + } + + if (haveDetectionPipelines() === false) { + setDetectionTestRunning(false) + toast.error("No detection pipelines found. Please deploy the Syslog (TCP) & Sigma pipelines first.") + return + } + + + // 1. Run a new pipeline which exits. + const detectionTest = `from {message: "<165>1 2025-10-06T12:34:56.789Z myhost.example.com myapp 1234 ID47 [huh eventSource=\\\"App\\\" EventID=\\\"4688\\\" NewProcessName=\\\"notepad.exe\\\" Context=\\\"Testing\\\"] This is a test log message"} | this = message.parse_syslog() | import` + + for (var pipelineKey in pipelines) { + const curPipeline = pipelines[pipelineKey] + if (curPipeline.definition === detectionTest && changePipelineState !== undefined) { + changePipelineState(curPipeline, "stop"); + } + } + + // 1. Submit it to run + // 2. Check executions if they happened recently~ + if (submitPipelineWrapper !== undefined) { + submitPipelineWrapper(detectionTest) + } + + for (var i = 0; i < 10; i++) { + setTimeout(() => { + handleLoadExecutions(detectionWorkflowId, true) + }, i * 5000) + } + + setTimeout(() => { + setDetectionTestRunning(false) + }, 60000) + } + + const haveDetectionPipelines = () => { + if (pipelines === undefined) { + toast.warn("No pipelines found. Please create the Syslog (TCP) & Sigma pipelines first.") + return false + } + + var foundCorrect = 0 + for (var pipelineKey in pipelines) { + const curPipeline = pipelines[pipelineKey] + //if (curPipeline?.definition?.includes("load_tcp") && curPipeline?.definition?.includes("import")) { + // foundCorrect += 1 + //} + + if (curPipeline?.definition?.includes("sigma") && curPipeline?.definition?.includes("export")) { + foundCorrect += 1 + } + } + + if (foundCorrect >= 1) { + return true + } + + return false + } + + return ( + + ) +} + +export default RunDetectionTest diff --git a/frontend/src/components/RunsOverTimeWidget.jsx b/frontend/src/components/RunsOverTimeWidget.jsx new file mode 100644 index 00000000..c1d09113 --- /dev/null +++ b/frontend/src/components/RunsOverTimeWidget.jsx @@ -0,0 +1,326 @@ +import React, { useEffect, useMemo, useState } from 'react'; +import { Typography, ToggleButton, ToggleButtonGroup, Select, MenuItem, FormControl, InputLabel, Box } from '@mui/material'; +import { BarChart, BarSeries, Bar, GridlineSeries, Gridline, TooltipArea, ChartTooltip, LinearYAxis, LinearYAxisTickSeries, LinearYAxisTickLabel } from 'reaviz'; +import theme from '../theme.jsx'; + +// Compact number formatter for axis ticks (e.g. 12,000 -> 12k, 12,000,000 -> 12M) +function formatCompactNumber(value) { + const n = Number(value) || 0; + const abs = Math.abs(n); + if (abs >= 1e9) return `${Math.round((n / 1e9) * 10) / 10}B`; + if (abs >= 1e6) return `${Math.round((n / 1e6) * 10) / 10}M`; + if (abs >= 1e3) return `${Math.round((n / 1e3) * 10) / 10}k`; + return `${n}`; +} + +const RunsOverTimeWidget = (props) => { + const { globalUrl, onLoadingChange, monthOverride, dummyMode } = props; + const [mode, setMode] = useState('workflows'); // 'apps' | 'workflows' + const [series, setSeries] = useState([]); + const [days, setDays] = useState(365); // aggregate to last 12 months by default + const [selectedMonth, setSelectedMonth] = useState(null); // Date representing first day of target month, or null for yearly view + const [loading, setLoading] = useState(false); + + // Helper: fetch time series for a specific statistics key + const fetchSeriesForKey = async (key) => { + try { + const urlA = `${globalUrl}/api/v1/stats/${encodeURIComponent(key)}?days=${encodeURIComponent(days)}`; + const doFetch = async (u) => { + const r = await fetch(u, { method: 'GET', credentials: 'include', headers: { 'Content-Type': 'application/json' } }); + if (!r.ok) return []; + const j = await r.json(); + return Array.isArray(j?.entries) ? j.entries : []; + }; + const a = await doFetch(urlA); + if (a.length > 0) return a; + + // Optional org route fallback if present globally + const orgId = (window && window.selectedOrganization && window.selectedOrganization.id) || null; + if (orgId) { + const urlB = `${globalUrl}/api/v1/orgs/${encodeURIComponent(orgId)}/stats/${encodeURIComponent(key)}?days=${encodeURIComponent(days)}`; + const b = await doFetch(urlB); + if (b.length > 0) return b; + } + + // Final fallback: old aggregate endpoint returning daily_statistics + const fallback = await fetch(`${globalUrl}/api/v1/stats`, { method: 'GET', credentials: 'include', headers: { 'Content-Type': 'application/json' } }); + if (fallback.ok) { + const data = await fallback.json(); + const daily = Array.isArray(data?.daily_statistics) ? data.daily_statistics : []; + const valField = key; + return daily.map((d) => ({ date: d?.date, value: Number(d?.[valField] || 0) })); + } + return []; + } catch (e) { + return []; + } + }; + + // Load and transform into monthly aggregation for last 12 months + const load = async (curMode) => { + setLoading(true); + try { + // Clear current series immediately to avoid any visual overlap while switching views + setSeries([]); + if (dummyMode) { + // Bring back the older dummy series with emphasis on earlier months + const now = new Date(); + const months = []; + for (let i = 11; i >= 0; i--) { + const dt = new Date(now.getFullYear(), now.getMonth() - i, 1); + months.push(new Date(dt.getFullYear(), dt.getMonth(), 1)); + } + const base = [20, 18, 22, 24, 23, 21, 15, 12, 9, 15, 24, 20]; + const dummy = months.map((m, idx) => ({ key: m, id: `${m.getFullYear()}-${m.getMonth()}`, data: base[idx] })); + setSeries(dummy); + return; + } + const key = curMode === 'apps' ? 'app_executions' : 'workflow_executions'; + const entries = await fetchSeriesForKey(key); + // Normalize variants: {Date, Value} or {date, value} + const normalized = (entries || []).map((d) => ({ + date: d?.Date ? new Date(d.Date) : (d?.date ? new Date(d.date) : new Date()), + value: Number(d?.Value ?? d?.value ?? 0), + })); + + // If a month is selected, show DAILY bars for that month + if (selectedMonth instanceof Date) { + const year = selectedMonth.getFullYear(); + const month = selectedMonth.getMonth(); + + // Build all days for selected month + const firstDay = new Date(year, month, 1); + const nextMonthFirst = new Date(year, month + 1, 1); + const numDays = Math.round((nextMonthFirst - firstDay) / (1000 * 60 * 60 * 24)); + + // Sum values per day (normalize time to midnight) + const byDayKey = (d) => `${d.getFullYear()}-${d.getMonth()}-${d.getDate()}`; + const sumPerDay = new Map(); + normalized.forEach((p) => { + if (!p?.date || Number.isNaN(p.value)) return; + const d = p.date; + if (d.getFullYear() !== year || d.getMonth() !== month) return; + const dayKey = byDayKey(new Date(d.getFullYear(), d.getMonth(), d.getDate())); + sumPerDay.set(dayKey, (sumPerDay.get(dayKey) || 0) + p.value); + }); + + const dailySeries = Array.from({ length: numDays }, (_, i) => { + const d = new Date(year, month, i + 1); + const k = byDayKey(d); + const v = sumPerDay.get(k) || 0; + return { key: d, id: k, data: v }; + }); + + // Ensure consistent ordering + setSeries(dailySeries.sort((a, b) => a.key - b.key)); + return; + } + + // Otherwise, show MONTHLY aggregation for last 12 months including current month + const now = new Date(); + const months = []; + for (let i = 11; i >= 0; i--) { + const dt = new Date(now.getFullYear(), now.getMonth() - i, 1); + months.push({ y: dt.getFullYear(), m: dt.getMonth(), key: new Date(dt.getFullYear(), dt.getMonth(), 1) }); + } + + const byMonthKey = (d) => `${d.getFullYear()}-${d.getMonth()}`; + const sumPerMonth = new Map(); + normalized.forEach((p) => { + if (!p?.date || Number.isNaN(p.value)) return; + const k = byMonthKey(new Date(p.date.getFullYear(), p.date.getMonth(), 1)); + sumPerMonth.set(k, (sumPerMonth.get(k) || 0) + p.value); + }); + + const monthlySeries = months.map((mm) => { + const k = `${mm.y}-${mm.m}`; + const v = sumPerMonth.get(k) || 0; + return { key: mm.key, id: k, data: v }; + }); + + setSeries(monthlySeries); + } finally { + setLoading(false); + } + }; + + // Apply month override (e.g. onboarding Explore Now) - consolidated with main load effect + useEffect(() => { + if (monthOverride instanceof Date) { + // Clear series immediately to prevent visual overlap + setSeries([]); + setSelectedMonth(new Date(monthOverride.getFullYear(), monthOverride.getMonth(), 1)); + setDays(370); + } + }, [monthOverride]); + + useEffect(() => { + load(mode); + }, [mode, globalUrl, days, selectedMonth, dummyMode]); + + // Notify parent on loading changes + useEffect(() => { + if (typeof onLoadingChange === 'function') { + onLoadingChange(loading); + } + }, [loading, onLoadingChange]); + + const barData = useMemo(() => ( + (Array.isArray(series) ? series : []).map((d) => { + const dt = new Date(d.key); + const label = selectedMonth instanceof Date + ? String(dt.getDate()) // day of month for daily view + : dt.toLocaleString('default', { month: 'short' }); + return { key: label, data: Number(d?.data || 0) }; + }) + ), [series, mode, selectedMonth]); + + // Build month dropdown options for last 12 months + const monthOptions = useMemo(() => { + const now = new Date(); + const opts = []; + for (let i = 0; i < 12; i++) { + const dt = new Date(now.getFullYear(), now.getMonth() - i, 1); + opts.push(dt); + } + return opts; + }, []); + + const tooltip = ( +
+ {data?.x ?? ''} + {data?.y ?? ''} +
+ )} + /> + } + />; + + return ( +
+
+ Runs over time ({mode === "workflows" ? "Workflows" : "Apps"}) + + v && setMode(v)} + sx={{ + height: 37, + backgroundColor: 'rgba(255,255,255,0.06)', + border: '1px solid rgba(255,255,255,0.22)', + borderRadius: '30px', + padding: '2px', + "& .MuiToggleButton-root": { + border: "none", + borderRadius: "30px", + color: "#fff", + padding: "6px 16px", + textTransform: "none", + fontSize: "14px", + "&.Mui-selected": { + backgroundColor: "#fff", + color: "#222", + fontWeight: "600", + "&:hover": { + backgroundColor: "#fff", + }, + }, + "&:hover": { + backgroundColor: "rgba(255, 255, 255, 0.2)", + }, + }, + }} + > + + + Workflows + + + + + Apps + + + + + View Month + + + +
+ +
+
+ } />} + gridlines={} />} + yAxis={ + formatCompactNumber(d)} /> + } + /> + } + /> + } + animated={false} + /> +
+
+
+ ); +}; + +export default RunsOverTimeWidget; + + diff --git a/frontend/src/components/SchedulesTab.jsx b/frontend/src/components/SchedulesTab.jsx index cc91db50..26734560 100644 --- a/frontend/src/components/SchedulesTab.jsx +++ b/frontend/src/components/SchedulesTab.jsx @@ -7,6 +7,7 @@ import { ListItem, ListItemText, Button, + ButtonGroup, Tooltip, IconButton, Dialog, @@ -14,15 +15,21 @@ import { DialogContent, DialogActions, TextField, + Chip, + CircularProgress, } from '@mui/material'; import { - FileCopy as FileCopyIcon, - OpenInNew as OpenInNewIcon, - Padding, + FileCopy as FileCopyIcon, + OpenInNew as OpenInNewIcon, + Refresh as RefreshIcon, + Delete as DeleteIcon, + Check as CheckIcon, } from "@mui/icons-material" +import { green, yellow, red } from '../views/AngularWorkflow.jsx' import { Box, Skeleton, Typography } from '@mui/material'; import { Context } from '../context/ContextApi.jsx'; +import RunDetectionTest from '../components/RunDetectionTest.jsx'; const SchedulesTab = memo((props) => { const {globalUrl, users, } = props; @@ -30,13 +37,58 @@ const SchedulesTab = memo((props) => { const [allSchedules, setAllSchedules] = React.useState([]); const [pipelines, setPipelines] = React.useState([]); const [showLoader, setShowLoader] = React.useState(true); + const [workflows, setWorkflows] = React.useState([]); const [pipelineModalOpen, setPipelineModalOpen] = React.useState(false); - const [newPipelineValue, setNewPipelineValue] = React.useState("export | sigma /tmp/sigma_rules | to SHUFFLE_WEBHOOK"); + const [newPipelineValue, setNewPipelineValue] = React.useState(`export | sigma "/tmp/sigma_rules" | to "SHUFFLE_WEBHOOK"`); + + const [ticketWebhook, setTicketWebhook] = React.useState(""); + const [detectionWorkflowId, setDetectionWorkflowId] = React.useState(""); const { themeMode, brandColor } = useContext(Context); const theme = getTheme(themeMode, brandColor); + const handleGetWorkflows = () => { + const url = `${globalUrl}/api/v1/workflows`; + fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }).then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for getting all workflows"); + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success !== false) { + setWorkflows(responseJson || []); + + for (var i = 0; i < responseJson?.length; i++) { + if (responseJson[i].background_processing === true && responseJson[i].name.toLowerCase().includes("ingest tickets") && responseJson[i].triggers !== undefined) { + + for (var triggerkey in responseJson[i].triggers) { + if (responseJson[i].triggers[triggerkey].trigger_type === "WEBHOOK") { + setDetectionWorkflowId(responseJson[i].id) + setTicketWebhook(`${globalUrl}/api/v1/hooks/webhook_${responseJson[i].triggers[triggerkey].id}`) + setNewPipelineValue(`export | sigma /tmp/sigma_rules | to ${globalUrl}/api/v1/hooks/webhook_${responseJson[i].triggers[triggerkey].id}`) + break; + } + } + } + } + } + }) + .catch((error) => { + toast(error.toString()); + }) + } + useEffect(() => { + handleGetWorkflows() if (allSchedules.length === 0 && webHooks.length === 0 && pipelines.length === 0) { handleGetAllTriggers() } @@ -58,8 +110,11 @@ const SchedulesTab = memo((props) => { environment: pipeline.environment, }; - if (state === "start") toast("starting the pipeline"); - else toast.info("Stopping the pipeline. This may take a few minutes to propagate.") + if (state === "start") { + toast("starting the pipeline") + } else { + toast.info("Stopping a pipeline. This may take a few minutes to propagate.") + } const url = `${globalUrl}/api/v1/triggers/pipeline`; fetch(url, { @@ -144,16 +199,65 @@ const SchedulesTab = memo((props) => { }, }} > - + Run a Tenzir pipeline - Alpha feature. Deploys to the first available Orborus location. Explore Tenzir Pipelines. The example below exports everything in the Tenzir database, runs Sigma rules on it, and forwards the results to a Shuffle webhook. + Alpha feature. Deploys to the first available Orborus location. Explore Tenzir Pipelines. The example below exports everything in the Tenzir database, runs Sigma rules on it, and forwards the results to a Shuffle webhook. - -
+ +
+ + { + setNewPipelineValue(`load_tcp "0.0.0.0:1514" { read_syslog } | import`) + }} + label={"Syslog Listener (TCP)"} + variant="outlined" + color="secondary" + style={{ + marginRight: 10, + }} + /> + + { + setNewPipelineValue(`load_udp "0.0.0.0:1514", insert_newlines=true | read_syslog | import`) + }} + label={"Syslog Listener (UDP)"} + variant="outlined" + color="secondary" + style={{ + marginRight: 10, + }} + /> + + { + setNewPipelineValue(`export live=true | sigma "/tmp/sigma_rules" | to "${ticketWebhook !== "" ? ticketWebhook : "SHUFFLE_WEBHOOK"}"`) + }} + label={"Sigma Rules"} + variant="outlined" + color="secondary" + style={{ + marginRight: 10, + }} + /> + + { + setNewPipelineValue(`export live=true | to_opensearch "localhost:9200", action="create", index="shuffle_logs", user="admin", passwd="PASSWORD"`) + }} + label={"Opensearch Ingest"} + variant="outlined" + color="secondary" + style={{ + marginRight: 10, + }} + /> + { minRows={4} required fullWidth={true} - defaultValue="export | sigma /tmp/sigma_rules | to SHUFFLE_WEBHOOK" - placeholder="export | sigma /tmp/sigma_rules | to SHUFFLE_WEBHOOK" + defaultValue={`export | sigma /tmp/sigma_rules | to ${ticketWebhook !== "" ? ticketWebhook : "SHUFFLE_WEBHOOK"}`} + value={newPipelineValue} + placeholder={`export | sigma /tmp/sigma_rules | to ${ticketWebhook !== "" ? ticketWebhook : "SHUFFLE_WEBHOOK"}`} id="environment_name" margin="normal" variant="outlined" @@ -174,7 +279,7 @@ const SchedulesTab = memo((props) => { />
- + @@ -232,18 +337,18 @@ const SchedulesTab = memo((props) => { }) .then((responseJson) => { if (!responseJson.success && pipelineConfig.type !== "delete") { - toast("Failed to set pipeline: " + responseJson.reason); + toast.error("Failed to set pipeline: " + responseJson.reason); } else { if (pipelineConfig.type === "create") { - toast("Pipeline will be created: " + responseJson.reason) + toast.success("Pipeline will be created. Page will autorefresh in a bit: " + responseJson.reason) setPipelineModalOpen(false) } else if (pipelineConfig.type === "stop") { - toast("Pipeline will be stopped: " + responseJson.reason) + toast.success("Pipeline will be stopped: " + responseJson.reason) setPipelineModalOpen(false) } else { - toast("Unknown pipeline type: " + pipelineConfig.type) + toast.info("Unknown pipeline type: " + pipelineConfig.type) } } @@ -274,12 +379,7 @@ const SchedulesTab = memo((props) => { // Just use this one? - const url = - globalUrl + - "/api/v1/workflows/" + - data["workflow_id"] + - "/schedule/" + - data.id; + const url = `${globalUrl}/api/v1/workflows/${data?.workflow_id}/schedule/${data.id}`; fetch(url, { method: "DELETE", credentials: "include", @@ -414,7 +514,7 @@ const SchedulesTab = memo((props) => { //toast(error.toString()); console.log("Get schedule error: ", error.toString()); }); - }; + } const startWebHook = (trigger) => { const hookname = trigger.info.name; @@ -490,8 +590,197 @@ const SchedulesTab = memo((props) => { Triggers are Automatic Workflow starters. Status: Schedules ({allSchedules.length}), Webhooks ({webHooks.length}), Pipelines ({pipelines.length}) +
+ Pipelines + + + Controls Pipelines on your Orborus Runners, e.g. for Log Ingestion, MQ subscriptions or Sigma Detections.{" "} + + Learn more + + + +
+ + + +
+
+ + + {["Status", "Command", "Environment", "Total Runs", "Actions"].map((header, index) => ( + + ))} + + {showLoader ? ( + [...Array(6)].map((_, rowIndex) => { + return ( + + {Array(5) + .fill() + .map((_, colIndex) => { + return ( + + + + ) + })} + + ) + } + ) + + ) : ( + pipelines?.length === 0 ? ( +
+ No pipelines found. +
+ + ):( + pipelines.map((pipeline, index) => { + var bgColor = themeMode === "dark" ? "#212121" : "#FFFFFF"; + if (index % 2 === 0) { + bgColor = themeMode === "dark" ? "#1A1A1A" : "#EAEAEA"; + } + + return ( + + + + + + + + { + const copyContent = `curl -XPOST http://localhost:5160/api/v0/pipeline/delete -H "Content-Type: application/json" -d '{"id":"${pipeline.id}"}' -v` + const copyText = navigator?.clipboard?.writeText(copyContent) + if (copyText) { + toast.success("Pipeline copied to clipboard") + } else { + toast.error("Failed to copy pipeline") + } + }}> + + + + + { + changePipelineState(pipeline, "stop"); + }}> + + + + + )} + /> + + ); + }) + ) + )} + + +
+ +
+
+
- + Schedules @@ -903,11 +1192,9 @@ const SchedulesTab = memo((props) => { style={{ textTransform: 'none', fontSize: 16, - color:webhook.status === "running" ? '#1a1a1a' : null, - backgroundColor: webhook.status === "running" ? '#ff8544' : null, width: 150, }} - color={webhook.status === "running" ? "secondary" : "primary"} + color={"secondary"} variant={webhook.status === "running" ? "contained" : "outlined"} disabled={webhook.status === "uninitialized"} onClick={() => { @@ -929,166 +1216,7 @@ const SchedulesTab = memo((props) => { )}
-
- Pipelines - - - Controls Pipelines on your Orborus Runners, e.g. for Log Ingestion, MQ subscriptions or Sigma Detections.{" "} - - Learn more - - - -
- - - -
-
- - - {["Command", "Environment", "Total Runs", "Actions"].map((header, index) => ( - - ))} - - {showLoader ? ( - [...Array(6)].map((_, rowIndex) => { - return ( - - {Array(5) - .fill() - .map((_, colIndex) => { - return ( - - - - ) - })} - - ) - } - ) - - ): ( - pipelines?.length === 0 ? ( -
- No pipeline trigger found -
- - ):( - pipelines.map((pipeline, index) => { - var bgColor = themeMode === "dark" ? "#212121" : "#FFFFFF"; - if (index % 2 === 0) { - bgColor = themeMode === "dark" ? "#1A1A1A" : "#EAEAEA"; - } - - return ( - - - - - - - - )} - /> - - ); - }) - ) - )} - -
+
@@ -1096,3 +1224,4 @@ const SchedulesTab = memo((props) => { }); export default SchedulesTab; + diff --git a/frontend/src/components/ShuffleCodeEditor1.jsx b/frontend/src/components/ShuffleCodeEditor1.jsx index 3424d3d2..8a26556c 100644 --- a/frontend/src/components/ShuffleCodeEditor1.jsx +++ b/frontend/src/components/ShuffleCodeEditor1.jsx @@ -147,6 +147,8 @@ const CodeEditor = (props) => { // Auto-indent JSON-like content (with safety hehe) const autoIndentContent = React.useCallback((content) => { + return content + // Safety checks :) if (!content || typeof content !== 'string' || content.trim().length === 0) { return content; @@ -173,6 +175,7 @@ const CodeEditor = (props) => { } }, []); + const [localcodedata, setlocalcodedata] = React.useState(codedata === undefined || codedata === null || codedata.length === 0 ? "" : codedata); // const {codelang, setcodelang} = props @@ -1832,6 +1835,7 @@ const CodeEditor = (props) => { display: 'flex', }} > +
{ File Editor ({localcodedata.length})
+ + + { + const indentedText = IndentJsonLikeString(localcodedata, 2) + if (indentedText !== undefined && indentedText !== null) { + setlocalcodedata(indentedText) + } else { + toast.warn("Could not indent the text. Please check the input format.", { autoClose: 5000 }) + } + }} + color="secondary" + > + + + +
:
{ width: 50, marginLeft: 100, }} - disabled={editorData === undefined || editorData.example === undefined || editorData.example === null || editorData.example.length === 0} + disabled={localcodedata === undefined || localcodedata === null || localcodedata.length === 0} onClick={() => { const indentedText = IndentJsonLikeString(localcodedata, 2) if (indentedText !== undefined && indentedText !== null) { diff --git a/frontend/src/components/SuccessFailedRunsWidget.jsx b/frontend/src/components/SuccessFailedRunsWidget.jsx new file mode 100644 index 00000000..a14bf6e6 --- /dev/null +++ b/frontend/src/components/SuccessFailedRunsWidget.jsx @@ -0,0 +1,1001 @@ +import React, { useEffect, useMemo, useState } from "react"; +import { + Typography, + FormControl, + InputLabel, + Select, + MenuItem, + ToggleButton, + ToggleButtonGroup, + Box, +} from "@mui/material"; +import { + AreaChart, + AreaSeries, + Area, + GridlineSeries, + Gridline, + ChartTooltip, + TooltipArea, + LinearXAxis, + LinearXAxisTickSeries, + LinearXAxisTickLabel, + LinearYAxis, + LinearYAxisTickSeries, + LinearYAxisTickLabel, +} from "reaviz"; +import theme from "../theme"; + +// KPI configuration constants +const RUN_MINUTES_SAVED_PER_WORKFLOW = 15; // minutes saved per workflow run +const RUN_DOLLARS_SAVED_PER_WORKFLOW = 25; // dollars saved per workflow run + +// Filters for status selection +const statusOptions = [ + { key: "ALL", label: "All" }, + { key: "FINISHED", label: "Success" }, + { key: "FAILED", label: "Failed" }, +]; + +// Response Object (Coming from backend) +// { +// "key": "app_executions", +// "value": 70, +// "date": "2025-10-15T19:36:03.928872+05:30" +// } + +// Date formatting helpers +// This is used to format the date in the format of YYYY-MM-DD +function formatDay(dateInput) { + try { + return new Date(dateInput).toISOString().slice(0, 10); + } catch { + return String(dateInput); + } +} + +// This is used to format the date in the format of YYYY-MM +function formatMonth(dateInput) { + const dt = new Date(dateInput); + const month = String(dt.getMonth() + 1).padStart(2, "0"); + return `${dt.getFullYear()}-${month}`; +} + +// Aggregate values by day or by month +// This is used for area chart toggle button (Daily / Monthly) +function bucketSeries(items, resolution) { + const map = new Map(); + for (const item of items) { + const key = + resolution === "monthly" ? formatMonth(item.key) : formatDay(item.key); + const value = Number(item.data || 0); + map.set(key, (map.get(key) || 0) + value); + } + return map; +} + +// Normalize API entries to a consistent structure +// for e.g, {date: "2025-10-15T19:36:03.928872+05:30", value: 70} +// will be normalized to {key: "2025-10-15", id: "2025-10-15", data: 70} +function normalizeEntries(arr) { + return (arr || []).map((d) => ({ + key: d?.date ? new Date(d.date) : new Date(), + id: d?.date || Math.random().toString(36).slice(2), + data: Number(d?.value ?? 0), + })); +} + +// Build continuous key sequence from start to end, aligned by resolution (Daily / Monthly) +function buildBackfilledKeys(allKeys, resolution) { + if (allKeys.length === 0) return allKeys; + + const start = new Date(allKeys[0]); + let end = new Date(allKeys[allKeys.length - 1]); + const today = new Date(); + + if (resolution === "monthly") { + const monthToday = new Date(today.getFullYear(), today.getMonth(), 1); + if (monthToday > end) end = monthToday; + } else { + // To ensure that the last day is included in the series + const dayToday = new Date( + today.getFullYear(), + today.getMonth(), + today.getDate() + ); + if (dayToday > end) end = dayToday; + } + + const addKey = (dt) => + resolution === "monthly" ? formatMonth(dt) : formatDay(dt); + const step = (dt) => { + if (resolution === "monthly") { + dt.setMonth(dt.getMonth() + 1); + dt.setDate(1); + } else { + dt.setDate(dt.getDate() + 1); + } + }; + + const sequence = []; + const cursor = new Date(start); + if (resolution === "monthly") cursor.setDate(1); + while (cursor <= end) { + sequence.push(addKey(cursor)); + step(cursor); + } + return sequence; +} + +// Ensure area series has at least two points +function ensureMinTwoPoints(arr) { + if (arr.length === 1) { + return [ + { key: 0, data: arr[0].data }, + { key: 1, data: arr[0].data }, + ]; + } + return arr; +} + +// Compact number formatter for axis ticks (e.g. 12,000 -> 12k, 12000000 -> 12M, 12000000000 -> 12B) +function formatCompactNumber(value) { + const n = Number(value) || 0; + const abs = Math.abs(n); + if (abs >= 1e9) return `${Math.round((n / 1e9) * 10) / 10}B`; + if (abs >= 1e6) return `${Math.round((n / 1e6) * 10) / 10}M`; + if (abs >= 1e3) return `${Math.round((n / 1e3) * 10) / 10}k`; + return `${n}`; +} + +// Compute X axis ticks and label formatter +function computeTicks(allKeys, days, resolution) { + const maxTicks = 12; + const xInterval = Math.max(1, Math.floor(allKeys.length / maxTicks)); + let tickValues = Array.from( + { length: Math.ceil(allKeys.length / xInterval) }, + (_, i) => i * xInterval + ); + const lastIdx = allKeys.length - 1; + if (lastIdx >= 0 && tickValues[tickValues.length - 1] !== lastIdx) { + tickValues = [...tickValues, lastIdx]; + } + + // Format the label for the x axis + // if resolution is monthly, it will return the date in the format of YYYY-MM + // if resolution is daily, it will return the date in the format of YYYY-MM-DD + // if days is greater than 90, it will return the date in the format of YYYY-MM-DD + // else it will return the date in the format of MM-DD + const formatLabel = (idx) => { + const key = allKeys[idx]; + if (!key) return ""; + if (resolution === "monthly") return key; + if (days > 90) return key; + const parts = key.split("-"); + if (parts.length >= 3) return `${parts[1]}-${parts[2]}`; + return key; + }; + + return { tickValues, formatLabel }; +} + +// Compute upper bound for Y axis with padding +// Just to ensure that the area chart is not touching the top of the chart +function computePaddedMax(okArr, failArr) { + const rawMaxOk = okArr.reduce((m, p) => Math.max(m, Number(p?.data || 0)), 0); + const rawMaxFail = failArr.reduce( + (m, p) => Math.max(m, Number(p?.data || 0)), + 0 + ); + return Math.max(1, Math.ceil(Math.max(rawMaxOk, rawMaxFail) * 1.1 + 1)); +} + +// Build grouped series and matching color scheme based on filter selection +// This is used to build the grouped series and matching color scheme based on filter selection (Success / Failed / All) +function buildGroupedSeries(selectedStatus, okArr, failArr) { + let grouped = []; + let scheme = []; + + if (selectedStatus === "ALL") { + if (failArr.length > okArr.length) { + grouped = [ + { key: "Successful Runs", data: okArr }, + { key: "Failed Runs", data: failArr }, + ]; + scheme = ["#ef4444", "#22c55e"]; + } else { + grouped = [ + { key: "Failed Runs", data: failArr }, + { key: "Successful Runs", data: okArr }, + ]; + scheme = ["#22c55e", "#ef4444"]; + } + } else if (selectedStatus === "FAILED") { + grouped = [{ key: "Failed Runs", data: failArr }]; + scheme = ["#ef4444"]; + } else { + grouped = [{ key: "Successful Runs", data: okArr }]; + scheme = ["#22c55e"]; + } + + return { grouped, scheme }; +} + +const SuccessFailedRunsWidget = (props) => { + const { globalUrl, workflows, onControlsChange, onLoadingChange, onTotalsChange, overrideDays, dummyMode } = props; + + const [mode, setMode] = useState("workflows"); // 'workflows' | 'apps' + const [days, setDays] = useState(30); + const daysOptions = [5, 10, 15, 30, 60, 90, 180, 230, 365]; + + const [resolution, setResolution] = useState("daily"); // 'daily' | 'monthly' + const [selectedWorkflow, setSelectedWorkflow] = useState("ALL"); + const [selectedStatus, setSelectedStatus] = useState("ALL"); + const [seriesOk, setSeriesOk] = useState([]); + const [seriesFail, setSeriesFail] = useState([]); + const [loading, setLoading] = useState(false); + const [wfTotals, setWfTotals] = useState({ ok: 0, fail: 0, activeDays: 0 }); + + useEffect(() => { + try { + if (typeof onTotalsChange !== "function") return; + const totalOk = Math.max(0, Number(wfTotals.ok) || 0); + const totalFail = Math.max(0, Number(wfTotals.fail) || 0); + const totalRuns = totalOk + totalFail; + const activeDays = Math.max(0, Number(wfTotals.activeDays) || 0); + const timeSavedMinutes = totalRuns * RUN_MINUTES_SAVED_PER_WORKFLOW; + const moneySavedDollars = totalRuns * RUN_DOLLARS_SAVED_PER_WORKFLOW; + // Do not trigger parent updates when switching mode to avoid page blink + onTotalsChange({ days, totalRuns, successRuns: totalOk, failedRuns: totalFail, activeDays, timeSavedMinutes, moneySavedDollars }); + } catch { + onTotalsChange({ days, totalRuns: 0, successRuns: 0, failedRuns: 0, activeDays: 0, timeSavedMinutes: 0, moneySavedDollars: 0 }); + } + }, [wfTotals, days, onTotalsChange]); + + const workflowItems = useMemo(() => { + const base = [{ id: "ALL", name: "All Workflows" }]; + if (!Array.isArray(workflows)) return base; + return base.concat( + workflows + .filter((w) => w?.id && w?.name) + .map((w) => ({ id: w.id, name: w.name })) + ); + }, [workflows]); + + const fetchSeriesForKey = async (key) => { + try { + + const urlA = `${globalUrl}/api/v1/stats/${encodeURIComponent(key)}?days=${encodeURIComponent(days)}`; + const doFetch = async (u) => { + const r = await fetch(u, { method: "GET", credentials: "include" }); + if (!r.ok) return []; + const j = await r.json(); + return Array.isArray(j?.entries) ? j.entries : []; + }; + const a = await doFetch(urlA); + if (a.length > 0) return a; + + // Optional orgId fallback if exposed globally in app + // const orgId = (window && window.selectedOrganization && window.selectedOrganization.id) || null; + // if (orgId) { + // const urlB = `${globalUrl}/api/v1/orgs/${encodeURIComponent(orgId)}/stats/${encodeURIComponent(key)}?days=${encodeURIComponent(days)}`; + // const b = await doFetch(urlB); + // if (b.length > 0) return b; + // } + // return []; + } catch (e) { + return []; + } + }; + + const fetchSeries = async () => { + setLoading(true); + try { + if (dummyMode) { + // 10-day wave with a couple of bumps for a more dynamic preview + const today = new Date(); + const mk = (n, v) => ({ key: new Date(today.getFullYear(), today.getMonth(), today.getDate() - n), id: `${n}`, data: v }); + + // Success shows two bumps (days -8..-6 and -2..0) + const okVals = [7, 4, 9, 4, 6, 9, 7, 5, 8, 6]; // oldest -> newest + const failVals = [1, 0, 1, 2, 1, 1, 0, 1, 2, 1]; // small, non-zero noise + + const okSeries = okVals.map((v, idx) => mk(okVals.length - 1 - idx, v)); + const failSeries = failVals.map((v, idx) => mk(failVals.length - 1 - idx, v)); + + setSeriesOk(okSeries); + setSeriesFail(failSeries); + return; + } + + const successKey = + mode === "workflows" + ? "workflow_executions_finished" + : "app_executions"; + const failedKey = + mode === "workflows" + ? "workflow_executions_failed" + : "app_executions_failed"; + const [succ, fail] = await Promise.all([ + fetchSeriesForKey(successKey), + fetchSeriesForKey(failedKey), + ]); + + let okSeries = normalizeEntries(succ); + let failSeries = normalizeEntries(fail); + + // Fallback: if empty, derive from /api/v1/stats daily_statistics + if (okSeries.length === 0 && failSeries.length === 0) { + const resp = await fetch(`${globalUrl}/api/v1/stats`, { + method: "GET", + credentials: "include", + headers: { "Content-Type": "application/json" }, + }); + if (resp.ok) { + const data = await resp.json(); + const fieldOk = + mode === "workflows" + ? "workflow_executions_finished" + : "app_executions"; + const fieldFail = + mode === "workflows" + ? "workflow_executions_failed" + : "app_executions_failed"; + const list = Array.isArray(data?.daily_statistics) + ? data.daily_statistics + : []; + const cutoff = new Date(); + cutoff.setDate(cutoff.getDate() - days); + okSeries = list + .filter(Boolean) + .map((d) => ({ + key: new Date(d?.date || Date.now()), + id: d?.date || Math.random().toString(36).slice(2), + data: Number(d?.[fieldOk] || 0), + })) + .filter((p) => p.key >= cutoff); + failSeries = list + .filter(Boolean) + .map((d) => ({ + key: new Date(d?.date || Date.now()), + id: `f-${d?.date || Math.random().toString(36).slice(2)}`, + data: Number(d?.[fieldFail] || 0), + })) + .filter((p) => p.key >= cutoff); + } + } + + setSeriesOk(okSeries); + setSeriesFail(failSeries); + } catch (e) { + setSeriesOk([]); + setSeriesFail([]); + } finally { + setLoading(false); + } + }; + + // Only fetch on first render and when days window or mode changes + const firstLoadRef = React.useRef(true); + useEffect(() => { + if (firstLoadRef.current) { + firstLoadRef.current = false; + fetchSeries(); + return; + } + fetchSeries(); + }, [dummyMode, days, globalUrl, mode]); + + // Apply external days override (e.g. after onboarding completes) + useEffect(() => { + if (typeof overrideDays === 'number' && overrideDays > 0 && overrideDays !== days) { + setDays(overrideDays); + } + }, [overrideDays]); + +// For the KPIs : Time saved and Money saved + useEffect(() => { + let aborted = false; + const run = async () => { + // Skip fetching/storing real stats while onboarding preview is shown + // if (dummyMode) { + // if (!aborted) setWfTotals({ ok: 0, fail: 0, activeDays: 0 }); + // return; + // } + try { + const totalEntries = await fetchSeriesForKey("workflow_executions"); + const series = normalizeEntries(totalEntries); + const dayKey = (d) => { + try { return new Date(d?.date || d?.key).toISOString().slice(0,10); } catch { return null; } + }; + const dayTotals = new Map(); + for (const it of series) { + const k = dayKey(it); if (!k) continue; dayTotals.set(k, (dayTotals.get(k) || 0) + (Number(it?.data)||0)); + } + const activeDays = Array.from(dayTotals.values()).filter(v => v > 0).length; + const ok = series.reduce((s, p) => s + (Number(p?.data)||0), 0); + if (!aborted) setWfTotals({ ok, fail: 0, activeDays }); + } catch { + if (!aborted) setWfTotals({ ok: 0, fail: 0, activeDays: 0 }); + } + }; + run(); + return () => { aborted = true; }; + }, [globalUrl, days, dummyMode, overrideDays]); + + // Notify parent about loading state changes + useEffect(() => { + if (typeof onLoadingChange === "function") { + onLoadingChange(loading); + } + }, [loading, onLoadingChange]); + + // Build filters UI once here; optionally render externally via onControlsChange + const controlsNode = React.useMemo(() => ( +
+ {/* + Workflow + + */} + + Filter + + + + Last + + + v && setMode(v)} + sx={{ + height: 37, + backgroundColor: 'rgba(255,255,255,0.06)', + border: '1px solid rgba(255,255,255,0.22)', + borderRadius: '30px', + padding: '2px', + "& .MuiToggleButton-root": { + border: "none", + borderRadius: "30px", + color: "#fff", + padding: "6px 16px", + textTransform: "none", + fontSize: "14px", + "&.Mui-selected": { + backgroundColor: "#fff", + color: "#222", + fontWeight: "600", + "&:hover": { + backgroundColor: "#fff", + }, + }, + "&:hover": { + backgroundColor: "rgba(255, 255, 255, 0.2)", + }, + }, + }} + > + + + Workflows + + + + + + Apps + + + + { + if (!v) return; + setResolution(v); + if (v === 'monthly' && days !== 180) { + setDays(180); + } else if (v === 'daily' && days !== 30) { + setDays(30); + } + }} + sx={{ + height: 37, + backgroundColor: 'rgba(255,255,255,0.06)', + border: '1px solid rgba(255,255,255,0.22)', + borderRadius: '30px', + padding: '2px', + "& .MuiToggleButton-root": { + border: "none", + borderRadius: "30px", + color: "#fff", + padding: "6px 16px", + textTransform: "none", + fontSize: "14px", + "&.Mui-selected": { + backgroundColor: "#fff", + color: "#222", + fontWeight: "600", + "&:hover": { + backgroundColor: "#fff", + }, + }, + "&:hover": { + backgroundColor: "rgba(255, 255, 255, 0.2)", + }, + }, + }} + > + + + Daily + + + + + + Monthly + + + +
+ ), + [selectedStatus, mode, days, resolution, workflowItems] + ); + + useEffect(() => { + if (typeof onControlsChange === "function") { + onControlsChange(controlsNode); + return () => { + onControlsChange(null); + }; + } + }, [onControlsChange, controlsNode]); + + return ( +
+ {/* Top controls row: title left, all filters on the right */} +
+ {/* Title moved inside the area chart card */} + + {!onControlsChange && ( + <> + {controlsNode} + + )} +
+ + {/* Content row: area chart (left) + ring gauges (right) in separate sub-cards */} +
+
+ Successful vs Failed Runs ({mode === "workflows" ? "Workflows" : "Apps"}) + {(() => { + // Build unified timeline by day or by month + const useOk = Array.isArray(seriesOk) ? seriesOk : []; + const useFail = Array.isArray(seriesFail) ? seriesFail : []; + + const okMap = bucketSeries(useOk, resolution); + const failMap = bucketSeries(useFail, resolution); + let allKeys = Array.from( + new Set([...okMap.keys(), ...failMap.keys()]) + ); + allKeys.sort((a, b) => new Date(a) - new Date(b)); + allKeys = buildBackfilledKeys(allKeys, resolution); + + if (allKeys.length === 0) { + return ( +
+ + No data available + +
+ ); + } + + let okArr = allKeys.map((k, i) => ({ + key: i, + data: okMap.get(k) || 0, + })); + let failArr = allKeys.map((k, i) => ({ + key: i, + data: failMap.get(k) || 0, + })); + okArr = ensureMinTwoPoints(okArr); + failArr = ensureMinTwoPoints(failArr); + + const { grouped, scheme } = buildGroupedSeries( + selectedStatus, + okArr, + failArr + ); + const { tickValues, formatLabel } = computeTicks( + allKeys, + days, + resolution + ); + const paddedMax = computePaddedMax(okArr, failArr); + + return ( + formatCompactNumber(d)} />} + /> + } + /> + } + xAxis={ + formatLabel(Number(d))} + /> + } + tickValues={tickValues} + /> + } + /> + } + gridlines={} />} + series={ + + } + colorScheme={scheme} + tooltip={ + { + const idx = Math.max(0, Number(d?.x ?? 0)); + const rows = (grouped || []).map( + (seriesItem, i) => { + const point = Array.isArray(seriesItem?.data) + ? seriesItem.data[ + Math.min( + idx, + seriesItem.data.length - 1 + ) + ] + : null; + const value = Number(point?.data || 0); + return { + label: seriesItem?.key, + value, + color: scheme[scheme.length - 1 - i], + }; + } + ); + return ( +
+
+ {formatLabel(idx)} +
+
+ {rows.reverse().map((r) => ( +
+ + + {r.label} + + + {r.value} + +
+ ))} +
+
+ ); + }} + /> + } + /> + } + /> + } + /> + ); + })()} +
+
+
+ Successful Runs +
+
+ Failed Runs +
+
+
+ + X: {resolution === "monthly" ? "Date (month)" : "Date (MM-DD)"} + + |Y: Runs +
+
+
+ + {/* Ring gauges */} +
+
+ + {mode === "workflows" ? "Workflow" : "App"} Success Rates + +
+ {(() => { + const totalOk = (seriesOk || []).reduce( + (s, p) => s + (p?.data || 0), + 0 + ); + const totalFail = (seriesFail || []).reduce( + (s, p) => s + (p?.data || 0), + 0 + ); + const total = totalOk + totalFail; + const okPct = total > 0 ? Math.round((totalOk / total) * 100) : 0; + const failPct = + total > 0 ? Math.round((totalFail / total) * 100) : 0; + return ( + <> + + + + ); + })()} +
+
+
+
+
+ ); +}; + +export default SuccessFailedRunsWidget; + +// Lightweight SVG ring to avoid RadialGauge runtime issues +function Ring({ title, color, bg, percent }) { + const stroke = 9; + const r = 60; + const c = 2 * Math.PI * r; + const filled = (Math.max(0, Math.min(100, Number(percent) || 0)) / 100) * c; + + return ( +
+
+ + + + + {Math.round(Math.max(0, Math.min(100, Number(percent) || 0)))}% + + + {title} +
+
+ ); +} + +// (Old) custom area/line removed in favor of Reaviz AreaChart grouped + +function LegendDot({ color }) { + return ( + + ); +} diff --git a/frontend/src/components/WorkflowGrid.jsx b/frontend/src/components/WorkflowGrid.jsx index 218cc060..63ea3abb 100644 --- a/frontend/src/components/WorkflowGrid.jsx +++ b/frontend/src/components/WorkflowGrid.jsx @@ -20,6 +20,7 @@ import { Zoom, Chip, } from '@mui/material'; +import { useDebouncedCallback } from "../utils/useDebouncedCallback"; import WorkflowPaper from "../components/WorkflowPaper.jsx" import WorkflowPaperNew from "../components/WorkflowPaperNew.jsx" @@ -172,6 +173,7 @@ const AppGrid = props => { // value={currentRefinement} const SearchBox = ({currentRefinement, refine, isSearchStalled} ) => { var defaultSearch = "" + const [inputValue, setInputValue] = useState("") useEffect(() => { if (window !== undefined && window.location !== undefined && window.location.search !== undefined && window.location.search !== null) { const urlSearchParams = new URLSearchParams(window.location.search) @@ -185,6 +187,12 @@ const AppGrid = props => { } }, []) + useEffect(() => { + setInputValue(currentRefinement || defaultSearch || "") + }, [currentRefinement]) + + const debouncedRefine = useDebouncedCallback((value) => refine(value), 300) + if (localMessage !== inputsearch && inputsearch !== undefined && inputsearch !== null && inputsearch.length > 0) { //setLocalMessage(inputsearch) refine(inputsearch) @@ -217,12 +225,14 @@ const AppGrid = props => { autoComplete='off' type="search" color="primary" - value={currentRefinement} + value={inputValue} placeholder="Find Workflows..." id="shuffle_search_field" onChange={(event) => { removeQuery("q") - refine(event.currentTarget.value) + const value = event.currentTarget.value + setInputValue(value) + debouncedRefine(value) }} onKeyDown={(event) => { if(event.key === "Enter") { diff --git a/frontend/src/utils/useDebouncedCallback.jsx b/frontend/src/utils/useDebouncedCallback.jsx new file mode 100644 index 00000000..aafa23c9 --- /dev/null +++ b/frontend/src/utils/useDebouncedCallback.jsx @@ -0,0 +1,25 @@ +import { useRef, useEffect, useCallback } from "react"; + +export const useDebouncedCallback = (callback, delay = 300) => { + const timeoutRef = useRef(null); + const savedCallback = useRef(callback); + + useEffect(() => { + savedCallback.current = callback; + }, [callback]); + + useEffect(() => () => { + if (timeoutRef.current) clearTimeout(timeoutRef.current); + }, []); + + return useCallback((...args) => { + if (timeoutRef.current) clearTimeout(timeoutRef.current); + timeoutRef.current = setTimeout(() => { + savedCallback.current(...args); + }, delay); + }, [delay]); +}; + +export default useDebouncedCallback; + + diff --git a/frontend/src/views/AgentUI.jsx b/frontend/src/views/AgentUI.jsx index fab6d012..2360a426 100644 --- a/frontend/src/views/AgentUI.jsx +++ b/frontend/src/views/AgentUI.jsx @@ -1,5 +1,6 @@ import React, { useState, useEffect, useContext, memo } from "react"; import { Context } from "../context/ContextApi.jsx"; +import AuthenticationModal from "../components/AuthenticationModal.jsx"; import { useNavigate, Link, useLocation } from "react-router-dom"; import { getTheme } from "../theme.jsx"; import { toast } from "react-toastify" @@ -21,12 +22,16 @@ import { import { CheckCircle as CheckCircleIcon, + Check as CheckIcon, HourglassDisabled as HourglassDisabledIcon, RestartAlt as RestartAltIcon, ExpandMore as ExpandMoreIcon, ExpandLess as ExpandLessIcon, Send as SendIcon, Error as ErrorIcon, + Close as CloseIcon, + OpenInNew as OpenInNewIcon, + Refresh as RefreshIcon, } from '@mui/icons-material' import { @@ -43,21 +48,26 @@ const AgentUI = (props) => { const [data, setData] = useState({}) const [openIndexes, setOpenIndexes] = useState([]) const [disableButtons, setDisableButtons] = useState(false) + const [apps, setApps] = useState([]) + const [appAuth, setAppAuth] = useState([]) - const [originalStartTime, setOriginalStartTime] = useState(0) - const [latestEndTime, setLatestEndTime] = useState(0) const [showAgentStarter, setShowAgentStarter] = useState(false) const [actionInput, setActionInput] = useState("") + const [questionAnswers, setQuestionAnswers] = useState({}) const {themeMode} = useContext(Context) const theme = getTheme(themeMode) const navigate = useNavigate(); + document.title = "Shuffle AI Agents" + const agentWrapperStyle = { width: 1000, height: 1000, margin: "auto", paddingTop: 100, + paddingBottom: 1000, + backgroundColor: theme.palette.backgroundColor, } if (data.input === undefined || data.input === null) { @@ -75,7 +85,22 @@ const AgentUI = (props) => { } if (node_id === undefined || node_id === null || node_id === "") { - return + // Look for AI agent + /* + for (var key in execution_data.results) { + const item = execution_data.results[key] + if (item?.action?.app_name !== "AI Agent") { + continue + } + + node_id = item?.action?.id + break + } + */ + + if (node_id === undefined || node_id === null || node_id === "") { + return + } } var found = false @@ -150,15 +175,22 @@ const AgentUI = (props) => { if (responseJson.success !== false) { if (responseJson.status === "EXECUTING") { // Recursively looking for updates until it's not executing anymore - setTimeout(() => { - GetExecution(execution_id, node_id, authorization) - }, 3000) + //setTimeout(() => { + // GetExecution(execution_id, node_id, authorization) + //}, 3000) } else { setDisableButtons(false) - setDisableButtons(false) } - setExecution(responseJson) + try { + if (JSON.stringify(responseJson) !== JSON.stringify(execution)) { + setExecution(responseJson) + } + } catch(e) { + console.log("Error comparing executions: ", e) + setExecution(responseJson) + } + findNodeData(responseJson, node_id) } else { setDisableButtons(false) @@ -216,12 +248,53 @@ const AgentUI = (props) => { } GetExecution(execution.execution_id, agentActionResult.action.id, execution.authorization) + setTimeout(() => { + GetExecution(execution.execution_id, agentActionResult.action.id, execution.authorization) + }, 10000) }) .catch((error) => { toast.error("Error: " + error) }) } + const getAppAuth = () => { + const url = `${globalUrl}/api/v1/apps/authentication` + fetch(url, { + method: "GET", + credentials: "include", + }) + .then((response) => { + return response.json() + }) + .then((responseJson) => { + if (responseJson.success !== false) { + setAppAuth(responseJson) + } + }) + .catch((error) => { + toast.error("Error in auth load: " + error) + }) + } + + const getApps = () => { + const url = `${globalUrl}/api/v1/apps` + fetch(url, { + method: "GET", + credentials: "include", + }) + .then((response) => { + return response.json() + }) + .then((responseJson) => { + if (responseJson.success !== false) { + setApps(responseJson) + } + }) + .catch((error) => { + toast.error("Error in app load: " + error) + }) + } + useEffect(() => { const params = new URLSearchParams(window.location.search) const executionId = params.get("execution_id") @@ -233,9 +306,15 @@ const AgentUI = (props) => { setShowAgentStarter(true) //toast.warn("No execution ID or node ID provided. Please provide execution_id and node_id in the URL.") } + + getApps() + getAppAuth() }, []) - const maxTimelineWidth = 150 + const maxTimelineWidth = 300 + + var latestEndTime = 0 + var originalStartTime = 0 const TimelineItem = (props) => { const { item, index } = props; const [hovered, setHovered] = useState(false); @@ -258,12 +337,96 @@ const AgentUI = (props) => { const categoryStyle = { - width: 20, - height: 20, + width: 25, + height: 25, marginRight: 10, + borderRadius: 5, } - const parsedCategory = item.category === "singul" ? + + const validate = validateJson(item.details) + const itemStartTime = item.start_time + var itemEndTime = item.end_time + if (item.category === "agent" && itemStartTime !== undefined && itemStartTime !== originalStartTime && (itemStartTime < originalStartTime || originalStartTime === 0)) { + console.log("Rerender 1: ", itemStartTime, originalStartTime) + originalStartTime = itemStartTime + } + + if (itemEndTime !== undefined && itemEndTime > latestEndTime) { + console.log("Rerender 2") + latestEndTime = itemEndTime + } + + if (itemEndTime === undefined || itemEndTime === null) { + // Set it to now + itemEndTime = latestEndTime + } + + if (item.category == "agent" && itemEndTime === 0) { + // Right now -> .toLocaleString() support + itemEndTime = Date.now() / 1000 + + // + + if (itemEndTime > latestEndTime) { + latestEndTime = itemEndTime + } + } + + const totalDuration = latestEndTime - originalStartTime + var currentDuration = itemStartTime - itemEndTime + var timelineMarginLeft = ((itemStartTime - originalStartTime) / totalDuration) * maxTimelineWidth + //var timelineMarginLeft = 0 + + // Calculate how long the div should be + var timelineWidth = ((itemEndTime - itemStartTime) / totalDuration) * maxTimelineWidth + + //console.log("CURRENT DURATION (1): ", currentDuration, itemStartTime, itemEndTime, originalStartTime, latestEndTime, totalDuration, timelineMarginLeft, timelineWidth) + if (totalDuration === currentDuration) { + timelineMarginLeft = 0 + timelineWidth = maxTimelineWidth + } + + // Just for simplicity's sake + if (currentDuration < -1000000 || currentDuration > 1000000) { + currentDuration = 0 + } + + if (currentDuration < 0) { + currentDuration = currentDuration * -1 + } + + const defaultTopPadding = 10 + const open = openIndexes.includes(index) + + var questions = [] + if (item?.details?.action === "finish" || item.category == "finish" || item?.details?.action == "finalise") { + item.type = "finalise" + item.category = "finalise" + item.label = item?.details?.reason || item.label + + } else if (item?.category === "ask" || item?.details?.action === "ask") { + + item.type = "question" + item.category = "ask" + item.label = item?.details?.reason || item.label + + for (var fieldKey in item?.details?.fields) { + const field = item?.details?.fields[fieldKey] + if (field?.key !== "question") { + continue + } + + questions.push({ + "question": field?.value, + "index": questions.length + 1, + }) + } + } else if (item?.details?.action === "api" && item?.details?.tool?.length > 0) { + item.label = item?.details?.reason || item.label + } + + var parsedCategory = item.category === "singul" ? @@ -273,38 +436,167 @@ const AgentUI = (props) => { :
- - const validate = validateJson(item.details) - const itemStartTime = item.start_time - var itemEndTime = item.end_time - if (itemStartTime !== undefined && itemStartTime !== originalStartTime && (itemStartTime < originalStartTime || originalStartTime === 0)) { - console.log("Rerender 1") - //setOriginalStartTime(itemStartTime) + + var showAuthentication = false + var selectedApp = {} + if (item?.details?.tool !== undefined && item?.details?.tool !== null && item?.details?.tool?.length > 0 && item?.details?.tool !== "singul" && item?.details?.tool !== item?.details?.action) { + + // Find the app and inject the image + const toolName = item.details.tool.toLowerCase().replaceAll(" ", "_").replaceAll("-", "_") + for (var appKey in apps) { + const app = apps[appKey] + + const appname = app.name.toLowerCase().replaceAll(" ", "_").replaceAll("-", "_") + if (appname !== toolName) { + continue + } + + if (app.large_image === undefined || app.large_image === null || app.large_image.length === 0) { + break + } + + selectedApp = app + + // Override the category + //item.category = app.name + //item.label = item?.details?.reason || item.label + parsedCategory = + + + + + break + } } - if (itemEndTime !== undefined && itemEndTime > latestEndTime) { - console.log("Rerender 2") - setLatestEndTime(itemEndTime) + if (!showAuthentication) { + if (item?.details?.run_details?.raw_response !== undefined && item?.details?.run_details?.raw_response !== null && item?.details?.run_details?.raw_response?.includes("app_authentication")) { + showAuthentication = true + } } - if (itemEndTime === undefined || itemEndTime === null) { - // Set it to now - itemEndTime = latestEndTime + var questionSubmitDisabled = questions.length === 0 ? true : false + for (var qKey in questions) { + const q = questions[qKey] + if (questionAnswers[q.question] === undefined || questionAnswers[q.question] === null || questionAnswers[q.question] === "") { + //console.log("EMPTY QUESTION: ", q) + questionSubmitDisabled = true + break + } else { + questionSubmitDisabled = false + } } - const totalDuration = latestEndTime - originalStartTime - const currentDuration = itemStartTime - itemEndTime - var timelineMarginLeft = ((itemStartTime - originalStartTime) / totalDuration) * maxTimelineWidth - var timelineWidth = ((itemEndTime - itemStartTime) / totalDuration) * maxTimelineWidth + const barColor = item.status === "FINISHED" ? green : + item.status === "FAILURE" || item.status == "ABORTED" ? red : + item.status === "RUNNING" || item.status === "" ? theme.palette.main : + theme.palette.surfaceColor - if (totalDuration === currentDuration) { - timelineMarginLeft = 0 - timelineWidth = maxTimelineWidth + const rerunAgentButton = + + + { + e.preventDefault() + e.stopPropagation() + + toast.info("Attempting to rerun everything.") + setDisableButtons(true) + + if (item?.details === undefined || item?.details === null || item?.details?.input === undefined || item?.details?.input === null) { + toast.error("No decision details found to rerun. Cannot proceed. Please go back to your workflow or /agents to start over.") + } else { + //console.log("DETAILS: ", item?.details) + for (var messagekey in item?.details?.input?.messages) { + const message = item?.details?.input?.messages[messagekey] + if (message.role === "user") { + setActionInput(message.content) + setDisableButtons(true) + + submitInput(message.content) + //toast.info("Rerun started. Please wait a few seconds and this page should refresh automatically.") + break + } + } + } + }} + > + + + + + + + const rerunButton = + + + { + e.preventDefault() + e.stopPropagation() + + //toast.info("Attempting to rerun this decision by itself.") + setDisableButtons(true) + RerunDecision(item.details) + }} + > + + + + + + const submitQuestions = (decisionId, questionAnswers) => { + console.log("Submitting questions: ", decisionId, questionAnswers) + if (decisionId === undefined || decisionId === null || decisionId === "") { + toast.error("No decision ID provided. Cannot submit answers.") + return + } + + if (Object.keys(questionAnswers).length === 0) { + toast.error("No answers provided. Cannot submit empty answers.") + return + } + + // Loop qu + var newArgument = {} + for (var key in questionAnswers) { + const answer = questionAnswers[key] + newArgument["question_"+(answer.index)] = answer.value + } + + const params = new URLSearchParams(window.location.search) + const executionId = params.get("execution_id") + const nodeId = params.get("node_id") + const authorization = params.get("authorization") + + const url = `${globalUrl}/api/v1/workflows/${executionId}/run?reference_execution=${executionId}&authorization=${authorization}&answer=true¬e=${encodeURIComponent(JSON.stringify(newArgument))}&agentic=true&decision_id=${decisionId}` + console.log("PARSED URL: ", url) + fetch(url, { + method: "GET", + credentials: "include", + }) + .then((response) => { + return response.json() + }) + .then((responseJson) => { + if (responseJson.success !== false) { + setTimeout(() => { + GetExecution(execution.execution_id, agentActionResult.action.id, execution.authorization) + }, 500) + + toast.success("Successfully submitted answers! The agent should continue shortly.") + } else { + toast.warn("Failed to submit answers. Please try again or contact support@shuffler.io if this persists..") + } + }) + .catch((error) => { + toast.error("Problem with submitting: " + error) + }) } - const defaultTopPadding = 10 - const open = openIndexes.includes(index) - return (
{ }} onMouseEnter={() => { if (!hovered) { - console.log("HOVER") + //console.log("HOVER") setHovered(true) } }} @@ -364,41 +656,50 @@ const AgentUI = (props) => {
{parsedCategory}
+ {/*
- {/* To ISO string from unix time */} - {new Date(item.start_time * 1000).toLocaleString()} + {item?.start_time !== undefined && item?.start_time !== null && item?.start_time !== 0 ? + new Date(item.start_time * 1000).toLocaleString() + : + null + } +
+ */}
{item.label}
- +
- {currentDuration !== 0 && !isNaN(timelineMarginLeft) && !isNaN(timelineWidth) && timelineWidth > 0 ? + {currentDuration != 0 && !isNaN(timelineMarginLeft) && !isNaN(timelineWidth) && timelineWidth > 0 ?
- : null} + minHeight: 10, + maxHeight: 10, + borderRadius: theme.palette.borderRadius, + }}> +
+ : + + + }
@@ -407,24 +708,68 @@ const AgentUI = (props) => { maxWidth: 100, display: "flex", }}> - - - { - e.preventDefault() - e.stopPropagation() + {item.category === "ask" ? + + {rerunButton} + {/* + + + { + e.preventDefault() + e.stopPropagation() - toast.info("Attempting to rerun this decision by itself.") - setDisableButtons(true) - RerunDecision(item.details) - }} - > - - + toast.info("Approving this step.") + }} + > + + + + + + + { + e.preventDefault() + e.stopPropagation() + + toast.info("Stopping on this step.") + }} + > + + + + + */} + + + + { + e.preventDefault() + e.stopPropagation() + + //http://localhost:3002/forms/aadfe022-fe93-431c-8634-de42dd7440ac?authorization=9357f6a6-7d59-44be-ad66-be27657369ac&reference_execution=0726378d-b501-470f-b850-f7fb48cd8ca4&source_node=de446bcf-ad37-4337-9f72-e069c7425fac&backend_url=https://ec4245cd2941.ngrok-free.app + const newurl = `/forms/${execution?.workflow?.id}?authorization=${execution.authorization}&reference_execution=${execution.execution_id}&source_node=${agentActionResult?.action?.id}&decision_id=${item.details.run_details.id}&backend_url=${globalUrl}` + window.open(newurl, '_blank', 'noopener,noreferrer'); + }} + > + + + + - + : + item.category === "agent" ? + rerunAgentButton + : + rerunButton + } {
+ + {showAuthentication && selectedApp.id !== undefined ? +
+ +
+ : null} + + {questions?.length > 0 && item?.status === "RUNNING" ? +
+ {questions.map((q, questionIndex) => { + return ( +
+ + {`${q.question}`} + + + { + console.log("Change: ", e.target.value) + try { + questionAnswers[q.question] = { + "index": questionIndex, + "value": e.target.value, + } + + setQuestionAnswers({...questionAnswers, }) + } catch (e) { + toast.warn("Something went wrong. Please contact support@shuffler.io. Details: " + e) + } + }} + + /> +
+ ) + })} + + +
+ : null} + {open ?
@@ -479,7 +887,12 @@ const AgentUI = (props) => { const TimelineRender = (props) => { const { agent_data } = props; - const actionResult = execution?.results?.length > 0 ? execution.results[0] : execution + var actionResult = execution?.results?.length > 0 ? execution.results[0] : execution + const validate = validateJson(actionResult?.result) + if (validate.valid === true) { + actionResult.result = validate.result + } + var timelineItems = [ { "label": "AI Agent 2", @@ -493,6 +906,27 @@ const AgentUI = (props) => { }, ] + // Setting up the initial item + if (agent_data?.started_at === undefined && execution?.started_at !== undefined) { + timelineItems[0].start_time = execution?.started_at + } + + if (agent_data?.completed_at === undefined && execution?.completed_at !== undefined) { + timelineItems[0].end_time = execution?.completed_at + } + + // Always prioritise the execution status first + // agent (RUNNING) = workflow (EXECUTING) + if (execution?.status !== undefined) { + timelineItems[0].status = execution?.status + } + + if (actionResult?.result?.status !== undefined && actionResult?.result?.status !== null && actionResult?.result?.status?.length > 0) { + if (timelineItems[0].status !== "FINISHED" && timelineItems[0].status !== "ABORTED" && timelineItems[0].status !== "FAILURE") { + timelineItems[0].status = actionResult?.result?.status + } + } + // Autofixer for result lol if ((agent_data?.decisions === undefined || agent_data?.decisions === null)) { const verifiedInput = validateJson(actionResult?.result) @@ -500,6 +934,7 @@ const AgentUI = (props) => { agent_data.decisions = verifiedInput.result?.decisions setAgentActionResult(actionResult) + } } @@ -516,13 +951,13 @@ const AgentUI = (props) => { } var newTimelineItem = { - "label": item.action, + "label": item?.action, "type": "decision", - "category": item.category, + "category": item?.category, - "status": item.run_details.status, - "start_time": item.run_details.started_at, - "end_time": item.run_details.completed_at, + "status": item?.run_details?.status, + "start_time": item?.run_details?.started_at, + "end_time": item?.run_details?.completed_at, } newTimelineItem.details = item @@ -577,6 +1012,14 @@ const AgentUI = (props) => { setAgentRequestLoading(true) //setShowAgentStarter(false); //GetExecution(execution?.execution_id, execution?.node_id, execution?.authorization); + // + setData({}) + setExecution(null) + setAgentRequestLoading(true) + setShowAgentStarter(true) + setActionInput(inputText) + + setAgentActionResult(null) if (inputText === undefined || inputText === null || inputText === "") { toast.error("Please provide a valid input for the AI Agent.") @@ -606,7 +1049,7 @@ const AgentUI = (props) => { }, { "name":"action", - "value":"list_tickets" + "value":"list_tickets,API" } ]} @@ -637,18 +1080,38 @@ const AgentUI = (props) => { } + const handleKeyDown = (e) => { + const isCmdEnter = e.metaKey && e.key === "Enter"; // macOS + const isCtrlEnter = e.ctrlKey && e.key === "Enter"; // Windows/Linux + if (isCmdEnter || isCtrlEnter) { + e.preventDefault() + submitInput(actionInput) + } + } + return (
+ {showAgentStarter ? - { - e.preventDefault(); - submitInput(actionInput); - }}> + { + e.preventDefault(); + submitInput(actionInput); + }} + > +
@@ -661,7 +1124,7 @@ const AgentUI = (props) => { style={{width: 450, marginRight: 20, marginTop: 30, }} multiline minRows={2} - defaultValue={execution?.execution_id || ""} + defaultValue={actionInput || ""} onChange={(e) => { setActionInput(e.target.value) }} @@ -704,6 +1167,22 @@ const AgentUI = (props) => { + + + + + + {buttonState === "timeline" ? : diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index e1adeb09..61b3a3c3 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -22,6 +22,7 @@ import { CodeHandler, Img, OuterLink, } from "../views/Docs.jsx"; import { InstantSearch, Configure, connectSearchBox, connectHits, Index } from 'react-instantsearch-dom'; import algoliasearch from 'algoliasearch/lite'; +import useDebouncedCallback from "../utils/useDebouncedCallback.jsx"; import { Zoom, Fade, @@ -275,7 +276,7 @@ export const triggers = [ { "name": "alertinfo", "example": "", - "value": "Do you want to continue the workflow? Start parameters: $exec", + "value": "## Stop or continue?\n\nDetails: $exec", }, { "name": "options", @@ -1259,6 +1260,24 @@ const AngularWorkflow = (defaultprops) => { "multiline": true, }] }, + /* + // An attempt at handling APIs directly. This ~kind of works + { + "name": "API", + "description": "Attempts to take your fields and run an API call with them, whatever they are", + "label": "Custom Action", + "example": "{\"source_data\": \"{\\\"event\\\": \\\"login\\\", \\\"user\\\": \\\"john_doe\\\", \\\"timestamp\\\": \\\"2023-10-01T12:00:00Z\\\"}\", \"standard\": \"OCSF\"}", + "parameters": [ + { + "name": "fields", + "value": "", + "description": "A JSON object with the fields to send to the API. Example: {\"url\": \"hello\", \"key2\": \"value2\"}", + "required": true, + "multiline": true, + } + ] + }, + */ { "name": "Translate standard", "description": "Translates your JSON data into a standard formats, then stores it in the Shuffle Datastore", @@ -11589,10 +11608,16 @@ const AngularWorkflow = (defaultprops) => { }; const handleDragStop = (e, app) => { + if (cy === undefined || cy == null) { + console.log("Cytoscape not initialized") + return + } + var currentnode = cy.getElementById(newNodeId); if (currentnode === undefined || currentnode === null || currentnode.length === 0) { - return; + console.log("No current node found") + return } if (parsedApp === undefined || parsedApp === null || parsedApp.data === undefined || parsedApp.data === null) { @@ -12411,11 +12436,20 @@ const AngularWorkflow = (defaultprops) => { }; const SearchBox = ({ currentRefinement, refine, isSearchStalled, }) => { + const debouncedRefine = useDebouncedCallback(refine, 500) + const lastRefinedRef = useRef(currentRefinement) + + const safeRefine = (value) => { + if (value === lastRefinedRef.current) return + lastRefinedRef.current = value + debouncedRefine(value) + } + if (document !== undefined) { const appsearchValue = document.getElementById("appsearch") if (appsearchValue !== undefined && appsearchValue !== null) { if (appsearchValue.value !== undefined && appsearchValue.value !== null && appsearchValue.value.length > 0) { - refine(appsearchValue.value) + safeRefine(appsearchValue.value) } } } @@ -12448,8 +12482,7 @@ const AngularWorkflow = (defaultprops) => { //if (event.currentTarget.value.length > 0 && !searchOpen) { // setSearchOpen(true) //} - - refine(event.currentTarget.value) + safeRefine(event.currentTarget.value) }} limit={5} /> @@ -14725,7 +14758,7 @@ const AngularWorkflow = (defaultprops) => { zIndex: 10000, }} > - Conditions can't be used for loops [ .# ]{" "} + PS: Conditions can't be used for loops [ .# ]. Use the filters list action.{" "} { const shownErrors = !isMobile && workflow.errors !== undefined && workflow.errors !== null && workflow.errors.length > 0 && showErrors && (!workflow.public || userdata.support === true) ?
{ style={{ float: "right", marginTop: 20, }} // Max 5 days in the past - disabled={userdata.region_url !== "https://shuffler.io" || executionData.started_at < (Math.floor(Date.now() / 1000) - 432000)} + disabled={executionData.started_at < (Math.floor(Date.now() / 1000) - 432000)} onClick={() => { toast("Opening logs in a new tab") setTimeout(() => { - window.open(`/api/v1/workflows/search/${executionData.execution_id}`, "_blank") + window.open(`${globalUrl}/api/v1/workflows/search/${executionData.execution_id}`, "_blank") }, 250) }} > diff --git a/frontend/src/views/ApiExplorerWrapper.jsx b/frontend/src/views/ApiExplorerWrapper.jsx index d104d43d..81d3cd39 100644 --- a/frontend/src/views/ApiExplorerWrapper.jsx +++ b/frontend/src/views/ApiExplorerWrapper.jsx @@ -1494,13 +1494,13 @@ const ApiExplorerWrapper = (props) => { />
+ style={{ + backgroundColor: theme.palette.inputColor, + padding: 15, + borderRadius: theme.palette?.borderRadius, + marginBottom: 30, + }} + > There is no Shuffle-specific documentation for this app yet outside of the general description above. Documentation is written for each api, and is a community effort. We hope to see your contribution! diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 414a7af0..26f1179b 100755 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -8,6 +8,7 @@ import { Typography, FormControlLabel, Button, + ButtonGroup, Divider, Select, MenuItem, @@ -2680,11 +2681,11 @@ const AppCreator = (defaultprops) => { setErrorCode(responseJson.reason); if (responseJson?.details !== undefined && responseJson?.details !== null) { - toast.error("Failed to build - contact support@shuffler.io: " + responseJson.details, { + toast.error("Failed to build - contact support@shuffler.io:\n\n" + responseJson.details, { autoClose: 60000 }) } else { - toast.error("Failed to build: " + responseJson.reason, { + toast.error("Failed to build: \n\n" + responseJson?.reason, { autoClose: 10000 }) } @@ -2930,7 +2931,7 @@ const AppCreator = (defaultprops) => { Query -
+ {index === extraAuth.length - 1 ? ( -
+ ); })} @@ -3431,13 +3432,13 @@ const AppCreator = (defaultprops) => { const ActionPaper = (props) => { const { data, index } = props - const [updater, setUpdater] = useState("tmp"); - const [actionsModalOpen, setActionsModalOpen] = useState(false); - const [urlPath, setUrlPath] = useState(""); - const [fileUploadEnabled, setFileUploadEnabled] = useState(false); - const [currentActionMethod, setCurrentActionMethod] = useState(actionNonBodyRequest[0]) - const [extraBodyFields, setExtraBodyFields] = useState([]); - const [urlPathQueries, setUrlPathQueries] = useState([]); + const [updater, setUpdater] = useState("tmp"); + const [actionsModalOpen, setActionsModalOpen] = useState(false); + const [urlPath, setUrlPath] = useState(""); + const [fileUploadEnabled, setFileUploadEnabled] = useState(false); + const [currentActionMethod, setCurrentActionMethod] = useState(actionNonBodyRequest[0]) + const [extraBodyFields, setExtraBodyFields] = useState([]); + const [urlPathQueries, setUrlPathQueries] = useState([]); const [currentAction, setCurrentAction] = useState({ name: "", file_field: "", @@ -3454,6 +3455,10 @@ const AppCreator = (defaultprops) => { required_bodyfields: [], }); + useEffect(() => { + console.log("Queries: ", urlPathQueries) + }, [urlPathQueries]) + const findBodyParams = (body) => { const regex = /\${(\w+)}/g; const found = body.match(regex); @@ -3462,7 +3467,7 @@ const AppCreator = (defaultprops) => { } else { setExtraBodyFields(found); } - }; + }; const UrlPathParameters = () => { const values = getCurrentPaths(urlPath); @@ -3495,28 +3500,27 @@ const AppCreator = (defaultprops) => { ) : null; }; - const HandleIndividualChip = (props) => { - const { chipData, index } = props; - const [chipRequired, setChipRequired] = useState(currentAction.required_bodyfields !== undefined ? currentAction.required_bodyfields.includes(chipData) : false); + const { chipData, index } = props; + const [chipRequired, setChipRequired] = useState(currentAction.required_bodyfields !== undefined ? currentAction.required_bodyfields.includes(chipData) : false); const parsedChip = chipData.startsWith("${") && chipData.endsWith("}") ? chipData.substring(2, chipData.length - 1) : chipData - return ( - - { + return ( + + { if (chipRequired) { currentAction["required_bodyfields"].splice(currentAction["required_bodyfields"].indexOf(chipData), 1) } else { @@ -3524,27 +3528,28 @@ const AppCreator = (defaultprops) => { } setCurrentAction(currentAction); - setChipRequired(!chipRequired); - }} - /> - - ); - }; - - const setActionField = (field, value) => { - currentAction[field] = value - setCurrentAction(currentAction) - - //setUrlPathQueries(currentAction.queries) + setChipRequired(!chipRequired); + }} + /> + + ); }; - const addPathQuery = () => { + const setActionField = (field, value) => { + currentAction[field] = value + setCurrentAction(currentAction) + + //setUrlPathQueries(currentAction.queries) + }; + + const addPathQuery = () => { urlPathQueries.push({ name: "", required: true, example: "", }); if (updater === "addupdater") { setUpdater("updater"); } else { setUpdater("addupdater"); } + setUrlPathQueries(urlPathQueries); }; @@ -3555,6 +3560,7 @@ const AppCreator = (defaultprops) => { } else { setUpdater("flipupdater"); } + setUrlPathQueries(urlPathQueries); }; @@ -3573,7 +3579,7 @@ const AppCreator = (defaultprops) => { } }; - const loopQueries = urlPathQueries.length === 0 ? null : ( + const loopQueries = urlPathQueries.length === 0 ? null : (
{ return (
-
- - Click required to flip - - } - onBlur={(e) => { - console.log("IN BLUR: ", e.target.value); - urlPathQueries[queryIndex].name = e.target.value.replaceAll("=", ""); - setUrlPathQueries(urlPathQueries); - }} - style={{flex: 3}} - InputProps={{ - style: { - color: theme.palette.text.primary, - }, - }} - /> - { - urlPathQueries[queryIndex].example = e.target.value.replaceAll( - "=", - "" - ) - - setUrlPathQueries(urlPathQueries) - }} - style={{flex: 2}} - InputProps={{ - style: { - color: theme.palette.text.primary, - }, - }} - /> -
+
+ { + urlPathQueries[queryIndex].name = e.target.value.replaceAll("=", "") + setUrlPathQueries(urlPathQueries) + }} + style={{flex: 3}} + InputProps={{ + style: { + color: theme.palette.text.primary, + }, + }} + /> + { + // E.g. for Jira -> JQL -> requires = in param + urlPathQueries[queryIndex].example = e.target.value.replaceAll("=","=") + setUrlPathQueries(urlPathQueries) + }} + style={{flex: 2}} + InputProps={{ + style: { + color: theme.palette.text.primary, + }, + }} + /> +
{ @@ -3654,7 +3651,7 @@ const AppCreator = (defaultprops) => { deletePathQuery(queryIndex); }} > - +
); @@ -4106,22 +4103,22 @@ const AppCreator = (defaultprops) => { if (request.header !== undefined && request.header !== null) { var headers = []; for (let [key, value] of Object.entries(request.header)) { - if (value === undefined) { - if (key.includes(":")) { - const keysplit = key.split(":") - key = keysplit[0].trim() - value = keysplit[1].trim() + if (value === undefined) { + if (key.includes(":")) { + const keysplit = key.split(":") + key = keysplit[0].trim() + value = keysplit[1].trim() - } else if (key.includes("=")) { - const keysplit = key.split("=") - key = keysplit[0].trim() - value = keysplit[1].trim() + } else if (key.includes("=")) { + const keysplit = key.split("=") + key = keysplit[0].trim() + value = keysplit[1].trim() - } else { - toast("Removed key: ", key) - continue - } - } + } else { + toast("Removed key: ", key) + continue + } + } if ( parameterName !== undefined && @@ -4392,9 +4389,8 @@ const AppCreator = (defaultprops) => { variant={urlPath.length > 0 ? "contained" : "outlined"} style={{ }} onClick={() => { - //console.log(urlPathQueries) - //console.log(urlPath) console.log(currentAction); + const errors = getActionErrors(); addActionToView(errors); setActionsModalOpen(false); @@ -4460,7 +4456,7 @@ const AppCreator = (defaultprops) => { return ( - {newActionModal} + {newActionModal} {error} diff --git a/frontend/src/views/AppExplorer.jsx b/frontend/src/views/AppExplorer.jsx index 1f434a83..38e7cc4f 100644 --- a/frontend/src/views/AppExplorer.jsx +++ b/frontend/src/views/AppExplorer.jsx @@ -3075,7 +3075,7 @@ const buttonBackground = "linear-gradient(to right, #f86a3e, #f34079)"; const appEnding = app?.public === true ? app?.app_version : app?.id - return `curl -L \ \\\n "${globalUrl}/api/v1/download_docker_image?image=frikky/shuffle:${app?.name.toLowerCase().replaceAll(' ', '_')}_${appEnding}" \\\n -H \"Authorization: Bearer APIKEY" \\\n -o image.zip; \\\n docker load -i image.zip` + return `curl -L \ \\\n "${globalUrl}/api/v1/download_docker_image?image=frikky/shuffle:${app?.name.toLowerCase().replaceAll(' ', '_')}_${appEnding}" \\\n -H \"Authorization: Bearer APIKEY" \\\n -o image.zip; \\\n docker load -i image.zip${!app?.public ? ` \\\n docker tag frikky/shuffle:${app?.name.toLowerCase().replaceAll(' ', '_')}_${appEnding} frikky/shuffle:${app?.name.toLowerCase().replaceAll(' ', '_')}_${app.app_version}` : ``}` } const renderedActionOptions = deduplicateByName(( @@ -3405,7 +3405,7 @@ const buttonBackground = "linear-gradient(to right, #f86a3e, #f34079)";
+
@@ -2365,6 +2417,7 @@ const Apps2 = (props) => {
+ ); }; diff --git a/frontend/src/views/DashboardViews.jsx b/frontend/src/views/DashboardViews.jsx index 0748e80a..98d03344 100644 --- a/frontend/src/views/DashboardViews.jsx +++ b/frontend/src/views/DashboardViews.jsx @@ -8,7 +8,7 @@ import { useNavigate, Link, useParams } from "react-router-dom"; import { ToastContainer, toast } from "react-toastify" import Draggable from "react-draggable"; -import DashboardBarchart, { LoadStats } from '../components/DashboardBarchart.jsx'; +import { LoadStats } from '../components/LineChartWrapper.jsx'; import { Autocomplete, @@ -828,9 +828,14 @@ const Dashboard = (props) => { }
- diff --git a/frontend/src/views/Docs.jsx b/frontend/src/views/Docs.jsx index 8b4772c6..ac1babe2 100755 --- a/frontend/src/views/Docs.jsx +++ b/frontend/src/views/Docs.jsx @@ -400,6 +400,10 @@ const Docs = (defaultprops) => { if (propkey === "app_creation") { navigate('/docs/apps#app-creation-introduction') } + + if (propkey === "api") { + navigate('/docs/API') + } } @@ -690,7 +694,8 @@ const Docs = (defaultprops) => { const Heading = (props) => { const [hover, setHover] = useState(false); - var id = props.children[0].toLowerCase().toString() + + var id = (props.children?.[0] ?? props.children ?? '').toString().toLowerCase(); if (props.level <= 3) { id = props.children[0].toLowerCase().toString().replaceAll(" ", "-"); } diff --git a/frontend/src/views/LoginPage.jsx b/frontend/src/views/LoginPage.jsx index e470255d..e710c612 100755 --- a/frontend/src/views/LoginPage.jsx +++ b/frontend/src/views/LoginPage.jsx @@ -476,7 +476,7 @@ const LoginPage = props => { return; } - window.location.pathname = "/workflows" + window.location.pathname = "/new-dashboard" }, 2000); } @@ -658,7 +658,7 @@ const LoginPage = props => { } } - window.location.pathname = "/workflows" + window.location.pathname = "/new-dashboard" }, 2000); } }) diff --git a/frontend/src/views/NewDashboard.jsx b/frontend/src/views/NewDashboard.jsx new file mode 100644 index 00000000..bbe32310 --- /dev/null +++ b/frontend/src/views/NewDashboard.jsx @@ -0,0 +1,325 @@ +import React, { useEffect, useState, useContext, useRef, useCallback } from 'react'; +import { + Typography, + Grid, + Paper, + Box, + Stack, + Chip, + Avatar, + Divider, + Select, + MenuItem, + Tooltip, + CircularProgress, +} from '@mui/material'; +import { toast } from "react-toastify"; + + +import { + TrendingFlat as TrendingFlatIcon, + TrendingUp as TrendingUpIcon, + TrendingDown as TrendingDownIcon, + TaskAlt as TaskAltIcon, + SuccessFailed as SuccessFailedIcon, + RunsOverTime as RunsOverTimeIcon, + ErrorOutline as ErrorOutlineIcon, +} from '@mui/icons-material'; + +import SuccessFailedRunsWidget from '../components/SuccessFailedRunsWidget.jsx'; +import RunsOverTimeWidget from '../components/RunsOverTimeWidget.jsx'; +import DashboardOnboarding from '../components/DashboardOnboarding.jsx'; +import { Context } from '../context/ContextApi.jsx'; +import { useNavigate } from 'react-router-dom'; + +const NewDashboard = (props) => { + const { globalUrl, serverside, userdata } = props; + + const { leftSideBarOpenByClick } = useContext(Context); + const [sfwControls, setSfwControls] = useState(null); + const [loadingSfw, setLoadingSfw] = useState(true); + const [loadingRot, setLoadingRot] = useState(true); + const [loadingNoti, setLoadingNoti] = useState(true); + const [showOverlay, setShowOverlay] = useState(true); + const [totals, setTotals] = useState({ days: 30, mode: 'workflows', totalRuns: 0, successRuns: 0, failedRuns: 0, activeDays: 0, timeSavedMinutes: 0, moneySavedDollars: 0 }); + const [notifications, setNotifications] = useState([]); + const [onboardingOpen, setOnboardingOpen] = useState(() => { + try { + return localStorage.getItem("dashboard_onboarding_complete") === "true" ? false : true; + } catch { + return true; + } + + }) + const [overrideDays, setOverrideDays] = useState(undefined); + const [rotMonthOverride, setRotMonthOverride] = useState(undefined); + const [isProdStatusOn, setIsProdStatusOn] = useState(false) + + const isCloud = + serverside === true || typeof window === "undefined" + ? true + : window.location.host === "localhost:3002" || + window.location.host === "shuffler.io" || + window.location.host === "localhost:5002"; + + const navigate = useNavigate(); + const handleSfwControls = useCallback((node) => { + setSfwControls(node); + }, []); + + const formatCurrencyCompact = (value) => { + const n = Math.max(0, Number(value) || 0); + const abs = Math.abs(n); + const fmt = (x, suffix) => `${(Math.round(x * 10) / 10).toString().replace(/\.0$/, '')}${suffix}`; + if (abs >= 1e9) return `$${fmt(n / 1e9, 'B')}`; + if (abs >= 1e6) return `$${fmt(n / 1e6, 'M')}`; + if (abs >= 1e3) return `$${fmt(n / 1e3, 'k')}`; + return `$${Math.round(n).toLocaleString()}`; + }; + + const formatTimeDisplay = (mins) => { + const totalMins = Math.max(0, Math.round(mins || 0)); + if (totalMins < 60) return { display: `${totalMins}m`, title: `${totalMins} minutes` }; + const totalHours = Math.floor(totalMins / 60); + if (totalHours >= 24) { + const days = Math.floor(totalHours / 24); + return { display: `${days}d`, title: `${totalHours} hours` }; + } + return { display: `${totalHours}h`, title: `${totalHours} hours` }; + }; + + const timeFmt = formatTimeDisplay(totals.timeSavedMinutes); + + const unreadCount = notifications.filter(n => n && n.read === false).length; + const readCount = notifications.filter(n => n && n.read === true).length; + + // Current values + // 1 Workflow run = 15 minutes + // 1 Workflow run = $25 + + const STATIC_TIME_PERCENT = 'TBD' + const STATIC_MONEY_PERCENT = 'TBD' + const kpis = [ + { value: timeFmt.display, title: timeFmt.title, label: 'Time saved', icon: , percentage: STATIC_TIME_PERCENT, color: '#5cc879', disabled: true}, + { value: formatCurrencyCompact(totals.moneySavedDollars), label: 'Money saved', icon: , percentage: STATIC_MONEY_PERCENT, color: '#5cc879', disabled: true, }, + { value: String(unreadCount), label: 'Total errors', icon: , percentage: "", color: '#f87171' }, + { value: String(readCount), label: 'Errors resolved', icon: , percentage: "", color: '#5cc879' }, + ]; + + const getGreeting = () => { + try { + const hour = new Date().getHours(); + if (hour < 5) return 'Good night'; + if (hour < 12) return 'Good morning'; + if (hour < 18) return 'Good afternoon'; + return 'Good evening'; + } catch { + return 'Hey'; + } + }; + + const displayName = userdata !== undefined && userdata?.username !== undefined ? userdata?.username?.split('@')[0]?.charAt(0)?.toUpperCase() + userdata?.username?.split('@')[0]?.slice(1) : 'User'; + + useEffect(() => { + let t; + const anyLoading = loadingSfw || loadingRot || loadingNoti; + if (anyLoading) { + t = setShowOverlay(true); + } else { + setShowOverlay(false); + } + return () => { if (t) clearTimeout(t); }; + }, [loadingSfw, loadingRot, loadingNoti]); + + // Auto-open onboarding when there aren't enough active days of stats + useEffect(() => { + try { + const alreadyDone = localStorage.getItem("dashboard_onboarding_complete") === "true"; + if (alreadyDone) { + setOnboardingOpen(false); + return; + } + const active = Number(totals?.activeDays || 0); + setOnboardingOpen(active < 5); + } catch { + setOnboardingOpen(true); + } + }, [totals?.activeDays]); + + // Load notifications + useEffect(() => { + const loadNotifications = async () => { + try { + const resp = await fetch(`${globalUrl}/api/v1/notifications`, { + method: 'GET', + credentials: 'include', + headers: { 'Content-Type': 'application/json' }, + }); + if (resp.status !== 200) { + setNotifications([]); + return; + } + const data = await resp.json(); + const list = Array.isArray(data?.notifications) ? data.notifications : (Array.isArray(data) ? data : []); + setNotifications(list.filter(Boolean)); + } catch (e) { + setNotifications([]); + } finally { + setLoadingNoti(false); + } + }; + + loadNotifications(); + }, [globalUrl]); + + // useEffect(() => { + // // Lightweight workflows list for selector in success/failed widget + // const loadWorkflows = async () => { + // try { + // const resp = await fetch(`${globalUrl}/api/v1/workflows`, { + // method: 'GET', + // credentials: 'include', + // headers: { 'Content-Type': 'application/json' }, + // }); + // if (resp.status !== 200) { + // return; + // } + // const data = await resp.json(); + // const list = Array.isArray(data?.workflows) ? data.workflows : (Array.isArray(data) ? data : []); + // const normalized = list.filter(Boolean).map((w, idx) => ({ id: w?.id || w?.ID || `${idx}`, name: w?.name || w?.Name || `Workflow ${idx+1}` })); + // setWorkflows(normalized); + // } catch (e) { + // // ignore + // } + // }; + + // loadWorkflows(); + // }, [globalUrl]); + + useEffect(() => { + const orgId = userdata?.active_org?.id; + if (!orgId) { + return; + } + + let fetched = false; + fetch(`${globalUrl}/api/v1/orgs/${orgId}`, { + method: "GET", + credentials: "include", + headers: { "Content-Type": "application/json" }, + }) + .then((response) => (response.ok ? response.json() : null)) + .then((org) => { + if (!fetched && org) { + if (!isCloud) { + if (org?.cloud_sync && org?.subscriptions[0]?.name?.toLowerCase().includes("enterprise") && org?.subscriptions[0]?.active) { + setIsProdStatusOn(true); + } else if (org?.subscriptions[0]?.name?.toLowerCase().includes("enterprise") && org?.subscriptions[0]?.active) { + setIsProdStatusOn(true); + } else { + setIsProdStatusOn(false); + } + } + } + }) + .catch(() => {}); + + return () => { + fetched = true; + }; + }, [userdata?.active_org?.id, globalUrl]); + + return ( +
+ { + setOnboardingOpen(false) + }} + setOnboardingOpen={setOnboardingOpen} + isProdStatusOn={isProdStatusOn} + isCloud={isCloud} + onExplore={() => { + // Ensure overrides are set before closing modal + setOverrideDays(5); + setRotMonthOverride(new Date(new Date().getFullYear(), new Date().getMonth(), 1)); + + // Close modal immediately to trigger data fetching + setOnboardingOpen(false); + }} + headerTitle="Unlock your Dashboard" + headerSubtitle="Complete these steps to start seeing insights." + /> + {showOverlay && ( +
+
+ + Loading dashboard… +
+
+ )} + {/* Header / Greeting */} + + {`${getGreeting()}, ${displayName ?? 'User'}!`} + <> + {sfwControls} + + + + {/* KPI cards */} + + {kpis.map((kpi) => ( + + + { + if (kpi.label.toLowerCase().includes('total errors')) { + // navigate to notifications page + navigate('/admin?admin_tab=notifications'); + } + }} + + > + + + {kpi.value} + {kpi.label} + + + {kpi.icon} + {kpi.percentage} + + + + + + ))} + + + {/* Success/Failed widget uses its own internal sub-cards; make wrapper transparent */} + + + + + {/* Runs over time section */} + + + +
+ ); +}; + +export default NewDashboard; + + diff --git a/frontend/src/views/RunWorkflow.jsx b/frontend/src/views/RunWorkflow.jsx index 654646b7..cf8e9ad8 100644 --- a/frontend/src/views/RunWorkflow.jsx +++ b/frontend/src/views/RunWorkflow.jsx @@ -72,6 +72,7 @@ const RunWorkflow = (defaultprops) => { const [executionLoading, setExecutionLoading] = useState(false); const [executionData, setExecutionData] = React.useState({}); const [executionRunning, setExecutionRunning] = useState(false); + const [disableButtons, setDisableButtons] = useState(false); const [workflowQuestion, setWorkflowQuestion] = useState(""); const [selectedOrganization, setSelectedOrganization] = React.useState(undefined); const [apps, setApps] = React.useState([]); @@ -84,12 +85,14 @@ const RunWorkflow = (defaultprops) => { const [workflows, setWorkflows] = React.useState([]) const [boxWidth, setBoxWidth] = React.useState(500) const [inputQuestions, setInputQuestions] = React.useState([]) + const [agentic, setAgentic] = React.useState(false) const searchParams = new URLSearchParams(window.location.search) const answer = searchParams.get("answer") const execution_id = searchParams.get("reference_execution") const authorization = searchParams.get("authorization") const sourceNode = searchParams.get("source_node") + const decisionId = searchParams.get("decision_id") // ONLY for agentic workflows const backendUrl = searchParams.get("backend_url") || globalUrl useEffect(() => { @@ -162,11 +165,8 @@ const RunWorkflow = (defaultprops) => { } } - // Used to swap from login to register. True = login, false = register - // Error messages etc const [executionInfo, setExecutionInfo] = useState(""); - const handleValidateForm = (executionArgument) => { // Check if every field exists if (executionArgument === undefined || executionArgument === null) { @@ -184,9 +184,12 @@ const RunWorkflow = (defaultprops) => { } } - //console.log("EXEC: ", executionArgument) + // FIXME: Error with User Input + Required arg (?) + // Somehow validation is not happening as it should, and it just checks all + // questions if none are selected for (var key in executionArgument) { if (executionArgument[key] === undefined || executionArgument[key] === null || executionArgument[key] === "") { + console.log("Unanswered, required question: ", key) return false } } @@ -334,17 +337,18 @@ const RunWorkflow = (defaultprops) => { } const validate = validateJson(executionData.result) - return (
{workflowQuestion !== "" ? null : - +
} {workflowQuestion !== "" ? null : validate.valid === false ?
- + {validate?.result !== undefined && validate?.result !== null && validate?.result.length > 0 ? + + : null } { stop() setMessage("") - setExecutionLoading(true) setExecutionData({}) setExecutionInfo("") + setTimeout(() => { + setExecutionLoading(true) + }, 2500) + var data = { "execution_argument": executionArgument, "execution_source": "form", @@ -462,6 +469,14 @@ const RunWorkflow = (defaultprops) => { fetchBody.body = JSON.stringify(data) } + if (agentic === true) { + if (url.includes("?")) { + url += `&agentic=true&decision_id=${decisionId}` + } else { + url += `?agentic=true&decision_id=${decisionId}` + } + } + // IF there is an execution argument, we should use it fetch(url, fetchBody) .then((response) => { @@ -480,25 +495,30 @@ const RunWorkflow = (defaultprops) => { } } - if ((response.status === 401 || response.status === 403) && authorization === undefined || authorization === null || authorization.length === 0) { - toast(`This form is not available for you to run. If you this is an error, contact ${supportEmail} with a link to this form`) - } + //if ((response.status === 401 || response.status === 403) && authorization === undefined || authorization === null || authorization?.length === 0) { + // toast(`This form is not available for you to run. If you this is an error, contact ${supportEmail} with a link to this form (2)`) + //} return response.json() }) .then(responseJson => { + //if (responseJson.success === true) { + // setDisableButtons(true) + //} + setExecutionLoading(false) - if (responseJson.execution_id !== undefined && responseJson.execution_id !== null && responseJson.execution_id.length > 0) { + if (responseJson?.execution_id !== undefined && responseJson?.execution_id !== null && responseJson?.execution_id?.length > 0) { navigate(`?execution_id=${responseJson.execution_id}`) } if (responseJson.success === false) { + console.log("Failed sending execution request") - if (responseJson.reason !== undefined && responseJson.reason !== null) { + if (responseJson?.reason !== undefined && responseJson?.reason !== null) { if (responseJson?.reason?.toLowerCase().includes("already clicked")) { - setMessage("Already answered. You may close this window (2).") + setMessage("This form has been answered. You may close this window.") } else { - toast.warn(responseJson.reason) + toast.warn(responseJson?.reason) } } @@ -520,11 +540,17 @@ const RunWorkflow = (defaultprops) => { setExecutionRequest(responseJson) start() } + + // If execution_id or authorization, add them to the URL + if (responseJson?.execution_id !== undefined && responseJson?.execution_id !== null && responseJson?.execution_id?.length > 0 && responseJson?.authorization !== undefined && responseJson?.authorization !== null && responseJson?.authorization?.length > 0) { + navigate(`?execution_id=${responseJson.execution_id}&authorization=${responseJson.authorization}`) + } } }) .catch(error => { //setExecutionInfo("Error in workflow startup: " + error) - toast.warn("Error submitting form. Please try again.") + console.log("Error starting workflow: ", error) + toast.warn(`Error submitting form. Please try again: ${error}`) stop() setMessage("") @@ -597,8 +623,8 @@ const RunWorkflow = (defaultprops) => { if (realtimeMarkdown !== undefined && realtimeMarkdown !== null && realtimeMarkdown.length > 0) { const newmarkdown = realtimeMarkdown.replace(`{{ ${workflow_id} }}`, "", -1) setRealtimeMarkdown(newmarkdown) - } else if (inputWorkflow.form_control.input_markdown !== undefined && inputWorkflow.form_control.input_markdown !== null && inputWorkflow.form_control.input_markdown.length > 0) { - const newmarkdown = inputWorkflow.form_control.input_markdown.replace(`{{ ${workflow_id} }}`, "", -1) + } else if (inputWorkflow?.form_control?.input_markdown !== undefined && inputWorkflow?.form_control?.input_markdown !== null && inputWorkflow?.form_control?.input_markdown.length > 0) { + const newmarkdown = inputWorkflow?.form_control?.input_markdown.replace(`{{ ${workflow_id} }}`, "", -1) setRealtimeMarkdown(newmarkdown) } } @@ -608,10 +634,10 @@ const RunWorkflow = (defaultprops) => { console.log("Get workflow error: ", error.toString()) if (realtimeMarkdown !== undefined && realtimeMarkdown !== null && realtimeMarkdown.length > 0) { - const newmarkdown = inputWorkflow.form_control.input_markdown.replace(`{{ ${workflow_id} }}`, "", -1) + const newmarkdown = inputWorkflow?.form_control?.input_markdown.replace(`{{ ${workflow_id} }}`, "", -1) setRealtimeMarkdown(newmarkdown) - } else if (inputWorkflow.form_control.input_markdown !== undefined && inputWorkflow.form_control.input_markdown !== null && inputWorkflow.form_control.input_markdown.length > 0) { - const newmarkdown = inputWorkflow.form_control.input_markdown.replace(`{{ ${workflow_id} }}`, "", -1) + } else if (inputWorkflow?.form_control?.input_markdown !== undefined && inputWorkflow?.form_control?.input_markdown !== null && inputWorkflow?.form_control?.input_markdown.length > 0) { + const newmarkdown = inputWorkflow?.form_control?.input_markdown.replace(`{{ ${workflow_id} }}`, "", -1) setRealtimeMarkdown(newmarkdown) } }) @@ -646,6 +672,7 @@ const RunWorkflow = (defaultprops) => { trig.parameters = [] } + newexec = {} for (var paramkey in trig.parameters) { const param = trig.parameters[paramkey] if (param.name !== "input_questions") { @@ -683,6 +710,7 @@ const RunWorkflow = (defaultprops) => { } } + console.log("Setting exec arg: ", newexec) setExecutionArgument(newexec) } @@ -733,10 +761,10 @@ const RunWorkflow = (defaultprops) => { setInputQuestions(workflow.input_questions) } - if (workflow.form_control.input_markdown !== undefined && workflow.form_control.input_markdown !== null && workflow.form_control.input_markdown.length > 0) { + if (workflow?.form_control?.input_markdown !== undefined && workflow?.form_control?.input_markdown !== null && workflow?.form_control?.input_markdown.length > 0) { // Look for {{ uuid }} format, and try to run that workflow with their account // This is a hack, but a fun one. - var newmarkdown = workflow.form_control.input_markdown.replace("", "") + var newmarkdown = workflow?.form_control?.input_markdown.replace("", "") const uuidRegex = /{{\s[a-f0-9-]+\s}}/g const found = newmarkdown.match(uuidRegex) @@ -784,8 +812,8 @@ const RunWorkflow = (defaultprops) => { } } - if (workflow.status !== "WAITING") { - setMessage("Already answered. You may close this window (3).") + if (workflow.status === "EXECUTING" || workflow.status === "SUCCESS" || workflow.status === "ABORTED" || workflow.status === "STOPPED" || workflow.status === "FAILURE" || workflow.status === "FINISHED") { + setMessage("Already handled. You may close this window.") } } @@ -806,13 +834,17 @@ const RunWorkflow = (defaultprops) => { console.log("Status not 200 for workflows :O!"); } - if ((response.status === 401 || response.status === 403) && authorization === undefined || authorization === null || authorization.length === 0) { - toast(`This form is not available to you. If you think this is an error, please contact ${supportEmail} with the URL.`) - } + //if (response.status >= 400 && authorization === undefined || authorization === null || authorization.length === 0) { + // toast.warn(`This form may not be available to you. If you think this is an error, please contact ${supportEmail} with the URL.`) + //} return response.json() }) .then((responseJson) => { + if (responseJson.success === false) { + return + } + // Not sure why this is necessary. if (responseJson.isValid === undefined) { responseJson.isValid = true; @@ -1008,14 +1040,78 @@ const RunWorkflow = (defaultprops) => { return response.json(); }) .then((responseJson) => { - if (responseJson.success == false) { + if (responseJson?.success == false) { return } - if (execution_id !== undefined && execution_id !== null && authorization !== undefined && authorization !== null && execution_id.length > 0 && authorization.length > 0 && (workflow.id === undefined || workflow.id === null || workflow.id.length === 0) && responseJson.workflow !== undefined && responseJson.workflow !== null) { + if (execution_id !== undefined && execution_id !== null && authorization !== undefined && authorization !== null && execution_id.length > 0 && authorization.length > 0 && disableButtons === false && responseJson?.status !== "" && responseJson?.status !== "WAITING") { + setDisableButtons(true) + } + //if (execution_id !== undefined && execution_id !== null && authorization !== undefined && authorization !== null && execution_id.length > 0 && authorization.length > 0 && (workflow.id === undefined || workflow.id === null || workflow.id.length === 0) && responseJson.workflow !== undefined && responseJson.workflow !== null) { + if (execution_id !== undefined && execution_id !== null && authorization !== undefined && authorization !== null && execution_id.length > 0 && authorization.length > 0 && responseJson.workflow !== undefined && responseJson.workflow !== null) { setupSourcenode(responseJson.workflow, sourceNode) setWorkflow(responseJson.workflow) + + //const decisionId = searchParams.get("decision_id") // ONLY for agentic workflows + // Check for decision_id in url + if (decisionId?.length > 0 && responseJson?.workflow?.actions?.length > 0 && sourceNode?.length > 0 && responseJson?.results?.length > 0) { + console.log("Setting workflow: ", responseJson.workflow, ", EXEC RESULTS: ", responseJson.results) + + setAgentic(true) + + for (var resultkey in responseJson.results) { + const result = responseJson.results[resultkey] + if (result.action.id !== sourceNode) { + continue + } + + const validated = validateJson(result.result) + if (!validated.valid) { + console.log("Error parsing result: ", validated.error) + continue + } + + var parsedresult = validated.result + console.log("PARSED RES: ", parsedresult) + if (parsedresult?.decisions?.length > 0) { + var newexec = executionArgument + if (newexec === undefined || newexec === null || Object.keys(newexec).length === 0) { + newexec = {} + } + + for (var decisionkey in parsedresult?.decisions) { + const decision = parsedresult.decisions[decisionkey] + if (decision?.run_details?.id !== decisionId) { + continue + } + + for (var fieldkey in decision?.fields) { + const field = decision.fields[fieldkey] + if (field.key === "question" && !inputQuestions.find(q => q.name=== field.value)) { + console.log("QUESTION: ", field) + const newquestion = { + "name": field.value, + "value": field.key+"_"+fieldkey, + } + + inputQuestions.push(newquestion) + + newexec[newquestion.value] = "" + } + } + } + + setInputQuestions([...inputQuestions] ) + console.log("EXEC: ", newexec) + setExecutionArgument(newexec) + + responseJson.workflow.input_questions = inputQuestions + setWorkflow(responseJson?.workflow) + setDisableButtons(false) + } + } + } } @@ -1031,12 +1127,12 @@ const RunWorkflow = (defaultprops) => { localStorage.setItem(storageKey, JSON.stringify(value)) } - if (realtimeMarkdown !== undefined && realtimeMarkdown !== null && realtimeMarkdown.length > 0) { + if (realtimeMarkdown !== undefined && realtimeMarkdown !== null && realtimeMarkdown?.length > 0) { const newmarkdown = realtimeMarkdown.replace(`{{ ${responseJson.workflow.id} }}`, responseJson.result, -1) setRealtimeMarkdown(newmarkdown) - } else if (workflow.form_control.input_markdown !== undefined && workflow.form_control.input_markdown !== null && workflow.form_control.input_markdown.length > 0) { - const newmarkdown = workflow.form_control.input_markdown.replace(`{{ ${responseJson.workflow.id} }}`, responseJson.result, -1) + } else if (workflow?.form_control?.input_markdown !== undefined && workflow?.form_control?.input_markdown !== null && workflow?.form_control?.input_markdown.length > 0) { + const newmarkdown = workflow?.form_control?.input_markdown.replace(`{{ ${responseJson.workflow.id} }}`, responseJson.result, -1) setRealtimeMarkdown(newmarkdown) } @@ -1072,7 +1168,6 @@ const RunWorkflow = (defaultprops) => { getWorkflow(props.match.params.key, sourceNode) if (execution_id !== undefined && execution_id !== null && authorization !== undefined && authorization !== null) { - console.log("Get execution: ", execution_id) fetchUpdates(execution_id, authorization, true) } @@ -1136,13 +1231,13 @@ const RunWorkflow = (defaultprops) => { const buttonStyle = {borderRadius: 25, height: 50, fontSize: 18, backgroundImage: handleValidateForm(executionArgument) || executionLoading ? buttonBackground : "grey", color: "white"} // Check if all fields are filled in? - var disabledButtons = executionLoading || executionRunning || message.length > 0 + var disabledButtons = executionLoading || executionRunning || message.length > 0 || disableButtons if (disabledButtons === false && workflow.input_questions !== undefined && workflow.input_questions !== null && workflow.input_questions.length > 0) { // Check field values //disabledButtons = handleValidateForm(executionArgument) } - const organization = selectedOrganization !== undefined && selectedOrganization !== null ? selectedOrganization.name : "Unknown" + const organization = selectedOrganization !== undefined && selectedOrganization !== null ? selectedOrganization.name : "" const contact = selectedOrganization !== undefined && selectedOrganization !== null && selectedOrganization.org !== undefined && selectedOrganization.org !== null? selectedOrganization.org : "support@shuffler.io" //const contact = selectedOrganization !== undefined && selectedOrganization !== null && selectedOrganization.contact !== undefined && selectedOrganization.contact !== null? selectedOrganization.contact : "support@shuffler.io" @@ -1321,12 +1416,12 @@ const RunWorkflow = (defaultprops) => {
- Loading Form Details... + Loading Details...
:
- {workflowQuestion !== "" || (workflow.form_control.input_markdown !== undefined && workflow.form_control.input_markdown !== null && workflow.form_control.input_markdown.length > 0) ? + {workflowQuestion !== "" || (workflow?.form_control?.input_markdown !== undefined && workflow?.form_control?.input_markdown !== null && workflow?.form_control?.input_markdown.length > 0) ?
{ }} rehypePlugins={[rehypeRaw]} > - {workflowQuestion !== "" ? workflowQuestion : realtimeMarkdown !== undefined && realtimeMarkdown !== null && realtimeMarkdown.length > 0 ? realtimeMarkdown : workflow.form_control.input_markdown} + {workflowQuestion !== "" ? workflowQuestion : realtimeMarkdown !== undefined && realtimeMarkdown !== null && realtimeMarkdown.length > 0 ? realtimeMarkdown : workflow?.form_control?.input_markdown}
: null} {onSubmit(e)}} style={{margin: "25px 0px 15px 0px",}}> - {workflowQuestion !== "" || (workflow.form_control.input_markdown !== undefined && workflow.form_control.input_markdown !== null && workflow.form_control.input_markdown.length > 0) ? null : + {workflowQuestion !== "" || (workflow?.form_control?.input_markdown !== undefined && workflow?.form_control?.input_markdown !== null && workflow?.form_control?.input_markdown.length > 0) ? null :
{/* { {organization} - + {organization?.length > 0 && + + } {disabledButtons && message.length > 0 ? null : - + {message} } @@ -1412,6 +1509,11 @@ const RunWorkflow = (defaultprops) => { executionArgument[multiChoiceOptions[0]] = multiChoiceOptions[1] } + const parsedLabel = question?.value?.startsWith("question_") ? + "" + : + question?.value?.charAt(0)?.toUpperCase() + question?.value?.slice(1) + return (
@@ -1457,7 +1559,7 @@ const RunWorkflow = (defaultprops) => { backgroundColor: theme.palette.inputColor, marginTop: 5, }} - label={question?.value?.charAt(0)?.toUpperCase() + question?.value?.slice(1)} + label={parsedLabel} required disabled={disabledButtons} @@ -1542,7 +1644,7 @@ const RunWorkflow = (defaultprops) => { : - {disabledButtons ? "Already answered. You may close this window." : ""} + {disabledButtons ? "Question answered. You may close this window." : ""} } @@ -1565,10 +1667,13 @@ const RunWorkflow = (defaultprops) => { textTransform: "none", }} onClick={() => { - setButtonClicked("FINISHED") - setExecutionData({ - status: "FINISHED", - }) + // Timeout 2500 just in case + setTimeout(() => { + setButtonClicked("FINISHED") + setExecutionData({ + status: "FINISHED", + }) + }, 2500) onSubmit(null, execution_id, authorization, true) }}> @@ -1586,16 +1691,24 @@ const RunWorkflow = (defaultprops) => { flex: 1, textTransform: "none", }} onClick={() => { - setButtonClicked("ABORTED") - setExecutionData({ - status: "ABORTED", - }) + setTimeout(() => { + setButtonClicked("ABORTED") + setExecutionData({ + status: "ABORTED", + }) + }, 2500) onSubmit(null, execution_id, authorization, false) }}> Stop
+ + {handleValidateForm(executionArgument) === false && disabledButtons === false ? + + All required questions have not been answered yet. + + : null} :
diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 32859549..702092da 100755 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -477,7 +477,7 @@ export const HandleJsonCopy = (base, copy, base_node_name) => { //var newitem = JSON.parse(base); var newitem = validateJson(base).result - var to_be_copied = "$" + base_node_name.toLowerCase().replaceAll(" ", "_"); + var to_be_copied = "$" + base_node_name?.toLowerCase()?.replaceAll(" ", "_"); for (let copykey in copy.namespace) { if (copy.namespace[copykey].includes("Results for")) { continue; @@ -742,7 +742,7 @@ const DropzoneWrapper = memo(({ onDrop, WorkflowView }) => { const Workflows = (props) => { const { globalUrl, isLoggedIn, isLoaded, userdata, checkLogin } = props; - document.title = "Shuffle - Workflows"; + document.title = "Workflows - Shuffle"; let navigate = useNavigate(); const classes = useStyles(theme) diff --git a/frontend/src/views/Workflows2.jsx b/frontend/src/views/Workflows2.jsx index 86c28e18..2732279a 100644 --- a/frontend/src/views/Workflows2.jsx +++ b/frontend/src/views/Workflows2.jsx @@ -4,16 +4,6 @@ import { useLocation, useNavigate, Link } from "react-router-dom"; import ReactDOM from "react-dom" import { getTheme } from "../theme.jsx"; -// Material UI Icons -import Add from '@mui/icons-material/Add'; -import Search from '@mui/icons-material/Search'; -import ClearIcon from '@mui/icons-material/Clear'; -import QueryStatsIcon from '@mui/icons-material/QueryStats'; -import GridOnIcon from '@mui/icons-material/GridOn'; -import ListIcon from '@mui/icons-material/List'; -import PublishIcon from '@mui/icons-material/Publish'; -import GetAppIcon from '@mui/icons-material/GetApp'; - // Material UI & Components import { makeStyles } from "@mui/styles"; import { Navigate } from "react-router-dom"; @@ -67,6 +57,7 @@ import { // Material UI Icons import { + ContentCopy as ContentCopyIcon, Close as CloseIcon, Compare as CompareIcon, Maximize as MaximizeIcon, @@ -105,6 +96,12 @@ import { AutoAwesome as AutoAwesomeIcon, BarChart as BarChartIcon, Lock as LockIcon, + Clear as ClearIcon, + QueryStats as QueryStatsIcon, + GridOn as GridOnIcon, + List as ListIcon, + Publish as PublishIcon, + GetApp as GetAppIcon, } from "@mui/icons-material"; // Additional Components @@ -209,10 +206,10 @@ export const GetIconInfo = (action) => { key: "compare", values: ["compare", "convert", "to", "filter", "translate", "parse"], }, - { key: "assets", values: ["cmdb", "assets", "asset", "cmdb", "inventory", "host", "hosts", "device", "devices"] }, + { key: "assets", values: ["cmdb", "assets", "asset", "cmdb", "inventory", "host", "hosts", "device", "devices", "app",] }, { key: "close", values: ["close", "stop", "cancel", "block"] }, { key: "communication", values: ["communication", "comms", "email", "mail",] }, - { key: "eradication", values: ["eradication", "edr", "xdr"] }, + { key: "eradication", values: ["eradication", "edr", "xdr", "sigma", "yara",] }, { key: "iam", values: ["iam", "identity", "access", "auth", "authentication", "authorization", "oauth", "sso", "openid"] }, { key: "intel", values: ["intel", "feed", "threat intel", "threat intelligence", "ti", "t.i.", "t.i", "ti.", "rule", "technique", "tactic", "techniques", "tactics", "ioc", "indicator",] }, { key: "network", values: ["network", "net", "networking", "firewall", "proxy", "vpn", "sdwan", "sd-wan"] }, @@ -235,6 +232,7 @@ export const GetIconInfo = (action) => { values: [ "api", "password", + "passwd", "protect", ], } @@ -820,6 +818,7 @@ const Workflows2 = (props) => { const [highlightIds, setHighlightIds] = React.useState([]) const [apps, setApps] = React.useState([]); + const [isProdStatusOn, setIsProdStatusOn] = React.useState(false); const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); @@ -835,10 +834,45 @@ const Workflows2 = (props) => { setCurrTab(1); } else if (tabParam === 'all_workflows' && currTab !== 2) { setCurrTab(2); - } + } else if (tabParam === 'background_processes' && currTab !== 4) { + setCurrTab(4); + } } }, [location.search]); + useEffect(() => { + const orgId = userdata?.active_org?.id; + if (!orgId) { + return; + } + + let fetched = false; + fetch(`${globalUrl}/api/v1/orgs/${orgId}`, { + method: "GET", + credentials: "include", + headers: { "Content-Type": "application/json" }, + }) + .then((response) => (response.ok ? response.json() : null)) + .then((org) => { + if (!fetched && org) { + if (!isCloud) { + if (org?.cloud_sync && org?.subscriptions[0]?.name?.toLowerCase().includes("enterprise") && org?.subscriptions[0]?.active) { + setIsProdStatusOn(true); + } else if (org?.subscriptions[0]?.name?.toLowerCase().includes("enterprise") && org?.subscriptions[0]?.active) { + setIsProdStatusOn(true); + } else { + setIsProdStatusOn(false); + } + } + } + }) + .catch(() => {}); + + return () => { + fetched = true; + }; + }, [userdata?.active_org?.id, globalUrl]); + const handleTabChange = (event, newValue) => { setCurrTab(newValue); @@ -853,10 +887,15 @@ const Workflows2 = (props) => { 1: 'my_workflows', 2: 'all_workflows', 3: 'backup_apps', + 4: 'background_processes', }; const queryParams = new URLSearchParams(location.search); queryParams.set('tab', tabMapping[newValue]); + if (newValue === 4) { + setShowExecutionStats(true) + setView("grid") + } navigate(`${location.pathname}?${queryParams.toString()}`); }; @@ -1553,7 +1592,7 @@ const Workflows2 = (props) => { sx: { borderRadius: theme?.palette?.DialogStyle?.borderRadius, border: theme?.palette?.DialogStyle?.border, - minWidth: '440px', + minWidth: 440, fontFamily: theme?.typography?.fontFamily, backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, zIndex: 1000, @@ -1566,11 +1605,11 @@ const Workflows2 = (props) => { } }} > - +
Are you sure you want to delete {selectedWorkflowId.length > 0 ? filteredWorkflows.find((w) => w.id === selectedWorkflowId)?.name : `${selectedWorkflowIndexes.length} workflow${selectedWorkflowIndexes.length === 1 ? '' : 's'}`}?
- Other workflows relying on {selectedWorkflowIndexes.length > 0 ? "them" : "it"} one will stop working + Other workflows relying on {selectedWorkflowIndexes.length > 0 ? "them" : "it"} one will stop working.
{ credentials: "include", }) .then((response) => { + setIsLoadingWorkflow(false) if (response.status !== 200) { console.log("Status not 200 for workflows :O!: ", response.status); @@ -1956,6 +1996,7 @@ const Workflows2 = (props) => { } }) .catch((error) => { + setIsLoadingWorkflow(false) toast(error.toString()); }); } @@ -2949,6 +2990,8 @@ const Workflows2 = (props) => { triggerfound = true image = wfTriggers[0].large_image + trigger.status = trigger?.status?.toLowerCase() + relevantTrigger = trigger if (trigger?.status === "running") { imageStyle.border = `3px solid ${green}` @@ -2962,6 +3005,8 @@ const Workflows2 = (props) => { triggerfound = true image = wfTriggers[1].large_image + trigger.status = trigger?.status?.toLowerCase() + relevantTrigger = trigger if (trigger?.status === "running") { imageStyle.border = `3px solid ${green}` @@ -3032,12 +3077,35 @@ const Workflows2 = (props) => { } } + const isPublicWorkflow = data?.objectId === undefined || data?.objectId === null + const foundImage = !isPublicWorkflow ? "" : data?.image_url === undefined || data?.image_url === null || data?.image_url === "" ? data?.image : data?.image_url + const foundTimeline = workflowTimelines.find((timeline) => timeline.id === data.id) return ( -
+
+ {isPublicWorkflow && foundImage?.length > 0 ? + image { + if (isCloud) { + navigate(`/workflows/${data.objectID}`) + } else { + window.open(`https://shuffler.io/workflows/${data.objectID}`, "_blank") + } + }} + /> + : null} + - {selectedCategory !== "" ?
{ }} /> - : null} + : null} { {currTab === 2 ? null : -
{ @@ -3183,6 +3250,7 @@ const Workflows2 = (props) => { + {appGroup.length > 0 ?
@@ -3437,7 +3505,7 @@ const Workflows2 = (props) => { {showExecutionStats === true && foundTimeline !== undefined && foundTimeline?.timeline?.length > 0 && -
+
{ }} /> {
-
+ {currTab === 4 ? null : +
- {currTab === 2 ? ( - - ) : ( - - // - // - // ), - onKeyDown: (e) => { - // Prevent default behavior for Enter and Backspace - if (e.key === 'Enter' || e.key === 'Backspace') { - e.preventDefault(); - e.stopPropagation(); - e.target.focus(); - } - }, - }} - clearInputOnBlur={false} - sx={{ - // Container styling - '& .MuiOutlinedInput-root': { - height: "fit-content", - borderRadius: '4px', - color: theme.palette.textFieldStyle.color, - backgroundColor: theme.palette.textFieldStyle.backgroundColor, - '& fieldset': { - borderColor: 'rgba(255, 255, 255, 0.23)', - }, - '&:hover fieldset': { - borderColor: 'rgba(255, 255, 255, 0.4)', - }, - }, + {currTab === 2 ? ( + + ) : + ( + + // + // + // ), + onKeyDown: (e) => { + // Prevent default behavior for Enter and Backspace + if (e.key === 'Enter' || e.key === 'Backspace') { + e.preventDefault(); + e.stopPropagation(); + e.target.focus(); + } + }, + }} + clearInputOnBlur={false} + sx={{ + // Container styling + '& .MuiOutlinedInput-root': { + height: "fit-content", + borderRadius: '4px', + color: theme.palette.textFieldStyle.color, + backgroundColor: theme.palette.textFieldStyle.backgroundColor, + '& fieldset': { + borderColor: 'rgba(255, 255, 255, 0.23)', + }, + '&:hover fieldset': { + borderColor: 'rgba(255, 255, 255, 0.4)', + }, + }, - // Adjust chip container to center vertically - '& .MuiInputBase-root': { - display: 'flex', - flexWrap: 'wrap', - gap: '4px', - fontSize: 18, - padding: '4px 8px', - alignItems: 'center', - height: "fit-content", // Match height - backgroundColor: theme.palette.textFieldStyle.backgroundColor, - color: theme.palette.textFieldStyle.color - }, + // Adjust chip container to center vertically + '& .MuiInputBase-root': { + display: 'flex', + flexWrap: 'wrap', + gap: '4px', + fontSize: 18, + padding: '4px 8px', + alignItems: 'center', + height: "fit-content", // Match height + backgroundColor: theme.palette.textFieldStyle.backgroundColor, + color: theme.palette.textFieldStyle.color + }, - // Rest of the styling remains the same... - }} - value={filters} - onChange={(chips) => { - setFilters(chips); - const remainingCategories = chips.map(chip => { - const match = chip.match(/\d+\.\s+(\w+)/i); - return match ? match[1] : chip; - }).filter(category => { - return usecases.some(usecase => - usecase.name.toLowerCase().includes(category.toLowerCase()) - ); - }); + // Rest of the styling remains the same... + }} + value={filters} + onChange={(chips) => { + setFilters(chips); + const remainingCategories = chips.map(chip => { + const match = chip.match(/\d+\.\s+(\w+)/i); + return match ? match[1] : chip; + }).filter(category => { + return usecases.some(usecase => + usecase.name.toLowerCase().includes(category.toLowerCase()) + ); + }); - setSelectedCategory(remainingCategories); - findWorkflow(chips); + setSelectedCategory(remainingCategories); + findWorkflow(chips); - }} - //onAdd={(chip) => { - // console.log("ADd: ", chip); - // addFilter(chip); - //}} - //onDelete={(_, index) => { - // console.log("Remove: ", index); - // removeFilter(index); - //}} - /> - )} + }} + //onAdd={(chip) => { + // console.log("ADd: ", chip); + // addFilter(chip); + //}} + //onDelete={(_, index) => { + // console.log("Remove: ", index); + // removeFilter(index); + //}} + /> + )} - { - currTab !== 2 && ( - selected.length ? selected.join(', ') : 'All Categories'} + > + + All Categories + + {usecases.map((usecase, index) => { + if (usecase?.name === "5. Verify") { + return null; + } - const percentDone = usecase.matches.length > 0 ? parseInt(usecase.matches.length / usecase.list.length * 100) : 0 - if (percentDone === 0) { - usecase = findMatches(usecase, workflows) - } + const percentDone = usecase.matches.length > 0 ? parseInt(usecase.matches.length / usecase.list.length * 100) : 0 + if (percentDone === 0) { + usecase = findMatches(usecase, workflows) + } - const category = usecase?.name.split(" ")[1] - return ( - { - if (!filters.includes(usecase?.name.toLowerCase())) { - addFilter(usecase.name) - } else { - removeFilter(filters.indexOf(usecase?.name.toLowerCase())) - } - }} - sx={{ - padding: "12px 16px", - borderBottom: index === usecases.length - 2 ? "none" : "1px solid rgba(255,255,255,0.05)", - "&:hover": { - backgroundColor: "rgba(255,255,255,0.1)" - }, - }} - > -
- -
- - {category} - - - {usecase?.matches.length}/{usecase?.list.length} - -
-
- - ) - })} - - ) - } - { - currTab === 2 && ( - - ) - } + const category = usecase?.name.split(" ")[1] + return ( + { + if (!filters.includes(usecase?.name.toLowerCase())) { + addFilter(usecase.name) + } else { + removeFilter(filters.indexOf(usecase?.name.toLowerCase())) + } + }} + sx={{ + padding: "12px 16px", + borderBottom: index === usecases.length - 2 ? "none" : "1px solid rgba(255,255,255,0.05)", + "&:hover": { + backgroundColor: "rgba(255,255,255,0.1)" + }, + }} + > +
+ +
+ + {category} + + + {usecase?.matches.length}/{usecase?.list.length} + +
+
+ + ) + })} + + ) + } + { + currTab === 2 && ( + + ) + } -
-
+
+
- - { + + { - const newView = !showExecutionStats - localStorage.setItem("showExecutionStats", newView) - setShowExecutionStats(!showExecutionStats) - }} - disabled={currTab === 2} - > - - - + const newView = !showExecutionStats + localStorage.setItem("showExecutionStats", newView) + setShowExecutionStats(!showExecutionStats) + }} + disabled={currTab === 2} + > + + + - - navigate("/workflows/debug")} - disabled={currTab === 2} - > - - - + + navigate("/workflows/debug")} + disabled={currTab === 2} + > + + + - - { - const newView = view === "grid" ? "list" : "grid"; - localStorage.setItem("workflowView", newView); - setView(newView); + + { + const newView = view === "grid" ? "list" : "grid"; + localStorage.setItem("workflowView", newView); + setView(newView); - if (view === "grid") { - setCurrTab(0) - } - }} - disabled={currTab === 2} - > - {view === "grid" ? - : - - } - - + if (view === "grid") { + setCurrTab(0) + } + }} + disabled={currTab === 2} + > + {view === "grid" ? + : + + } + + - - upload.click()} - disabled={currTab === 2} - > - {submitLoading ? - : - - } - - + + upload.click()} + disabled={currTab === 2} + > + {submitLoading ? + : + + } + + - (upload = ref)} - onChange={importFiles} - /> + (upload = ref)} + onChange={importFiles} + /> - - exportAllWorkflows(workflows)} - > - - - -
- -
+ + exportAllWorkflows(workflows)} + > + + + +
+ +
+
+ } + + {!isCloud ? ( +
{ + navigate("/admin?admin_tab=billingstats") + }} + > + + + {isProdStatusOn ? "Production" : "NOT Production"} + +
+ ) : null} - -
+ + { (isLoadingWorkflow && currTab !== 2) ? ( ) : ( view === "grid" && currTab !== 2 ? ( <> -
{ + if (data.triggers.length === 0) { + return null + } + + var foundWebhook = "" + var foundtrigger = {} + for (var triggerKey in data.triggers) { + if (data.triggers[triggerKey].trigger_type === "WEBHOOK") { + foundWebhook = `${globalUrl}/api/v1/hooks/webhook_${data.triggers[triggerKey].id}` + foundtrigger = data.triggers[triggerKey] + break + } + } + + if (foundWebhook === "") { + return null + } + + var webhookName = `` + if (data?.name?.toLowerCase().includes("ingest tickets")) { + webhookName = "Send your Tickets, Alerts, Cases and Detections here. This will ingest them into Shuffle." + } + + return ( +
{ + // Find the relevant workflow paper and highlight it + const foundElement = document.getElementById(`workflowbox-${data.id}`) + if (foundElement) { + foundElement.style.border = `3px solid ${theme.palette.primary.main}` + } + }} + onMouseLeave={() => { + const foundElement = document.getElementById(`workflowbox-${data.id}`) + if (foundElement) { + foundElement.style.border = null + } + }} + > + + {webhookName} + + + + webhook + + + { + if (navigator.clipboard === undefined) { + toast("Your browser doesn't support clipboard copying, please copy manually.", { type: "error" }); + } else { + navigator.clipboard.writeText(foundWebhook); + } + }} + style={{ + color: theme.palette.textFieldStyle.color, + backgroundColor: theme.palette.platformColor, + marginRight: 10, + borderRadius: 4, + }} + id="copy_webhook_url_button" + > + + + + + ), + style: { + color: theme.palette.textFieldStyle.color, + backgroundColor: theme.palette.textFieldStyle.backgroundColor, + } + }} + /> +
+ ) + })} + +
0 { @@ -799,6 +800,48 @@ func buildEnvVars(envMap map[string]string) []corev1.EnvVar { return envVars } +func buildResourcesFromEnv() corev1.ResourceRequirements { + requests := corev1.ResourceList{} + limits := corev1.ResourceList{} + + type item struct { + env string + resourceName corev1.ResourceName + resourceList corev1.ResourceList + } + + items := []item{ + // kubernetes requests + {env: "SHUFFLE_WORKER_CPU_REQUEST", resourceName: corev1.ResourceCPU, resourceList: requests}, + {env: "SHUFFLE_WORKER_MEMORY_REQUEST", resourceName: corev1.ResourceMemory, resourceList: requests}, + {env: "SHUFFLE_WORKER_EPHEMERAL_STORAGE_REQUEST", resourceName: corev1.ResourceEphemeralStorage, resourceList: requests}, + // kubernetes limits + {env: "SHUFFLE_WORKER_CPU_LIMIT", resourceName: corev1.ResourceCPU, resourceList: limits}, + {env: "SHUFFLE_WORKER_MEMORY_LIMIT", resourceName: corev1.ResourceMemory, resourceList: limits}, + {env: "SHUFFLE_WORKER_EPHEMERAL_STORAGE_LIMIT", resourceName: corev1.ResourceEphemeralStorage, resourceList: limits}, + } + + for _, it := range items { + if value := strings.TrimSpace(os.Getenv(it.env)); value != "" { + if quantity, err := resource.ParseQuantity(value); err == nil { + it.resourceList[it.resourceName] = quantity + } else { + log.Printf("[WARNING] Cannot parse %s=%q as resource quantity: %v", it.env, value, err) + } + } + } + + rr := corev1.ResourceRequirements{} + if len(requests) > 0 { + rr.Requests = requests + } + if len(limits) > 0 { + rr.Limits = limits + } + + return rr +} + func handleBackendImageDownload(ctx context.Context, images string) error { // Replicate images with lowercase, as the name may be wrong @@ -1042,6 +1085,20 @@ func deployK8sWorker(image string, identifier string, env []string) error { env = append(env, fmt.Sprintf("IS_KUBERNETES=true")) env = append(env, fmt.Sprintf("KUBERNETES_NAMESPACE=%s", os.Getenv("KUBERNETES_NAMESPACE"))) + // app resource env + for _, k := range []string{ + "SHUFFLE_APP_CPU_REQUEST", + "SHUFFLE_APP_MEMORY_REQUEST", + "SHUFFLE_APP_EPHEMERAL_STORAGE_REQUEST", + "SHUFFLE_APP_CPU_LIMIT", + "SHUFFLE_APP_MEMORY_LIMIT", + "SHUFFLE_APP_EPHEMERAL_STORAGE_LIMIT", + } { + if v := os.Getenv(k); v != "" { + env = append(env, fmt.Sprintf("%s=%s", k, v)) + } + } + if len(os.Getenv("KUBERNETES_SERVICE_HOST")) > 0 { env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_HOST=%s", os.Getenv("KUBERNETES_SERVICE_HOST"))) } @@ -1089,10 +1146,18 @@ func deployK8sWorker(image string, identifier string, env []string) error { env = append(env, fmt.Sprintf("SHUFFLE_APP_CONTAINER_SECURITY_CONTEXT=%s", appContainerSecurityContext)) } + if len(os.Getenv("SHUFFLE_APP_MOUNT_TMP_VOLUME")) > 0 { + env = append(env, fmt.Sprintf("SHUFFLE_APP_MOUNT_TMP_VOLUME=%s", os.Getenv("SHUFFLE_APP_MOUNT_TMP_VOLUME"))) + } + if len(os.Getenv("SHUFFLE_LOGS_DISABLED")) > 0 { env = append(env, fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED"))) } + if len(os.Getenv("SHUFFLE_APP_REPLICAS")) > 0 { + env = append(env, fmt.Sprintf("SHUFFLE_APP_REPLICAS=%s", os.Getenv("SHUFFLE_APP_REPLICAS"))) + } + clientset, _, err := shuffle.GetKubernetesClient() if err != nil { log.Printf("[ERROR] Error getting kubernetes client:", err) @@ -1206,6 +1271,7 @@ func deployK8sWorker(image string, identifier string, env []string) error { Image: kubernetesImage, Env: buildEnvVars(envMap), SecurityContext: containerSecurityContext, + Resources: buildResourcesFromEnv(), //ImagePullPolicy: "Never", ImagePullPolicy: corev1.PullIfNotPresent, @@ -1337,6 +1403,7 @@ func deployK8sWorker(image string, identifier string, env []string) error { return err } + svcAppProtocol := "http" service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: identifier, @@ -1346,9 +1413,10 @@ func deployK8sWorker(image string, identifier string, env []string) error { Selector: matchLabels, Ports: []corev1.ServicePort{ { - Protocol: "TCP", - Port: 33333, - TargetPort: intstr.FromInt(33333), + Protocol: "TCP", + AppProtocol: &svcAppProtocol, + Port: 33333, + TargetPort: intstr.FromInt(33333), }, }, Type: corev1.ServiceTypeClusterIP, @@ -1571,12 +1639,12 @@ func initializeImages() { if appSdkVersion == "" { appSdkVersion = "latest" - log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %#v", appSdkVersion) + log.Printf("[INFO] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %#v", appSdkVersion) } if workerVersion == "" { workerVersion = "latest" - log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %#v", workerVersion) + log.Printf("[INFO] SHUFFLE_WORKER_VERSION not defined. Defaulting to %#v", workerVersion) } if baseimageregistry == "" { @@ -1586,12 +1654,12 @@ func initializeImages() { if len(os.Getenv("REGISTRY_URL")) > 0 { baseimageregistry = os.Getenv("REGISTRY_URL") } else { - os.Setenv("REGISTRY_URL", baseimageregistry) + // os.Setenv("REGISTRY_URL", baseimageregistry) } os.Setenv("SHUFFLE_BASE_IMAGE_REGISTRY", baseimageregistry) - log.Printf("[WARNING] Setting baseimageregistry to %#v", baseimageregistry) + log.Printf("[INFO] Setting baseimageregistry to %#v", baseimageregistry) } if baseimagename == "" { @@ -1600,7 +1668,7 @@ func initializeImages() { baseimagename = "frikky/shuffle" // Dockerhub os.Setenv("SHUFFLE_BASE_IMAGE_NAME", baseimagename) - log.Printf("[WARNING] Setting baseimagename to %#v", baseimagename) + log.Printf("[INFO] Setting baseimagename to %#v", baseimagename) } // Old sane default overrides: @@ -2063,12 +2131,29 @@ func main() { } } - log.Println("[INFO] Setting up execution environment") + // Auto enables pipelines IF they are not mentioned + if len(os.Getenv("SHUFFLE_SKIP_PIPELINES")) == 0 { + os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") + os.Setenv("SHUFFLE_PIPELINE_ENABLED", "true") + } + if os.Getenv("SHUFFLE_SKIP_PIPELINES") != "true" && os.Getenv("SHUFFLE_PIPELINE_ENABLED") != "false" { + // Run in 15 seconds in a goroutine + go func() { + time.Sleep(15 * time.Second) + log.Printf("[INFO] Auto-downloading Sigma rules during startup") + ruleType := "sigma" + err := handleFileCategoryChange(ruleType) + if err != nil { + log.Printf("[WARNING] Failed downloading %s rules: %s", ruleType, err) + } + }() + } + + log.Println("[INFO] Setting up execution environment for env '%s'", environment) // //FIXME if baseUrl == "" { baseUrl = "https://shuffler.io" - //baseUrl = "http://localhost:5001" } if len(orborusUuid) == 0 { @@ -2229,7 +2314,6 @@ func main() { fullUrl += "?amount=50" } - if isKubernetes == "true" { log.Printf("[INFO] Finished configuring kubernetes environment. Connecting to %s", fullUrl) } else { @@ -2432,9 +2516,10 @@ func main() { for _, incRequest := range executionRequests.Data { // Looking for specific jobs - if incRequest.Type == "PIPELINE_CREATE" || incRequest.Type == "PIPELINE_START" || incRequest.Type == "PIPELINE_STOP" || incRequest.Type == "PIPELINE_DELETE" { + if incRequest.Type == "PIPELINE_CREATE" || incRequest.Type == "PIPELINE_START" || incRequest.Type == "PIPELINE_STOP" || incRequest.Type == "PIPELINE_DELETE" || incRequest.Type == "PIPELINE_UPDATE" { + log.Printf("[INFO] Handling pipeline request from backend: '%s' with argument '%s'", incRequest.Type, incRequest.ExecutionArgument) - os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") + //os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") tenzirDisabled = false // Running NEW or editing pipelines @@ -2457,9 +2542,9 @@ func main() { } else if incRequest.Type == "CATEGORY_UPDATE" { os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - tenzirDisabled = false - err = handleFileCategoryChange() + tenzirDisabled = false + err = handleFileCategoryChange("sigma") if err != nil { log.Printf("[ERROR] Failed to download the file category: %s", err) } @@ -2469,7 +2554,7 @@ func main() { } else if incRequest.Type == "DISABLE_SIGMA_FOLDER" { log.Printf("[INFO] Got job to disable sigma rules") - err = removeFileCategory() + err = removeFileCategory("sigma") if err != nil { log.Printf("[ERROR] Failed to disable the sigma rules: %s", err) } @@ -2501,7 +2586,7 @@ func main() { log.Printf("[INFO] Got job to start tenzir") // Manual command = overrides to allow starting of Tenzir from the frontend anyway. - os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") + //os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") tenzirDisabled = false // Removed either way @@ -2511,8 +2596,8 @@ func main() { if err != nil { if strings.Contains(fmt.Sprintf("%s", err), "node available") { // Disabling until UI is updated - os.Setenv("SHUFFLE_SKIP_PIPELINES", "true") - tenzirDisabled = true + //os.Setenv("SHUFFLE_SKIP_PIPELINES", "true") + //tenzirDisabled = true log.Printf("[ERROR] Failed to start tenzir, reason: %s", err) err = shuffle.CreateOrgNotification( @@ -2578,6 +2663,8 @@ func main() { executionRequests.Data = executionRequests.Data[0:allowed] } } else if swarmControlMode && (swarmConfig == "run" || swarmConfig == "swarm") { + // any reason it is not maxConcurrency instead of + // hardcoded 50? if len(executionRequests.Data) > 50 { executionRequests.Data = executionRequests.Data[0:50] } @@ -2758,7 +2845,7 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { // no need of execution arguments for STOP and DELETE if (incRequest.Type != "PIPELINE_STOP" && incRequest.Type != "PIPELINE_DELETE") && len(incRequest.ExecutionArgument) == 0 { - log.Printf("[ERROR] No execution argument found for pipeline create. Skipping") + log.Printf("[ERROR] No execution argument found for pipeline type %s. Skipping", incRequest.Type) return errors.New("no execution argument found for pipeline create. Skipping") } @@ -2769,6 +2856,7 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { } command := incRequest.ExecutionArgument + pipelines = []shuffle.PipelineInfo{} if incRequest.Type == "PIPELINE_CREATE" { log.Printf("[INFO] Should delete -> recreate new pipeline with id %#v", identifier) //err := deployPipeline(image, identifier, command) @@ -2778,19 +2866,16 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { return err } } else if incRequest.Type == "PIPELINE_DELETE" || incRequest.Type == "PIPELINE_STOP" { - { - log.Printf("[INFO] Should delete pipeline %#v", identifier) - pipelineId, err := searchPipeline(identifier) - if err != nil { - log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) - return err - } + pipelineId := incRequest.ExecutionId + log.Printf("[INFO] Should delete pipeline %#v. PipelineID: %s", identifier, pipelineId) + //pipelineId, err := searchPipeline(identifier) + //if err != nil { + //} - err = deletePipeline(pipelineId) - if err != nil { - log.Printf("[ERROR] Failed Deleting Pipeline %s", err) - return err - } + err = deletePipeline(pipelineId) + if err != nil { + log.Printf("[ERROR] Failed Deleting Pipeline %s", err) + return err } /* @@ -2817,19 +2902,24 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { if err != nil { if err.Error() == "no existing pipeline found with name" { log.Printf("[INFO] Starting a new pipeline with command '%s' and identifier '%s'", command, identifier) - _, CreateErr := createPipeline(command, identifier) - return CreateErr + var createErr error + pipelineId, createErr = createPipeline(command, identifier) + if createErr != nil { + return createErr + } + } else { + log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) + return err } - - log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) - return err } + + log.Printf("[INFO] Starting existing pipeline with ID %s", pipelineId) _, err = updatePipelineState(command, pipelineId, "start") if err != nil { log.Printf("[ERROR] Failed to start Pipeline: %s reason:%s ", pipelineId, err) return err } else { - log.Printf("[INFO] Successfully started the Pipeline: %s", pipelineId) + log.Printf("[INFO] Successfully started pipeline: %s", pipelineId) } } else { @@ -2859,27 +2949,38 @@ func deployTenzirNode() error { ctx := context.Background() cacheKey := "tenzir-key" - - imageName := "frikky/shuffle:tenzir" - containerName := "tenzir-node" - containerStartOptions := container.StartOptions{} _, err = shuffle.GetCache(ctx, cacheKey) if err == nil { return nil } + imageName := "frikky/shuffle:tenzir" + if os.Getenv("TENZIR_IMAGE_NAME") != "" { + imageName = os.Getenv("TENZIR_IMAGE_NAME") + log.Printf("[INFO] Using custom Tenzir image name: %s", imageName) + } + + containerName := "tenzir-node" + containerStartOptions := container.StartOptions{} + containerInfo, err := dockercli.ContainerInspect(ctx, containerName) if err != nil { if dockerclient.IsErrNotFound(err) { // Create network if it doesn't exist networkName := "tenzir-network" - networkSubnet := "192.168.1.0/24" - networkGateway := "192.168.1.1" + networkSubnet := "192.168.102.0/24" + networkGateway := "192.168.102.1" err = createNetworkIfNotExists(ctx, networkName, networkSubnet, networkGateway) if err != nil { - log.Printf("[ERROR] Failed to create network: %s", err) - return err + log.Printf("[ERROR] Failed to create network %s: %s", networkName, err) + //return err + } + + // Trying to connect orborus to the tenzir network as well + err = dockercli.NetworkConnect(ctx, networkName, containerId, nil) + if err != nil { + log.Printf("[ERROR] Error connecting tenzir container to network: %s", err) } // Check if image exists @@ -2908,7 +3009,7 @@ func deployTenzirNode() error { } } else { if !containerInfo.State.Running { - log.Printf("[DEBUG] Tenzir Node exists but is not running. Restarting it.") + log.Printf("[DEBUG] Tenzir Node exists, but is not running. Restarting it.") err := dockercli.ContainerStart(ctx, containerName, containerStartOptions) if err != nil { log.Printf("[ERROR] Failed to start Tenzir Node container: %v", err) @@ -2934,6 +3035,7 @@ func deployTenzirNode() error { if err != nil { log.Printf("[WARNING] Failed marshalling execution: %s", err) } + err = shuffle.SetCache(ctx, cacheKey, cacheData, 1) if err != nil { log.Printf("[WARNING] Failed updating cache for tenzir: %s", err) @@ -2957,8 +3059,8 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri Healthcheck: healthconfig, ExposedPorts: nat.PortSet{ "5160/tcp": struct{}{}, - "514/udp": struct{}{}, - "514/tcp": struct{}{}, + "1514/udp": struct{}{}, + "1514/tcp": struct{}{}, }, Entrypoint: []string{containerName}, Env: []string{}, @@ -3004,26 +3106,28 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri hostConfig := &container.HostConfig{ PortBindings: nat.PortMap{ - "514/tcp": []nat.PortBinding{{HostPort: "514"}}, - "514/udp": []nat.PortBinding{{HostPort: "514"}}, + "1514/tcp": []nat.PortBinding{{HostPort: "1514"}}, + "1514/udp": []nat.PortBinding{{HostPort: "1514"}}, "5160/tcp": []nat.PortBinding{{HostPort: "5160"}}, }, Mounts: []mount.Mount{ { Type: "bind", Source: tenzirStorageFolder, - Target: "/var/lib/tenzir/", - }, - { - Type: "bind", - Source: tenzirStorageFolder, - Target: "/var/log/tenzir/", - }, - { - Type: "bind", - Source: tenzirStorageFolder, - Target: "/var/cache/tenzir/", + Target: "/tmp", }, + /* + { + Type: "bind", + Source: tenzirStorageFolder, + Target: "/var/log/tenzir/", + }, + { + Type: "bind", + Source: tenzirStorageFolder, + Target: "/var/cache/tenzir/", + }, + */ }, VolumeDriver: "local", RestartPolicy: container.RestartPolicy{ @@ -3031,6 +3135,12 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri }, } + if os.Getenv("SHUFFLE_DISABLE_SYSLOG") == "true" { + hostConfig.PortBindings = nat.PortMap{ + "5160/tcp": []nat.PortBinding{{HostPort: "5160"}}, + } + } + if skipPipelineMount { hostConfig.Mounts = []mount.Mount{} } @@ -3039,7 +3149,7 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri EndpointsConfig: map[string]*network.EndpointSettings{ "tenzir-network": { IPAMConfig: &network.EndpointIPAMConfig{ - IPv4Address: "192.168.1.100", + IPv4Address: "192.168.102.100", }, }, }, @@ -3093,12 +3203,19 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri return err } - log.Printf("[INFO] Successfully deployed Tenzir Node! Setting up default syslog listener on UDP 514") + log.Printf("[INFO] Successfully deployed Tenzir Node! Setting up default syslog listener on TCP/1514 AND UDP/1514") - command := "from udp://0.0.0.0:514 read syslog | import" - _, err = createPipeline(command, "default-syslog-514") + command := `from "tcp://0.0.0.0:1514" { read_syslog } | import` + _, err = createPipeline(command, "default-syslog-tcp-514") if err != nil { - log.Printf("[ERROR] Failed to create default syslog pipeline: %s", err) + log.Printf("[ERROR] Failed to create tcp syslog pipeline: %s", err) + return nil + } + + command = `load_udp "0.0.0.0:1514", insert_newlines=true | read_syslog | import` + _, err = createPipeline(command, "default-syslog-udp-514") + if err != nil { + log.Printf("[ERROR] Failed to create udp syslog pipeline: %s", err) return nil } @@ -3143,7 +3260,7 @@ func createNetworkIfNotExists(ctx context.Context, networkName, subnet, gateway } func checkTenzirNode() error { - if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "true" { + if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "true" && os.Getenv("SHUFFLE_PIPELINE_ENABLED") == "false" { return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") } @@ -3216,21 +3333,18 @@ func createPipeline(command, identifier string) (string, error) { //command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | sigma /var/lib/tenzir/rule.yaml" //command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | import" + // Make sure to escape them + //if strings.Contains(command, "/") { + // command = strings.ReplaceAll("\\\"", "", command) + // command = strings.ReplaceAll(command, "\"", "") + //} + requestBody := map[string]interface{}{ - "definition": command, - "name": identifier, - "hidden": false, - "autostart": map[string]bool{ - "created": true, - "completed": false, - "failed": false, - }, - "autodelete": map[string]bool{ - "completed": false, - "failed": false, - "stopped": false, - }, + "definition": command, + "name": identifier, + "hidden": false, "retry_delay": "500.0ms", + "unstoppable": true, } requestBodyJSON, err := json.Marshal(requestBody) @@ -3240,18 +3354,18 @@ func createPipeline(command, identifier string) (string, error) { } forwardData := bytes.NewBuffer(requestBodyJSON) - req, err := http.NewRequest( forwardMethod, url, forwardData, ) + if err != nil { log.Printf("[ERROR] Failed to create HTTP request: %s", err) return "", err } - req.Header.Set("Content-Type", "application/json") + req.Header.Set("Content-Type", "application/json") client := &http.Client{Timeout: 10 * time.Second} resp, err := client.Do(req) if err != nil { @@ -3266,7 +3380,9 @@ func createPipeline(command, identifier string) (string, error) { } if strings.Contains(string(body), "error") { - log.Printf("[ERROR] Pipeline creation response (%d): %s", resp.StatusCode, string(body)) + log.Printf("[ERROR] Pipeline creation error resp (%d): %s", resp.StatusCode, string(body)) + } else { + log.Printf("[DEBUG] Pipeline creation debug (%d): %s", resp.StatusCode, string(body)) } defer resp.Body.Close() @@ -3292,37 +3408,39 @@ func createPipeline(command, identifier string) (string, error) { return "", errors.New("Pipeline ID not found or empty in the response. See error logs.") } - id := response.ID - return id, nil + return response.ID, nil } func updatePipelineState(command, pipelineId, action string) (string, error) { url := fmt.Sprintf("%s/api/v0/pipeline/update", pipelineUrl) forwardMethod := "POST" - requestBody := map[string]interface{}{ - "id": pipelineId, - "definition": command, - "action": action, - "autostart": map[string]bool{ - "created": true, - "completed": true, - "failed": true, - }, - "autodelete": map[string]bool{ - "completed": false, - "failed": false, - "stopped": false, - }, + "id": pipelineId, + "action": action, + + /* + "autostart": map[string]bool{ + "created": true, + "completed": false, + "failed": false, + }, + "autodelete": map[string]bool{ + "completed": false, + "failed": false, + "stopped": false, + }, + */ } requestBodyJSON, err := json.Marshal(requestBody) if err != nil { return "", err } - forwardData := bytes.NewBuffer(requestBodyJSON) + log.Printf("[INFO] Updating pipeline %s with action %s to ensure it starts. Body: %s", pipelineId, action, string(requestBodyJSON)) + + forwardData := bytes.NewBuffer(requestBodyJSON) req, err := http.NewRequest( forwardMethod, url, @@ -3405,7 +3523,7 @@ func deletePipeline(pipelineId string) error { log.Printf("[INFO] Pipeline with ID: %s deleted successfully", pipelineId) - pipelines = []shuffle.PipelineInfoMini{} + pipelines = []shuffle.PipelineInfo{} return nil } @@ -3471,15 +3589,22 @@ func searchPipeline(identifier string) (string, error) { return "", errors.New("no existing pipeline found with name") } -func handleFileCategoryChange() error { - apiEndpoint := baseUrl + "/api/v1/files/namespaces/sigma" +func handleFileCategoryChange(ruleType string) error { + apiEndpoint := fmt.Sprintf("%s/api/v1/files/namespaces/%s", baseUrl, ruleType) req, err := http.NewRequest("GET", apiEndpoint, nil) if err != nil { return err } if len(pipelineApikey) == 0 { - return errors.New("Shuffle API-key not set for Pipelines: SHUFFLE_PIPELINE_AUTH=") + //var auth = os.Getenv("AUTH") + //var org = os.Getenv("ORG") + + if len(auth) > 0 && len(org) > 0 { + pipelineApikey = auth + } else { + return errors.New("Shuffle API-key not set for Pipelines: SHUFFLE_PIPELINE_AUTH=") + } } req.Header.Add("Authorization", "Bearer "+pipelineApikey) @@ -3512,14 +3637,13 @@ func handleFileCategoryChange() error { } //log.Println("[DEBUG] ZIP file downloaded successfully.") - tenzirStorageFolder := os.Getenv("SHUFFLE_STORAGE_FOLDER") if len(tenzirStorageFolder) == 0 { tenzirStorageFolder = "/tmp/" } tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/") - sigmaPath := fmt.Sprintf("%s/sigma_rules", tenzirStorageFolder) + sigmaPath := fmt.Sprintf("%s/%s_rules", tenzirStorageFolder, ruleType) err = extractZIP("files.zip", sigmaPath) if err != nil { log.Printf("[ERROR] Failed to extract ZIP file: %s", err) @@ -3544,12 +3668,12 @@ func extractZIP(zipFile, destDir string) error { } log.Printf("[DEBUG] Total size of the ZIP file: %d bytes", totalSize) - defer r.Close() if err := os.MkdirAll(destDir, 0755); err != nil { return err } + log.Printf("[DEBUG] Total files to extract: %d", len(r.File)) for _, f := range r.File { // Fix path traversal if strings.Contains(f.Name, "..") { @@ -3570,16 +3694,15 @@ func extractFile(f *zip.File, destDir string) error { if err != nil { return err } + defer rc.Close() - path := filepath.Join(destDir, f.Name) - out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) if err != nil { return err } - defer out.Close() + defer out.Close() _, err = io.Copy(out, rc) return err } @@ -3608,7 +3731,7 @@ func copyToTenzir(srcPath, destPath string) error { return nil } -func removeFileCategory() error { +func removeFileCategory(ruleType string) error { tenzirStorageFolder := os.Getenv("SHUFFLE_STORAGE_FOLDER") if len(tenzirStorageFolder) == 0 { tenzirStorageFolder = "/tmp/" @@ -3617,18 +3740,19 @@ func removeFileCategory() error { tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/") //sigmaPath := "/var/lib/tenzir/sigma_rules/*" - sigmaPath := fmt.Sprintf("%s/sigma_rules", tenzirStorageFolder) + rulePath := fmt.Sprintf("%s/%s_rules", tenzirStorageFolder, ruleType) - err := os.RemoveAll(sigmaPath) + err := os.RemoveAll(rulePath) if err != nil { - return fmt.Errorf("Error removing category files in %s: %v", sigmaPath, err) + return fmt.Errorf("Error removing category files in %s: %v", rulePath, err) } - log.Printf("[INFO] Removed all local category data in %s", sigmaPath) + log.Printf("[INFO] Removed all local category data in %s", rulePath) return nil } +// curl https://get.tenzir.app | sh func removeFile(fileName string) error { containerName := "tenzir-node" srcPath := fmt.Sprintf("/var/lib/tenzir/sigma_rules/%s", fileName) @@ -3658,7 +3782,7 @@ func removePath(containerName, path string) error { func sendPipelineHealthStatus() (shuffle.LakeConfig, error) { pipelinePayload := shuffle.LakeConfig{ Enabled: false, - Pipelines: []shuffle.PipelineInfoMini{}, + Pipelines: []shuffle.PipelineInfo{}, } if tenzirDisabled { @@ -3670,18 +3794,9 @@ func sendPipelineHealthStatus() (shuffle.LakeConfig, error) { if len(pipelines) == 0 || randint == 0 { pipelineDef, err := listPipelines() - if err == nil { - for _, pipeline := range pipelineDef { - pipelinePayload.Pipelines = append(pipelinePayload.Pipelines, shuffle.PipelineInfoMini{ - ID: pipeline.ID, - Name: pipeline.Name, - Definition: pipeline.Definition, - TotalRuns: pipeline.TotalRuns, - CreatedAt: pipeline.CreatedAt, - }) - } - - pipelines = pipelinePayload.Pipelines + if err == nil || len(pipelines) > 0 { + pipelines = pipelineDef + pipelinePayload.Pipelines = pipelines } } else { pipelinePayload.Pipelines = pipelines @@ -3694,7 +3809,7 @@ func sendPipelineHealthStatus() (shuffle.LakeConfig, error) { log.Printf("[ERROR] Tenzir node connection problem: %s", err) } else { - tenzirDisabled = true + //tenzirDisabled = true log.Printf("[WARNING] Disabling pipelines: %s. You will need to restart the Orborus to fix this.", err) } @@ -3995,7 +4110,7 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, // Specific to debugging if len(workerServerUrl) == 0 { - if debug { + if debug { log.Printf("[INFO] Using default worker server url as previous is invalid: %s. Swapping to shuffle-workers:33333", streamUrl) } } @@ -4107,6 +4222,10 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, return nil } +// 0x0elliot: +// let's never increase worker replicas. +// in our tests, workers replicas mattered a lot less. +// edge-case: subflows are helped with when worker replicas are higher. func AutoScale(ctx context.Context) { if os.Getenv("SHUFFLE_SCALE_REPLICAS") != "" { return diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 0821bc1c..c0cd00f7 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -11,7 +11,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.15 + github.com/shuffle/shuffle-shared v0.9.30 github.com/shuffle/singul v0.0.16 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 6dd41de4..13c507d3 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -2,7 +2,7 @@ package main import ( "github.com/shuffle/shuffle-shared" - "github.com/shuffle/singul/pkg" + singul "github.com/shuffle/singul/pkg" "bytes" "context" @@ -43,6 +43,7 @@ import ( //k8s deps appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" @@ -610,9 +611,9 @@ func deployk8sApp(image string, identifier string, env []string) error { // use deployment instead of pod // then expose a service similarly. - // number of replicas can be set to os.Getenv("SHUFFLE_SCALE_REPLICAS") + // number of replicas can be set to os.Getenv("SHUFFLE_APP_REPLICAS") replicaNumber := 1 - replicaNumberStr := os.Getenv("SHUFFLE_SCALE_REPLICAS") + replicaNumberStr := os.Getenv("SHUFFLE_APP_REPLICAS") if len(replicaNumberStr) > 0 { tmpInt, err := strconv.Atoi(replicaNumberStr) if err != nil { @@ -652,6 +653,7 @@ func deployk8sApp(image string, identifier string, env []string) error { }, }, SecurityContext: containerSecurityContext, + Resources: buildResourcesFromEnv(), }, }, DNSPolicy: corev1.DNSClusterFirst, @@ -662,6 +664,27 @@ func deployk8sApp(image string, identifier string, env []string) error { }, } + if os.Getenv("SHUFFLE_APP_MOUNT_TMP_VOLUME") == "true" { + deployment.Spec.Template.Spec.Volumes = append( + deployment.Spec.Template.Spec.Volumes, + corev1.Volume{ + Name: "tmp", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + ) + + deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append( + deployment.Spec.Template.Spec.Containers[0].VolumeMounts, + corev1.VolumeMount{ + Name: "tmp", + ReadOnly: false, + MountPath: "/tmp", + }, + ) + } + if len(os.Getenv("REGISTRY_URL")) > 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 { log.Printf("[INFO] Setting image pull policy to Always as private registry is used.") //containerAttachment.ImagePullPolicy = corev1.PullAlways @@ -676,6 +699,7 @@ func deployk8sApp(image string, identifier string, env []string) error { return err } + svcAppProtocol := "http" service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -685,9 +709,10 @@ func deployk8sApp(image string, identifier string, env []string) error { Selector: matchLabels, Ports: []corev1.ServicePort{ { - Protocol: "TCP", - Port: 80, - TargetPort: intstr.FromInt(deployport), + Protocol: "TCP", + AppProtocol: &svcAppProtocol, + Port: 80, + TargetPort: intstr.FromInt(deployport), }, }, Type: corev1.ServiceTypeClusterIP, @@ -2436,6 +2461,49 @@ func buildEnvVars(envMap map[string]string) []corev1.EnvVar { } return envVars } + +func buildResourcesFromEnv() corev1.ResourceRequirements { + requests := corev1.ResourceList{} + limits := corev1.ResourceList{} + + type item struct { + env string + resourceName corev1.ResourceName + resourceList corev1.ResourceList + } + + items := []item{ + // kubernetes requests + {env: "SHUFFLE_APP_CPU_REQUEST", resourceName: corev1.ResourceCPU, resourceList: requests}, + {env: "SHUFFLE_APP_MEMORY_REQUEST", resourceName: corev1.ResourceMemory, resourceList: requests}, + {env: "SHUFFLE_APP_EPHEMERAL_STORAGE_REQUEST", resourceName: corev1.ResourceEphemeralStorage, resourceList: requests}, + // kubernetes limits + {env: "SHUFFLE_APP_CPU_LIMIT", resourceName: corev1.ResourceCPU, resourceList: limits}, + {env: "SHUFFLE_APP_MEMORY_LIMIT", resourceName: corev1.ResourceMemory, resourceList: limits}, + {env: "SHUFFLE_APP_EPHEMERAL_STORAGE_LIMIT", resourceName: corev1.ResourceEphemeralStorage, resourceList: limits}, + } + + for _, it := range items { + if value := strings.TrimSpace(os.Getenv(it.env)); value != "" { + if quantity, err := resource.ParseQuantity(value); err == nil { + it.resourceList[it.resourceName] = quantity + } else { + log.Printf("[WARNING] Cannot parse %s=%q as resource quantity: %v", it.env, value, err) + } + } + } + + rr := corev1.ResourceRequirements{} + if len(requests) > 0 { + rr.Requests = requests + } + if len(limits) > 0 { + rr.Limits = limits + } + + return rr +} + func getWorkerBackendExecution(auth string, executionId string) (*shuffle.WorkflowExecution, error) { backendUrl := os.Getenv("BASE_URL") if len(backendUrl) == 0 { @@ -2446,9 +2514,9 @@ func getWorkerBackendExecution(auth string, executionId string) (*shuffle.Workfl streamResultUrl := fmt.Sprintf("%s/api/v1/streams/results", backendUrl) topClient := shuffle.GetExternalClient(backendUrl) - requestData := shuffle.ActionResult { + requestData := shuffle.ActionResult{ Authorization: auth, - ExecutionId: executionId, + ExecutionId: executionId, } data, err := json.Marshal(requestData) @@ -2708,7 +2776,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } if setExecution || workflowExecution.Status == "FINISHED" || workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" { - if debug { + if debug { log.Printf("[DEBUG][%s] Running setexec with status %s and %d/%d results", workflowExecution.ExecutionId, workflowExecution.Status, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)) } @@ -2726,7 +2794,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || os.Getenv("SHUFFLE_SWARM_CONFIG") == "swarm" { finished := shuffle.ValidateFinished(ctx, -1, *workflowExecution) if !finished { - if debug { + if debug { log.Printf("[DEBUG][%s] Handling next node since it's not finished!", workflowExecution.ExecutionId) } @@ -3188,8 +3256,6 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl log.Printf("[DEBUG] Deploying service for %s to swarm on port %d", name, deployport) //containerName := fmt.Sprintf("shuffle-worker-%s", parsedUuid) - - // Check if the image exists or not - just in case _, _, err := dockercli.ImageInspectWithRaw(context.Background(), image) if err != nil { @@ -3202,8 +3268,8 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl } _, err := dockercli.ImagePull( - context.Background(), - image, + context.Background(), + image, dockerimage.PullOptions{}, ) if err != nil { @@ -3212,7 +3278,6 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl } } - if len(baseimagename) == 0 || baseimagename == "/" { baseimagename = "frikky/shuffle" //var baseimagename = "frikky/shuffle" @@ -3245,7 +3310,7 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl // Max scale as well nodeCount := uint64(1) - if inputReplicas > 0 && inputReplicas < 100 { + if inputReplicas > 0 && inputReplicas < 100 { if replicas != uint64(inputReplicas) { log.Printf("[DEBUG] Overwriting replicas to %d/node as inputReplicas is set to %d", inputReplicas, inputReplicas) } @@ -3265,7 +3330,6 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl nodeCount = 1 } - replicatedJobs := uint64(replicas * nodeCount) log.Printf("[DEBUG] Deploying app with name %s with image %s", name, image) @@ -3395,12 +3459,12 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl } // Retry deploying the service (once) - if !retry { + if !retry { return deploySwarmService(dockercli, name, image, deployport, -1, true) } } - // For port mapping. + // For port mapping. if strings.Contains(fmt.Sprintf("%s", err), "InvalidArgument") && strings.Contains(fmt.Sprintf("%s", err), "is already in use") { //log.Printf("\n\n[WARNING] Port %d is already allocated. Trying to deploy on next port.\n\n", deployport) @@ -3427,7 +3491,6 @@ func findAppInfo(image, name string, redeploy bool) (int, error) { // chance of being successful time.Sleep(time.Duration(rand.Intn(1500)) * time.Millisecond) - highest := baseport exposedPort := -1 @@ -3531,10 +3594,10 @@ func findAppInfo(image, name string, redeploy bool) (int, error) { time.Sleep(time.Duration(rand.Intn(4)+8) * time.Second) replicas := service.Spec.Mode.Replicated.Replicas err = deploySwarmService( - dockercli, - name, - image, - exposedPort, + dockercli, + name, + image, + exposedPort, int64(*replicas), false, ) @@ -3624,7 +3687,7 @@ func findAppInfoKubernetes(image, name string, env []string) error { for _, deployment := range deployments.Items { if deployment.Name == name { - if debug { + if debug { log.Printf("[DEBUG] Found deployment %s - no need to deploy another", name) } @@ -3650,7 +3713,6 @@ func initSwarmNetwork() error { mtu := 1500 options["com.docker.network.driver.mtu"] = fmt.Sprintf("%d", mtu) - ingressOptions := network.CreateOptions{ Driver: "overlay", Attachable: false, @@ -3734,11 +3796,9 @@ func initSwarmNetwork() error { log.Printf("[WARNING] Swarm Executions network may already exist: %s", err) } - return nil + return nil } - - /*** ENDREMOVE ***/ func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, action *shuffle.Action, workflowExecution *shuffle.WorkflowExecution, image string, attempts int64) error { @@ -3829,6 +3889,12 @@ func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, // Checking as LATE as possible, ensuring we don't rerun what's already ran // ctx = context.Background() + + // Sleep between 0 and 250 ms for randomness so no same worker check at same time (same as cloud) + rand.Seed(time.Now().UnixNano()) + randMs := rand.Intn(250) + time.Sleep(time.Duration(randMs) * time.Millisecond) + newExecId := fmt.Sprintf("%s_%s", workflowExecution.ExecutionId, action.ID) _, err = shuffle.GetCache(ctx, newExecId) if err == nil { @@ -3889,18 +3955,18 @@ func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, } // Try redeployment - attempts += 1 + attempts += 1 if attempts < 2 { - // Check the service and fix it. + // Check the service and fix it. if isKubernetes == "true" { - log.Printf("[WARNING] App Redeployment in K8s isn't fully supported yet, but should be done for app %s with image %s.", appName, image) + log.Printf("[WARNING] App Redeployment in K8s isn't fully supported yet, but should be done for app %s with image %s.", appName, image) } else { _, err = findAppInfo(image, appName, true) if err != nil { log.Printf("[ERROR][%s] Error re-deploying app %s: %s", workflowExecution.ExecutionId, appName, err) } - return sendAppRequest(ctx, incomingUrl, appName, port, action, workflowExecution, image, attempts) + return sendAppRequest(ctx, incomingUrl, appName, port, action, workflowExecution, image, attempts) } } @@ -3926,7 +3992,7 @@ func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, log.Printf("[ERROR] Failed reading app request body body: %s", err) return err } else { - if debug { + if debug { log.Printf("[DEBUG][%s] NEWRESP (from app): %s", workflowExecution.ExecutionId, string(body)) } } @@ -4326,7 +4392,7 @@ func checkStandaloneRun() { // Initial loop etc func main() { - // Testing swarm auto-replacements. This also tests ports + // Testing swarm auto-replacements. This also tests ports // in rapid succession checkStandaloneRun() @@ -4334,7 +4400,7 @@ func main() { debug = true log.Printf("[INFO] Disabled cleanup due to debug mode (DEBUG=true)") - cleanupEnv = "false" + cleanupEnv = "false" } /*** STARTREMOVE ***/ @@ -4784,7 +4850,7 @@ func runWebserver(listener net.Listener) { if strings.ToLower(os.Getenv("SHUFFLE_SWARM_CONFIG")) == "run" || strings.ToLower(os.Getenv("SHUFFLE_APP_REPLICAS")) == "" { // go AutoScaleApps(ctx, dockercli, maxExecutionsPerMinute) } - if strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" { + if strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" || strings.ToLower(os.Getenv("DEBUG_MEMORY")) == "true" { r.HandleFunc("/debug/pprof/", pprof.Index) r.HandleFunc("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP) r.HandleFunc("/debug/pprof/profile", pprof.Profile) @@ -4818,6 +4884,12 @@ func runWebserver(listener net.Listener) { } } +// 0x0elliot: +// IF we had to rewrite this, we will focus on ONLY auto scale for apps. +// i recommend we target executions/minute (?) as a metric. +// edge-case: subflows are helped with when worker replicas are higher. +// i kind of never want to scale down. at least, not now. +// also, algorithm is very broken. executions/worker func AutoScaleApps(ctx context.Context, client *dockerclient.Client, maxExecutionsPerMinute int) { ticker := time.NewTicker(1 * time.Second)