From 212d1a879aa1367dbad823063cf0ad19a711ee62 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Mon, 10 Jun 2024 10:49:38 +0000 Subject: [PATCH 01/46] added a function to handle file category change --- functions/onprem/orborus/orborus.go | 167 +++++++++++++++++++++++++--- 1 file changed, 149 insertions(+), 18 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 847cd29e..faa435bf 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -10,7 +10,7 @@ package main import ( "github.com/shuffle/shuffle-shared" - + "archive/zip" "bytes" "context" "encoding/json" @@ -29,6 +29,7 @@ import ( "strings" "sync" "time" + "path/filepath" //"os/signal" //"syscall" @@ -2010,6 +2011,13 @@ func main() { } toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + } else if incRequest.Type == "CATEGORY_CHANGE" { + err := handleFileCategoryChange() + if err != nil { + log.Printf("[ERROR] Failed to download the file category: %s", err) + } + + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) } else if incRequest.Type == "DOCKER_IMAGE_DOWNLOAD" { log.Printf("[INFO] Should delete -> download new image %#v", incRequest.ExecutionArgument) @@ -2021,7 +2029,11 @@ func main() { } toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - } else { + } else if incRequest.Type == "CATEGORY_UPDATE" { + handleFileCategoryChange() + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + + }else { newrequests = append(newrequests, incRequest) } } @@ -2627,24 +2639,24 @@ func createPipeline(command, identifier string) (string, error) { log.Printf("[INFO] an existing pipeline found with ID: %s. it will be deleted", pipelineId) toBeDeleted = true } - if strings.Contains(command, "shuffler.io") { + // if strings.Contains(command, "shuffler.io") { - } else { - var scheme string - if strings.Contains(command, "http://") { - scheme = "http://" - } else if strings.Contains(command, "https://") { - scheme = "https://" - } + // } else { + // var scheme string + // if strings.Contains(command, "http://") { + // scheme = "http://" + // } else if strings.Contains(command, "https://") { + // scheme = "https://" + // } - startIndex := strings.Index(command, scheme) - if startIndex != -1 { - endIndex := startIndex + len(scheme) - endIndex += strings.Index(command[endIndex:], "/") - - command = command[:startIndex] + baseUrl + command[endIndex:] - } - } + // startIndex := strings.Index(command, scheme) + // if startIndex != -1 { + // endIndex := startIndex + len(scheme) + // endIndex += strings.Index(command[endIndex:], "/") + + // command = command[:startIndex] + baseUrl + command[endIndex:] + // } + // } requestBody := map[string]interface{}{ "definition": command, "name": identifier, @@ -2870,6 +2882,125 @@ func searchPipeline(identifier string) (string, error) { return "", errors.New("no existing pipeline found with name") } +func handleFileCategoryChange() error{ + apiEndpoint := "https://expert-acorn-v6vg4j4j5w7q2wg6g-5001.app.github.dev/api/v1/files/namespaces/hari" + apiKey := "23e57313-5f0f-4a20-bddd-a9059c980adf" + + req, err := http.NewRequest("GET", apiEndpoint, nil) + if err != nil { + return err + } + + req.Header.Add("Authorization", "Bearer "+apiKey) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return err + } + + out, err := os.Create("files.zip") + if err != nil { + return err + } + + defer out.Close() + defer os.Remove("files.zip") + + _, err = io.Copy(out, resp.Body) + if err != nil { + return err + } + + fmt.Println("ZIP file downloaded successfully.") + + err = extractZIP("files.zip", "unzipped_files") + if err != nil { + return err + } + + destPath := "/var/lib/tenzir/unzipped_files" + + err = copyToTenzir("unzipped_files", destPath) + if err != nil { + return err + } + + fmt.Println("Files copied to container successfully.") + return nil +} + +func extractZIP(zipFile, destDir string) error { + r, err := zip.OpenReader(zipFile) + if err != nil { + return err + } + defer r.Close() + + if err := os.MkdirAll(destDir, 0755); err != nil { + return err + } + + for _, f := range r.File { + err := extractFile(f, destDir) + if err != nil { + return err + } + } + + return nil +} + +func extractFile(f *zip.File, destDir string) error { + rc, err := f.Open() + if err != nil { + return err + } + defer rc.Close() + + path := filepath.Join(destDir, f.Name) + + out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, rc) + return err +} + +func copyToTenzir(srcPath, destPath string) error { + containerName := "tenzir-node" + + // Check if the extracted_files directory exists in the container + checkCmd := exec.Command("docker", "exec", containerName, "test", "-d", destPath) + if err := checkCmd.Run(); err == nil { + rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", "-rf", destPath) + if err := rmCmd.Run(); err != nil { + return fmt.Errorf("error removing existing directory in container: %v", err) + } + } + + // Copy the new directory to the container + cpCmd := exec.Command("docker", "cp", srcPath, fmt.Sprintf("%s:%s", containerName, destPath)) + var out bytes.Buffer + cpCmd.Stdout = &out + cpCmd.Stderr = &out + + err := cpCmd.Run() + if err != nil { + return fmt.Errorf("error copying files: %v, output: %s", err, out.String()) + } + + return nil +} + // func savePipelineData(pipelineId, identifier, status string) error { // url := fmt.Sprintf("%s/api/v1/triggers/pipeline/save", baseUrl) From 94779e66136f2408678aa59ff0c62d857bc3df0e Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Tue, 11 Jun 2024 05:59:11 +0000 Subject: [PATCH 02/46] initial UI implementation for managing detection rules --- frontend/src/views/Detection.jsx | 58 ++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 frontend/src/views/Detection.jsx diff --git a/frontend/src/views/Detection.jsx b/frontend/src/views/Detection.jsx new file mode 100644 index 00000000..c2711b45 --- /dev/null +++ b/frontend/src/views/Detection.jsx @@ -0,0 +1,58 @@ +import React from 'react'; +import { Container, Box, TextField, Switch, Card, CardContent, IconButton, Typography, Button } from '@mui/material'; +import EditIcon from '@mui/icons-material/Edit'; +import { styled } from '@mui/system'; + +const ConnectedButton = styled(Button)({ + backgroundColor: 'red', + color: 'white', +}); + +const RuleCard = ({ ruleName, description }) => ( + + +
+ {ruleName} +
+ + + + +
+
+ {description} + + + {/* we need icons here ??? */} + + + + +); + +const Detection = () => { + return ( + + + + + Group 1 Title + + Not Connected to SIEM + + + + + Global disable/enable + + + + + + + + + ); +}; + +export default Detection; From e1d0857fd76aa693e14bffcf0ca2423518d892d1 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Tue, 11 Jun 2024 16:38:28 +0000 Subject: [PATCH 03/46] made the ui to look good --- frontend/src/App.jsx | 6 ++ frontend/src/views/Detection.jsx | 148 +++++++++++++++++++++++++------ 2 files changed, 126 insertions(+), 28 deletions(-) diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 639b80bd..27517625 100755 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -15,6 +15,7 @@ import HealthPage from "./components/HealthPage.jsx"; import theme from "./theme"; import Apps from "./views/Apps"; import AppCreator from "./views/AppCreator"; +import Dectection from "./views/Detection.jsx"; import Welcome from "./views/Welcome.jsx"; import Dashboard from "./views/Dashboard.jsx"; @@ -414,6 +415,11 @@ const App = (message, props) => { /> } /> + } + /> ( +const disableRule = (fileId) => { + +} + + +const RuleCard = ({ ruleName, description, ...otherProps }) => { + const [additionalProps, setAdditionalProps] = React.useState(otherProps); + return ( -
- {ruleName} -
- +
+ {ruleName} +
+ - +
-
- {description} - - +
+ + {description} + + + {/* we need icons here ??? */} - -); + ) +} -const Detection = () => { +const Detection = (props) => { + const {globalUrl} = props; + const [ruleInfo, setRuleInfo] = React.useState([]); + + const getSigmaInfo = () => { + const url = globalUrl + "/api/v1/files/detection/sigma_rules" + + fetch(url, { + method: "GET", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) .then((response) => + response.json().then((responseJson) => { + if (responseJson["success"] === false) { + toast("Failed to get sigma rules"); + } else { + setRuleInfo(responseJson); + } + + }), + ) + .catch((error) => { + console.log("Error in geting sigma files: ", error); + }); + } + + React.useEffect(() => { + getSigmaInfo() +}, []); + return ( - - + + Group 1 Title - Not Connected to SIEM + + Not Connected to SIEM + - + - - Global disable/enable + + + Global disable/enable + - - - + {ruleInfo.length > 0 && + ruleInfo.map((card) => ( + + ))} ); From c9da78c7812afb30773fcc65a184cb4cbf293fcc Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Wed, 12 Jun 2024 04:48:34 +0000 Subject: [PATCH 04/46] added toggle rule function to enable or disable the rule --- frontend/src/views/Detection.jsx | 149 ++++++++++++++++++------------- 1 file changed, 87 insertions(+), 62 deletions(-) diff --git a/frontend/src/views/Detection.jsx b/frontend/src/views/Detection.jsx index e542c5f9..79cb71b7 100644 --- a/frontend/src/views/Detection.jsx +++ b/frontend/src/views/Detection.jsx @@ -19,86 +19,109 @@ const ConnectedButton = styled(Button)({ color: "white", }); -const disableRule = (fileId) => { - -} - - -const RuleCard = ({ ruleName, description, ...otherProps }) => { +const RuleCard = ({ ruleName, description, file_id, globalUrl, ...otherProps }) => { const [additionalProps, setAdditionalProps] = React.useState(otherProps); + + const handleSwitchChange = (event) => { + const isEnabled = event.target.checked; + toggleRule(file_id, !isEnabled, globalUrl, () => { + setAdditionalProps((prevProps) => ({ + ...prevProps, + is_enabled: isEnabled, + })); + }); + }; + return ( - - -
- {ruleName} -
- - - - + +
+ {ruleName} +
+ + + + +
-
- - {description} - - - - {/* we need icons here ??? */} - - - - ) -} + + {description} + + + + ); +}; + +const toggleRule = (fileId, isCurrentlyEnabled, globalUrl, callback) => { + const action = isCurrentlyEnabled ? "disable" : "enable"; + const url = `${globalUrl}/api/v1/files/${fileId}/${action}_rule`; + + fetch(url, { + method: "PUT", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => + response.json().then((responseJson) => { + if (responseJson["success"] === false) { + toast(`Failed to ${action} the rule`); + } else { + toast(`Rule ${action}d successfully`); + callback(); + } + }) + ) + .catch((error) => { + console.log(`Error in ${action}ing the rule: `, error); + toast(`An error occurred while ${action}ing the rule`); + }); +}; const Detection = (props) => { - const {globalUrl} = props; + const { globalUrl } = props; const [ruleInfo, setRuleInfo] = React.useState([]); const getSigmaInfo = () => { - const url = globalUrl + "/api/v1/files/detection/sigma_rules" - + const url = globalUrl + "/api/v1/files/detection/sigma_rules"; + fetch(url, { method: "GET", credentials: "include", headers: { "Content-Type": "application/json", }, - }) .then((response) => - response.json().then((responseJson) => { - if (responseJson["success"] === false) { - toast("Failed to get sigma rules"); - } else { + }) + .then((response) => + response.json().then((responseJson) => { + if (responseJson["success"] === false) { + toast("Failed to get sigma rules"); + } else { setRuleInfo(responseJson); - } - - }), - ) - .catch((error) => { - console.log("Error in geting sigma files: ", error); - }); - } + } + }) + ) + .catch((error) => { + console.log("Error in getting sigma files: ", error); + toast("An error occurred while fetching sigma rules"); + }); + }; React.useEffect(() => { - getSigmaInfo() -}, []); - + getSigmaInfo(); + }, []); + return ( @@ -136,9 +159,11 @@ const Detection = (props) => { {ruleInfo.length > 0 && ruleInfo.map((card) => ( ))} From a7d2a5d676ae161f52c454ac4c2f454f0ab476e7 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Wed, 12 Jun 2024 04:50:59 +0000 Subject: [PATCH 05/46] endpoints for getting sigma rule info and to disable and enable the rules --- backend/go-app/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index d24cbc64..1cc3c771 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5122,6 +5122,9 @@ func initHandlers() { r.HandleFunc("/api/v1/files/{fileId}", shuffle.HandleGetFileMeta).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}", shuffle.HandleDeleteFile).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/files", shuffle.HandleGetFiles).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/files/detection/sigma_rules", shuffle.HandleGetSigmaRules).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}/disable_rule", shuffle.HandleToggleRule).Methods("PUT", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}/enable_rule", shuffle.HandleToggleRule).Methods("PUT", "OPTIONS") // Introduced in 0.9.21 to handle notifications for e.g. failed Workflow r.HandleFunc("/api/v1/notifications", shuffle.HandleCreateNotification).Methods("POST", "OPTIONS") From 366b8438a1257dca5605e616d4b917833670cac4 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Wed, 12 Jun 2024 07:21:37 +0000 Subject: [PATCH 06/46] feat : support enabling and disabling sigma rules --- functions/onprem/orborus/orborus.go | 86 +++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index faa435bf..0654169f 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -3001,6 +3001,92 @@ func copyToTenzir(srcPath, destPath string) error { return nil } +func manageSigmaRule(fileName, action string) error { + containerName := "tenzir-node" + srcPath := "" + destPath := "" + + switch action { + case "disable": + srcPath = fmt.Sprintf("/var/lib/tenzir/sigma_files/%s", fileName) + destPath = "/var/lib/tenzir/disabled_rules" + case "enable": + srcPath = fmt.Sprintf("/var/lib/tenzir/disabled_rules/%s", fileName) + destPath = "/var/lib/tenzir/sigma_files" + default: + return fmt.Errorf("invalid action: %s", action) + } + + checkSrcCmd := exec.Command("docker", "exec", containerName, "test", "-f", srcPath) + if err := checkSrcCmd.Run(); err != nil { + return fmt.Errorf("source file does not exist: %v", err) + } + + checkDestCmd := exec.Command("docker", "exec", containerName, "test", "-d", destPath) + if err := checkDestCmd.Run(); err != nil { + mkdirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mkdir", "-p", destPath) + if err := mkdirCmd.Run(); err != nil { + return fmt.Errorf("error creating destination directory in container: %v", err) + } + } + + // Move the file to the destination directory or shall we copy it and then remove the file from source dir + mvCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mv", srcPath, destPath) + if err := mvCmd.Run(); err != nil { + return fmt.Errorf("error moving file: %v", err) + } + + return nil +} + +func manageSigmaFolder(action string) error { + containerName := "tenzir-node" + sigmaPath := "/var/lib/tenzir/sigma_files" + disabledPath := "/var/lib/tenzir/disabled_sigma" + + if action == "disable" { + + checkSigmaCmd := exec.Command("docker", "exec", containerName, "test", "-d", sigmaPath) + if err := checkSigmaCmd.Run(); err != nil { + return fmt.Errorf("sigma_files directory does not exist: %v", err) + } + + // Rename sigma_files to disabled_sigma + renameCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mv", sigmaPath, disabledPath) + if err := renameCmd.Run(); err != nil { + return fmt.Errorf("error renaming sigma_files to disabled_sigma: %v", err) + } + + // Create a new sigma_files directory + createCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mkdir", sigmaPath) + if err := createCmd.Run(); err != nil { + return fmt.Errorf("error creating new sigma_files directory: %v", err) + } + } else if action == "enable" { + + checkDisabledCmd := exec.Command("docker", "exec", containerName, "test", "-d", disabledPath) + if err := checkDisabledCmd.Run(); err != nil { + return fmt.Errorf("disabled_sigma directory does not exist: %v", err) + } + + removeCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", "-rf", sigmaPath) + if err := removeCmd.Run(); err != nil { + return fmt.Errorf("error removing existing sigma_files directory: %v", err) + } + + // Rename disabled_sigma back to sigma_files + renameBackCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mv", disabledPath, sigmaPath) + if err := renameBackCmd.Run(); err != nil { + return fmt.Errorf("error renaming disabled_sigma back to sigma_files: %v", err) + } + } else { + return fmt.Errorf("invalid action: %s", action) + } + + return nil +} + + // func savePipelineData(pipelineId, identifier, status string) error { // url := fmt.Sprintf("%s/api/v1/triggers/pipeline/save", baseUrl) From 813b641486611cde53679fdc7d732bcc910c5c37 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Wed, 12 Jun 2024 11:01:51 +0000 Subject: [PATCH 07/46] support for enabling and disabling sigma rules --- functions/onprem/orborus/orborus.go | 57 ++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 0654169f..979a6dc6 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2011,13 +2011,6 @@ func main() { } toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - } else if incRequest.Type == "CATEGORY_CHANGE" { - err := handleFileCategoryChange() - if err != nil { - log.Printf("[ERROR] Failed to download the file category: %s", err) - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) } else if incRequest.Type == "DOCKER_IMAGE_DOWNLOAD" { log.Printf("[INFO] Should delete -> download new image %#v", incRequest.ExecutionArgument) @@ -2029,11 +2022,47 @@ func main() { } toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - } else if incRequest.Type == "CATEGORY_UPDATE" { - handleFileCategoryChange() + + } else if incRequest.Type == "CATEGORY_UPDATE" { + err := handleFileCategoryChange() + if err != nil { + log.Printf("[ERROR] Failed to download the file category: %s", err) + } + + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + + } else if incRequest.Type == "DISABLE_SIGMA_FILE" { + fileName := incRequest.ExecutionArgument + err = manageSigmaRule(fileName, "disable") + if err != nil { + log.Printf("[ERROR] Failed to disable the sigma file %s, reason: %s", fileName, err) + } + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - }else { + } else if incRequest.Type == "ENABLE_SIGMA_FILE" { + fileName := incRequest.ExecutionArgument + err = manageSigmaRule(fileName, "enable") + if err != nil { + log.Printf("[ERROR] Failed to enable the sigma file %s, reason: %s",fileName, err) + } + + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + } else if incRequest.Type == "DISABLE_SIGMA_RULES" { + err := manageSigmaFolder("disable") + if err != nil { + log.Printf("[ERROR] Failed to disable the sigma rules: %s", err) + } + + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + } else if incRequest.Type == "ENABLE_SIGMA_RULES" { + err := manageSigmaFolder("enable") + if err != nil { + log.Printf("[ERROR] Failed to enable the sigma rules: %s", err) + } + + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + } else { newrequests = append(newrequests, incRequest) } } @@ -2883,7 +2912,7 @@ func searchPipeline(identifier string) (string, error) { } func handleFileCategoryChange() error{ - apiEndpoint := "https://expert-acorn-v6vg4j4j5w7q2wg6g-5001.app.github.dev/api/v1/files/namespaces/hari" + apiEndpoint := baseUrl+"/api/v1/files/namespaces/sigma" apiKey := "23e57313-5f0f-4a20-bddd-a9059c980adf" req, err := http.NewRequest("GET", apiEndpoint, nil) @@ -2919,14 +2948,14 @@ func handleFileCategoryChange() error{ fmt.Println("ZIP file downloaded successfully.") - err = extractZIP("files.zip", "unzipped_files") + err = extractZIP("files.zip", "sigma_rules") if err != nil { return err } - destPath := "/var/lib/tenzir/unzipped_files" + destPath := "/var/lib/tenzir/sigma_rules" - err = copyToTenzir("unzipped_files", destPath) + err = copyToTenzir("sigma_rules", destPath) if err != nil { return err } From 2318d92c3a87460fdc0f0b7ecbddeb0877120582 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Wed, 12 Jun 2024 16:32:28 +0000 Subject: [PATCH 08/46] renamed the group the name --- frontend/src/views/Detection.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/views/Detection.jsx b/frontend/src/views/Detection.jsx index 79cb71b7..c28ffc05 100644 --- a/frontend/src/views/Detection.jsx +++ b/frontend/src/views/Detection.jsx @@ -134,7 +134,7 @@ const Detection = (props) => { }} > - Group 1 Title + Sigma Detection Rules Not Connected to SIEM From 809fc2c202c53d445e4d28a6098cf56dc2836491 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Thu, 13 Jun 2024 07:00:22 +0000 Subject: [PATCH 09/46] rewriting the component structure --- frontend/src/App.jsx | 4 +- frontend/src/views/Detection.jsx | 111 +--------------------- frontend/src/views/DetectionDashboard.jsx | 102 ++++++++++++++++++++ frontend/src/views/EditRules.jsx | 46 +++++++++ frontend/src/views/RuleCard.jsx | 82 ++++++++++++++++ 5 files changed, 235 insertions(+), 110 deletions(-) create mode 100644 frontend/src/views/DetectionDashboard.jsx create mode 100644 frontend/src/views/EditRules.jsx create mode 100644 frontend/src/views/RuleCard.jsx diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 27517625..668cb557 100755 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -15,7 +15,7 @@ import HealthPage from "./components/HealthPage.jsx"; import theme from "./theme"; import Apps from "./views/Apps"; import AppCreator from "./views/AppCreator"; -import Dectection from "./views/Detection.jsx"; +import DetectionDashBoard from "./views/DetectionDashboard.jsx"; import Welcome from "./views/Welcome.jsx"; import Dashboard from "./views/Dashboard.jsx"; @@ -418,7 +418,7 @@ const App = (message, props) => { } + element={} /> { - const [additionalProps, setAdditionalProps] = React.useState(otherProps); - - const handleSwitchChange = (event) => { - const isEnabled = event.target.checked; - toggleRule(file_id, !isEnabled, globalUrl, () => { - setAdditionalProps((prevProps) => ({ - ...prevProps, - is_enabled: isEnabled, - })); - }); - }; - - return ( - - -
- {ruleName} -
- - - - -
-
- - {description} - - - - ); -}; - -const toggleRule = (fileId, isCurrentlyEnabled, globalUrl, callback) => { - const action = isCurrentlyEnabled ? "disable" : "enable"; - const url = `${globalUrl}/api/v1/files/${fileId}/${action}_rule`; - - fetch(url, { - method: "PUT", - credentials: "include", - headers: { - "Content-Type": "application/json", - }, - }) - .then((response) => - response.json().then((responseJson) => { - if (responseJson["success"] === false) { - toast(`Failed to ${action} the rule`); - } else { - toast(`Rule ${action}d successfully`); - callback(); - } - }) - ) - .catch((error) => { - console.log(`Error in ${action}ing the rule: `, error); - toast(`An error occurred while ${action}ing the rule`); - }); -}; - -const Detection = (props) => { - const { globalUrl } = props; - const [ruleInfo, setRuleInfo] = React.useState([]); - - const getSigmaInfo = () => { - const url = globalUrl + "/api/v1/files/detection/sigma_rules"; - - fetch(url, { - method: "GET", - credentials: "include", - headers: { - "Content-Type": "application/json", - }, - }) - .then((response) => - response.json().then((responseJson) => { - if (responseJson["success"] === false) { - toast("Failed to get sigma rules"); - } else { - setRuleInfo(responseJson); - } - }) - ) - .catch((error) => { - console.log("Error in getting sigma files: ", error); - toast("An error occurred while fetching sigma rules"); - }); - }; - - React.useEffect(() => { - getSigmaInfo(); - }, []); - +const Detection = ({ globalUrl, ruleInfo, openEditBar }) => { return ( @@ -164,6 +58,7 @@ const Detection = (props) => { description={card.description} file_id={card.file_id} globalUrl={globalUrl} + openEditBar={() => openEditBar(card)} {...card} /> ))} diff --git a/frontend/src/views/DetectionDashboard.jsx b/frontend/src/views/DetectionDashboard.jsx new file mode 100644 index 00000000..50771d63 --- /dev/null +++ b/frontend/src/views/DetectionDashboard.jsx @@ -0,0 +1,102 @@ +import React, { useState, useEffect } from "react"; +import { Container} from "@mui/material"; +import { toast } from "react-toastify"; +import Detection from "./Detection"; +import EditComponent from "./EditRules"; + +const getSigmaInfo = (globalUrl, setRuleInfo) => { + const url = globalUrl + "/api/v1/files/detection/sigma_rules"; + + fetch(url, { + method: "GET", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => + response.json().then((responseJson) => { + if (responseJson["success"] === false) { + toast("Failed to get sigma rules"); + } else { + setRuleInfo(responseJson); + } + }) + ) + .catch((error) => { + console.log("Error in getting sigma files: ", error); + toast("An error occurred while fetching sigma rules"); + }); +}; + +const DetectionDashBoard = (props) => { + const { globalUrl } = props; + const [ruleInfo, setRuleInfo] = useState([]); + const [selectedRule, setSelectedRule] = useState(null); + const [fileData, setFileData] = useState("") + + useEffect(() => { + getSigmaInfo(globalUrl, setRuleInfo); + }, [globalUrl]); + + const openEditBar = (rule) => { + setSelectedRule(rule); + getFileContent(rule.file_id) + }; + + const handleSave = (updatedContent) => { + toast("this will be saved"); + setSelectedRule(null); // Close the edit bar after saving + }; + + const getFileContent = (file_id) => { + fetch(globalUrl + "/api/v1/files/" + file_id + "/content", { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for file :O!"); + return ""; + } + return response.text(); + }) + .then((respdata) => { + if (respdata.length === 0) { + toast("Failed getting file. Is it deleted?"); + return; + } + return respdata + }) + .then((responseData) => { + + setFileData(responseData); + }) + .catch((error) => { + toast(error.toString()); + }); + }; + + return ( + + {selectedRule ? ( + + ) : null} + + + ); +}; + +export default DetectionDashBoard; diff --git a/frontend/src/views/EditRules.jsx b/frontend/src/views/EditRules.jsx new file mode 100644 index 00000000..501bb6cc --- /dev/null +++ b/frontend/src/views/EditRules.jsx @@ -0,0 +1,46 @@ +import React, { useState } from 'react'; +import { Box, Typography, Button, Switch, TextField } from '@mui/material'; + +const EditComponent = ({ ruleName, description, content, setContent, lastEdited, editedBy, onSave }) => { + + const handleSave = () => { + onSave(content); + }; + + return ( + + + {ruleName} + + + + + + {description} + + + Last edited: {lastEdited} + + + Edited By: {editedBy} + + + setContent(e.target.value)} + variant="outlined" + fullWidth + /> + + + + + + ); +}; + +export default EditComponent; diff --git a/frontend/src/views/RuleCard.jsx b/frontend/src/views/RuleCard.jsx new file mode 100644 index 00000000..e166ce6b --- /dev/null +++ b/frontend/src/views/RuleCard.jsx @@ -0,0 +1,82 @@ +import React from "react"; +import { + Card, + CardContent, + IconButton, + Typography, + Switch, +} from "@mui/material"; +import EditIcon from "@mui/icons-material/Edit"; +import { toast } from "react-toastify"; + +const RuleCard = ({ ruleName, description, file_id, globalUrl, openEditBar, ...otherProps }) => { + const [additionalProps, setAdditionalProps] = React.useState(otherProps); + + const handleSwitchChange = (event) => { + const isEnabled = event.target.checked; + toggleRule(file_id, !isEnabled, globalUrl, () => { + setAdditionalProps((prevProps) => ({ + ...prevProps, + is_enabled: isEnabled, + })); + }); + }; + + return ( + + +
+ {ruleName} +
+ openEditBar({ ruleName, description, file_id, ...additionalProps })}> + + + +
+
+ + {description} + + + + ); +}; + +const toggleRule = (fileId, isCurrentlyEnabled, globalUrl, callback) => { + const action = isCurrentlyEnabled ? "disable" : "enable"; + const url = `${globalUrl}/api/v1/files/${fileId}/${action}_rule`; + + fetch(url, { + method: "PUT", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => + response.json().then((responseJson) => { + if (responseJson["success"] === false) { + toast(`Failed to ${action} the rule`); + } else { + toast(`Rule ${action}d successfully`); + callback(); + } + }) + ) + .catch((error) => { + console.log(`Error in ${action}ing the rule: `, error); + toast(`An error occurred while ${action}ing the rule`); + }); +}; + +export default RuleCard; From e90b698f84405219d7e8efd0544c197f49f08c58 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Thu, 13 Jun 2024 09:54:50 +0000 Subject: [PATCH 10/46] fixing bugs in rule editing --- frontend/src/views/Detection.jsx | 38 ++++++++++++++--------- frontend/src/views/DetectionDashboard.jsx | 11 +++++-- frontend/src/views/EditRules.jsx | 7 ++--- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/frontend/src/views/Detection.jsx b/frontend/src/views/Detection.jsx index 67b6a2ed..63567584 100644 --- a/frontend/src/views/Detection.jsx +++ b/frontend/src/views/Detection.jsx @@ -7,7 +7,7 @@ import { Typography, Button, } from "@mui/material"; -import RuleCard from "./RuleCard"; +import RuleCard from "./RuleCard"; import { styled } from "@mui/system"; const ConnectedButton = styled(Button)({ @@ -18,7 +18,7 @@ const ConnectedButton = styled(Button)({ const Detection = ({ globalUrl, ruleInfo, openEditBar }) => { return ( - + { - {ruleInfo.length > 0 && - ruleInfo.map((card) => ( - openEditBar(card)} - {...card} - /> - ))} + + {ruleInfo.length > 0 && + ruleInfo.map((card) => ( + openEditBar(card)} + {...card} + /> + ))} + ); diff --git a/frontend/src/views/DetectionDashboard.jsx b/frontend/src/views/DetectionDashboard.jsx index 50771d63..bd8a2813 100644 --- a/frontend/src/views/DetectionDashboard.jsx +++ b/frontend/src/views/DetectionDashboard.jsx @@ -39,6 +39,12 @@ const DetectionDashBoard = (props) => { getSigmaInfo(globalUrl, setRuleInfo); }, [globalUrl]); + useEffect(() => { + if (ruleInfo.length > 0) { + openEditBar(ruleInfo[0]); + } + }, [ruleInfo]); + const openEditBar = (rule) => { setSelectedRule(rule); getFileContent(rule.file_id) @@ -50,6 +56,7 @@ const DetectionDashBoard = (props) => { }; const getFileContent = (file_id) => { + setFileData(""); fetch(globalUrl + "/api/v1/files/" + file_id + "/content", { method: "GET", headers: { @@ -82,10 +89,10 @@ const DetectionDashBoard = (props) => { }; return ( - + {selectedRule ? ( + - {ruleName} - - - + {ruleName} {description} From f8c786b9ecea2577d8d6f4591462a76882557efc Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Thu, 13 Jun 2024 09:56:31 +0000 Subject: [PATCH 11/46] removing unnessary dependencies --- frontend/src/views/DetectionDashboard.jsx | 1 - frontend/src/views/EditRules.jsx | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/frontend/src/views/DetectionDashboard.jsx b/frontend/src/views/DetectionDashboard.jsx index bd8a2813..382d0dc8 100644 --- a/frontend/src/views/DetectionDashboard.jsx +++ b/frontend/src/views/DetectionDashboard.jsx @@ -52,7 +52,6 @@ const DetectionDashBoard = (props) => { const handleSave = (updatedContent) => { toast("this will be saved"); - setSelectedRule(null); // Close the edit bar after saving }; const getFileContent = (file_id) => { diff --git a/frontend/src/views/EditRules.jsx b/frontend/src/views/EditRules.jsx index 27788135..a433ebc3 100644 --- a/frontend/src/views/EditRules.jsx +++ b/frontend/src/views/EditRules.jsx @@ -1,5 +1,5 @@ -import React, { useState } from 'react'; -import { Box, Typography, Button, Switch, TextField } from '@mui/material'; +import React from 'react'; +import { Box, Typography, Button, TextField } from '@mui/material'; const EditComponent = ({ ruleName, description, content, setContent, lastEdited, editedBy, onSave }) => { From f081e907b3efc6d56971ea67f7587479ac2f2608 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Fri, 14 Jun 2024 04:45:20 +0000 Subject: [PATCH 12/46] made the file disabling logic in orborus easy --- frontend/src/App.jsx | 6 +-- functions/onprem/orborus/orborus.go | 57 +++++++++-------------------- 2 files changed, 20 insertions(+), 43 deletions(-) diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 668cb557..ef77004a 100755 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -416,9 +416,9 @@ const App = (message, props) => { } /> } + exact + path="/detections/sigma" + element={} /> Date: Fri, 14 Jun 2024 05:04:24 +0000 Subject: [PATCH 13/46] made the sigma rule disable logic better and simple --- functions/onprem/orborus/orborus.go | 67 ++++++----------------------- 1 file changed, 12 insertions(+), 55 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 2eef9435..71c9955d 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2033,7 +2033,7 @@ func main() { } else if incRequest.Type == "DISABLE_SIGMA_FILE" { fileName := incRequest.ExecutionArgument - err = disableSigmaRule(fileName) + err = removeFile(fileName) if err != nil { log.Printf("[ERROR] Failed to disable the sigma file %s, reason: %s", fileName, err) } @@ -2041,18 +2041,11 @@ func main() { toBeRemoved.Data = append(toBeRemoved.Data, incRequest) } else if incRequest.Type == "DISABLE_SIGMA_RULES" { - err := manageSigmaFolder("disable") + err := removeAllFiles() if err != nil { log.Printf("[ERROR] Failed to disable the sigma rules: %s", err) } - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - } else if incRequest.Type == "ENABLE_SIGMA_RULES" { - err := manageSigmaFolder("enable") - if err != nil { - log.Printf("[ERROR] Failed to enable the sigma rules: %s", err) - } - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) } else { newrequests = append(newrequests, incRequest) @@ -3029,7 +3022,7 @@ func copyToTenzir(srcPath, destPath string) error { return nil } -func disableSigmaRule(fileName string) error { +func removeFile(fileName string) error { containerName := "tenzir-node" srcPath := fmt.Sprintf("/var/lib/tenzir/sigma_rules/%s", fileName) @@ -3038,57 +3031,21 @@ func disableSigmaRule(fileName string) error { return fmt.Errorf("source file does not exist: %v", err) } - rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", srcPath) - if err := rmCmd.Run(); err != nil { - return fmt.Errorf("error removing file: %v", err) - } - - return nil + return removePath(containerName, srcPath) } -func manageSigmaFolder(action string) error { +func removeAllFiles() error { containerName := "tenzir-node" - sigmaPath := "/var/lib/tenzir/sigma_rules" + sigmaPath := "/var/lib/tenzir/sigma_rules/*" - if action == "disable" { + return removePath(containerName, sigmaPath) +} - checkSigmaCmd := exec.Command("docker", "exec", containerName, "test", "-d", sigmaPath) - if err := checkSigmaCmd.Run(); err != nil { - return fmt.Errorf("sigma_files directory does not exist: %v", err) - } - - // Rename sigma_files to disabled_sigma - renameCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mv", sigmaPath, disabledPath) - if err := renameCmd.Run(); err != nil { - return fmt.Errorf("error renaming sigma_files to disabled_sigma: %v", err) - } - - // Create a new sigma_files directory - createCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mkdir", sigmaPath) - if err := createCmd.Run(); err != nil { - return fmt.Errorf("error creating new sigma_files directory: %v", err) - } - } else if action == "enable" { - - checkDisabledCmd := exec.Command("docker", "exec", containerName, "test", "-d", disabledPath) - if err := checkDisabledCmd.Run(); err != nil { - return fmt.Errorf("disabled_sigma directory does not exist: %v", err) - } - - removeCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", "-rf", sigmaPath) - if err := removeCmd.Run(); err != nil { - return fmt.Errorf("error removing existing sigma_files directory: %v", err) - } - - // Rename disabled_sigma back to sigma_files - renameBackCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mv", disabledPath, sigmaPath) - if err := renameBackCmd.Run(); err != nil { - return fmt.Errorf("error renaming disabled_sigma back to sigma_files: %v", err) - } - } else { - return fmt.Errorf("invalid action: %s", action) +func removePath(containerName, path string) error { + rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", "-r", path) + if err := rmCmd.Run(); err != nil { + return fmt.Errorf("error removing path: %v", err) } - return nil } From 29f294f2ef8e0dbc7ea8c21fdaa6b02f0b71a9cc Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Tue, 18 Jun 2024 12:43:19 +0000 Subject: [PATCH 14/46] removing the hard coded ui --- frontend/src/views/AngularWorkflow.jsx | 186 ++++++------------------- 1 file changed, 45 insertions(+), 141 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 70d9b761..00bc68de 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1437,7 +1437,7 @@ const releaseToConnectLabel = "Release to Connect" }); }; - const handleKafkaSubmit = (trigger) => { + const handleCommandSubmit = (trigger) => { if (trigger.trigger_type !== "PIPELINE") { toast("Unable to save the configuration"); return; @@ -1445,38 +1445,18 @@ const releaseToConnectLabel = "Release to Connect" trigger.parameters = [] - const topic = document.getElementById('topic')?.value; - const bootstrapServers = document.getElementById('bootstrap_servers')?.value; - const groupId = document.getElementById('group_id')?.value; - //const autoOffsetReset = document.getElementById('auto_offset_reset')?.value; + const command = document.getElementById('sigma')?.value - if(topic) { + if(command) { trigger.parameters.push({ - name: "topic", - value: topic - }); + name: "command", + value: command + }) } else { - toast("please enter the topic name"); + toast("Please enter the comamnd"); return; } - if (bootstrapServers) { - trigger.parameters.push({ - name: "bootstrap_servers", - value: bootstrapServers - }); - } else { - toast("please enter bootstrap server details"); - return; - } - - if (groupId) { - trigger.parameters.push({ - name: "group_id", - value: groupId - }); - } - // if (autoOffsetReset) { // trigger.parameters.push({ // name: "auto_offset_reset", @@ -15366,14 +15346,18 @@ const releaseToConnectLabel = "Release to Connect"
{ - // setSelectedOption("Syslog listener") - // setTenzirConfigModalOpen(true); - }} + if(selectedTrigger.status === "running"){ + toast("please stop the trigger to edit the configuration"); + return; + } else { + setSelectedOption("Syslog listener"); + setTenzirConfigModalOpen(true); + }}} style={{ border: "1px solid rgba(255,255,255,0.3)", borderRadius: theme.palette.borderRadius, padding: 10, - cursor: "not-allowed", + cursor: "pointer", marginTop: 5, display: "flex", alignItems: "center", @@ -15386,7 +15370,6 @@ const releaseToConnectLabel = "Release to Connect" onChange={() => setSelectedOption("Syslog listener")} value={"Syslog listener"} name="option" - disabled={true} /> } label="Start Syslog listener" @@ -15396,14 +15379,18 @@ const releaseToConnectLabel = "Release to Connect"
{ - // setSelectedOption("Sigma Rulesearch") - // setTenzirConfigModalOpen(true); - }} + if(selectedTrigger.status === "running"){ + toast("please stop the trigger to edit the configuration"); + return; + } else { + setSelectedOption("Sigma Rulesearch"); + setTenzirConfigModalOpen(true); + }}} style={{ border: "1px solid rgba(255,255,255,0.3)", borderRadius: theme.palette.borderRadius, padding: 10, - cursor: "not-allowed", + cursor: "pointer", marginTop: 5, display: "flex", alignItems: "center", @@ -15416,7 +15403,6 @@ const releaseToConnectLabel = "Release to Connect" onChange={() => setSelectedOption("Sigma Rulesearch")} value={"Sigma Rulesearch"} name="option" - disabled={true} /> } label="Run Sigma Rulesearch" @@ -15463,39 +15449,7 @@ const releaseToConnectLabel = "Release to Connect" disabled={selectedTrigger.status === "running"} onClick={() => { - const topic = (selectedTrigger?.parameters?.find(param => param.name === "topic")?.value) || '' - const bootstrapServers = (selectedTrigger?.parameters?.find(param => param.name === "bootstrap_servers")?.value) || '' - const groupId = (selectedTrigger?.parameters?.find(param => param.name === "group_id")?.value) || '' - // const autoOffsetReset = (selectedTrigger?.parameters?.find(param => param.name === "auto_offset_reset")?.value) || '' - let command = "from kafka" - - if(topic) { - command = `${command} -t ${topic}` - } else { - toast("please enter the topic name") - return; - } - if(bootstrapServers) { - command = `${command} -e -o stored -X bootstrap.servers=${bootstrapServers}` - } else { - toast("please enter the bootstrap servers details") - return; - } - - if(groupId) { - command = `${command},group.id=${groupId}` - } else { - command = `${command},group.id=${selectedTrigger.id}` - } - // if(autoOffsetReset) { - // command = `${command},auto.offset.reset=${autoOffsetReset}` - // } else { - // command = `${command},auto.offset.reset=earliest` - - // } - command = `${command},auto.offset.reset=earliest` - command = `${command},client.id=${selectedTrigger.id},enable.auto.commit=true,auto.commit.interval.ms=1` - command = `${command} read json | to ${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` + const command = (selectedTrigger?.parameters?.find(param => param.name === "command")?.value) || '' const pipelineConfig = { command: command, @@ -21218,8 +21172,8 @@ const releaseToConnectLabel = "Release to Connect" pointerEvents: "auto", color: "white", minWidth: 600, - minHeight: 450, - maxHeight: 450, + minHeight: 200, + maxHeight: 200, padding: 15, overflow: "hidden", zIndex: 10012, @@ -21236,75 +21190,25 @@ const releaseToConnectLabel = "Release to Connect" overflowY: "auto", overflowX: isMobile ? "auto" : "hidden", }} - > - -
Configuration options for {selectedOption}
- + > - {selectedOption === "Kafka Queue" && ( - <> - Topic - param.name === "topic")?.value) || ''} - /> - bootstrap.servers - param.name === "bootstrap_servers")?.value) || ''} - /> - group.id - param.name === "group_id")?.value) || ''} - /> - {/* auto.offest.reset - param.name === "auto_offset_reset")?.value) || ''} - /> */} - - )} + + command + param.name === "command")?.value) || ''} + /> +
@@ -84,19 +81,19 @@ const RuleCard = ({ ruleName, description, file_id, globalUrl, folderDisabled, . + isCloud={isCloud} + expansionModalOpen={openCodeEditor} + setExpansionModalOpen={setOpenCodeEditor} + setcodedata={setFileData} + codedata={fileData} + isFileEditor={true} + key={fileData} // https://reactjs.org/docs/reconciliation.html#recursing-on-children + runUpdateText={UpdateText} + /> ); -}; +} const toggleRule = (fileId, isCurrentlyEnabled, globalUrl, callback) => { const action = isCurrentlyEnabled ? "disable" : "enable"; From 2cc0d3d57639a9b6677870e00b49d50ce4fe5078 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Tue, 18 Jun 2024 16:11:29 +0000 Subject: [PATCH 20/46] fixing the entire directory getting deleted instead of contents inside it --- functions/onprem/orborus/orborus.go | 30 +++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index f1459c00..9c84c03d 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2024,7 +2024,13 @@ func main() { toBeRemoved.Data = append(toBeRemoved.Data, incRequest) } else if incRequest.Type == "CATEGORY_UPDATE" { - err := handleFileCategoryChange() + + err := deployTenzirNode() + if err != nil{ + log.Printf("[ERROR] failed to deploy the pipeline, reason: %s", err) + } + + err = handleFileCategoryChange() if err != nil { log.Printf("[ERROR] Failed to download the file category: %s", err) } @@ -2033,6 +2039,11 @@ func main() { } else if incRequest.Type == "DISABLE_SIGMA_FILE" { fileName := incRequest.ExecutionArgument + err := deployTenzirNode() + if err != nil{ + log.Printf("[ERROR] failed to deploy the pipeline, reason: %s", err) + } + err = removeFile(fileName) if err != nil { log.Printf("[ERROR] Failed to disable the sigma file %s, reason: %s", fileName, err) @@ -2041,7 +2052,13 @@ func main() { toBeRemoved.Data = append(toBeRemoved.Data, incRequest) } else if incRequest.Type == "DISABLE_SIGMA_FOLDER" { - err := removeAllFiles() + + err := deployTenzirNode() + if err != nil{ + log.Printf("[ERROR] failed to deploy the pipeline, reason: %s", err) + } + + err = removeAllFiles() if err != nil { log.Printf("[ERROR] Failed to disable the sigma rules: %s", err) } @@ -3021,7 +3038,12 @@ func removeAllFiles() error { containerName := "tenzir-node" sigmaPath := "/var/lib/tenzir/sigma_rules/*" - return removePath(containerName, sigmaPath) + cmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("rm -rf %s", sigmaPath)) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error removing files: %v, output: %s", err, output) + } + return nil } func removeFile(fileName string) error { @@ -3038,7 +3060,7 @@ func removeFile(fileName string) error { func removePath(containerName, path string) error { rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", "-rf", path) - output, err := rmCmd.CombinedOutput() + output, err := rmCmd.CombinedOutput() if err != nil { return fmt.Errorf("error removing path: %v, output: %s", err, output) } From 5620d8a76b97a2fad1a9ceb092a4e6b81011f569 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Tue, 18 Jun 2024 16:11:52 +0000 Subject: [PATCH 21/46] adding the trigger url to the command --- frontend/src/views/AngularWorkflow.jsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 00bc68de..aca59723 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -15449,7 +15449,8 @@ const releaseToConnectLabel = "Release to Connect" disabled={selectedTrigger.status === "running"} onClick={() => { - const command = (selectedTrigger?.parameters?.find(param => param.name === "command")?.value) || '' + let command = (selectedTrigger?.parameters?.find(param => param.name === "command")?.value) || '' + command = `${command} | to ${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` const pipelineConfig = { command: command, From 3595881a349daab527cfbb5971d7954f81203426 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Tue, 18 Jun 2024 17:11:16 +0000 Subject: [PATCH 22/46] trying to parse the json logs properly --- backend/go-app/main.go | 63 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 8bb75ae9..713abda8 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1978,7 +1978,6 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { } func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { - if request.Method != "POST" { request.Method = "POST" } @@ -1999,7 +1998,7 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { location := strings.Split(request.URL.String(), "/") var pipelineId string - + if location[1] == "api" { if len(location) <= 4 { log.Printf("[INFO] Couldn't handle location. Too short in pipeline: %d", len(location)) @@ -2013,7 +2012,7 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { userAgent := request.Header.Get("User-Agent") if strings.Contains(strings.ToLower(userAgent), "microsoftpreview") || strings.Contains(strings.ToLower(userAgent), "googlebot") { - log.Printf("[AUDIT] Blocking googlebot and microsoftbot for pielines. UA: '%s'", userAgent) + log.Printf("[AUDIT] Blocking googlebot and microsoftbot for pipelines. UA: '%s'", userAgent) resp.WriteHeader(400) resp.Write([]byte(`{"success": false, "reason": "Google/Microsoft preview bots not allowed. Please change the useragent."}`)) return @@ -2058,7 +2057,23 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { return } - parsedBody := shuffle.GetExecutionbody(body) + // Parse concatenated JSON logs + jsonList, err := parseConcatenatedJSONLogs(string(body)) + if err != nil { + log.Printf("[DEBUG] JSON parsing error: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } +// fix this + parsedBody, err := string(jsonList) + if err != nil { + log.Printf("[ERROR] Failed to marshal jsonList: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false}`)) + return + } + newBody := shuffle.ExecutionStruct{ Start: pipeline.StartNode, ExecutionSource: "pipeline", @@ -2093,8 +2108,7 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { } if len(pipeline.StartNode) == 0 { - log.Printf("[WARNING] No start node for pipeline %s - running with workflow default.", pipeline.TriggerId) - + log.Printf("[WARNING] No start node for pipeline %s - running with workflow default.") } newRequest := &http.Request{ @@ -2115,6 +2129,43 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, executionResp))) } +func parseConcatenatedJSONLogs(logs string) ([]map[string]interface{}, error) { + var jsonList []map[string]interface{} + var currentObject []rune + var depth int + + for _, char := range logs { + if char == '{' { + depth++ + } + if char == '}' { + depth-- + } + + currentObject = append(currentObject, char) + + // When depth is 0, it means we have a complete JSON object but will this work ?? + if depth == 0 && len(currentObject) > 0 { + var jsonObject map[string]interface{} + err := json.Unmarshal([]byte(string(currentObject)), &jsonObject) + if err != nil { + log.Printf("[WARNING] JSON unmarshal error: %s. Skipping this object.", err) + } else { + jsonList = append(jsonList, jsonObject) + } + currentObject = nil + } + } + + currentObject = []rune(strings.TrimSpace(string(currentObject))) + if len(currentObject) > 0 { + log.Printf("[WARNING] Incomplete JSON object found: %s. Skipping this object.", string(currentObject)) + } + + return jsonList, nil +} + + func executeCloudAction(action shuffle.CloudSyncJob, apikey string) error { data, err := json.Marshal(action) if err != nil { From ad92e5e72272e15363001cd1e5d5fb0c3c854111 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Fri, 21 Jun 2024 10:28:49 +0000 Subject: [PATCH 23/46] made the pipeline to parse json logs --- backend/go-app/main.go | 51 ++++++++++------------------- functions/onprem/orborus/orborus.go | 15 ++++++--- 2 files changed, 27 insertions(+), 39 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 713abda8..d5fd7c6b 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -2065,8 +2065,8 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": false}`)) return } -// fix this - parsedBody, err := string(jsonList) + + parsedBody, err := json.Marshal(jsonList) if err != nil { log.Printf("[ERROR] Failed to marshal jsonList: %s", err) resp.WriteHeader(500) @@ -2077,7 +2077,7 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { newBody := shuffle.ExecutionStruct{ Start: pipeline.StartNode, ExecutionSource: "pipeline", - ExecutionArgument: parsedBody, + ExecutionArgument: string(parsedBody), } workflow, err := shuffle.GetWorkflow(ctx, pipeline.WorkflowId) @@ -2130,42 +2130,25 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { } func parseConcatenatedJSONLogs(logs string) ([]map[string]interface{}, error) { - var jsonList []map[string]interface{} - var currentObject []rune - var depth int + var jsonList []map[string]interface{} + decoder := json.NewDecoder(strings.NewReader(logs)) - for _, char := range logs { - if char == '{' { - depth++ - } - if char == '}' { - depth-- - } + for decoder.More() { + var jsonObject map[string]interface{} + if err := decoder.Decode(&jsonObject); err != nil { + log.Printf("[WARNING] JSON decoding error: %s. Skipping this object.", err) + continue + } + jsonList = append(jsonList, jsonObject) + } - currentObject = append(currentObject, char) + if err := decoder.Decode(&struct{}{}); err != io.EOF { + return nil, fmt.Errorf("error after decoding all JSON objects: %v", err) + } - // When depth is 0, it means we have a complete JSON object but will this work ?? - if depth == 0 && len(currentObject) > 0 { - var jsonObject map[string]interface{} - err := json.Unmarshal([]byte(string(currentObject)), &jsonObject) - if err != nil { - log.Printf("[WARNING] JSON unmarshal error: %s. Skipping this object.", err) - } else { - jsonList = append(jsonList, jsonObject) - } - currentObject = nil - } - } - - currentObject = []rune(strings.TrimSpace(string(currentObject))) - if len(currentObject) > 0 { - log.Printf("[WARNING] Incomplete JSON object found: %s. Skipping this object.", string(currentObject)) - } - - return jsonList, nil + return jsonList, nil } - func executeCloudAction(action shuffle.CloudSyncJob, apikey string) error { data, err := json.Marshal(action) if err != nil { diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 9c84c03d..b2ec0e80 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2462,7 +2462,7 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) return err } - _, err = updatePipelineState(pipelineId, "stop") + _, err = updatePipelineState(command, pipelineId, "stop") if err != nil { log.Printf("[ERROR] Failed to stop Pipeline: %s reason:%s ", pipelineId, err) return err @@ -2482,7 +2482,7 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) return err } - _, err = updatePipelineState(pipelineId, "start") + _, err = updatePipelineState(command, pipelineId, "start") if err != nil { log.Printf("[ERROR] Failed to start Pipeline: %s reason:%s ", pipelineId, err) return err @@ -2689,7 +2689,11 @@ func createPipeline(command, identifier string) (string, error) { // } // } - command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | sigma /var/lib/tenzir/rule.yaml | to https://shuffler.io/api/v1/hooks/webhook_d295c43a-e322-4afc-9a59-af167ae7c190" + //command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | sigma /var/lib/tenzir/rule.yaml" + //command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | import" + //command = "export | to https://expert-acorn-v6vg4j4j5w7q2wg6g-5001.app.github.dev/api/v1/hooks/webhook_623eab3f-0af4-4d40-abb9-699d9a493411" + log.Printf("[HARI] this is the command %s", command) + requestBody := map[string]interface{}{ "definition": command, "name": identifier, @@ -2697,7 +2701,7 @@ func createPipeline(command, identifier string) (string, error) { "autostart": map[string]bool{ "created": true, "completed": false, - "failed": true, + "failed": false, }, "autodelete": map[string]bool{ "completed": false, @@ -2763,13 +2767,14 @@ func createPipeline(command, identifier string) (string, error) { return id, nil } -func updatePipelineState(pipelineId, action string) (string, error) { +func updatePipelineState(command, pipelineId, action string) (string, error) { url := fmt.Sprintf("%s/api/v0/pipeline/update", tenzirUrl) forwardMethod := "POST" requestBody := map[string]interface{}{ "id": pipelineId, + "definition": command, "action": action, "autostart": map[string]bool{ "created": true, From ab3c755ad3ca7ebf96060f66ce245ccf436d9f13 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Sun, 23 Jun 2024 09:13:08 +0000 Subject: [PATCH 24/46] sending tenzir health check status to backend --- backend/go-app/main.go | 41 ++++++++++++ functions/onprem/orborus/orborus.go | 98 ++++++++++++++--------------- 2 files changed, 90 insertions(+), 49 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index d5fd7c6b..6fd0ff22 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -2149,6 +2149,46 @@ func parseConcatenatedJSONLogs(logs string) ([]map[string]interface{}, error) { return jsonList, nil } +func handleTenzirHealthUpdate(resp http.ResponseWriter, request *http.Request) { + if request.Method != "POST" { + request.Method = "POST" + } + + type HealthUpdate struct { + Status string `json:"status"` + } + + var healthUpdate HealthUpdate + err := json.NewDecoder(request.Body).Decode(&healthUpdate) + if err != nil { + resp.WriteHeader(http.StatusBadRequest) + fmt.Fprintf(resp, "Failed to decode JSON: %v", err) + return + } + ctx := context.Background() + status := healthUpdate.Status + + result, err := shuffle.GetDisabledRules(ctx) + if (err != nil && err.Error() != "rules doesn't exist") || err == nil { + result.IsTenzirActive = status + result.LastActive = time.Now().Unix() + + err = shuffle.StoreDisabledRules(ctx, *result) + if err != nil { + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false}`)) + return + } + + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) + return + } + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false}`)) + return +} + func executeCloudAction(action shuffle.CloudSyncJob, apikey string) error { data, err := json.Marshal(action) if err != nil { @@ -5091,6 +5131,7 @@ func initHandlers() { r.HandleFunc("/api/v1/triggers/pipeline", shuffle.HandleNewPipelineRegister).Methods("POST", "OPTIONS") //r.HandleFunc("/api/v1/triggers/pipeline/save", shuffle.HandleSavePipelineInfo).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/pipelines/{key}", handlePipelineCallback).Methods("POST", "GET", "PATCH", "PUT", "DELETE", "OPTIONS") + r.HandleFunc("/api/v1/pipelines/tenzir_node_health", handleTenzirHealthUpdate).Methods("POST","OPTIONS") r.HandleFunc("/api/v1/triggers", shuffle.HandleGetTriggers).Methods("GET", "OPTIONS") //r.HandleFunc("/api/v1/triggers/gmail/routing", handleGmailRouting).Methods("POST", "OPTIONS") diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index b2ec0e80..49bb2bca 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1806,6 +1806,12 @@ func main() { log.Printf("[WARNING] Defaulting to environment name %s. Set environment variable ENVIRONMENT_NAME to change. This should be the same as in the frontend action.", environment) } + if tenzirUrl == "" { + tenzirUrl = "http://localhost:5160" + log.Printf("[WARNING] SHUFFLE_TENZIR_URL not set, falling back to default URL: %s",tenzirUrl) + } + + // FIXME - during init, BUILD and/or LOAD worker and app_sdk // Build/load app_sdk so it can be loaded as 127.0.0.1:5000/walkoff_app_sdk log.Printf("[INFO] Setting up Docker environment. Downloading worker and App SDK!") @@ -2254,7 +2260,7 @@ func main() { } } - + _ = sendTenzirHealthStatus() time.Sleep(time.Duration(sleepTime) * time.Second) } } @@ -2414,11 +2420,6 @@ func main() { // docker run tenzir/tenzir:latest 'from http://192.168.86.44:5002/api/v1/orgs/7e9b9007-5df2-4b47-bca5-c4d267ef2943/cache/CIDR%20ranges?type=text&authorization=cec9d01f-09b2-4419-8a0a-76c6046e3fef read lines | to http://192.168.86.44:5002/api/v1/hooks/webhook_665ace5f-f27b-496a-a365-6e07eb61078c write lines' func handlePipeline(incRequest shuffle.ExecutionRequest) error { - if tenzirUrl == "" { - tenzirUrl = "http://localhost:5160" - log.Printf("[WARNING] SHUFFLE_TENZIR_URL not set, falling back to default URL: %s", tenzirUrl) - } - err := deployTenzirNode() if err != nil { log.Printf("[ERROR] failed to deploy the pipeline, reason: %s", err) @@ -3072,53 +3073,52 @@ func removePath(containerName, path string) error { return nil } -// func savePipelineData(pipelineId, identifier, status string) error { +func sendTenzirHealthStatus() error { + var status string + url := fmt.Sprintf("%s/api/v1/triggers/pipeline/tenzir_node_health", baseUrl) + err := checkTenzirNode() + if err != nil { + return err + } else { + status = "active" + } -// url := fmt.Sprintf("%s/api/v1/triggers/pipeline/save", baseUrl) -// identifierWithoutPrefix := strings.TrimPrefix(identifier, "shuffle-") + forwardMethod := "POST" + payload := map[string]interface{}{ + "status": status, + } + payloadBytes, err := json.Marshal(payload) + if err != nil { + log.Printf("[ERROR] Failed to marshal payload: %s", err) + return err + } + forwardData := bytes.NewBuffer(payloadBytes) + req, err := http.NewRequest( + forwardMethod, + url, + forwardData, + ) + if err != nil { + log.Printf("[ERROR] Failed to create HTTP request: %s", err) + return err + } + req.Header.Set("Content-Type", "application/json") -// forwardMethod := "PUT" + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Do(req) + if err != nil { + log.Printf("[ERROR] Failed to send HTTP request: %s", err) + return err + } + defer resp.Body.Close() -// payload := map[string]interface{}{ -// "pipeline_id": pipelineId, -// "trigger_id": identifierWithoutPrefix, -// "status": status, -// } + if resp.StatusCode != 200 { + log.Printf("[ERROR] Received non-successful HTTP status code: %d", resp.StatusCode) + return fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode) + } -// payloadBytes, err := json.Marshal(payload) -// if err != nil { -// log.Printf("[ERROR] Failed to marshal payload: %s", err) -// return err -// } - -// forwardData := bytes.NewBuffer(payloadBytes) - -// req, err := http.NewRequest( -// forwardMethod, -// url, -// forwardData, -// ) -// if err != nil { -// log.Printf("[ERROR] Failed to create HTTP request: %s", err) -// return err -// } -// req.Header.Set("Content-Type", "application/json") - -// client := &http.Client{Timeout: 10 * time.Second} -// resp, err := client.Do(req) -// if err != nil { -// log.Printf("[ERROR] Failed to send HTTP request: %s", err) -// return err -// } -// defer resp.Body.Close() - -// if resp.StatusCode != 200 { -// log.Printf("[ERROR] Received non-successful HTTP status code: %d", resp.StatusCode) -// return fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode) -// } - -// return nil -// } + return nil +} // Is this ok to do with Docker? idk :) func getRunningWorkers(ctx context.Context, workerTimeout int) int { From ecfb78e82d1f7027265c47b20a00e3ec1e805857 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Sun, 23 Jun 2024 09:16:03 +0000 Subject: [PATCH 25/46] showing the tenzir active status in the UI --- frontend/src/views/Detection.jsx | 50 +++++++++++++++++++------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/frontend/src/views/Detection.jsx b/frontend/src/views/Detection.jsx index 80d2f821..63ca8ac4 100644 --- a/frontend/src/views/Detection.jsx +++ b/frontend/src/views/Detection.jsx @@ -11,14 +11,13 @@ import { toast } from "react-toastify"; import RuleCard from "./RuleCard"; import { styled } from "@mui/system"; -const ConnectedButton = styled(Button)({ - backgroundColor: "red", +const ConnectedButton = styled(Button)(({ theme, isConnected }) => ({ + backgroundColor: isConnected ? "green" : "red", color: "white", -}); +})); -const handleDirectoryChange = ( folderDisabled, setFolderDisabled, globalUrl) => { - - const action = folderDisabled ? "enable_folder" : "disable_folder" +const handleDirectoryChange = (folderDisabled, setFolderDisabled, globalUrl) => { + const action = folderDisabled ? "enable_folder" : "disable_folder"; const url = `${globalUrl}/api/v1/files/detection/${action}`; fetch(url, { @@ -31,8 +30,8 @@ const handleDirectoryChange = ( folderDisabled, setFolderDisabled, globalUrl) => .then((response) => response.json().then((responseJson) => { if (responseJson["success"] === true) { - if (action === "enable_folder") setFolderDisabled(false); - else setFolderDisabled(true); + if (action === "enable_folder") setFolderDisabled(false); + else setFolderDisabled(true); } else { //toast(`failed to disable rule`); } @@ -42,13 +41,19 @@ const handleDirectoryChange = ( folderDisabled, setFolderDisabled, globalUrl) => console.log(`Error in ${action} the rule: `, error); toast(`An error occurred while ${action} the rule`); }); +}; -} - -const Detection = ({ globalUrl, ruleInfo, folderDisabled, setFolderDisabled, openEditBar }) => { +const Detection = ({ + globalUrl, + ruleInfo, + folderDisabled, + setFolderDisabled, + openEditBar, + isTenzirActive, +}) => { return ( - + Sigma Detection Rules - - Not Connected to SIEM + + {isTenzirActive ? "Connected to SIEM" : "Not Connected to SIEM"} Global disable/enable - handleDirectoryChange(folderDisabled, setFolderDisabled, globalUrl)} - /> + + handleDirectoryChange(folderDisabled, setFolderDisabled, globalUrl) + } + /> Date: Sun, 23 Jun 2024 16:49:17 +0000 Subject: [PATCH 26/46] adding health check for tenzir --- backend/go-app/main.go | 2 +- functions/onprem/orborus/orborus.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 6fd0ff22..202aa891 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5131,7 +5131,6 @@ func initHandlers() { r.HandleFunc("/api/v1/triggers/pipeline", shuffle.HandleNewPipelineRegister).Methods("POST", "OPTIONS") //r.HandleFunc("/api/v1/triggers/pipeline/save", shuffle.HandleSavePipelineInfo).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/pipelines/{key}", handlePipelineCallback).Methods("POST", "GET", "PATCH", "PUT", "DELETE", "OPTIONS") - r.HandleFunc("/api/v1/pipelines/tenzir_node_health", handleTenzirHealthUpdate).Methods("POST","OPTIONS") r.HandleFunc("/api/v1/triggers", shuffle.HandleGetTriggers).Methods("GET", "OPTIONS") //r.HandleFunc("/api/v1/triggers/gmail/routing", handleGmailRouting).Methods("POST", "OPTIONS") @@ -5200,6 +5199,7 @@ func initHandlers() { r.HandleFunc("/api/v1/files/detection/sigma_rules", shuffle.HandleGetSigmaRules).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/detection/{fileId}/{action}", shuffle.HandleToggleRule).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/files/detection/{action}", shuffle.HandleFolderToggle).Methods("PUT", "OPTIONS") + r.HandleFunc("/api/v1/detection/siem/node_health", handleTenzirHealthUpdate).Methods("POST","OPTIONS") // Introduced in 0.9.21 to handle notifications for e.g. failed Workflow r.HandleFunc("/api/v1/notifications", shuffle.HandleCreateNotification).Methods("POST", "OPTIONS") diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 49bb2bca..17a76611 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1913,6 +1913,7 @@ func main() { log.Printf("[INFO] Waiting for executions at %s with Environment %#v", fullUrl, environment) hasStarted := false for { + _ = sendTenzirHealthStatus() if req.Method == "POST" { // Should find data to send (memory etc.) @@ -2260,7 +2261,6 @@ func main() { } } - _ = sendTenzirHealthStatus() time.Sleep(time.Duration(sleepTime) * time.Second) } } @@ -2580,9 +2580,9 @@ func deployTenzirNode() error { } func checkTenzirNode() error { - retries := 20 - retryInterval := 3 * time.Second - url := fmt.Sprintf("%s/api/v0/ping", tenzirUrl) + retries := 5 + retryInterval := 3 * time.Second + url := fmt.Sprintf("%s/api/v0/ping",tenzirUrl) forwardMethod := "POST" client := http.Client{} @@ -3075,7 +3075,7 @@ func removePath(containerName, path string) error { func sendTenzirHealthStatus() error { var status string - url := fmt.Sprintf("%s/api/v1/triggers/pipeline/tenzir_node_health", baseUrl) + url := fmt.Sprintf("%s/api/v1/detection/siem/node_health", baseUrl) err := checkTenzirNode() if err != nil { return err @@ -3116,7 +3116,7 @@ func sendTenzirHealthStatus() error { log.Printf("[ERROR] Received non-successful HTTP status code: %d", resp.StatusCode) return fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode) } - + log.Printf("this is send successfully") return nil } From e24550a64106775a7d7a5ae7f6069e9cab838bbe Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Sun, 23 Jun 2024 16:49:53 +0000 Subject: [PATCH 27/46] refactored and adding support for searching --- frontend/src/views/Detection.jsx | 132 ++++++++++++++++++++-- frontend/src/views/DetectionDashboard.jsx | 9 +- 2 files changed, 130 insertions(+), 11 deletions(-) diff --git a/frontend/src/views/Detection.jsx b/frontend/src/views/Detection.jsx index 63ca8ac4..8b6214a1 100644 --- a/frontend/src/views/Detection.jsx +++ b/frontend/src/views/Detection.jsx @@ -1,4 +1,4 @@ -import React from "react"; +import React, { useState, useRef } from "react"; import { Container, Box, @@ -7,6 +7,7 @@ import { Typography, Button, } from "@mui/material"; +import { Publish as PublishIcon } from "@mui/icons-material"; import { toast } from "react-toastify"; import RuleCard from "./RuleCard"; import { styled } from "@mui/system"; @@ -51,6 +52,95 @@ const Detection = ({ openEditBar, isTenzirActive, }) => { + const [searchQuery, setSearchQuery] = useState(""); + const uploadRef = useRef(null); + + const uploadFiles = (files) => { + for (const key in files) { + try { + const filename = files[key].name; + const filedata = new FormData(); + filedata.append("shuffle_file", files[key]); + + if (typeof files[key] === "object") { + handleCreateFile(filename, filedata); + } + } catch (e) { + console.log("Error in dropzone: ", e); + } + } + + setTimeout(() => { + // Additional logic if needed + }, 2500); + }; + + const handleCreateFile = (filename, file) => { + const data = { + filename: filename, + org_id: "default", + workflow_id: "global", + namespace: "sigma", + }; + + fetch(globalUrl + "/api/v1/files/create", { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + body: JSON.stringify(data), + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for apps :O!"); + return; + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success === true) { + handleFileUpload(responseJson.id, file); + } else { + toast("Failed to upload file ", filename); + } + }) + .catch((error) => { + toast("Failed to upload file ", filename); + console.log(error.toString()); + }); + }; + + const handleFileUpload = (file_id, file) => { + fetch(`${globalUrl}/api/v1/files/${file_id}/upload`, { + method: "POST", + credentials: "include", + body: file, + }) + .then((response) => { + if (response.status !== 200 && response.status !== 201) { + console.log("Status not 200 for apps :O!"); + toast("File was created, but failed to upload."); + return; + } + + return response.json(); + }) + .then((responseJson) => { + // Handle the response as needed + }) + .catch((error) => { + toast(error.toString()); + }); + }; + + const filteredRules = ruleInfo.filter((rule) => + rule.title.toLowerCase().includes(searchQuery.toLowerCase()) || + rule.description.toLowerCase().includes(searchQuery.toLowerCase()) + ); + return ( @@ -65,10 +155,7 @@ const Detection = ({ Sigma Detection Rules - + {isTenzirActive ? "Connected to SIEM" : "Not Connected to SIEM"} @@ -80,7 +167,36 @@ const Detection = ({ mb: 2, }} > - + + setSearchQuery(e.target.value)} + /> + + { + uploadFiles(event.target.files); + }} + /> + Global disable/enable @@ -102,8 +218,8 @@ const Detection = ({ p: 1, }} > - {ruleInfo.length > 0 && - ruleInfo.map((card) => ( + {filteredRules.length > 0 && + filteredRules.map((card) => ( { +const getSigmaInfo = (globalUrl, setRuleInfo, setFolderDisabled, setIsTenzirActive) => { const url = globalUrl + "/api/v1/files/detection/sigma_rules"; fetch(url, { @@ -21,6 +21,8 @@ const getSigmaInfo = (globalUrl, setRuleInfo, setFolderDisabled) => { } else { setRuleInfo(responseJson.sigma_info); setFolderDisabled(responseJson.folder_disabled); + setIsTenzirActive(responseJson.is_tenzir_active); + } }) ) @@ -35,11 +37,12 @@ const DetectionDashBoard = (props) => { const [ruleInfo, setRuleInfo] = useState([]); const [selectedRule, setSelectedRule] = useState(null); const [fileData, setFileData] = React.useState(""); + const [isTenzirActive, setIsTenzirActive] = React.useState(false); const [folderDisabled, setFolderDisabled] = useState(false); useEffect(() => { - getSigmaInfo(globalUrl, setRuleInfo, setFolderDisabled); + getSigmaInfo(globalUrl, setRuleInfo, setFolderDisabled, setIsTenzirActive); }, [folderDisabled]); useEffect(() => { @@ -103,7 +106,7 @@ const DetectionDashBoard = (props) => { onSave={handleSave} /> ) : null} */} - + ); }; From 073b9f3f790355f6c7d87028efdb0edcbc4b81bd Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Mon, 24 Jun 2024 20:28:32 +0530 Subject: [PATCH 28/46] adding a select option for the sigma rules --- frontend/src/views/AngularWorkflow.jsx | 226 +++++++++++++++---------- 1 file changed, 140 insertions(+), 86 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index aca59723..9b7755ac 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -534,6 +534,7 @@ const AngularWorkflow = (defaultprops) => { const [listCache, setListCache] = React.useState([]); const [selectedOption, setSelectedOption] = React.useState(""); const [tenzirConfigModalOpen, setTenzirConfigModalOpen] = React.useState(false); + const [rules, setRules] = React.useState([]); const [distributedFromParent, setDistributedFromParent] = React.useState("") const [suborgWorkflows, setSuborgWorkflows] = React.useState([]) @@ -992,6 +993,12 @@ const releaseToConnectLabel = "Release to Connect" } }, [authenticationModalOpen]) + useEffect(() =>{ + if (tenzirConfigModalOpen === false) return; + + getSigmaInfo(); + },[tenzirConfigModalOpen]) + const listOrgCache = (orgId) => { fetch(`${globalUrl}/api/v1/orgs/${orgId}/list_cache`, { method: "GET", @@ -8308,6 +8315,32 @@ const releaseToConnectLabel = "Release to Connect" }); }; + const getSigmaInfo = () => { + const url = globalUrl + "/api/v1/files/detection/sigma_rules"; + + fetch(url, { + method: "GET", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => + response.json().then((responseJson) => { + if (responseJson["success"] === false) { + toast("Failed to get sigma rules"); + } else { + setRules(responseJson.sigma_info); + + } + }) + ) + .catch((error) => { + console.log("Error in getting sigma files: ", error); + toast("An error occurred while fetching sigma rules"); + }); + }; + const parsedHeight = isMobile ? bodyHeight - appBarSize * 4 : bodyHeight - appBarSize - 50 const appViewStyle = { marginLeft: 5, @@ -21160,95 +21193,116 @@ const releaseToConnectLabel = "Release to Connect" ) : null; - const tenzirConfigModal = tenzirConfigModalOpen ? ( - { + if (!tenzirConfigModalOpen) return null; + + const [loading, setLoading] = useState(true); + const [selectedRules, setSelectedRules] = useState([]); + + const handleRuleChange = (event) => { + setSelectedRules(event.target.value); + }; + + const handleSelectAll = () => { + const allEnabledRules = rules.filter(rule => rule.is_enabled).map(rule => rule.file_id); + setSelectedRules(allEnabledRules); + }; + + const handleClose = () => { + setTenzirConfigModalOpen(false); + }; + + const handleSubmit = () => { + const selectedRuleFiles = rules + .filter(rule => selectedRules.includes(rule.file_id)); + + console.log('Selected Rule Files:', selectedRuleFiles); + console.log('Selected Rule File Names:', selectedRuleFiles.map(rule => rule.file_id)); + + setTenzirConfigModalOpen(false); + }; + + + const enabledSigmaInfo = rules.filter(rule => rule.is_enabled); + + + {loading ? ( + + ) : ( +
-
- - - command - param.name === "command")?.value) || ''} - /> + + {selectedOption === 'sigmaRule' && ( + <> + + Select Sigma Rules + + + + + )} + + + + + +
+ )} - - - - - -
- - { - setTenzirConfigModalOpen(false); - }} - > - - - - ) : null; + + + + +} From 6873a2202498eaa82c330c57819344afd9bd5e6c Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Tue, 25 Jun 2024 12:00:03 +0530 Subject: [PATCH 29/46] saving the selected rules to the trigger --- frontend/src/views/AngularWorkflow.jsx | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 9b7755ac..2e3bb393 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -21215,14 +21215,29 @@ const releaseToConnectLabel = "Release to Connect" const handleSubmit = () => { const selectedRuleFiles = rules .filter(rule => selectedRules.includes(rule.file_id)); + + if (selectedTrigger.trigger_type !== "PIPELINE") { + toast("Unable to save the configuration"); + return; + } + + selectedTrigger.parameters = selectedRuleFiles; + console.log('Selected Rule Files:', selectedRuleFiles); console.log('Selected Rule File Names:', selectedRuleFiles.map(rule => rule.file_id)); setTenzirConfigModalOpen(false); }; - + useEffect(()=>{ + if (selectedTrigger.trigger_type !== "PIPELINE") { + //toast("Unable to save the configuration"); + return; + } + setSelectedRules(selectedTrigger.parameters); + },[]) + const enabledSigmaInfo = rules.filter(rule => rule.is_enabled); Date: Tue, 25 Jun 2024 12:01:37 +0530 Subject: [PATCH 30/46] Merge branch '2.0.0' of github.com:satti-hari-krishna-reddy/Shuffle into 2.0.0 --- frontend/src/views/AngularWorkflow.jsx | 181 +++++++++++++------------ 1 file changed, 92 insertions(+), 89 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 2e3bb393..cc4d0b40 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -15416,7 +15416,7 @@ const releaseToConnectLabel = "Release to Connect" toast("please stop the trigger to edit the configuration"); return; } else { - setSelectedOption("Sigma Rulesearch"); + setSelectedOption("SigmaRule"); setTenzirConfigModalOpen(true); }}} style={{ @@ -15432,8 +15432,8 @@ const releaseToConnectLabel = "Release to Connect" setSelectedOption("Sigma Rulesearch")} + checked={selectedOption === "SigmaRule"} + onChange={() => setSelectedOption("SigmaRule")} value={"Sigma Rulesearch"} name="option" /> @@ -21193,25 +21193,23 @@ const releaseToConnectLabel = "Release to Connect" ) : null; - const tenzirConfigModal = () => { - if (!tenzirConfigModalOpen) return null; - - const [loading, setLoading] = useState(true); + const TenzirConfigModal = () => { + const [loading, setLoading] = useState(false); const [selectedRules, setSelectedRules] = useState([]); - + const handleRuleChange = (event) => { setSelectedRules(event.target.value); }; - + const handleSelectAll = () => { const allEnabledRules = rules.filter(rule => rule.is_enabled).map(rule => rule.file_id); setSelectedRules(allEnabledRules); }; - + const handleClose = () => { setTenzirConfigModalOpen(false); }; - + const handleSubmit = () => { const selectedRuleFiles = rules .filter(rule => selectedRules.includes(rule.file_id)); @@ -21239,85 +21237,90 @@ const releaseToConnectLabel = "Release to Connect" },[]) const enabledSigmaInfo = rules.filter(rule => rule.is_enabled); - - - {loading ? ( - - ) : ( -
- - {selectedOption === 'sigmaRule' && ( - <> - - Select Sigma Rules - - - - - )} - - - - - -
- )} - - - - - -} + {loading ? ( + + ) : ( +
+ + {selectedOption === 'SigmaRule' && ( + <> + + Select Sigma Rules + + + + + )} + + + + + +
+ )} + + + + + + ); + } + @@ -21897,7 +21900,7 @@ const releaseToConnectLabel = "Release to Connect" {codePopoutModal} {workflowRevisions} {authenticationModal} - {tenzirConfigModal} + {} {/*editWorkflowModal*/} {authgroupModal} {executionArgumentModal} From 6ec94335649e951c6c9cd84c00d3db9d775344cb Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Tue, 25 Jun 2024 12:35:49 +0000 Subject: [PATCH 31/46] making the select rules option work for pipelines --- backend/go-app/main.go | 4 ++ frontend/src/views/AngularWorkflow.jsx | 93 ++++++++++++++++++-------- 2 files changed, 70 insertions(+), 27 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 202aa891..274f1145 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5199,7 +5199,11 @@ func initHandlers() { r.HandleFunc("/api/v1/files/detection/sigma_rules", shuffle.HandleGetSigmaRules).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/detection/{fileId}/{action}", shuffle.HandleToggleRule).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/files/detection/{action}", shuffle.HandleFolderToggle).Methods("PUT", "OPTIONS") + r.HandleFunc("/api/v1/detection/siem/node_health", handleTenzirHealthUpdate).Methods("POST","OPTIONS") + r.HandleFunc("/api/v1/detection/{triggerId}/selected_rules", shuffle.HandleGetSelectedRules).Methods("GET","OPTIONS") + r.HandleFunc("/api/v1/detection/{triggerId}/selected_rules/save", shuffle.HandleSaveSelectedRules).Methods("POST","OPTIONS") + // Introduced in 0.9.21 to handle notifications for e.g. failed Workflow r.HandleFunc("/api/v1/notifications", shuffle.HandleCreateNotification).Methods("POST", "OPTIONS") diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index cc4d0b40..f3acbf27 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -21194,9 +21194,37 @@ const releaseToConnectLabel = "Release to Connect" ) : null; const TenzirConfigModal = () => { - const [loading, setLoading] = useState(false); + const [loading, setLoading] = useState(true); const [selectedRules, setSelectedRules] = useState([]); + useEffect(() => { + if (tenzirConfigModalOpen) { + try { + const url = globalUrl + "/api/v1/detection/" + selectedTrigger.id + "/selected_rules" + fetch(url, { + method: "GET", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then(response => response.json()) + .then(data => { + const savedRules = data.selected_rules.map(rule => rule.file_id); + setSelectedRules(savedRules); + setLoading(false); + }) + .catch(error => { + console.error('Error fetching selected rules:', error); + setLoading(false); + }); + } catch (error) { + console.error('Error:', error); + setLoading(false); + } + } + }, [tenzirConfigModalOpen]); + const handleRuleChange = (event) => { setSelectedRules(event.target.value); }; @@ -21211,31 +21239,43 @@ const releaseToConnectLabel = "Release to Connect" }; const handleSubmit = () => { - const selectedRuleFiles = rules - .filter(rule => selectedRules.includes(rule.file_id)); - - if (selectedTrigger.trigger_type !== "PIPELINE") { - toast("Unable to save the configuration"); - return; - } - - selectedTrigger.parameters = selectedRuleFiles; - - - console.log('Selected Rule Files:', selectedRuleFiles); - console.log('Selected Rule File Names:', selectedRuleFiles.map(rule => rule.file_id)); - - setTenzirConfigModalOpen(false); - }; - - useEffect(()=>{ - if (selectedTrigger.trigger_type !== "PIPELINE") { - //toast("Unable to save the configuration"); - return; + const selectedRuleFiles = rules.filter(rule => selectedRules.includes(rule.file_id)); + + const payload = { + selected_rules: selectedRuleFiles.map(rule => ({ + file_name: rule.file_name, + title: rule.title, + description: rule.description, + file_id: rule.file_id, + is_enabled: rule.is_enabled, + })) + }; + + try { + const url = globalUrl + "/api/v1/detection/" + selectedTrigger.id + "/selected_rules/save" + fetch(url, { + method: 'POST', + credentials: "include", + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(payload), + }) + .then(response => response.json()) + .then(data => { + console.log('Rules saved successfully:', data); + setTenzirConfigModalOpen(false); + }) + .catch(error => { + console.error('Error saving selected rules:', error); + toast("Unable to save the configuration"); + }); + } catch (error) { + console.error('Error:', error); + toast("Unable to save the configuration"); } - setSelectedRules(selectedTrigger.parameters); - },[]) - + }; + const enabledSigmaInfo = rules.filter(rule => rule.is_enabled); if (!tenzirConfigModalOpen) return null; @@ -21319,8 +21359,7 @@ const releaseToConnectLabel = "Release to Connect" ); - } - + }; From c6b124da3adf5bd40342e7ad3acfbe3480439c80 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Tue, 25 Jun 2024 21:30:43 +0530 Subject: [PATCH 32/46] reverting select rules and adding the kafka ui back --- frontend/src/views/AngularWorkflow.jsx | 318 +++++++++++++------------ 1 file changed, 172 insertions(+), 146 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index f3acbf27..f4604e15 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -535,6 +535,7 @@ const AngularWorkflow = (defaultprops) => { const [selectedOption, setSelectedOption] = React.useState(""); const [tenzirConfigModalOpen, setTenzirConfigModalOpen] = React.useState(false); const [rules, setRules] = React.useState([]); + const [sigmaFilesNames, setSigmaFileNames] = React.useState("") const [distributedFromParent, setDistributedFromParent] = React.useState("") const [suborgWorkflows, setSuborgWorkflows] = React.useState([]) @@ -993,12 +994,6 @@ const releaseToConnectLabel = "Release to Connect" } }, [authenticationModalOpen]) - useEffect(() =>{ - if (tenzirConfigModalOpen === false) return; - - getSigmaInfo(); - },[tenzirConfigModalOpen]) - const listOrgCache = (orgId) => { fetch(`${globalUrl}/api/v1/orgs/${orgId}/list_cache`, { method: "GET", @@ -1473,6 +1468,59 @@ const releaseToConnectLabel = "Release to Connect" setTenzirConfigModalOpen(false); }; + + const handleSubmit = (trigger) => { + if (trigger.trigger_type !== "PIPELINE") { + toast("Unable to save the configuration"); + return; + } + if (selectedOption == "kafka Queue") { + trigger.parameters = [] + + const topic = document.getElementById('topic')?.value + const bootstrapServers = document.getElementById('bootstrap_servers')?.value + const groupId = document.getElementById('group_id')?.value + const autoOffsetReset = document.getElementById('auto_offset_reset')?.value; + + if(topic) { + trigger.parameters.push({ + name: "topic", + value: topic + }) + } else { + toast("Please enter the topic name"); + return; + } + + if (bootstrapServers) { + trigger.parameters.push({ + name: "bootstrap_servers", + value: bootstrapServers + }); + } else { + toast("please enter bootstrap server details"); + return; + } + + if (groupId) { + trigger.parameters.push({ + name: "group_id", + value: groupId + }); + } + + if (autoOffsetReset) { + trigger.parameters.push({ + name: "auto_offset_reset", + value: autoOffsetReset + }); + } + + setTenzirConfigModalOpen(false); + } + + }; + const handleColoring = (actionId, status, label) => { if (cy === undefined) { @@ -15417,7 +15465,18 @@ const releaseToConnectLabel = "Release to Connect" return; } else { setSelectedOption("SigmaRule"); - setTenzirConfigModalOpen(true); + const command = `export | sigma /var/lib/tenzir/sigma_rules | to ${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` + const pipelineConfig = { + command: command, + name: selectedTrigger.label, + type: "create", + environment: selectedTrigger.environment, + workflow_id: workflow.id, + trigger_id: selectedTrigger.id, + start_node: "", + }; + submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig); + }}} style={{ border: "1px solid rgba(255,255,255,0.3)", @@ -21194,90 +21253,6 @@ const releaseToConnectLabel = "Release to Connect" ) : null; const TenzirConfigModal = () => { - const [loading, setLoading] = useState(true); - const [selectedRules, setSelectedRules] = useState([]); - - useEffect(() => { - if (tenzirConfigModalOpen) { - try { - const url = globalUrl + "/api/v1/detection/" + selectedTrigger.id + "/selected_rules" - fetch(url, { - method: "GET", - credentials: "include", - headers: { - "Content-Type": "application/json", - }, - }) - .then(response => response.json()) - .then(data => { - const savedRules = data.selected_rules.map(rule => rule.file_id); - setSelectedRules(savedRules); - setLoading(false); - }) - .catch(error => { - console.error('Error fetching selected rules:', error); - setLoading(false); - }); - } catch (error) { - console.error('Error:', error); - setLoading(false); - } - } - }, [tenzirConfigModalOpen]); - - const handleRuleChange = (event) => { - setSelectedRules(event.target.value); - }; - - const handleSelectAll = () => { - const allEnabledRules = rules.filter(rule => rule.is_enabled).map(rule => rule.file_id); - setSelectedRules(allEnabledRules); - }; - - const handleClose = () => { - setTenzirConfigModalOpen(false); - }; - - const handleSubmit = () => { - const selectedRuleFiles = rules.filter(rule => selectedRules.includes(rule.file_id)); - - const payload = { - selected_rules: selectedRuleFiles.map(rule => ({ - file_name: rule.file_name, - title: rule.title, - description: rule.description, - file_id: rule.file_id, - is_enabled: rule.is_enabled, - })) - }; - - try { - const url = globalUrl + "/api/v1/detection/" + selectedTrigger.id + "/selected_rules/save" - fetch(url, { - method: 'POST', - credentials: "include", - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(payload), - }) - .then(response => response.json()) - .then(data => { - console.log('Rules saved successfully:', data); - setTenzirConfigModalOpen(false); - }) - .catch(error => { - console.error('Error saving selected rules:', error); - toast("Unable to save the configuration"); - }); - } catch (error) { - console.error('Error:', error); - toast("Unable to save the configuration"); - } - }; - - const enabledSigmaInfo = rules.filter(rule => rule.is_enabled); - if (!tenzirConfigModalOpen) return null; return ( @@ -21302,65 +21277,116 @@ const releaseToConnectLabel = "Release to Connect" }, }} > - {loading ? ( - - ) : ( -
- - {selectedOption === 'SigmaRule' && ( - <> - - Select Sigma Rules - - - - - )} - - - - - -
- )} + +
Configuration options for Kafka
+ + + {selectedOption === "Kafka Queue" ? ( +
+ Topic + param.name === "topic", + )?.value || "" + } + /> + bootstrap.servers + param.name === "bootstrap_servers", + )?.value || "" + } + /> + group.id + param.name === "group_id", + )?.value || "" + } + /> + auto.offest.reset + param.name === "auto_offset_reset", + )?.value || "" + } + /> +
+ ) : null}{" "} + - - - + + + + ); }; - + const SuggestionBoxUi = () => { From a738be78bbed9afe11141f1156723d6ec8e47e03 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Tue, 25 Jun 2024 16:36:59 +0000 Subject: [PATCH 33/46] fixing few typos --- frontend/src/views/AngularWorkflow.jsx | 45 ++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index f4604e15..7a16c6e9 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1474,7 +1474,7 @@ const releaseToConnectLabel = "Release to Connect" toast("Unable to save the configuration"); return; } - if (selectedOption == "kafka Queue") { + if (selectedOption === "Kafka Queue") { trigger.parameters = [] const topic = document.getElementById('topic')?.value @@ -15541,8 +15541,41 @@ const releaseToConnectLabel = "Release to Connect" disabled={selectedTrigger.status === "running"} onClick={() => { - let command = (selectedTrigger?.parameters?.find(param => param.name === "command")?.value) || '' - command = `${command} | to ${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` + if (selectedOption === "Kafka Queue"){ + + const topic = (selectedTrigger?.parameters?.find(param => param.name === "topic")?.value) || '' + const bootstrapServers = (selectedTrigger?.parameters?.find(param => param.name === "bootstrap_servers")?.value) || '' + const groupId = (selectedTrigger?.parameters?.find(param => param.name === "group_id")?.value) || '' + const autoOffsetReset = (selectedTrigger?.parameters?.find(param => param.name === "auto_offset_reset")?.value) || '' + let command = "from kafka" + + if(topic) { + command = `${command} -t ${topic}` + } else { + toast("please enter the topic name") + return; + } + if(bootstrapServers) { + command = `${command} -e -o stored -X bootstrap.servers=${bootstrapServers}` + } else { + toast("please enter the bootstrap servers details") + return; + } + + if(groupId) { + command = `${command},group.id=${groupId}` + } else { + command = `${command},group.id=${selectedTrigger.id}` + } + if(autoOffsetReset) { + command = `${command},auto.offset.reset=${autoOffsetReset}` + } else { + command = `${command},auto.offset.reset=earliest` + + } + command = `${command},auto.offset.reset=earliest` + command = `${command},client.id=${selectedTrigger.id},enable.auto.commit=true,auto.commit.interval.ms=1` + command = `${command} read json | to ${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` const pipelineConfig = { command: command, @@ -15554,7 +15587,7 @@ const releaseToConnectLabel = "Release to Connect" start_node: "", }; submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig); - }} + }}} color="primary" > Start @@ -21268,8 +21301,8 @@ const releaseToConnectLabel = "Release to Connect" pointerEvents: "auto", color: "white", minWidth: 600, - minHeight: 200, - maxHeight: 200, + minHeight: 550, + maxHeight: 550, padding: 15, overflow: "hidden", zIndex: 10012, From d4ce234fa9c0d3dd59e7596ea9cd1e30dbefab47 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Wed, 26 Jun 2024 16:34:19 +0530 Subject: [PATCH 34/46] adding endpoint option for syslog --- frontend/src/views/AngularWorkflow.jsx | 61 +++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 7a16c6e9..2ed1f493 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1517,6 +1517,20 @@ const releaseToConnectLabel = "Release to Connect" } setTenzirConfigModalOpen(false); + } else if (selectedOption === "Syslog listener") { + trigger.parameters = [] + + const endpoint = document.getElementById('endpoint')?.value + + if(endpoint) { + trigger.parameters.push({ + name: "endpoint", + value: endpoint + }) + } else { + toast("Please enter your endpoint"); + return; + } } }; @@ -15587,7 +15601,28 @@ const releaseToConnectLabel = "Release to Connect" start_node: "", }; submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig); - }}} + } else if (selectedOption === "Syslog listener"){ + let command = "" + const endpoint = (selectedTrigger?.parameters?.find(param => param.name === "endpoint")?.value) || '' + if(endpoint) { + command = `from tcp://${endpoint} | read syslog | import` + } else { + toast("please enter the topic name") + return; + } + + const pipelineConfig = { + command: command, + name: selectedTrigger.label, + type: "create", + environment: selectedTrigger.environment, + workflow_id: workflow.id, + trigger_id: selectedTrigger.id, + start_node: "", + }; + submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig); + } + }} color="primary" > Start @@ -21394,6 +21429,30 @@ const releaseToConnectLabel = "Release to Connect" />
) : null}{" "} + +{selectedOption === "Syslog listener" ? ( +
+ End Point + param.name === "endpoint", + )?.value || "" + } + /> +
+ ) : null}{" "} From 78b6d9e0673c2f86c62f73e99b13311fd87239e2 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Thu, 27 Jun 2024 05:22:49 +0000 Subject: [PATCH 35/46] bug fixes --- backend/go-app/main.go | 2 +- frontend/src/views/Detection.jsx | 4 ++-- frontend/src/views/DetectionDashboard.jsx | 2 +- functions/onprem/orborus/orborus.go | 4 +--- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 274f1145..7864a427 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -2169,7 +2169,7 @@ func handleTenzirHealthUpdate(resp http.ResponseWriter, request *http.Request) { status := healthUpdate.Status result, err := shuffle.GetDisabledRules(ctx) - if (err != nil && err.Error() != "rules doesn't exist") || err == nil { + if (err != nil && err.Error() == "rules doesn't exist") || err == nil { result.IsTenzirActive = status result.LastActive = time.Now().Unix() diff --git a/frontend/src/views/Detection.jsx b/frontend/src/views/Detection.jsx index 8b6214a1..f7e2d135 100644 --- a/frontend/src/views/Detection.jsx +++ b/frontend/src/views/Detection.jsx @@ -136,7 +136,7 @@ const Detection = ({ }); }; - const filteredRules = ruleInfo.filter((rule) => + const filteredRules = ruleInfo?.filter((rule) => rule.title.toLowerCase().includes(searchQuery.toLowerCase()) || rule.description.toLowerCase().includes(searchQuery.toLowerCase()) ); @@ -218,7 +218,7 @@ const Detection = ({ p: 1, }} > - {filteredRules.length > 0 && + {filteredRules?.length > 0 && filteredRules.map((card) => ( { }, [folderDisabled]); useEffect(() => { - if (ruleInfo.length > 0) { + if (ruleInfo?.length > 0) { openEditBar(ruleInfo[0]); } }, [ruleInfo]); diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 17a76611..d08f2bed 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -102,6 +102,7 @@ var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME") var orborusLabel = os.Getenv("SHUFFLE_ORBORUS_LABEL") var memcached = os.Getenv("SHUFFLE_MEMCACHED") var tenzirUrl = os.Getenv("SHUFFLE_TENZIR_URL") +var apiKey = os.Getenv("AUTH_FOR_ORBORUS") var executionIds = []string{} var namespacemade = false // For K8s @@ -2923,8 +2924,6 @@ func searchPipeline(identifier string) (string, error) { func handleFileCategoryChange() error{ apiEndpoint := baseUrl+"/api/v1/files/namespaces/sigma" - apiKey := "12e7150e-1e03-4834-a839-de4688f50ad0" - req, err := http.NewRequest("GET", apiEndpoint, nil) if err != nil { return err @@ -3116,7 +3115,6 @@ func sendTenzirHealthStatus() error { log.Printf("[ERROR] Received non-successful HTTP status code: %d", resp.StatusCode) return fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode) } - log.Printf("this is send successfully") return nil } From 4708d7d9078924a97813ebb9f92dac0572b09e61 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Thu, 27 Jun 2024 11:59:31 +0000 Subject: [PATCH 36/46] adding api key to use for orborus to download files --- .env | 1 + 1 file changed, 1 insertion(+) diff --git a/.env b/.env index 298128cc..5d6b0288 100755 --- a/.env +++ b/.env @@ -40,6 +40,7 @@ BACKEND_HOSTNAME=shuffle-backend BACKEND_PORT=5001 FRONTEND_PORT=3001 FRONTEND_PORT_HTTPS=3443 +AUTH_FOR_ORBORUS = # CHANGE THIS IF YOU WANT GOOD LOCAL EXECUTIONS: OUTER_HOSTNAME=shuffle-backend From dbbe54a8c6362c7f8b82dd48af3913e52f8fde19 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Wed, 3 Jul 2024 11:09:05 +0530 Subject: [PATCH 37/46] adding an endpoint to spin up tenzir node --- backend/go-app/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 7864a427..4ac495de 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5200,6 +5200,7 @@ func initHandlers() { r.HandleFunc("/api/v1/files/detection/{fileId}/{action}", shuffle.HandleToggleRule).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/files/detection/{action}", shuffle.HandleFolderToggle).Methods("PUT", "OPTIONS") + r.HandleFunc("/api/v1/detection/siem/connect", shuffle.HandleConnectSiem).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/detection/siem/node_health", handleTenzirHealthUpdate).Methods("POST","OPTIONS") r.HandleFunc("/api/v1/detection/{triggerId}/selected_rules", shuffle.HandleGetSelectedRules).Methods("GET","OPTIONS") r.HandleFunc("/api/v1/detection/{triggerId}/selected_rules/save", shuffle.HandleSaveSelectedRules).Methods("POST","OPTIONS") From ccad70a2ee4cca6f1d862b7e9309b07ff85f86c3 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Wed, 3 Jul 2024 11:09:52 +0530 Subject: [PATCH 38/46] orborus can now start the tenzir node based on the request --- functions/onprem/orborus/orborus.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index d08f2bed..391a1620 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2072,7 +2072,16 @@ func main() { } toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - } else { + } else if incRequest.Type == "START_TENZIR" { + + err := deployTenzirNode() + if err != nil{ + log.Printf("[ERROR] failed to deploy the pipeline, reason: %s", err) + } + + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + + } else { newrequests = append(newrequests, incRequest) } } From d7741db32f8e19d21dbca7a25b5cf2ab6701d413 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Wed, 3 Jul 2024 11:10:13 +0530 Subject: [PATCH 39/46] adding connect to siem button --- frontend/src/views/Detection.jsx | 58 ++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/frontend/src/views/Detection.jsx b/frontend/src/views/Detection.jsx index f7e2d135..3f909758 100644 --- a/frontend/src/views/Detection.jsx +++ b/frontend/src/views/Detection.jsx @@ -10,12 +10,7 @@ import { import { Publish as PublishIcon } from "@mui/icons-material"; import { toast } from "react-toastify"; import RuleCard from "./RuleCard"; -import { styled } from "@mui/system"; - -const ConnectedButton = styled(Button)(({ theme, isConnected }) => ({ - backgroundColor: isConnected ? "green" : "red", - color: "white", -})); +import CircularProgress from "@material-ui/core/CircularProgress"; const handleDirectoryChange = (folderDisabled, setFolderDisabled, globalUrl) => { const action = folderDisabled ? "enable_folder" : "disable_folder"; @@ -54,6 +49,7 @@ const Detection = ({ }) => { const [searchQuery, setSearchQuery] = useState(""); const uploadRef = useRef(null); + const [loading, setLoading] = useState(false); const uploadFiles = (files) => { for (const key in files) { @@ -136,6 +132,41 @@ const Detection = ({ }); }; + const handleConnectClick = () => { + if (!isTenzirActive) { + setLoading(true); + const url = `${globalUrl}/api/v1/detection/siem/connect`; + + fetch(url, { + method: "GET", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => + response.json().then((responseJson) => { + if (responseJson["success"] === true) { + setTimeout(() => { + setLoading(false); + window.location.reload(); + }, 5000); + } else { + setLoading(false); + toast("Failed to connect to SIEM"); + } + }) + ) + .catch((error) => { + setLoading(false); + console.log(`Error in connecting to SIEM: `, error); + toast("An error occurred while connecting to SIEM"); + }); + } else { + console.log("Already connected to SIEM"); + } + }; + const filteredRules = ruleInfo?.filter((rule) => rule.title.toLowerCase().includes(searchQuery.toLowerCase()) || rule.description.toLowerCase().includes(searchQuery.toLowerCase()) @@ -155,9 +186,14 @@ const Detection = ({ Sigma Detection Rules - - {isTenzirActive ? "Connected to SIEM" : "Not Connected to SIEM"} - + setSearchQuery(e.target.value)} /> -
+ const defaultEnvironment = environments.find( + (env) => env.default && env.Name.toLowerCase() !== "cloud" + ); + + if (selectedTrigger.environment === "onprem" && defaultEnvironment !== undefined) { + selectedTrigger.environment = defaultEnvironment.Name + setSelectedTrigger(selectedTrigger) } + const PipelineSidebar = Object.getOwnPropertyNames(selectedTrigger).length === 0 || workflow.triggers[selectedTriggerIndex] === undefined && selectedTrigger.trigger_type !== "SCHEDULE" ? null :

From 295bcf07a450078c71f0fd72469a63659da4034e Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Tue, 16 Jul 2024 20:25:13 +0530 Subject: [PATCH 41/46] adding a new endpoint for downloading files from a repo --- backend/go-app/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 4ac495de..8f247fc8 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5188,6 +5188,7 @@ func initHandlers() { // PS: For cloud, this has to use cloud storage. // https://developer.box.com/reference/get-files-id-content/ r.HandleFunc("/api/v1/files/download_remote", shuffle.HandleDownloadRemoteFiles).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v2/files/download_remote", shuffle.HandleDownloadRemoteFiles2).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/files/namespaces/{namespace}", shuffle.HandleGetFileNamespace).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}/content", shuffle.HandleGetFileContent).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/create", shuffle.HandleCreateFile).Methods("POST", "OPTIONS") From 94749e679e66dca9421d20071068456d13094d93 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Tue, 16 Jul 2024 20:26:11 +0530 Subject: [PATCH 42/46] making the pipeline side bar to auto select the defualt env initially --- frontend/src/views/AngularWorkflow.jsx | 96 ++++++++++---------------- 1 file changed, 38 insertions(+), 58 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index b74a7afd..c3ce2581 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -8290,11 +8290,11 @@ const releaseToConnectLabel = "Release to Connect" toast("Pipeline deleted!") return } - + if (trigger.parameters){ trigger.parameters.push({ name: data.name, value: data.command, - }); + });} if (data.type === "stop") trigger.status = "stopped"; else trigger.status = "running"; @@ -8302,7 +8302,6 @@ const releaseToConnectLabel = "Release to Connect" setSelectedTrigger(trigger); setWorkflow(workflow); - console.log("Should set the status to running and save"); saveWorkflow(workflow); } }) @@ -15347,7 +15346,7 @@ const releaseToConnectLabel = "Release to Connect" (env) => env.default && env.Name.toLowerCase() !== "cloud" ); - if (selectedTrigger.environment === "onprem" && defaultEnvironment !== undefined) { + if (selectedTrigger.trigger_type === "PIPELINE" && selectedTrigger.environment === "onprem" && defaultEnvironment !== undefined) { selectedTrigger.environment = defaultEnvironment.Name setSelectedTrigger(selectedTrigger) } @@ -15450,11 +15449,25 @@ const releaseToConnectLabel = "Release to Connect" key="syslogListener" onClick={() => { if(selectedTrigger.status === "running"){ - toast("please stop the trigger to edit the configuration"); + //toast("please stop the trigger to edit the configuration"); return; } else { setSelectedOption("Syslog listener"); - setTenzirConfigModalOpen(true); + const url = `${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` + const command = `from tcp://192.168.1.100:5162 read syslog | import` + const pipelineConfig = { + command: command, + name: selectedTrigger.label, + type: "create", + environment: selectedTrigger.environment, + workflow_id: workflow.id, + trigger_id: selectedTrigger.id, + start_node: "", + url:url, + }; + submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig); + + }}} style={{ border: "1px solid rgba(255,255,255,0.3)", @@ -15470,12 +15483,15 @@ const releaseToConnectLabel = "Release to Connect" control={ setSelectedOption("Syslog listener")} + onChange={() => { + if (selectedTrigger.status !== "running"){ + setSelectedOption("Syslog listener")}} + } value={"Syslog listener"} name="option" /> } - label="Start Syslog listener" + label= {selectedOption === "Syslog listener" && selectedTrigger.status === "running" ? "listening at 192.168.1.100:5162" : "Start Syslog listener"} />

@@ -15483,11 +15499,12 @@ const releaseToConnectLabel = "Release to Connect" key="sigmaRulesearch" onClick={() => { if(selectedTrigger.status === "running"){ - toast("please stop the trigger to edit the configuration"); + // toast("please stop the trigger to edit the configuration"); return; } else { setSelectedOption("SigmaRule"); - const command = `export | sigma /var/lib/tenzir/sigma_rules | to ${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` + const url = `${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` + const command = `export | sigma /var/lib/tenzir/sigma_rules | to ${url}` const pipelineConfig = { command: command, name: selectedTrigger.label, @@ -15496,6 +15513,7 @@ const releaseToConnectLabel = "Release to Connect" workflow_id: workflow.id, trigger_id: selectedTrigger.id, start_node: "", + url:url, }; submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig); @@ -15514,7 +15532,9 @@ const releaseToConnectLabel = "Release to Connect" control={ setSelectedOption("SigmaRule")} + onChange={() => { + if (selectedTrigger.status !== "running"){ + setSelectedOption("SigmaRule")}}} value={"Sigma Rulesearch"} name="option" /> @@ -15547,7 +15567,9 @@ const releaseToConnectLabel = "Release to Connect" control={ setSelectedOption("Kafka Queue")} + onChange={() => { + if (selectedTrigger.status !== "running"){ + setSelectedOption("Kafka Queue")}}} value={"Kafka Queue"} name="option" /> @@ -15564,7 +15586,7 @@ const releaseToConnectLabel = "Release to Connect" onClick={() => { if (selectedOption === "Kafka Queue"){ - + const url = `${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` const topic = (selectedTrigger?.parameters?.find(param => param.name === "topic")?.value) || '' const bootstrapServers = (selectedTrigger?.parameters?.find(param => param.name === "bootstrap_servers")?.value) || '' const groupId = (selectedTrigger?.parameters?.find(param => param.name === "group_id")?.value) || '' @@ -15594,10 +15616,10 @@ const releaseToConnectLabel = "Release to Connect" } else { command = `${command},auto.offset.reset=earliest` - } + } command = `${command},auto.offset.reset=earliest` command = `${command},client.id=${selectedTrigger.id},enable.auto.commit=true,auto.commit.interval.ms=1` - command = `${command} read json | to ${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` + command = `${command} read json | to ${url}` const pipelineConfig = { command: command, @@ -15607,28 +15629,9 @@ const releaseToConnectLabel = "Release to Connect" workflow_id: workflow.id, trigger_id: selectedTrigger.id, start_node: "", + url: url, }; submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig); - } else if (selectedOption === "Syslog listener"){ - let command = "" - const endpoint = (selectedTrigger?.parameters?.find(param => param.name === "endpoint")?.value) || '' - if(endpoint) { - command = `from tcp://${endpoint} | read syslog | import` - } else { - toast("please enter the topic name") - return; - } - - const pipelineConfig = { - command: command, - name: selectedTrigger.label, - type: "create", - environment: selectedTrigger.environment, - workflow_id: workflow.id, - trigger_id: selectedTrigger.id, - start_node: "", - }; - submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig); } }} color="primary" @@ -21438,29 +21441,6 @@ const releaseToConnectLabel = "Release to Connect"
) : null}{" "} -{selectedOption === "Syslog listener" ? ( -
- End Point - param.name === "endpoint", - )?.value || "" - } - /> -
- ) : null}{" "} From d6b78612df49b9a415c640d71c53ee13775bb3bf Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Tue, 16 Jul 2024 20:27:09 +0530 Subject: [PATCH 43/46] making enable / disable button to work only when connected to the siem --- frontend/src/views/Detection.jsx | 100 ++----------- frontend/src/views/DetectionDashboard.jsx | 174 ++++++++++++++-------- frontend/src/views/RuleCard.jsx | 78 +++++----- 3 files changed, 164 insertions(+), 188 deletions(-) diff --git a/frontend/src/views/Detection.jsx b/frontend/src/views/Detection.jsx index 3f909758..c86eeba2 100644 --- a/frontend/src/views/Detection.jsx +++ b/frontend/src/views/Detection.jsx @@ -1,4 +1,4 @@ -import React, { useState, useRef } from "react"; +import React, { useState } from "react"; import { Container, Box, @@ -7,12 +7,16 @@ import { Typography, Button, } from "@mui/material"; -import { Publish as PublishIcon } from "@mui/icons-material"; import { toast } from "react-toastify"; import RuleCard from "./RuleCard"; import CircularProgress from "@material-ui/core/CircularProgress"; -const handleDirectoryChange = (folderDisabled, setFolderDisabled, globalUrl) => { +const handleDirectoryChange = (folderDisabled, setFolderDisabled, globalUrl, isTenzirActive) => { + + if (!isTenzirActive) { + toast("connect to siem first for global enable/disable to work"); + return; + } const action = folderDisabled ? "enable_folder" : "disable_folder"; const url = `${globalUrl}/api/v1/files/detection/${action}`; @@ -44,94 +48,11 @@ const Detection = ({ ruleInfo, folderDisabled, setFolderDisabled, - openEditBar, isTenzirActive, }) => { const [searchQuery, setSearchQuery] = useState(""); - const uploadRef = useRef(null); const [loading, setLoading] = useState(false); - const uploadFiles = (files) => { - for (const key in files) { - try { - const filename = files[key].name; - const filedata = new FormData(); - filedata.append("shuffle_file", files[key]); - - if (typeof files[key] === "object") { - handleCreateFile(filename, filedata); - } - } catch (e) { - console.log("Error in dropzone: ", e); - } - } - - setTimeout(() => { - // Additional logic if needed - }, 2500); - }; - - const handleCreateFile = (filename, file) => { - const data = { - filename: filename, - org_id: "default", - workflow_id: "global", - namespace: "sigma", - }; - - fetch(globalUrl + "/api/v1/files/create", { - method: "POST", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - credentials: "include", - body: JSON.stringify(data), - }) - .then((response) => { - if (response.status !== 200) { - console.log("Status not 200 for apps :O!"); - return; - } - - return response.json(); - }) - .then((responseJson) => { - if (responseJson.success === true) { - handleFileUpload(responseJson.id, file); - } else { - toast("Failed to upload file ", filename); - } - }) - .catch((error) => { - toast("Failed to upload file ", filename); - console.log(error.toString()); - }); - }; - - const handleFileUpload = (file_id, file) => { - fetch(`${globalUrl}/api/v1/files/${file_id}/upload`, { - method: "POST", - credentials: "include", - body: file, - }) - .then((response) => { - if (response.status !== 200 && response.status !== 201) { - console.log("Status not 200 for apps :O!"); - toast("File was created, but failed to upload."); - return; - } - - return response.json(); - }) - .then((responseJson) => { - // Handle the response as needed - }) - .catch((error) => { - toast(error.toString()); - }); - }; - const handleConnectClick = () => { if (!isTenzirActive) { setLoading(true); @@ -150,7 +71,7 @@ const Detection = ({ setTimeout(() => { setLoading(false); window.location.reload(); - }, 5000); + }, 15000); } else { setLoading(false); toast("Failed to connect to SIEM"); @@ -240,8 +161,9 @@ const Detection = ({ - handleDirectoryChange(folderDisabled, setFolderDisabled, globalUrl) + handleDirectoryChange(folderDisabled, setFolderDisabled, globalUrl, isTenzirActive) } + disabled={!isTenzirActive} /> @@ -263,7 +185,7 @@ const Detection = ({ file_id={card.file_id} globalUrl={globalUrl} folderDisabled={folderDisabled} - openEditBar={() => openEditBar(card)} + isTenzirActive={isTenzirActive} {...card} /> ))} diff --git a/frontend/src/views/DetectionDashboard.jsx b/frontend/src/views/DetectionDashboard.jsx index e09814e4..4b2b1b06 100644 --- a/frontend/src/views/DetectionDashboard.jsx +++ b/frontend/src/views/DetectionDashboard.jsx @@ -1,68 +1,45 @@ import React, { useState, useEffect } from "react"; -import { Container} from "@mui/material"; +import { Container, CircularProgress, Typography } from "@mui/material"; import { toast } from "react-toastify"; import Detection from "./Detection"; -import EditComponent from "./EditRules"; - -const getSigmaInfo = (globalUrl, setRuleInfo, setFolderDisabled, setIsTenzirActive) => { - const url = globalUrl + "/api/v1/files/detection/sigma_rules"; - - fetch(url, { - method: "GET", - credentials: "include", - headers: { - "Content-Type": "application/json", - }, - }) - .then((response) => - response.json().then((responseJson) => { - if (responseJson["success"] === false) { - toast("Failed to get sigma rules"); - } else { - setRuleInfo(responseJson.sigma_info); - setFolderDisabled(responseJson.folder_disabled); - setIsTenzirActive(responseJson.is_tenzir_active); - - } - }) - ) - .catch((error) => { - console.log("Error in getting sigma files: ", error); - toast("An error occurred while fetching sigma rules"); - }); -}; const DetectionDashBoard = (props) => { const { globalUrl } = props; - const [ruleInfo, setRuleInfo] = useState([]); - const [selectedRule, setSelectedRule] = useState(null); - const [fileData, setFileData] = React.useState(""); - const [isTenzirActive, setIsTenzirActive] = React.useState(false); - + const [ruleInfo, setRuleInfo] = useState(null); + const [, setSelectedRule] = useState(null); + const [, setFileData] = useState(""); + const [isTenzirActive, setIsTenzirActive] = useState(false); const [folderDisabled, setFolderDisabled] = useState(false); + const [isLoading, setIsLoading] = useState(false); + const [importAttempts, setImportAttempts] = useState(0); + const maxImportAttempts = 2; useEffect(() => { - getSigmaInfo(globalUrl, setRuleInfo, setFolderDisabled, setIsTenzirActive); - }, [folderDisabled]); + const fetchTimeout = setTimeout(() => { + fetchSigmaInfo(); + }, 1000); // Delay by 1 second + + return () => clearTimeout(fetchTimeout); + }, [globalUrl]); useEffect(() => { - if (ruleInfo?.length > 0) { - openEditBar(ruleInfo[0]); + if (ruleInfo && ruleInfo.length === 0 && importAttempts < maxImportAttempts) { + importSigmaFromUrl(); } }, [ruleInfo]); const openEditBar = (rule) => { setSelectedRule(rule); - getFileContent(rule.file_id) + fetchFileContent(rule.file_id); }; const handleSave = (updatedContent) => { - toast("this will be saved"); + toast("This will be saved"); }; - const getFileContent = (file_id) => { + const fetchFileContent = (file_id) => { setFileData(""); - fetch(globalUrl + "/api/v1/files/" + file_id + "/content", { + fetch(`${globalUrl}/api/v1/files/${file_id}/content`, { method: "GET", headers: { "Content-Type": "application/json", @@ -77,38 +54,109 @@ const DetectionDashBoard = (props) => { } return response.text(); }) - .then((respdata) => { + .then((respdata) => { if (respdata.length === 0) { toast("Failed getting file. Is it deleted?"); return; } - return respdata - }) - .then((responseData) => { - - setFileData(responseData); + setFileData(respdata); }) .catch((error) => { toast(error.toString()); }); }; + const fetchSigmaInfo = () => { + const url = `${globalUrl}/api/v1/files/detection/sigma_rules`; + setIsLoading(true); + + fetch(url, { + method: "GET", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => response.json()) + .then((responseJson) => { + if (responseJson["success"] === false) { + toast("Failed to get sigma rules"); + } else { + setRuleInfo(responseJson.sigma_info || []); + setFolderDisabled(responseJson.folder_disabled); + setIsTenzirActive(responseJson.is_tenzir_active); + } + setIsLoading(false); + }) + .catch((error) => { + setIsLoading(false); + console.log("Error in getting sigma files: ", error); + toast("An error occurred while fetching sigma rules"); + setRuleInfo([]); + }); + }; + + const importSigmaFromUrl = () => { + setIsLoading(true); + setImportAttempts((prevAttempts) => prevAttempts + 1); + + const url = "https://github.com/satti-hari-krishna-reddy/shuffle_sigma"; + const folder = "sigma"; + + const parsedData = { + url: url, + path: folder, + field_3: "main", + }; + + toast(`Getting files from url ${url}. This may take a while if the repository is large. Please wait...`); + fetch(`${globalUrl}/api/v2/files/download_remote`, { + method: "POST", + mode: "cors", + headers: { + Accept: "application/json", + }, + body: JSON.stringify(parsedData), + credentials: "include", + }) + .then((response) => response.json()) + .then((responseJson) => { + if (responseJson.success) { + toast("Successfully loaded files from " + url); + fetchSigmaInfo(); // Fetch again after successful import + } else { + toast(responseJson.reason ? `Failed loading: ${responseJson.reason}` : "Failed loading"); + } + setIsLoading(false); + }) + .catch((error) => { + toast(error.toString()); + setIsLoading(false); + }); + }; + + if (isLoading && (!ruleInfo || ruleInfo.length === 0)) { + return ( + +
+ + Downloading rules, please wait... +
+ + ); + } + return ( - - {/* {selectedRule ? ( - - ) : null} */} - + + - ); + ); }; export default DetectionDashBoard; diff --git a/frontend/src/views/RuleCard.jsx b/frontend/src/views/RuleCard.jsx index fe334f85..b35d993a 100644 --- a/frontend/src/views/RuleCard.jsx +++ b/frontend/src/views/RuleCard.jsx @@ -10,7 +10,7 @@ import EditIcon from "@mui/icons-material/Edit"; import { toast } from "react-toastify"; import ShuffleCodeEditor from "../components/ShuffleCodeEditor1.jsx"; -const RuleCard = ({ ruleName, description, file_id, globalUrl, folderDisabled, ...otherProps }) => { +const RuleCard = ({ ruleName, description, file_id, globalUrl, folderDisabled, isTenzirActive, ...otherProps }) => { const [openCodeEditor, setOpenCodeEditor] = React.useState(false); const [fileData, setFileData] = React.useState(""); const [isEnabled, setIsEnabled] = React.useState(otherProps.is_enabled); @@ -22,6 +22,10 @@ const RuleCard = ({ ruleName, description, file_id, globalUrl, folderDisabled, . toast("enable the directory to enable individual rules"); return; } + if (!isTenzirActive) { + toast("connect to the siem to enable/disable the rule"); + return; + } const newIsEnabled = event.target.checked; toggleRule(file_id, !newIsEnabled, globalUrl, () => { setIsEnabled(newIsEnabled); @@ -73,6 +77,7 @@ const RuleCard = ({ ruleName, description, file_id, globalUrl, folderDisabled, . @@ -121,42 +126,43 @@ const toggleRule = (fileId, isCurrentlyEnabled, globalUrl, callback) => { toast(`An error occurred while ${action}ing the rule`); }); }; - const openEditBar = (file_id, setOpenCodeEditor, setFileData, globalUrl) => { - getFileContent(file_id, setFileData, globalUrl); + +const openEditBar = (file_id, setOpenCodeEditor, setFileData, globalUrl) => { + getFileContent(file_id, setFileData, globalUrl) + setOpenCodeEditor(true); - }; +}; - const getFileContent = (file_id, setFileData, globalUrl) => { - setFileData(""); - fetch(globalUrl + "/api/v1/files/" + file_id + "/content", { - method: "GET", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - credentials: "include", +const getFileContent = (file_id, setFileData, globalUrl) => { + setFileData(""); + fetch(globalUrl + "/api/v1/files/" + file_id + "/content", { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for file :O!"); + return ""; + } + return response.text(); }) - .then((response) => { - if (response.status !== 200) { - console.log("Status not 200 for file :O!"); - return ""; - } - return response.text(); - }) - .then((respdata) => { - if (respdata.length === 0) { - toast("Failed getting file. Is it deleted?"); - return; - } - return respdata - }) - .then((responseData) => { - - setFileData(responseData); - }) - .catch((error) => { - toast(error.toString()); - }); - }; - + .then((respdata) => { + if (respdata.length === 0) { + toast("Failed getting file. Is it deleted?"); + return; + } + return respdata + }) + .then((responseData) => { + + setFileData(responseData); + }) + .catch((error) => { + toast(error.toString()); + }); +}; export default RuleCard; From 9cf3f2fd6333ea46ce6dcbd1a519fd8552cb4c91 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Tue, 16 Jul 2024 20:28:00 +0530 Subject: [PATCH 44/46] a lot of things --- functions/onprem/orborus/orborus.go | 316 ++++++++++++++++++++++------ 1 file changed, 246 insertions(+), 70 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 391a1620..411571bb 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2052,14 +2052,28 @@ func main() { log.Printf("[ERROR] failed to deploy the pipeline, reason: %s", err) } - err = removeFile(fileName) + err = disableRule(fileName) if err != nil { log.Printf("[ERROR] Failed to disable the sigma file %s, reason: %s", fileName, err) } - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - } else if incRequest.Type == "DISABLE_SIGMA_FOLDER" { + } else if incRequest.Type == "ENABLE_SIGMA_FILE" { + fileName := incRequest.ExecutionArgument + err := deployTenzirNode() + if err != nil{ + log.Printf("[ERROR] failed to deploy the pipeline, reason: %s", err) + } + + err = enableRule(fileName) + if err != nil { + log.Printf("[ERROR] Failed to disable the sigma file %s, reason: %s", fileName, err) + } + + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + + } else if incRequest.Type == "DISABLE_SIGMA_FOLDER" { err := deployTenzirNode() if err != nil{ @@ -2526,9 +2540,19 @@ func deployTenzirNode() error { return nil } - containerInfo, err := dockercli.ContainerInspect(ctx, containerName) - if err != nil { - if dockerclient.IsErrNotFound(err) { + containerInfo, err := dockercli.ContainerInspect(ctx, containerName) + if err != nil { + if dockerclient.IsErrNotFound(err) { + // Create network if it doesn't exist + networkName := "tenzir-network" + networkSubnet := "192.168.1.0/24" + networkGateway := "192.168.1.1" + + err = createNetworkIfNotExists(ctx, networkName, networkSubnet, networkGateway) + if err != nil { + log.Printf("[ERROR] Failed to create network: %s", err) + return err + } // Check if image exists _, _, err := dockercli.ImageInspectWithRaw(ctx, imageName) @@ -2589,30 +2613,6 @@ func deployTenzirNode() error { return nil } -func checkTenzirNode() error { - retries := 5 - retryInterval := 3 * time.Second - url := fmt.Sprintf("%s/api/v0/ping",tenzirUrl) - forwardMethod := "POST" - - client := http.Client{} - req, err := http.NewRequest(forwardMethod, url, nil) - if err != nil { - log.Printf("[ERROR] Failed to create HTTP request: %s", err) - return err - } - - for i := 0; i < retries; i++ { - resp, err := client.Do(req) - if err == nil && resp.StatusCode == http.StatusOK { - return nil - } - time.Sleep(retryInterval) - } - - return fmt.Errorf("tenzir node is not available") -} - func createAndStartTenzirNode(ctx context.Context, containerName, imageName string, containerStartOptions container.StartOptions) error { healthconfig := &container.HealthConfig{ Test: []string{"tenzir --connection-timeout=30s --connection-retry-delay=1s 'api /ping'"}, @@ -2628,23 +2628,34 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri Entrypoint: []string{containerName}, } - hostConfig := &container.HostConfig{ - PortBindings: nat.PortMap{ - "5160/tcp": []nat.PortBinding{{HostPort: "5160"}}, - }, - Mounts: []mount.Mount{ - { - Type: mount.TypeVolume, - Source: containerName, - Target: "/var/lib/tenzir/", - }, - }, - VolumeDriver: "local", - } - _, err := dockercli.ContainerCreate(ctx, config, hostConfig, nil, nil, containerName) - if err != nil { - return err - } + hostConfig := &container.HostConfig{ + PortBindings: nat.PortMap{ + "5160/tcp": []nat.PortBinding{{HostPort: "5160"}}, + }, + Mounts: []mount.Mount{ + { + Type: mount.TypeVolume, + Source: containerName, + Target: "/var/lib/tenzir/", + }, + }, + VolumeDriver: "local", + } + + networkingConfig := &network.NetworkingConfig{ + EndpointsConfig: map[string]*network.EndpointSettings{ + "tenzir-network": { + IPAMConfig: &network.EndpointIPAMConfig{ + IPv4Address: "192.168.1.100", + }, + }, + }, + } + + _, err := dockercli.ContainerCreate(ctx, config, hostConfig, networkingConfig, nil, containerName) + if err != nil { + return err + } err = dockercli.ContainerStart(ctx, containerName, containerStartOptions) if err != nil { @@ -2653,16 +2664,76 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri } log.Printf("[INFO] Tenzir Node container started successfully") - log.Printf("[INFO] Waiting for Tenzir to become available ...") - err = checkTenzirNode() - if err != nil { - return err - } - log.Printf("[INFO] Successfully deployed Tenzir Node !") + log.Printf("[INFO] Waiting for Tenzir to become available ...") + err = checkTenzirNode() + if err != nil { + return err + } + log.Printf("[INFO] Successfully deployed Tenzir Node!") return nil } +func createNetworkIfNotExists(ctx context.Context, networkName, subnet, gateway string) error { + networks, err := dockercli.NetworkList(ctx, types.NetworkListOptions{}) + if err != nil { + return err + } + + for _, network := range networks { + if network.Name == networkName { + // Network exists + return nil + } + } + + ipamConfig := &network.IPAM{ + Config: []network.IPAMConfig{ + { + Subnet: subnet, + Gateway: gateway, + }, + }, + } + + networkCreate := types.NetworkCreate{ + CheckDuplicate: true, + Driver: "bridge", + IPAM: ipamConfig, + } + + _, err = dockercli.NetworkCreate(ctx, networkName, networkCreate) + if err != nil { + return err + } + + return nil +} + +func checkTenzirNode() error { + retries := 5 + retryInterval := 3 * time.Second + url := fmt.Sprintf("%s/api/v0/ping",tenzirUrl) + forwardMethod := "POST" + + client := http.Client{} + req, err := http.NewRequest(forwardMethod, url, nil) + if err != nil { + log.Printf("[ERROR] Failed to create HTTP request: %s", err) + return err + } + + for i := 0; i < retries; i++ { + resp, err := client.Do(req) + if err == nil && resp.StatusCode == http.StatusOK { + return nil + } + time.Sleep(retryInterval) + } + + return fmt.Errorf("tenzir node is not available") +} + func createPipeline(command, identifier string) (string, error) { toBeDeleted := false @@ -2702,8 +2773,6 @@ func createPipeline(command, identifier string) (string, error) { //command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | sigma /var/lib/tenzir/rule.yaml" //command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | import" - //command = "export | to https://expert-acorn-v6vg4j4j5w7q2wg6g-5001.app.github.dev/api/v1/hooks/webhook_623eab3f-0af4-4d40-abb9-699d9a493411" - log.Printf("[HARI] this is the command %s", command) requestBody := map[string]interface{}{ "definition": command, @@ -2931,8 +3000,8 @@ func searchPipeline(identifier string) (string, error) { return "", errors.New("no existing pipeline found with name") } -func handleFileCategoryChange() error{ - apiEndpoint := baseUrl+"/api/v1/files/namespaces/sigma" +func handleFileCategoryChange() error { + apiEndpoint := baseUrl + "/api/v1/files/namespaces/sigma" req, err := http.NewRequest("GET", apiEndpoint, nil) if err != nil { return err @@ -2943,12 +3012,12 @@ func handleFileCategoryChange() error{ client := &http.Client{} resp, err := client.Do(req) if err != nil { - return err + return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return err + return fmt.Errorf("received non-200 response: %s", resp.Status) } out, err := os.Create("files.zip") @@ -2961,10 +3030,10 @@ func handleFileCategoryChange() error{ _, err = io.Copy(out, resp.Body) if err != nil { - return err + return err } - fmt.Println("ZIP file downloaded successfully.") + log.Println("ZIP file downloaded successfully.") err = extractZIP("files.zip", "sigma_rules") if err != nil { @@ -2978,7 +3047,45 @@ func handleFileCategoryChange() error{ return err } - fmt.Println("Files copied to container successfully.") + log.Println("Files copied to container successfully.") + + checkDisabledDirCmd := exec.Command("docker", "exec", "tenzir-node", "sh", "-c", "test -d /var/lib/tenzir/disabled_rules") + if err := checkDisabledDirCmd.Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { + // Directory does not exist, nothing to do + log.Println("[DEBUG] /var/lib/tenzir/disabled_rules does not exist.") + return nil + } + + return fmt.Errorf("error checking disabled rules directory: %v", err) + } + + // List files in /var/lib/tenzir/disabled_rules + listFilesCmd := exec.Command("docker", "exec", "tenzir-node", "sh", "-c", "ls /var/lib/tenzir/disabled_rules") + output, err := listFilesCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error listing files in disabled rules directory: %v, output: %s", err, output) + } + + files := strings.Split(strings.TrimSpace(string(output)), "\n") + for _, file := range files { + disabledFilePath := fmt.Sprintf("/var/lib/tenzir/sigma_rules/%s", file) + checkFileCmd := exec.Command("docker", "exec", "tenzir-node", "sh", "-c", fmt.Sprintf("test -f %s", disabledFilePath)) + if err := checkFileCmd.Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { + log.Printf("[ERROR] File does not exist: %s, moving on.\n", disabledFilePath) + continue + } + return fmt.Errorf("error checking file: %v", err) + } + + deleteFileCmd := exec.Command("docker", "exec", "-u", "root", "tenzir-node", "sh", "-c", fmt.Sprintf("rm -f %s", disabledFilePath)) + if err := deleteFileCmd.Run(); err != nil { + return fmt.Errorf("error deleting file: %v", err) + } + log.Printf("[INFO] Deleted file: %s\n", disabledFilePath) + } + return nil } @@ -3025,7 +3132,6 @@ func extractFile(f *zip.File, destDir string) error { func copyToTenzir(srcPath, destPath string) error { containerName := "tenzir-node" - // Check if the sigma_rules directory exists in the container checkCmd := exec.Command("docker", "exec", containerName, "test", "-d", destPath) if err := checkCmd.Run(); err == nil { rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", "-rf", destPath) @@ -3034,7 +3140,6 @@ func copyToTenzir(srcPath, destPath string) error { } } - // Copy the new directory to the container cpCmd := exec.Command("docker", "cp", srcPath, fmt.Sprintf("%s:%s", containerName, destPath)) var out bytes.Buffer cpCmd.Stdout = &out @@ -3049,10 +3154,19 @@ func copyToTenzir(srcPath, destPath string) error { } func removeAllFiles() error { - containerName := "tenzir-node" - sigmaPath := "/var/lib/tenzir/sigma_rules/*" + containerName := "tenzir-node" + sigmaPath := "/var/lib/tenzir/sigma_rules/*" - cmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("rm -rf %s", sigmaPath)) + checkCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("ls %s", sigmaPath)) + checkOutput, checkErr := checkCmd.CombinedOutput() + if checkErr != nil { + if strings.Contains(string(checkOutput), "No such file or directory") { + return nil // nothing to delete + } + return fmt.Errorf("error checking files: %v, output: %s", checkErr, checkOutput) + } + + cmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("rm -rf %s", sigmaPath)) output, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("error removing files: %v, output: %s", err, output) @@ -3064,16 +3178,21 @@ func removeFile(fileName string) error { containerName := "tenzir-node" srcPath := fmt.Sprintf("/var/lib/tenzir/sigma_rules/%s", fileName) - checkSrcCmd := exec.Command("docker", "exec", containerName, "test", "-f", srcPath) + checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) if err := checkSrcCmd.Run(); err != nil { - return fmt.Errorf("source file does not exist: %v", err) + // If the file does not exist, simply return nil + if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { + log.Printf("[ERROR] No such file: %s, nothing to delete\n", srcPath) + return nil + } + return fmt.Errorf("error checking source file: %v", err) } return removePath(containerName, srcPath) } func removePath(containerName, path string) error { - rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", "-rf", path) + rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("rm -rf %s", path)) output, err := rmCmd.CombinedOutput() if err != nil { return fmt.Errorf("error removing path: %v, output: %s", err, output) @@ -3127,6 +3246,63 @@ func sendTenzirHealthStatus() error { return nil } +func disableRule(fileName string) error { + containerName := "tenzir-node" + srcPath := fmt.Sprintf("/var/lib/tenzir/sigma_rules/%s", fileName) + destDir := "/var/lib/tenzir/disabled_rules" + destPath := fmt.Sprintf("%s/%s", destDir, fileName) + + checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) + if err := checkSrcCmd.Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { + fmt.Printf("File does not exist: %s\n", srcPath) + return nil // Nothing to disable + } + return fmt.Errorf("error checking source file: %v", err) + } + + checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mkdir -p %s", destDir)) + if err := checkDestDirCmd.Run(); err != nil { + return fmt.Errorf("error ensuring destination directory exists: %v", err) + } + + moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mv %s %s", srcPath, destPath)) + if err := moveCmd.Run(); err != nil { + return fmt.Errorf("error moving file: %v", err) + } + + fmt.Printf("File %s moved to %s successfully.\n", fileName, destDir) + return nil +} + +func enableRule(fileName string) error { + containerName := "tenzir-node" + srcPath := fmt.Sprintf("/var/lib/tenzir/disabled_rules/%s", fileName) + destDir := "/var/lib/tenzir/sigma_rules" + destPath := fmt.Sprintf("%s/%s", destDir, fileName) + + checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) + if err := checkSrcCmd.Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { + fmt.Printf("File does not exist: %s\n", srcPath) + return nil // Nothing to enable + } + return fmt.Errorf("error checking source file: %v", err) + } + + checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mkdir -p %s", destDir)) + if err := checkDestDirCmd.Run(); err != nil { + return fmt.Errorf("error ensuring destination directory exists: %v", err) + } + moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mv %s %s", srcPath, destPath)) + if err := moveCmd.Run(); err != nil { + return fmt.Errorf("error moving file: %v", err) + } + + fmt.Printf("File %s moved to %s successfully.\n", fileName, destDir) + return nil +} + // Is this ok to do with Docker? idk :) func getRunningWorkers(ctx context.Context, workerTimeout int) int { //log.Printf("[DEBUG] Getting running workers with API version %s", dockerApiVersion) From b33c45543ac1fa4454514f45a5b014997a390319 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Mon, 22 Jul 2024 11:35:55 +0530 Subject: [PATCH 45/46] pointing to the correct url --- backend/go-app/main.go | 2 +- frontend/src/views/DetectionDashboard.jsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 8f247fc8..860a16ef 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5188,7 +5188,7 @@ func initHandlers() { // PS: For cloud, this has to use cloud storage. // https://developer.box.com/reference/get-files-id-content/ r.HandleFunc("/api/v1/files/download_remote", shuffle.HandleDownloadRemoteFiles).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v2/files/download_remote", shuffle.HandleDownloadRemoteFiles2).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/files/download_remote_enhanced", shuffle.HandleEnhancedDownloadRemoteFiles).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/files/namespaces/{namespace}", shuffle.HandleGetFileNamespace).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}/content", shuffle.HandleGetFileContent).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/create", shuffle.HandleCreateFile).Methods("POST", "OPTIONS") diff --git a/frontend/src/views/DetectionDashboard.jsx b/frontend/src/views/DetectionDashboard.jsx index 4b2b1b06..3d4eb871 100644 --- a/frontend/src/views/DetectionDashboard.jsx +++ b/frontend/src/views/DetectionDashboard.jsx @@ -110,7 +110,7 @@ const DetectionDashBoard = (props) => { }; toast(`Getting files from url ${url}. This may take a while if the repository is large. Please wait...`); - fetch(`${globalUrl}/api/v2/files/download_remote`, { + fetch(`${globalUrl}/api/v1/files/download_remote_enhanced`, { method: "POST", mode: "cors", headers: { From 2fe2772e3bd2f1906e16446c970a04ad9d7fbe9f Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Mon, 29 Jul 2024 14:13:24 +0530 Subject: [PATCH 46/46] track the sigma rules --- backend/go-app/main.go | 47 +++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 860a16ef..23486930 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -2122,6 +2122,9 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { if err == nil { resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s"}`, workflowExecution.ExecutionId))) + + // Track Sigma rules + trackSigmaRules(ctx, pipeline.OrgId, jsonList) return } @@ -2130,23 +2133,39 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { } func parseConcatenatedJSONLogs(logs string) ([]map[string]interface{}, error) { - var jsonList []map[string]interface{} - decoder := json.NewDecoder(strings.NewReader(logs)) + var jsonList []map[string]interface{} + decoder := json.NewDecoder(strings.NewReader(logs)) - for decoder.More() { - var jsonObject map[string]interface{} - if err := decoder.Decode(&jsonObject); err != nil { - log.Printf("[WARNING] JSON decoding error: %s. Skipping this object.", err) - continue - } - jsonList = append(jsonList, jsonObject) - } + for decoder.More() { + var jsonObject map[string]interface{} + if err := decoder.Decode(&jsonObject); err != nil { + log.Printf("[WARNING] JSON decoding error: %s. Skipping this object.", err) + continue + } + jsonList = append(jsonList, jsonObject) + } - if err := decoder.Decode(&struct{}{}); err != io.EOF { - return nil, fmt.Errorf("error after decoding all JSON objects: %v", err) - } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + return nil, fmt.Errorf("error after decoding all JSON objects: %v", err) + } - return jsonList, nil + return jsonList, nil +} + +func trackSigmaRules(ctx context.Context, orgId string, jsonList []map[string]interface{}) { + ruleCount := make(map[string]int) + for _, logEntry := range jsonList { + if rule, ok := logEntry["rule"].(map[string]interface{}); ok { + if ruleName, ok := rule["title"].(string); ok { + ruleCount[ruleName]++ + } + } + } + + for ruleName, count := range ruleCount { + shuffle.IncrementCache(ctx, orgId, ruleName, count) + log.Printf("[INFO] Rule %s incremented by %d", ruleName, count) + } } func handleTenzirHealthUpdate(resp http.ResponseWriter, request *http.Request) {