From 1f9a132b757af3f48aa021c1a7ff7c21635cd5bc Mon Sep 17 00:00:00 2001 From: monilprajapati Date: Thu, 4 Jul 2024 17:28:07 +0530 Subject: [PATCH 01/60] Added branch flip feature --- frontend/src/components/ParsedAction.jsx | 95 +++++++++++++++++++++--- frontend/src/views/AngularWorkflow.jsx | 59 ++++++++------- 2 files changed, 115 insertions(+), 39 deletions(-) diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 0d5ef733..9249361f 100755 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect, useLayoutEffect } from "react"; +import React, { useState, useEffect, useLayoutEffect, useMemo } from "react"; import { toast } from 'react-toastify'; import { makeStyles, createStyles } from "@mui/styles"; import theme from '../theme.jsx'; @@ -207,16 +207,16 @@ const ParsedAction = (props) => { } }, [expansionModalOpen]) - useEffect(() => { - setParamValues(selectedAction.parameters.map((param) => { - return { - name: param.name, - value: param.value, - } - })) - },[ - selectedAction, selectedApp,setNewSelectedAction, workflow, - ]) +// useEffect(() => { +// setParamValues(selectedAction.parameters?.map((param) => { +// return { +// name: param.name, +// value: param.value, +// } +// })) +// },[ +// selectedAction, selectedApp,setNewSelectedAction, workflow, +// ]) useEffect(() => { if (selectedAction.parameters === null || selectedAction.parameters === undefined) { @@ -565,6 +565,79 @@ const ParsedAction = (props) => { setActionlist(newActionList); }, [workflow.execution_variables, workflow.workflow_variables, workflowExecutions, workflow, selectedAction, listCache, getParents]); + + const memoizedParam = useMemo(() => { + let appActions = []; + if (getParents) { + const parents = getParents(selectedAction); + if (parents.length > 1) { + const labels = []; + for (let parentNode of parents) { + if (parentNode.label !== "Execution Argument" && !labels.includes(parentNode.label)) { + labels.push(parentNode.label); + let exampleData = parentNode.example ?? ""; + if (!exampleData && workflowExecutions.length > 0) { + for (let exec of workflowExecutions) { + const foundResult = exec.results?.find(result => result.action.id === parentNode.id); + if (foundResult) { + const valid = validateJson(foundResult.result); + if (valid.valid && valid.result.success !== false) { + exampleData = valid.result; + break; + } + } + } + } + appActions.push({ + type: "action", + id: parentNode.id, + name: parentNode.label, + autocomplete: parentNode.label.split(" ").join("_"), + example: exampleData, + }); + } + } + } + } + + let newParameters = selectedAction.parameters?.map((param) => { + let paramvalue = param.value; + if(paramvalue.includes("$")){ + let actions = workflow.actions?.map((action) => { + return "$"+action.label.toLowerCase(); + }) + if(actionlist.length > 0){ + let appParentActions = appActions?.map(action => "$" + action.name.toLowerCase()); + let notPresentAction = actions?.filter((action) => !appParentActions?.includes(action)) + console.log("ACTIONS: ", actions) + console.log("APP ACTIONS: ", appParentActions) + console.log("NOT PRESENT: ", notPresentAction) + notPresentAction?.forEach((action) => { + console.log("Not included Action: ", action) + if(paramvalue.includes(action)){ + paramvalue = paramvalue.replace(action, "") + paramvalue = paramvalue.replace(/^\s*[\r\n]/gm, ""); + } + }) + } + } + console.log("After removing param value: ", paramvalue) + return {...param, value: paramvalue} + }); + selectedAction.parameters = newParameters; + setSelectedAction(selectedAction); + return newParameters; + },[actionlist,selectedAction,workflow.actions,workflow,selectedApp,setNewSelectedAction]) + + useEffect(() => { + setParamValues(memoizedParam.map((param) => { + return { + name: param.name, + value: param.value, + } + })) + },[memoizedParam]) + useEffect(() => { selectedNameChange(appActionName) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index e1bb6737..48fd2ff2 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -126,7 +126,7 @@ import { ArrowForward as ArrowForwardIcon, } from "@mui/icons-material"; - +import SwapHorizIcon from '@mui/icons-material/SwapHoriz'; //import * as cytoscape from "cytoscape"; import cytoscape from "cytoscape"; @@ -11672,10 +11672,10 @@ const releaseToConnectLabel = "Release to Connect" : null} -
- {/* +
+ - + + + ); +}; + +export default EditComponent; diff --git a/frontend/src/views/RuleCard.jsx b/frontend/src/views/RuleCard.jsx new file mode 100644 index 00000000..e166ce6b --- /dev/null +++ b/frontend/src/views/RuleCard.jsx @@ -0,0 +1,82 @@ +import React from "react"; +import { + Card, + CardContent, + IconButton, + Typography, + Switch, +} from "@mui/material"; +import EditIcon from "@mui/icons-material/Edit"; +import { toast } from "react-toastify"; + +const RuleCard = ({ ruleName, description, file_id, globalUrl, openEditBar, ...otherProps }) => { + const [additionalProps, setAdditionalProps] = React.useState(otherProps); + + const handleSwitchChange = (event) => { + const isEnabled = event.target.checked; + toggleRule(file_id, !isEnabled, globalUrl, () => { + setAdditionalProps((prevProps) => ({ + ...prevProps, + is_enabled: isEnabled, + })); + }); + }; + + return ( + + +
+ {ruleName} +
+ openEditBar({ ruleName, description, file_id, ...additionalProps })}> + + + +
+
+ + {description} + + + + ); +}; + +const toggleRule = (fileId, isCurrentlyEnabled, globalUrl, callback) => { + const action = isCurrentlyEnabled ? "disable" : "enable"; + const url = `${globalUrl}/api/v1/files/${fileId}/${action}_rule`; + + fetch(url, { + method: "PUT", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => + response.json().then((responseJson) => { + if (responseJson["success"] === false) { + toast(`Failed to ${action} the rule`); + } else { + toast(`Rule ${action}d successfully`); + callback(); + } + }) + ) + .catch((error) => { + console.log(`Error in ${action}ing the rule: `, error); + toast(`An error occurred while ${action}ing the rule`); + }); +}; + +export default RuleCard; From e90b698f84405219d7e8efd0544c197f49f08c58 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Thu, 13 Jun 2024 09:54:50 +0000 Subject: [PATCH 15/60] fixing bugs in rule editing --- frontend/src/views/Detection.jsx | 38 ++++++++++++++--------- frontend/src/views/DetectionDashboard.jsx | 11 +++++-- frontend/src/views/EditRules.jsx | 7 ++--- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/frontend/src/views/Detection.jsx b/frontend/src/views/Detection.jsx index 67b6a2ed..63567584 100644 --- a/frontend/src/views/Detection.jsx +++ b/frontend/src/views/Detection.jsx @@ -7,7 +7,7 @@ import { Typography, Button, } from "@mui/material"; -import RuleCard from "./RuleCard"; +import RuleCard from "./RuleCard"; import { styled } from "@mui/system"; const ConnectedButton = styled(Button)({ @@ -18,7 +18,7 @@ const ConnectedButton = styled(Button)({ const Detection = ({ globalUrl, ruleInfo, openEditBar }) => { return ( - + { - {ruleInfo.length > 0 && - ruleInfo.map((card) => ( - openEditBar(card)} - {...card} - /> - ))} + + {ruleInfo.length > 0 && + ruleInfo.map((card) => ( + openEditBar(card)} + {...card} + /> + ))} + ); diff --git a/frontend/src/views/DetectionDashboard.jsx b/frontend/src/views/DetectionDashboard.jsx index 50771d63..bd8a2813 100644 --- a/frontend/src/views/DetectionDashboard.jsx +++ b/frontend/src/views/DetectionDashboard.jsx @@ -39,6 +39,12 @@ const DetectionDashBoard = (props) => { getSigmaInfo(globalUrl, setRuleInfo); }, [globalUrl]); + useEffect(() => { + if (ruleInfo.length > 0) { + openEditBar(ruleInfo[0]); + } + }, [ruleInfo]); + const openEditBar = (rule) => { setSelectedRule(rule); getFileContent(rule.file_id) @@ -50,6 +56,7 @@ const DetectionDashBoard = (props) => { }; const getFileContent = (file_id) => { + setFileData(""); fetch(globalUrl + "/api/v1/files/" + file_id + "/content", { method: "GET", headers: { @@ -82,10 +89,10 @@ const DetectionDashBoard = (props) => { }; return ( - + {selectedRule ? ( + - {ruleName} - - - + {ruleName} {description} From f8c786b9ecea2577d8d6f4591462a76882557efc Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Thu, 13 Jun 2024 09:56:31 +0000 Subject: [PATCH 16/60] removing unnessary dependencies --- frontend/src/views/DetectionDashboard.jsx | 1 - frontend/src/views/EditRules.jsx | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/frontend/src/views/DetectionDashboard.jsx b/frontend/src/views/DetectionDashboard.jsx index bd8a2813..382d0dc8 100644 --- a/frontend/src/views/DetectionDashboard.jsx +++ b/frontend/src/views/DetectionDashboard.jsx @@ -52,7 +52,6 @@ const DetectionDashBoard = (props) => { const handleSave = (updatedContent) => { toast("this will be saved"); - setSelectedRule(null); // Close the edit bar after saving }; const getFileContent = (file_id) => { diff --git a/frontend/src/views/EditRules.jsx b/frontend/src/views/EditRules.jsx index 27788135..a433ebc3 100644 --- a/frontend/src/views/EditRules.jsx +++ b/frontend/src/views/EditRules.jsx @@ -1,5 +1,5 @@ -import React, { useState } from 'react'; -import { Box, Typography, Button, Switch, TextField } from '@mui/material'; +import React from 'react'; +import { Box, Typography, Button, TextField } from '@mui/material'; const EditComponent = ({ ruleName, description, content, setContent, lastEdited, editedBy, onSave }) => { From f081e907b3efc6d56971ea67f7587479ac2f2608 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Fri, 14 Jun 2024 04:45:20 +0000 Subject: [PATCH 17/60] made the file disabling logic in orborus easy --- frontend/src/App.jsx | 6 +-- functions/onprem/orborus/orborus.go | 57 +++++++++-------------------- 2 files changed, 20 insertions(+), 43 deletions(-) diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 668cb557..ef77004a 100755 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -416,9 +416,9 @@ const App = (message, props) => { } /> } + exact + path="/detections/sigma" + element={} /> Date: Fri, 14 Jun 2024 05:04:24 +0000 Subject: [PATCH 18/60] made the sigma rule disable logic better and simple --- functions/onprem/orborus/orborus.go | 67 ++++++----------------------- 1 file changed, 12 insertions(+), 55 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 2eef9435..71c9955d 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2033,7 +2033,7 @@ func main() { } else if incRequest.Type == "DISABLE_SIGMA_FILE" { fileName := incRequest.ExecutionArgument - err = disableSigmaRule(fileName) + err = removeFile(fileName) if err != nil { log.Printf("[ERROR] Failed to disable the sigma file %s, reason: %s", fileName, err) } @@ -2041,18 +2041,11 @@ func main() { toBeRemoved.Data = append(toBeRemoved.Data, incRequest) } else if incRequest.Type == "DISABLE_SIGMA_RULES" { - err := manageSigmaFolder("disable") + err := removeAllFiles() if err != nil { log.Printf("[ERROR] Failed to disable the sigma rules: %s", err) } - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - } else if incRequest.Type == "ENABLE_SIGMA_RULES" { - err := manageSigmaFolder("enable") - if err != nil { - log.Printf("[ERROR] Failed to enable the sigma rules: %s", err) - } - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) } else { newrequests = append(newrequests, incRequest) @@ -3029,7 +3022,7 @@ func copyToTenzir(srcPath, destPath string) error { return nil } -func disableSigmaRule(fileName string) error { +func removeFile(fileName string) error { containerName := "tenzir-node" srcPath := fmt.Sprintf("/var/lib/tenzir/sigma_rules/%s", fileName) @@ -3038,57 +3031,21 @@ func disableSigmaRule(fileName string) error { return fmt.Errorf("source file does not exist: %v", err) } - rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", srcPath) - if err := rmCmd.Run(); err != nil { - return fmt.Errorf("error removing file: %v", err) - } - - return nil + return removePath(containerName, srcPath) } -func manageSigmaFolder(action string) error { +func removeAllFiles() error { containerName := "tenzir-node" - sigmaPath := "/var/lib/tenzir/sigma_rules" + sigmaPath := "/var/lib/tenzir/sigma_rules/*" - if action == "disable" { + return removePath(containerName, sigmaPath) +} - checkSigmaCmd := exec.Command("docker", "exec", containerName, "test", "-d", sigmaPath) - if err := checkSigmaCmd.Run(); err != nil { - return fmt.Errorf("sigma_files directory does not exist: %v", err) - } - - // Rename sigma_files to disabled_sigma - renameCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mv", sigmaPath, disabledPath) - if err := renameCmd.Run(); err != nil { - return fmt.Errorf("error renaming sigma_files to disabled_sigma: %v", err) - } - - // Create a new sigma_files directory - createCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mkdir", sigmaPath) - if err := createCmd.Run(); err != nil { - return fmt.Errorf("error creating new sigma_files directory: %v", err) - } - } else if action == "enable" { - - checkDisabledCmd := exec.Command("docker", "exec", containerName, "test", "-d", disabledPath) - if err := checkDisabledCmd.Run(); err != nil { - return fmt.Errorf("disabled_sigma directory does not exist: %v", err) - } - - removeCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", "-rf", sigmaPath) - if err := removeCmd.Run(); err != nil { - return fmt.Errorf("error removing existing sigma_files directory: %v", err) - } - - // Rename disabled_sigma back to sigma_files - renameBackCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mv", disabledPath, sigmaPath) - if err := renameBackCmd.Run(); err != nil { - return fmt.Errorf("error renaming disabled_sigma back to sigma_files: %v", err) - } - } else { - return fmt.Errorf("invalid action: %s", action) +func removePath(containerName, path string) error { + rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", "-r", path) + if err := rmCmd.Run(); err != nil { + return fmt.Errorf("error removing path: %v", err) } - return nil } From 29f294f2ef8e0dbc7ea8c21fdaa6b02f0b71a9cc Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Tue, 18 Jun 2024 12:43:19 +0000 Subject: [PATCH 19/60] removing the hard coded ui --- frontend/src/views/AngularWorkflow.jsx | 186 ++++++------------------- 1 file changed, 45 insertions(+), 141 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 70d9b761..00bc68de 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1437,7 +1437,7 @@ const releaseToConnectLabel = "Release to Connect" }); }; - const handleKafkaSubmit = (trigger) => { + const handleCommandSubmit = (trigger) => { if (trigger.trigger_type !== "PIPELINE") { toast("Unable to save the configuration"); return; @@ -1445,38 +1445,18 @@ const releaseToConnectLabel = "Release to Connect" trigger.parameters = [] - const topic = document.getElementById('topic')?.value; - const bootstrapServers = document.getElementById('bootstrap_servers')?.value; - const groupId = document.getElementById('group_id')?.value; - //const autoOffsetReset = document.getElementById('auto_offset_reset')?.value; + const command = document.getElementById('sigma')?.value - if(topic) { + if(command) { trigger.parameters.push({ - name: "topic", - value: topic - }); + name: "command", + value: command + }) } else { - toast("please enter the topic name"); + toast("Please enter the comamnd"); return; } - if (bootstrapServers) { - trigger.parameters.push({ - name: "bootstrap_servers", - value: bootstrapServers - }); - } else { - toast("please enter bootstrap server details"); - return; - } - - if (groupId) { - trigger.parameters.push({ - name: "group_id", - value: groupId - }); - } - // if (autoOffsetReset) { // trigger.parameters.push({ // name: "auto_offset_reset", @@ -15366,14 +15346,18 @@ const releaseToConnectLabel = "Release to Connect"
{ - // setSelectedOption("Syslog listener") - // setTenzirConfigModalOpen(true); - }} + if(selectedTrigger.status === "running"){ + toast("please stop the trigger to edit the configuration"); + return; + } else { + setSelectedOption("Syslog listener"); + setTenzirConfigModalOpen(true); + }}} style={{ border: "1px solid rgba(255,255,255,0.3)", borderRadius: theme.palette.borderRadius, padding: 10, - cursor: "not-allowed", + cursor: "pointer", marginTop: 5, display: "flex", alignItems: "center", @@ -15386,7 +15370,6 @@ const releaseToConnectLabel = "Release to Connect" onChange={() => setSelectedOption("Syslog listener")} value={"Syslog listener"} name="option" - disabled={true} /> } label="Start Syslog listener" @@ -15396,14 +15379,18 @@ const releaseToConnectLabel = "Release to Connect"
{ - // setSelectedOption("Sigma Rulesearch") - // setTenzirConfigModalOpen(true); - }} + if(selectedTrigger.status === "running"){ + toast("please stop the trigger to edit the configuration"); + return; + } else { + setSelectedOption("Sigma Rulesearch"); + setTenzirConfigModalOpen(true); + }}} style={{ border: "1px solid rgba(255,255,255,0.3)", borderRadius: theme.palette.borderRadius, padding: 10, - cursor: "not-allowed", + cursor: "pointer", marginTop: 5, display: "flex", alignItems: "center", @@ -15416,7 +15403,6 @@ const releaseToConnectLabel = "Release to Connect" onChange={() => setSelectedOption("Sigma Rulesearch")} value={"Sigma Rulesearch"} name="option" - disabled={true} /> } label="Run Sigma Rulesearch" @@ -15463,39 +15449,7 @@ const releaseToConnectLabel = "Release to Connect" disabled={selectedTrigger.status === "running"} onClick={() => { - const topic = (selectedTrigger?.parameters?.find(param => param.name === "topic")?.value) || '' - const bootstrapServers = (selectedTrigger?.parameters?.find(param => param.name === "bootstrap_servers")?.value) || '' - const groupId = (selectedTrigger?.parameters?.find(param => param.name === "group_id")?.value) || '' - // const autoOffsetReset = (selectedTrigger?.parameters?.find(param => param.name === "auto_offset_reset")?.value) || '' - let command = "from kafka" - - if(topic) { - command = `${command} -t ${topic}` - } else { - toast("please enter the topic name") - return; - } - if(bootstrapServers) { - command = `${command} -e -o stored -X bootstrap.servers=${bootstrapServers}` - } else { - toast("please enter the bootstrap servers details") - return; - } - - if(groupId) { - command = `${command},group.id=${groupId}` - } else { - command = `${command},group.id=${selectedTrigger.id}` - } - // if(autoOffsetReset) { - // command = `${command},auto.offset.reset=${autoOffsetReset}` - // } else { - // command = `${command},auto.offset.reset=earliest` - - // } - command = `${command},auto.offset.reset=earliest` - command = `${command},client.id=${selectedTrigger.id},enable.auto.commit=true,auto.commit.interval.ms=1` - command = `${command} read json | to ${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` + const command = (selectedTrigger?.parameters?.find(param => param.name === "command")?.value) || '' const pipelineConfig = { command: command, @@ -21218,8 +21172,8 @@ const releaseToConnectLabel = "Release to Connect" pointerEvents: "auto", color: "white", minWidth: 600, - minHeight: 450, - maxHeight: 450, + minHeight: 200, + maxHeight: 200, padding: 15, overflow: "hidden", zIndex: 10012, @@ -21236,75 +21190,25 @@ const releaseToConnectLabel = "Release to Connect" overflowY: "auto", overflowX: isMobile ? "auto" : "hidden", }} - > - -
Configuration options for {selectedOption}
- + > - {selectedOption === "Kafka Queue" && ( - <> - Topic - param.name === "topic")?.value) || ''} - /> - bootstrap.servers - param.name === "bootstrap_servers")?.value) || ''} - /> - group.id - param.name === "group_id")?.value) || ''} - /> - {/* auto.offest.reset - param.name === "auto_offset_reset")?.value) || ''} - /> */} - - )} + + command + param.name === "command")?.value) || ''} + /> +
@@ -84,19 +81,19 @@ const RuleCard = ({ ruleName, description, file_id, globalUrl, folderDisabled, . + isCloud={isCloud} + expansionModalOpen={openCodeEditor} + setExpansionModalOpen={setOpenCodeEditor} + setcodedata={setFileData} + codedata={fileData} + isFileEditor={true} + key={fileData} // https://reactjs.org/docs/reconciliation.html#recursing-on-children + runUpdateText={UpdateText} + /> ); -}; +} const toggleRule = (fileId, isCurrentlyEnabled, globalUrl, callback) => { const action = isCurrentlyEnabled ? "disable" : "enable"; From 2cc0d3d57639a9b6677870e00b49d50ce4fe5078 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Tue, 18 Jun 2024 16:11:29 +0000 Subject: [PATCH 25/60] fixing the entire directory getting deleted instead of contents inside it --- functions/onprem/orborus/orborus.go | 30 +++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index f1459c00..9c84c03d 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2024,7 +2024,13 @@ func main() { toBeRemoved.Data = append(toBeRemoved.Data, incRequest) } else if incRequest.Type == "CATEGORY_UPDATE" { - err := handleFileCategoryChange() + + err := deployTenzirNode() + if err != nil{ + log.Printf("[ERROR] failed to deploy the pipeline, reason: %s", err) + } + + err = handleFileCategoryChange() if err != nil { log.Printf("[ERROR] Failed to download the file category: %s", err) } @@ -2033,6 +2039,11 @@ func main() { } else if incRequest.Type == "DISABLE_SIGMA_FILE" { fileName := incRequest.ExecutionArgument + err := deployTenzirNode() + if err != nil{ + log.Printf("[ERROR] failed to deploy the pipeline, reason: %s", err) + } + err = removeFile(fileName) if err != nil { log.Printf("[ERROR] Failed to disable the sigma file %s, reason: %s", fileName, err) @@ -2041,7 +2052,13 @@ func main() { toBeRemoved.Data = append(toBeRemoved.Data, incRequest) } else if incRequest.Type == "DISABLE_SIGMA_FOLDER" { - err := removeAllFiles() + + err := deployTenzirNode() + if err != nil{ + log.Printf("[ERROR] failed to deploy the pipeline, reason: %s", err) + } + + err = removeAllFiles() if err != nil { log.Printf("[ERROR] Failed to disable the sigma rules: %s", err) } @@ -3021,7 +3038,12 @@ func removeAllFiles() error { containerName := "tenzir-node" sigmaPath := "/var/lib/tenzir/sigma_rules/*" - return removePath(containerName, sigmaPath) + cmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("rm -rf %s", sigmaPath)) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error removing files: %v, output: %s", err, output) + } + return nil } func removeFile(fileName string) error { @@ -3038,7 +3060,7 @@ func removeFile(fileName string) error { func removePath(containerName, path string) error { rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", "-rf", path) - output, err := rmCmd.CombinedOutput() + output, err := rmCmd.CombinedOutput() if err != nil { return fmt.Errorf("error removing path: %v, output: %s", err, output) } From 5620d8a76b97a2fad1a9ceb092a4e6b81011f569 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Tue, 18 Jun 2024 16:11:52 +0000 Subject: [PATCH 26/60] adding the trigger url to the command --- frontend/src/views/AngularWorkflow.jsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 00bc68de..aca59723 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -15449,7 +15449,8 @@ const releaseToConnectLabel = "Release to Connect" disabled={selectedTrigger.status === "running"} onClick={() => { - const command = (selectedTrigger?.parameters?.find(param => param.name === "command")?.value) || '' + let command = (selectedTrigger?.parameters?.find(param => param.name === "command")?.value) || '' + command = `${command} | to ${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` const pipelineConfig = { command: command, From 3595881a349daab527cfbb5971d7954f81203426 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Tue, 18 Jun 2024 17:11:16 +0000 Subject: [PATCH 27/60] trying to parse the json logs properly --- backend/go-app/main.go | 63 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 8bb75ae9..713abda8 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1978,7 +1978,6 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { } func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { - if request.Method != "POST" { request.Method = "POST" } @@ -1999,7 +1998,7 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { location := strings.Split(request.URL.String(), "/") var pipelineId string - + if location[1] == "api" { if len(location) <= 4 { log.Printf("[INFO] Couldn't handle location. Too short in pipeline: %d", len(location)) @@ -2013,7 +2012,7 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { userAgent := request.Header.Get("User-Agent") if strings.Contains(strings.ToLower(userAgent), "microsoftpreview") || strings.Contains(strings.ToLower(userAgent), "googlebot") { - log.Printf("[AUDIT] Blocking googlebot and microsoftbot for pielines. UA: '%s'", userAgent) + log.Printf("[AUDIT] Blocking googlebot and microsoftbot for pipelines. UA: '%s'", userAgent) resp.WriteHeader(400) resp.Write([]byte(`{"success": false, "reason": "Google/Microsoft preview bots not allowed. Please change the useragent."}`)) return @@ -2058,7 +2057,23 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { return } - parsedBody := shuffle.GetExecutionbody(body) + // Parse concatenated JSON logs + jsonList, err := parseConcatenatedJSONLogs(string(body)) + if err != nil { + log.Printf("[DEBUG] JSON parsing error: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } +// fix this + parsedBody, err := string(jsonList) + if err != nil { + log.Printf("[ERROR] Failed to marshal jsonList: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false}`)) + return + } + newBody := shuffle.ExecutionStruct{ Start: pipeline.StartNode, ExecutionSource: "pipeline", @@ -2093,8 +2108,7 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { } if len(pipeline.StartNode) == 0 { - log.Printf("[WARNING] No start node for pipeline %s - running with workflow default.", pipeline.TriggerId) - + log.Printf("[WARNING] No start node for pipeline %s - running with workflow default.") } newRequest := &http.Request{ @@ -2115,6 +2129,43 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, executionResp))) } +func parseConcatenatedJSONLogs(logs string) ([]map[string]interface{}, error) { + var jsonList []map[string]interface{} + var currentObject []rune + var depth int + + for _, char := range logs { + if char == '{' { + depth++ + } + if char == '}' { + depth-- + } + + currentObject = append(currentObject, char) + + // When depth is 0, it means we have a complete JSON object but will this work ?? + if depth == 0 && len(currentObject) > 0 { + var jsonObject map[string]interface{} + err := json.Unmarshal([]byte(string(currentObject)), &jsonObject) + if err != nil { + log.Printf("[WARNING] JSON unmarshal error: %s. Skipping this object.", err) + } else { + jsonList = append(jsonList, jsonObject) + } + currentObject = nil + } + } + + currentObject = []rune(strings.TrimSpace(string(currentObject))) + if len(currentObject) > 0 { + log.Printf("[WARNING] Incomplete JSON object found: %s. Skipping this object.", string(currentObject)) + } + + return jsonList, nil +} + + func executeCloudAction(action shuffle.CloudSyncJob, apikey string) error { data, err := json.Marshal(action) if err != nil { From ad92e5e72272e15363001cd1e5d5fb0c3c854111 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Fri, 21 Jun 2024 10:28:49 +0000 Subject: [PATCH 28/60] made the pipeline to parse json logs --- backend/go-app/main.go | 51 ++++++++++------------------- functions/onprem/orborus/orborus.go | 15 ++++++--- 2 files changed, 27 insertions(+), 39 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 713abda8..d5fd7c6b 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -2065,8 +2065,8 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": false}`)) return } -// fix this - parsedBody, err := string(jsonList) + + parsedBody, err := json.Marshal(jsonList) if err != nil { log.Printf("[ERROR] Failed to marshal jsonList: %s", err) resp.WriteHeader(500) @@ -2077,7 +2077,7 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { newBody := shuffle.ExecutionStruct{ Start: pipeline.StartNode, ExecutionSource: "pipeline", - ExecutionArgument: parsedBody, + ExecutionArgument: string(parsedBody), } workflow, err := shuffle.GetWorkflow(ctx, pipeline.WorkflowId) @@ -2130,42 +2130,25 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { } func parseConcatenatedJSONLogs(logs string) ([]map[string]interface{}, error) { - var jsonList []map[string]interface{} - var currentObject []rune - var depth int + var jsonList []map[string]interface{} + decoder := json.NewDecoder(strings.NewReader(logs)) - for _, char := range logs { - if char == '{' { - depth++ - } - if char == '}' { - depth-- - } + for decoder.More() { + var jsonObject map[string]interface{} + if err := decoder.Decode(&jsonObject); err != nil { + log.Printf("[WARNING] JSON decoding error: %s. Skipping this object.", err) + continue + } + jsonList = append(jsonList, jsonObject) + } - currentObject = append(currentObject, char) + if err := decoder.Decode(&struct{}{}); err != io.EOF { + return nil, fmt.Errorf("error after decoding all JSON objects: %v", err) + } - // When depth is 0, it means we have a complete JSON object but will this work ?? - if depth == 0 && len(currentObject) > 0 { - var jsonObject map[string]interface{} - err := json.Unmarshal([]byte(string(currentObject)), &jsonObject) - if err != nil { - log.Printf("[WARNING] JSON unmarshal error: %s. Skipping this object.", err) - } else { - jsonList = append(jsonList, jsonObject) - } - currentObject = nil - } - } - - currentObject = []rune(strings.TrimSpace(string(currentObject))) - if len(currentObject) > 0 { - log.Printf("[WARNING] Incomplete JSON object found: %s. Skipping this object.", string(currentObject)) - } - - return jsonList, nil + return jsonList, nil } - func executeCloudAction(action shuffle.CloudSyncJob, apikey string) error { data, err := json.Marshal(action) if err != nil { diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 9c84c03d..b2ec0e80 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2462,7 +2462,7 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) return err } - _, err = updatePipelineState(pipelineId, "stop") + _, err = updatePipelineState(command, pipelineId, "stop") if err != nil { log.Printf("[ERROR] Failed to stop Pipeline: %s reason:%s ", pipelineId, err) return err @@ -2482,7 +2482,7 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) return err } - _, err = updatePipelineState(pipelineId, "start") + _, err = updatePipelineState(command, pipelineId, "start") if err != nil { log.Printf("[ERROR] Failed to start Pipeline: %s reason:%s ", pipelineId, err) return err @@ -2689,7 +2689,11 @@ func createPipeline(command, identifier string) (string, error) { // } // } - command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | sigma /var/lib/tenzir/rule.yaml | to https://shuffler.io/api/v1/hooks/webhook_d295c43a-e322-4afc-9a59-af167ae7c190" + //command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | sigma /var/lib/tenzir/rule.yaml" + //command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | import" + //command = "export | to https://expert-acorn-v6vg4j4j5w7q2wg6g-5001.app.github.dev/api/v1/hooks/webhook_623eab3f-0af4-4d40-abb9-699d9a493411" + log.Printf("[HARI] this is the command %s", command) + requestBody := map[string]interface{}{ "definition": command, "name": identifier, @@ -2697,7 +2701,7 @@ func createPipeline(command, identifier string) (string, error) { "autostart": map[string]bool{ "created": true, "completed": false, - "failed": true, + "failed": false, }, "autodelete": map[string]bool{ "completed": false, @@ -2763,13 +2767,14 @@ func createPipeline(command, identifier string) (string, error) { return id, nil } -func updatePipelineState(pipelineId, action string) (string, error) { +func updatePipelineState(command, pipelineId, action string) (string, error) { url := fmt.Sprintf("%s/api/v0/pipeline/update", tenzirUrl) forwardMethod := "POST" requestBody := map[string]interface{}{ "id": pipelineId, + "definition": command, "action": action, "autostart": map[string]bool{ "created": true, From ab3c755ad3ca7ebf96060f66ce245ccf436d9f13 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Sun, 23 Jun 2024 09:13:08 +0000 Subject: [PATCH 29/60] sending tenzir health check status to backend --- backend/go-app/main.go | 41 ++++++++++++ functions/onprem/orborus/orborus.go | 98 ++++++++++++++--------------- 2 files changed, 90 insertions(+), 49 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index d5fd7c6b..6fd0ff22 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -2149,6 +2149,46 @@ func parseConcatenatedJSONLogs(logs string) ([]map[string]interface{}, error) { return jsonList, nil } +func handleTenzirHealthUpdate(resp http.ResponseWriter, request *http.Request) { + if request.Method != "POST" { + request.Method = "POST" + } + + type HealthUpdate struct { + Status string `json:"status"` + } + + var healthUpdate HealthUpdate + err := json.NewDecoder(request.Body).Decode(&healthUpdate) + if err != nil { + resp.WriteHeader(http.StatusBadRequest) + fmt.Fprintf(resp, "Failed to decode JSON: %v", err) + return + } + ctx := context.Background() + status := healthUpdate.Status + + result, err := shuffle.GetDisabledRules(ctx) + if (err != nil && err.Error() != "rules doesn't exist") || err == nil { + result.IsTenzirActive = status + result.LastActive = time.Now().Unix() + + err = shuffle.StoreDisabledRules(ctx, *result) + if err != nil { + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false}`)) + return + } + + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) + return + } + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false}`)) + return +} + func executeCloudAction(action shuffle.CloudSyncJob, apikey string) error { data, err := json.Marshal(action) if err != nil { @@ -5091,6 +5131,7 @@ func initHandlers() { r.HandleFunc("/api/v1/triggers/pipeline", shuffle.HandleNewPipelineRegister).Methods("POST", "OPTIONS") //r.HandleFunc("/api/v1/triggers/pipeline/save", shuffle.HandleSavePipelineInfo).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/pipelines/{key}", handlePipelineCallback).Methods("POST", "GET", "PATCH", "PUT", "DELETE", "OPTIONS") + r.HandleFunc("/api/v1/pipelines/tenzir_node_health", handleTenzirHealthUpdate).Methods("POST","OPTIONS") r.HandleFunc("/api/v1/triggers", shuffle.HandleGetTriggers).Methods("GET", "OPTIONS") //r.HandleFunc("/api/v1/triggers/gmail/routing", handleGmailRouting).Methods("POST", "OPTIONS") diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index b2ec0e80..49bb2bca 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1806,6 +1806,12 @@ func main() { log.Printf("[WARNING] Defaulting to environment name %s. Set environment variable ENVIRONMENT_NAME to change. This should be the same as in the frontend action.", environment) } + if tenzirUrl == "" { + tenzirUrl = "http://localhost:5160" + log.Printf("[WARNING] SHUFFLE_TENZIR_URL not set, falling back to default URL: %s",tenzirUrl) + } + + // FIXME - during init, BUILD and/or LOAD worker and app_sdk // Build/load app_sdk so it can be loaded as 127.0.0.1:5000/walkoff_app_sdk log.Printf("[INFO] Setting up Docker environment. Downloading worker and App SDK!") @@ -2254,7 +2260,7 @@ func main() { } } - + _ = sendTenzirHealthStatus() time.Sleep(time.Duration(sleepTime) * time.Second) } } @@ -2414,11 +2420,6 @@ func main() { // docker run tenzir/tenzir:latest 'from http://192.168.86.44:5002/api/v1/orgs/7e9b9007-5df2-4b47-bca5-c4d267ef2943/cache/CIDR%20ranges?type=text&authorization=cec9d01f-09b2-4419-8a0a-76c6046e3fef read lines | to http://192.168.86.44:5002/api/v1/hooks/webhook_665ace5f-f27b-496a-a365-6e07eb61078c write lines' func handlePipeline(incRequest shuffle.ExecutionRequest) error { - if tenzirUrl == "" { - tenzirUrl = "http://localhost:5160" - log.Printf("[WARNING] SHUFFLE_TENZIR_URL not set, falling back to default URL: %s", tenzirUrl) - } - err := deployTenzirNode() if err != nil { log.Printf("[ERROR] failed to deploy the pipeline, reason: %s", err) @@ -3072,53 +3073,52 @@ func removePath(containerName, path string) error { return nil } -// func savePipelineData(pipelineId, identifier, status string) error { +func sendTenzirHealthStatus() error { + var status string + url := fmt.Sprintf("%s/api/v1/triggers/pipeline/tenzir_node_health", baseUrl) + err := checkTenzirNode() + if err != nil { + return err + } else { + status = "active" + } -// url := fmt.Sprintf("%s/api/v1/triggers/pipeline/save", baseUrl) -// identifierWithoutPrefix := strings.TrimPrefix(identifier, "shuffle-") + forwardMethod := "POST" + payload := map[string]interface{}{ + "status": status, + } + payloadBytes, err := json.Marshal(payload) + if err != nil { + log.Printf("[ERROR] Failed to marshal payload: %s", err) + return err + } + forwardData := bytes.NewBuffer(payloadBytes) + req, err := http.NewRequest( + forwardMethod, + url, + forwardData, + ) + if err != nil { + log.Printf("[ERROR] Failed to create HTTP request: %s", err) + return err + } + req.Header.Set("Content-Type", "application/json") -// forwardMethod := "PUT" + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Do(req) + if err != nil { + log.Printf("[ERROR] Failed to send HTTP request: %s", err) + return err + } + defer resp.Body.Close() -// payload := map[string]interface{}{ -// "pipeline_id": pipelineId, -// "trigger_id": identifierWithoutPrefix, -// "status": status, -// } + if resp.StatusCode != 200 { + log.Printf("[ERROR] Received non-successful HTTP status code: %d", resp.StatusCode) + return fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode) + } -// payloadBytes, err := json.Marshal(payload) -// if err != nil { -// log.Printf("[ERROR] Failed to marshal payload: %s", err) -// return err -// } - -// forwardData := bytes.NewBuffer(payloadBytes) - -// req, err := http.NewRequest( -// forwardMethod, -// url, -// forwardData, -// ) -// if err != nil { -// log.Printf("[ERROR] Failed to create HTTP request: %s", err) -// return err -// } -// req.Header.Set("Content-Type", "application/json") - -// client := &http.Client{Timeout: 10 * time.Second} -// resp, err := client.Do(req) -// if err != nil { -// log.Printf("[ERROR] Failed to send HTTP request: %s", err) -// return err -// } -// defer resp.Body.Close() - -// if resp.StatusCode != 200 { -// log.Printf("[ERROR] Received non-successful HTTP status code: %d", resp.StatusCode) -// return fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode) -// } - -// return nil -// } + return nil +} // Is this ok to do with Docker? idk :) func getRunningWorkers(ctx context.Context, workerTimeout int) int { From ecfb78e82d1f7027265c47b20a00e3ec1e805857 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Sun, 23 Jun 2024 09:16:03 +0000 Subject: [PATCH 30/60] showing the tenzir active status in the UI --- frontend/src/views/Detection.jsx | 50 +++++++++++++++++++------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/frontend/src/views/Detection.jsx b/frontend/src/views/Detection.jsx index 80d2f821..63ca8ac4 100644 --- a/frontend/src/views/Detection.jsx +++ b/frontend/src/views/Detection.jsx @@ -11,14 +11,13 @@ import { toast } from "react-toastify"; import RuleCard from "./RuleCard"; import { styled } from "@mui/system"; -const ConnectedButton = styled(Button)({ - backgroundColor: "red", +const ConnectedButton = styled(Button)(({ theme, isConnected }) => ({ + backgroundColor: isConnected ? "green" : "red", color: "white", -}); +})); -const handleDirectoryChange = ( folderDisabled, setFolderDisabled, globalUrl) => { - - const action = folderDisabled ? "enable_folder" : "disable_folder" +const handleDirectoryChange = (folderDisabled, setFolderDisabled, globalUrl) => { + const action = folderDisabled ? "enable_folder" : "disable_folder"; const url = `${globalUrl}/api/v1/files/detection/${action}`; fetch(url, { @@ -31,8 +30,8 @@ const handleDirectoryChange = ( folderDisabled, setFolderDisabled, globalUrl) => .then((response) => response.json().then((responseJson) => { if (responseJson["success"] === true) { - if (action === "enable_folder") setFolderDisabled(false); - else setFolderDisabled(true); + if (action === "enable_folder") setFolderDisabled(false); + else setFolderDisabled(true); } else { //toast(`failed to disable rule`); } @@ -42,13 +41,19 @@ const handleDirectoryChange = ( folderDisabled, setFolderDisabled, globalUrl) => console.log(`Error in ${action} the rule: `, error); toast(`An error occurred while ${action} the rule`); }); +}; -} - -const Detection = ({ globalUrl, ruleInfo, folderDisabled, setFolderDisabled, openEditBar }) => { +const Detection = ({ + globalUrl, + ruleInfo, + folderDisabled, + setFolderDisabled, + openEditBar, + isTenzirActive, +}) => { return ( - + Sigma Detection Rules - - Not Connected to SIEM + + {isTenzirActive ? "Connected to SIEM" : "Not Connected to SIEM"} Global disable/enable - handleDirectoryChange(folderDisabled, setFolderDisabled, globalUrl)} - /> + + handleDirectoryChange(folderDisabled, setFolderDisabled, globalUrl) + } + /> Date: Sun, 23 Jun 2024 16:49:17 +0000 Subject: [PATCH 31/60] adding health check for tenzir --- backend/go-app/main.go | 2 +- functions/onprem/orborus/orborus.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 6fd0ff22..202aa891 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5131,7 +5131,6 @@ func initHandlers() { r.HandleFunc("/api/v1/triggers/pipeline", shuffle.HandleNewPipelineRegister).Methods("POST", "OPTIONS") //r.HandleFunc("/api/v1/triggers/pipeline/save", shuffle.HandleSavePipelineInfo).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/pipelines/{key}", handlePipelineCallback).Methods("POST", "GET", "PATCH", "PUT", "DELETE", "OPTIONS") - r.HandleFunc("/api/v1/pipelines/tenzir_node_health", handleTenzirHealthUpdate).Methods("POST","OPTIONS") r.HandleFunc("/api/v1/triggers", shuffle.HandleGetTriggers).Methods("GET", "OPTIONS") //r.HandleFunc("/api/v1/triggers/gmail/routing", handleGmailRouting).Methods("POST", "OPTIONS") @@ -5200,6 +5199,7 @@ func initHandlers() { r.HandleFunc("/api/v1/files/detection/sigma_rules", shuffle.HandleGetSigmaRules).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/detection/{fileId}/{action}", shuffle.HandleToggleRule).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/files/detection/{action}", shuffle.HandleFolderToggle).Methods("PUT", "OPTIONS") + r.HandleFunc("/api/v1/detection/siem/node_health", handleTenzirHealthUpdate).Methods("POST","OPTIONS") // Introduced in 0.9.21 to handle notifications for e.g. failed Workflow r.HandleFunc("/api/v1/notifications", shuffle.HandleCreateNotification).Methods("POST", "OPTIONS") diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 49bb2bca..17a76611 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1913,6 +1913,7 @@ func main() { log.Printf("[INFO] Waiting for executions at %s with Environment %#v", fullUrl, environment) hasStarted := false for { + _ = sendTenzirHealthStatus() if req.Method == "POST" { // Should find data to send (memory etc.) @@ -2260,7 +2261,6 @@ func main() { } } - _ = sendTenzirHealthStatus() time.Sleep(time.Duration(sleepTime) * time.Second) } } @@ -2580,9 +2580,9 @@ func deployTenzirNode() error { } func checkTenzirNode() error { - retries := 20 - retryInterval := 3 * time.Second - url := fmt.Sprintf("%s/api/v0/ping", tenzirUrl) + retries := 5 + retryInterval := 3 * time.Second + url := fmt.Sprintf("%s/api/v0/ping",tenzirUrl) forwardMethod := "POST" client := http.Client{} @@ -3075,7 +3075,7 @@ func removePath(containerName, path string) error { func sendTenzirHealthStatus() error { var status string - url := fmt.Sprintf("%s/api/v1/triggers/pipeline/tenzir_node_health", baseUrl) + url := fmt.Sprintf("%s/api/v1/detection/siem/node_health", baseUrl) err := checkTenzirNode() if err != nil { return err @@ -3116,7 +3116,7 @@ func sendTenzirHealthStatus() error { log.Printf("[ERROR] Received non-successful HTTP status code: %d", resp.StatusCode) return fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode) } - + log.Printf("this is send successfully") return nil } From e24550a64106775a7d7a5ae7f6069e9cab838bbe Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Sun, 23 Jun 2024 16:49:53 +0000 Subject: [PATCH 32/60] refactored and adding support for searching --- frontend/src/views/Detection.jsx | 132 ++++++++++++++++++++-- frontend/src/views/DetectionDashboard.jsx | 9 +- 2 files changed, 130 insertions(+), 11 deletions(-) diff --git a/frontend/src/views/Detection.jsx b/frontend/src/views/Detection.jsx index 63ca8ac4..8b6214a1 100644 --- a/frontend/src/views/Detection.jsx +++ b/frontend/src/views/Detection.jsx @@ -1,4 +1,4 @@ -import React from "react"; +import React, { useState, useRef } from "react"; import { Container, Box, @@ -7,6 +7,7 @@ import { Typography, Button, } from "@mui/material"; +import { Publish as PublishIcon } from "@mui/icons-material"; import { toast } from "react-toastify"; import RuleCard from "./RuleCard"; import { styled } from "@mui/system"; @@ -51,6 +52,95 @@ const Detection = ({ openEditBar, isTenzirActive, }) => { + const [searchQuery, setSearchQuery] = useState(""); + const uploadRef = useRef(null); + + const uploadFiles = (files) => { + for (const key in files) { + try { + const filename = files[key].name; + const filedata = new FormData(); + filedata.append("shuffle_file", files[key]); + + if (typeof files[key] === "object") { + handleCreateFile(filename, filedata); + } + } catch (e) { + console.log("Error in dropzone: ", e); + } + } + + setTimeout(() => { + // Additional logic if needed + }, 2500); + }; + + const handleCreateFile = (filename, file) => { + const data = { + filename: filename, + org_id: "default", + workflow_id: "global", + namespace: "sigma", + }; + + fetch(globalUrl + "/api/v1/files/create", { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + body: JSON.stringify(data), + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for apps :O!"); + return; + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success === true) { + handleFileUpload(responseJson.id, file); + } else { + toast("Failed to upload file ", filename); + } + }) + .catch((error) => { + toast("Failed to upload file ", filename); + console.log(error.toString()); + }); + }; + + const handleFileUpload = (file_id, file) => { + fetch(`${globalUrl}/api/v1/files/${file_id}/upload`, { + method: "POST", + credentials: "include", + body: file, + }) + .then((response) => { + if (response.status !== 200 && response.status !== 201) { + console.log("Status not 200 for apps :O!"); + toast("File was created, but failed to upload."); + return; + } + + return response.json(); + }) + .then((responseJson) => { + // Handle the response as needed + }) + .catch((error) => { + toast(error.toString()); + }); + }; + + const filteredRules = ruleInfo.filter((rule) => + rule.title.toLowerCase().includes(searchQuery.toLowerCase()) || + rule.description.toLowerCase().includes(searchQuery.toLowerCase()) + ); + return ( @@ -65,10 +155,7 @@ const Detection = ({ Sigma Detection Rules - + {isTenzirActive ? "Connected to SIEM" : "Not Connected to SIEM"} @@ -80,7 +167,36 @@ const Detection = ({ mb: 2, }} > - + + setSearchQuery(e.target.value)} + /> + + { + uploadFiles(event.target.files); + }} + /> + Global disable/enable @@ -102,8 +218,8 @@ const Detection = ({ p: 1, }} > - {ruleInfo.length > 0 && - ruleInfo.map((card) => ( + {filteredRules.length > 0 && + filteredRules.map((card) => ( { +const getSigmaInfo = (globalUrl, setRuleInfo, setFolderDisabled, setIsTenzirActive) => { const url = globalUrl + "/api/v1/files/detection/sigma_rules"; fetch(url, { @@ -21,6 +21,8 @@ const getSigmaInfo = (globalUrl, setRuleInfo, setFolderDisabled) => { } else { setRuleInfo(responseJson.sigma_info); setFolderDisabled(responseJson.folder_disabled); + setIsTenzirActive(responseJson.is_tenzir_active); + } }) ) @@ -35,11 +37,12 @@ const DetectionDashBoard = (props) => { const [ruleInfo, setRuleInfo] = useState([]); const [selectedRule, setSelectedRule] = useState(null); const [fileData, setFileData] = React.useState(""); + const [isTenzirActive, setIsTenzirActive] = React.useState(false); const [folderDisabled, setFolderDisabled] = useState(false); useEffect(() => { - getSigmaInfo(globalUrl, setRuleInfo, setFolderDisabled); + getSigmaInfo(globalUrl, setRuleInfo, setFolderDisabled, setIsTenzirActive); }, [folderDisabled]); useEffect(() => { @@ -103,7 +106,7 @@ const DetectionDashBoard = (props) => { onSave={handleSave} /> ) : null} */} - + ); }; From 073b9f3f790355f6c7d87028efdb0edcbc4b81bd Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Mon, 24 Jun 2024 20:28:32 +0530 Subject: [PATCH 33/60] adding a select option for the sigma rules --- frontend/src/views/AngularWorkflow.jsx | 226 +++++++++++++++---------- 1 file changed, 140 insertions(+), 86 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index aca59723..9b7755ac 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -534,6 +534,7 @@ const AngularWorkflow = (defaultprops) => { const [listCache, setListCache] = React.useState([]); const [selectedOption, setSelectedOption] = React.useState(""); const [tenzirConfigModalOpen, setTenzirConfigModalOpen] = React.useState(false); + const [rules, setRules] = React.useState([]); const [distributedFromParent, setDistributedFromParent] = React.useState("") const [suborgWorkflows, setSuborgWorkflows] = React.useState([]) @@ -992,6 +993,12 @@ const releaseToConnectLabel = "Release to Connect" } }, [authenticationModalOpen]) + useEffect(() =>{ + if (tenzirConfigModalOpen === false) return; + + getSigmaInfo(); + },[tenzirConfigModalOpen]) + const listOrgCache = (orgId) => { fetch(`${globalUrl}/api/v1/orgs/${orgId}/list_cache`, { method: "GET", @@ -8308,6 +8315,32 @@ const releaseToConnectLabel = "Release to Connect" }); }; + const getSigmaInfo = () => { + const url = globalUrl + "/api/v1/files/detection/sigma_rules"; + + fetch(url, { + method: "GET", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => + response.json().then((responseJson) => { + if (responseJson["success"] === false) { + toast("Failed to get sigma rules"); + } else { + setRules(responseJson.sigma_info); + + } + }) + ) + .catch((error) => { + console.log("Error in getting sigma files: ", error); + toast("An error occurred while fetching sigma rules"); + }); + }; + const parsedHeight = isMobile ? bodyHeight - appBarSize * 4 : bodyHeight - appBarSize - 50 const appViewStyle = { marginLeft: 5, @@ -21160,95 +21193,116 @@ const releaseToConnectLabel = "Release to Connect" ) : null; - const tenzirConfigModal = tenzirConfigModalOpen ? ( - { + if (!tenzirConfigModalOpen) return null; + + const [loading, setLoading] = useState(true); + const [selectedRules, setSelectedRules] = useState([]); + + const handleRuleChange = (event) => { + setSelectedRules(event.target.value); + }; + + const handleSelectAll = () => { + const allEnabledRules = rules.filter(rule => rule.is_enabled).map(rule => rule.file_id); + setSelectedRules(allEnabledRules); + }; + + const handleClose = () => { + setTenzirConfigModalOpen(false); + }; + + const handleSubmit = () => { + const selectedRuleFiles = rules + .filter(rule => selectedRules.includes(rule.file_id)); + + console.log('Selected Rule Files:', selectedRuleFiles); + console.log('Selected Rule File Names:', selectedRuleFiles.map(rule => rule.file_id)); + + setTenzirConfigModalOpen(false); + }; + + + const enabledSigmaInfo = rules.filter(rule => rule.is_enabled); + + + {loading ? ( + + ) : ( +
-
- - - command - param.name === "command")?.value) || ''} - /> + + {selectedOption === 'sigmaRule' && ( + <> + + Select Sigma Rules + + + + + )} + + + + + +
+ )} - - - - - -
- - { - setTenzirConfigModalOpen(false); - }} - > - - - - ) : null; + + + + +} From 6873a2202498eaa82c330c57819344afd9bd5e6c Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Tue, 25 Jun 2024 12:00:03 +0530 Subject: [PATCH 34/60] saving the selected rules to the trigger --- frontend/src/views/AngularWorkflow.jsx | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 9b7755ac..2e3bb393 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -21215,14 +21215,29 @@ const releaseToConnectLabel = "Release to Connect" const handleSubmit = () => { const selectedRuleFiles = rules .filter(rule => selectedRules.includes(rule.file_id)); + + if (selectedTrigger.trigger_type !== "PIPELINE") { + toast("Unable to save the configuration"); + return; + } + + selectedTrigger.parameters = selectedRuleFiles; + console.log('Selected Rule Files:', selectedRuleFiles); console.log('Selected Rule File Names:', selectedRuleFiles.map(rule => rule.file_id)); setTenzirConfigModalOpen(false); }; - + useEffect(()=>{ + if (selectedTrigger.trigger_type !== "PIPELINE") { + //toast("Unable to save the configuration"); + return; + } + setSelectedRules(selectedTrigger.parameters); + },[]) + const enabledSigmaInfo = rules.filter(rule => rule.is_enabled); Date: Tue, 25 Jun 2024 12:01:37 +0530 Subject: [PATCH 35/60] Merge branch '2.0.0' of github.com:satti-hari-krishna-reddy/Shuffle into 2.0.0 --- frontend/src/views/AngularWorkflow.jsx | 181 +++++++++++++------------ 1 file changed, 92 insertions(+), 89 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 2e3bb393..cc4d0b40 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -15416,7 +15416,7 @@ const releaseToConnectLabel = "Release to Connect" toast("please stop the trigger to edit the configuration"); return; } else { - setSelectedOption("Sigma Rulesearch"); + setSelectedOption("SigmaRule"); setTenzirConfigModalOpen(true); }}} style={{ @@ -15432,8 +15432,8 @@ const releaseToConnectLabel = "Release to Connect" setSelectedOption("Sigma Rulesearch")} + checked={selectedOption === "SigmaRule"} + onChange={() => setSelectedOption("SigmaRule")} value={"Sigma Rulesearch"} name="option" /> @@ -21193,25 +21193,23 @@ const releaseToConnectLabel = "Release to Connect" ) : null; - const tenzirConfigModal = () => { - if (!tenzirConfigModalOpen) return null; - - const [loading, setLoading] = useState(true); + const TenzirConfigModal = () => { + const [loading, setLoading] = useState(false); const [selectedRules, setSelectedRules] = useState([]); - + const handleRuleChange = (event) => { setSelectedRules(event.target.value); }; - + const handleSelectAll = () => { const allEnabledRules = rules.filter(rule => rule.is_enabled).map(rule => rule.file_id); setSelectedRules(allEnabledRules); }; - + const handleClose = () => { setTenzirConfigModalOpen(false); }; - + const handleSubmit = () => { const selectedRuleFiles = rules .filter(rule => selectedRules.includes(rule.file_id)); @@ -21239,85 +21237,90 @@ const releaseToConnectLabel = "Release to Connect" },[]) const enabledSigmaInfo = rules.filter(rule => rule.is_enabled); - - - {loading ? ( - - ) : ( -
- - {selectedOption === 'sigmaRule' && ( - <> - - Select Sigma Rules - - - - - )} - - - - - -
- )} - - - - - -} + {loading ? ( + + ) : ( +
+ + {selectedOption === 'SigmaRule' && ( + <> + + Select Sigma Rules + + + + + )} + + + + + +
+ )} + + + + + + ); + } + @@ -21897,7 +21900,7 @@ const releaseToConnectLabel = "Release to Connect" {codePopoutModal} {workflowRevisions} {authenticationModal} - {tenzirConfigModal} + {} {/*editWorkflowModal*/} {authgroupModal} {executionArgumentModal} From 6ec94335649e951c6c9cd84c00d3db9d775344cb Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Tue, 25 Jun 2024 12:35:49 +0000 Subject: [PATCH 36/60] making the select rules option work for pipelines --- backend/go-app/main.go | 4 ++ frontend/src/views/AngularWorkflow.jsx | 93 ++++++++++++++++++-------- 2 files changed, 70 insertions(+), 27 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 202aa891..274f1145 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5199,7 +5199,11 @@ func initHandlers() { r.HandleFunc("/api/v1/files/detection/sigma_rules", shuffle.HandleGetSigmaRules).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/detection/{fileId}/{action}", shuffle.HandleToggleRule).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/files/detection/{action}", shuffle.HandleFolderToggle).Methods("PUT", "OPTIONS") + r.HandleFunc("/api/v1/detection/siem/node_health", handleTenzirHealthUpdate).Methods("POST","OPTIONS") + r.HandleFunc("/api/v1/detection/{triggerId}/selected_rules", shuffle.HandleGetSelectedRules).Methods("GET","OPTIONS") + r.HandleFunc("/api/v1/detection/{triggerId}/selected_rules/save", shuffle.HandleSaveSelectedRules).Methods("POST","OPTIONS") + // Introduced in 0.9.21 to handle notifications for e.g. failed Workflow r.HandleFunc("/api/v1/notifications", shuffle.HandleCreateNotification).Methods("POST", "OPTIONS") diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index cc4d0b40..f3acbf27 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -21194,9 +21194,37 @@ const releaseToConnectLabel = "Release to Connect" ) : null; const TenzirConfigModal = () => { - const [loading, setLoading] = useState(false); + const [loading, setLoading] = useState(true); const [selectedRules, setSelectedRules] = useState([]); + useEffect(() => { + if (tenzirConfigModalOpen) { + try { + const url = globalUrl + "/api/v1/detection/" + selectedTrigger.id + "/selected_rules" + fetch(url, { + method: "GET", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then(response => response.json()) + .then(data => { + const savedRules = data.selected_rules.map(rule => rule.file_id); + setSelectedRules(savedRules); + setLoading(false); + }) + .catch(error => { + console.error('Error fetching selected rules:', error); + setLoading(false); + }); + } catch (error) { + console.error('Error:', error); + setLoading(false); + } + } + }, [tenzirConfigModalOpen]); + const handleRuleChange = (event) => { setSelectedRules(event.target.value); }; @@ -21211,31 +21239,43 @@ const releaseToConnectLabel = "Release to Connect" }; const handleSubmit = () => { - const selectedRuleFiles = rules - .filter(rule => selectedRules.includes(rule.file_id)); - - if (selectedTrigger.trigger_type !== "PIPELINE") { - toast("Unable to save the configuration"); - return; - } - - selectedTrigger.parameters = selectedRuleFiles; - - - console.log('Selected Rule Files:', selectedRuleFiles); - console.log('Selected Rule File Names:', selectedRuleFiles.map(rule => rule.file_id)); - - setTenzirConfigModalOpen(false); - }; - - useEffect(()=>{ - if (selectedTrigger.trigger_type !== "PIPELINE") { - //toast("Unable to save the configuration"); - return; + const selectedRuleFiles = rules.filter(rule => selectedRules.includes(rule.file_id)); + + const payload = { + selected_rules: selectedRuleFiles.map(rule => ({ + file_name: rule.file_name, + title: rule.title, + description: rule.description, + file_id: rule.file_id, + is_enabled: rule.is_enabled, + })) + }; + + try { + const url = globalUrl + "/api/v1/detection/" + selectedTrigger.id + "/selected_rules/save" + fetch(url, { + method: 'POST', + credentials: "include", + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(payload), + }) + .then(response => response.json()) + .then(data => { + console.log('Rules saved successfully:', data); + setTenzirConfigModalOpen(false); + }) + .catch(error => { + console.error('Error saving selected rules:', error); + toast("Unable to save the configuration"); + }); + } catch (error) { + console.error('Error:', error); + toast("Unable to save the configuration"); } - setSelectedRules(selectedTrigger.parameters); - },[]) - + }; + const enabledSigmaInfo = rules.filter(rule => rule.is_enabled); if (!tenzirConfigModalOpen) return null; @@ -21319,8 +21359,7 @@ const releaseToConnectLabel = "Release to Connect" ); - } - + }; From c6b124da3adf5bd40342e7ad3acfbe3480439c80 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Tue, 25 Jun 2024 21:30:43 +0530 Subject: [PATCH 37/60] reverting select rules and adding the kafka ui back --- frontend/src/views/AngularWorkflow.jsx | 318 +++++++++++++------------ 1 file changed, 172 insertions(+), 146 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index f3acbf27..f4604e15 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -535,6 +535,7 @@ const AngularWorkflow = (defaultprops) => { const [selectedOption, setSelectedOption] = React.useState(""); const [tenzirConfigModalOpen, setTenzirConfigModalOpen] = React.useState(false); const [rules, setRules] = React.useState([]); + const [sigmaFilesNames, setSigmaFileNames] = React.useState("") const [distributedFromParent, setDistributedFromParent] = React.useState("") const [suborgWorkflows, setSuborgWorkflows] = React.useState([]) @@ -993,12 +994,6 @@ const releaseToConnectLabel = "Release to Connect" } }, [authenticationModalOpen]) - useEffect(() =>{ - if (tenzirConfigModalOpen === false) return; - - getSigmaInfo(); - },[tenzirConfigModalOpen]) - const listOrgCache = (orgId) => { fetch(`${globalUrl}/api/v1/orgs/${orgId}/list_cache`, { method: "GET", @@ -1473,6 +1468,59 @@ const releaseToConnectLabel = "Release to Connect" setTenzirConfigModalOpen(false); }; + + const handleSubmit = (trigger) => { + if (trigger.trigger_type !== "PIPELINE") { + toast("Unable to save the configuration"); + return; + } + if (selectedOption == "kafka Queue") { + trigger.parameters = [] + + const topic = document.getElementById('topic')?.value + const bootstrapServers = document.getElementById('bootstrap_servers')?.value + const groupId = document.getElementById('group_id')?.value + const autoOffsetReset = document.getElementById('auto_offset_reset')?.value; + + if(topic) { + trigger.parameters.push({ + name: "topic", + value: topic + }) + } else { + toast("Please enter the topic name"); + return; + } + + if (bootstrapServers) { + trigger.parameters.push({ + name: "bootstrap_servers", + value: bootstrapServers + }); + } else { + toast("please enter bootstrap server details"); + return; + } + + if (groupId) { + trigger.parameters.push({ + name: "group_id", + value: groupId + }); + } + + if (autoOffsetReset) { + trigger.parameters.push({ + name: "auto_offset_reset", + value: autoOffsetReset + }); + } + + setTenzirConfigModalOpen(false); + } + + }; + const handleColoring = (actionId, status, label) => { if (cy === undefined) { @@ -15417,7 +15465,18 @@ const releaseToConnectLabel = "Release to Connect" return; } else { setSelectedOption("SigmaRule"); - setTenzirConfigModalOpen(true); + const command = `export | sigma /var/lib/tenzir/sigma_rules | to ${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` + const pipelineConfig = { + command: command, + name: selectedTrigger.label, + type: "create", + environment: selectedTrigger.environment, + workflow_id: workflow.id, + trigger_id: selectedTrigger.id, + start_node: "", + }; + submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig); + }}} style={{ border: "1px solid rgba(255,255,255,0.3)", @@ -21194,90 +21253,6 @@ const releaseToConnectLabel = "Release to Connect" ) : null; const TenzirConfigModal = () => { - const [loading, setLoading] = useState(true); - const [selectedRules, setSelectedRules] = useState([]); - - useEffect(() => { - if (tenzirConfigModalOpen) { - try { - const url = globalUrl + "/api/v1/detection/" + selectedTrigger.id + "/selected_rules" - fetch(url, { - method: "GET", - credentials: "include", - headers: { - "Content-Type": "application/json", - }, - }) - .then(response => response.json()) - .then(data => { - const savedRules = data.selected_rules.map(rule => rule.file_id); - setSelectedRules(savedRules); - setLoading(false); - }) - .catch(error => { - console.error('Error fetching selected rules:', error); - setLoading(false); - }); - } catch (error) { - console.error('Error:', error); - setLoading(false); - } - } - }, [tenzirConfigModalOpen]); - - const handleRuleChange = (event) => { - setSelectedRules(event.target.value); - }; - - const handleSelectAll = () => { - const allEnabledRules = rules.filter(rule => rule.is_enabled).map(rule => rule.file_id); - setSelectedRules(allEnabledRules); - }; - - const handleClose = () => { - setTenzirConfigModalOpen(false); - }; - - const handleSubmit = () => { - const selectedRuleFiles = rules.filter(rule => selectedRules.includes(rule.file_id)); - - const payload = { - selected_rules: selectedRuleFiles.map(rule => ({ - file_name: rule.file_name, - title: rule.title, - description: rule.description, - file_id: rule.file_id, - is_enabled: rule.is_enabled, - })) - }; - - try { - const url = globalUrl + "/api/v1/detection/" + selectedTrigger.id + "/selected_rules/save" - fetch(url, { - method: 'POST', - credentials: "include", - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(payload), - }) - .then(response => response.json()) - .then(data => { - console.log('Rules saved successfully:', data); - setTenzirConfigModalOpen(false); - }) - .catch(error => { - console.error('Error saving selected rules:', error); - toast("Unable to save the configuration"); - }); - } catch (error) { - console.error('Error:', error); - toast("Unable to save the configuration"); - } - }; - - const enabledSigmaInfo = rules.filter(rule => rule.is_enabled); - if (!tenzirConfigModalOpen) return null; return ( @@ -21302,65 +21277,116 @@ const releaseToConnectLabel = "Release to Connect" }, }} > - {loading ? ( - - ) : ( -
- - {selectedOption === 'SigmaRule' && ( - <> - - Select Sigma Rules - - - - - )} - - - - - -
- )} + +
Configuration options for Kafka
+ + + {selectedOption === "Kafka Queue" ? ( +
+ Topic + param.name === "topic", + )?.value || "" + } + /> + bootstrap.servers + param.name === "bootstrap_servers", + )?.value || "" + } + /> + group.id + param.name === "group_id", + )?.value || "" + } + /> + auto.offest.reset + param.name === "auto_offset_reset", + )?.value || "" + } + /> +
+ ) : null}{" "} + - - - + + + + ); }; - + const SuggestionBoxUi = () => { From a738be78bbed9afe11141f1156723d6ec8e47e03 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Tue, 25 Jun 2024 16:36:59 +0000 Subject: [PATCH 38/60] fixing few typos --- frontend/src/views/AngularWorkflow.jsx | 45 ++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index f4604e15..7a16c6e9 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1474,7 +1474,7 @@ const releaseToConnectLabel = "Release to Connect" toast("Unable to save the configuration"); return; } - if (selectedOption == "kafka Queue") { + if (selectedOption === "Kafka Queue") { trigger.parameters = [] const topic = document.getElementById('topic')?.value @@ -15541,8 +15541,41 @@ const releaseToConnectLabel = "Release to Connect" disabled={selectedTrigger.status === "running"} onClick={() => { - let command = (selectedTrigger?.parameters?.find(param => param.name === "command")?.value) || '' - command = `${command} | to ${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` + if (selectedOption === "Kafka Queue"){ + + const topic = (selectedTrigger?.parameters?.find(param => param.name === "topic")?.value) || '' + const bootstrapServers = (selectedTrigger?.parameters?.find(param => param.name === "bootstrap_servers")?.value) || '' + const groupId = (selectedTrigger?.parameters?.find(param => param.name === "group_id")?.value) || '' + const autoOffsetReset = (selectedTrigger?.parameters?.find(param => param.name === "auto_offset_reset")?.value) || '' + let command = "from kafka" + + if(topic) { + command = `${command} -t ${topic}` + } else { + toast("please enter the topic name") + return; + } + if(bootstrapServers) { + command = `${command} -e -o stored -X bootstrap.servers=${bootstrapServers}` + } else { + toast("please enter the bootstrap servers details") + return; + } + + if(groupId) { + command = `${command},group.id=${groupId}` + } else { + command = `${command},group.id=${selectedTrigger.id}` + } + if(autoOffsetReset) { + command = `${command},auto.offset.reset=${autoOffsetReset}` + } else { + command = `${command},auto.offset.reset=earliest` + + } + command = `${command},auto.offset.reset=earliest` + command = `${command},client.id=${selectedTrigger.id},enable.auto.commit=true,auto.commit.interval.ms=1` + command = `${command} read json | to ${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` const pipelineConfig = { command: command, @@ -15554,7 +15587,7 @@ const releaseToConnectLabel = "Release to Connect" start_node: "", }; submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig); - }} + }}} color="primary" > Start @@ -21268,8 +21301,8 @@ const releaseToConnectLabel = "Release to Connect" pointerEvents: "auto", color: "white", minWidth: 600, - minHeight: 200, - maxHeight: 200, + minHeight: 550, + maxHeight: 550, padding: 15, overflow: "hidden", zIndex: 10012, From d4ce234fa9c0d3dd59e7596ea9cd1e30dbefab47 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Wed, 26 Jun 2024 16:34:19 +0530 Subject: [PATCH 39/60] adding endpoint option for syslog --- frontend/src/views/AngularWorkflow.jsx | 61 +++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 7a16c6e9..2ed1f493 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1517,6 +1517,20 @@ const releaseToConnectLabel = "Release to Connect" } setTenzirConfigModalOpen(false); + } else if (selectedOption === "Syslog listener") { + trigger.parameters = [] + + const endpoint = document.getElementById('endpoint')?.value + + if(endpoint) { + trigger.parameters.push({ + name: "endpoint", + value: endpoint + }) + } else { + toast("Please enter your endpoint"); + return; + } } }; @@ -15587,7 +15601,28 @@ const releaseToConnectLabel = "Release to Connect" start_node: "", }; submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig); - }}} + } else if (selectedOption === "Syslog listener"){ + let command = "" + const endpoint = (selectedTrigger?.parameters?.find(param => param.name === "endpoint")?.value) || '' + if(endpoint) { + command = `from tcp://${endpoint} | read syslog | import` + } else { + toast("please enter the topic name") + return; + } + + const pipelineConfig = { + command: command, + name: selectedTrigger.label, + type: "create", + environment: selectedTrigger.environment, + workflow_id: workflow.id, + trigger_id: selectedTrigger.id, + start_node: "", + }; + submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig); + } + }} color="primary" > Start @@ -21394,6 +21429,30 @@ const releaseToConnectLabel = "Release to Connect" />
) : null}{" "} + +{selectedOption === "Syslog listener" ? ( +
+ End Point + param.name === "endpoint", + )?.value || "" + } + /> +
+ ) : null}{" "} From 78b6d9e0673c2f86c62f73e99b13311fd87239e2 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Thu, 27 Jun 2024 05:22:49 +0000 Subject: [PATCH 40/60] bug fixes --- backend/go-app/main.go | 2 +- frontend/src/views/Detection.jsx | 4 ++-- frontend/src/views/DetectionDashboard.jsx | 2 +- functions/onprem/orborus/orborus.go | 4 +--- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 274f1145..7864a427 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -2169,7 +2169,7 @@ func handleTenzirHealthUpdate(resp http.ResponseWriter, request *http.Request) { status := healthUpdate.Status result, err := shuffle.GetDisabledRules(ctx) - if (err != nil && err.Error() != "rules doesn't exist") || err == nil { + if (err != nil && err.Error() == "rules doesn't exist") || err == nil { result.IsTenzirActive = status result.LastActive = time.Now().Unix() diff --git a/frontend/src/views/Detection.jsx b/frontend/src/views/Detection.jsx index 8b6214a1..f7e2d135 100644 --- a/frontend/src/views/Detection.jsx +++ b/frontend/src/views/Detection.jsx @@ -136,7 +136,7 @@ const Detection = ({ }); }; - const filteredRules = ruleInfo.filter((rule) => + const filteredRules = ruleInfo?.filter((rule) => rule.title.toLowerCase().includes(searchQuery.toLowerCase()) || rule.description.toLowerCase().includes(searchQuery.toLowerCase()) ); @@ -218,7 +218,7 @@ const Detection = ({ p: 1, }} > - {filteredRules.length > 0 && + {filteredRules?.length > 0 && filteredRules.map((card) => ( { }, [folderDisabled]); useEffect(() => { - if (ruleInfo.length > 0) { + if (ruleInfo?.length > 0) { openEditBar(ruleInfo[0]); } }, [ruleInfo]); diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 17a76611..d08f2bed 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -102,6 +102,7 @@ var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME") var orborusLabel = os.Getenv("SHUFFLE_ORBORUS_LABEL") var memcached = os.Getenv("SHUFFLE_MEMCACHED") var tenzirUrl = os.Getenv("SHUFFLE_TENZIR_URL") +var apiKey = os.Getenv("AUTH_FOR_ORBORUS") var executionIds = []string{} var namespacemade = false // For K8s @@ -2923,8 +2924,6 @@ func searchPipeline(identifier string) (string, error) { func handleFileCategoryChange() error{ apiEndpoint := baseUrl+"/api/v1/files/namespaces/sigma" - apiKey := "12e7150e-1e03-4834-a839-de4688f50ad0" - req, err := http.NewRequest("GET", apiEndpoint, nil) if err != nil { return err @@ -3116,7 +3115,6 @@ func sendTenzirHealthStatus() error { log.Printf("[ERROR] Received non-successful HTTP status code: %d", resp.StatusCode) return fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode) } - log.Printf("this is send successfully") return nil } From 4708d7d9078924a97813ebb9f92dac0572b09e61 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Thu, 27 Jun 2024 11:59:31 +0000 Subject: [PATCH 41/60] adding api key to use for orborus to download files --- .env | 1 + 1 file changed, 1 insertion(+) diff --git a/.env b/.env index 298128cc..5d6b0288 100755 --- a/.env +++ b/.env @@ -40,6 +40,7 @@ BACKEND_HOSTNAME=shuffle-backend BACKEND_PORT=5001 FRONTEND_PORT=3001 FRONTEND_PORT_HTTPS=3443 +AUTH_FOR_ORBORUS = # CHANGE THIS IF YOU WANT GOOD LOCAL EXECUTIONS: OUTER_HOSTNAME=shuffle-backend From dbbe54a8c6362c7f8b82dd48af3913e52f8fde19 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Wed, 3 Jul 2024 11:09:05 +0530 Subject: [PATCH 42/60] adding an endpoint to spin up tenzir node --- backend/go-app/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 7864a427..4ac495de 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5200,6 +5200,7 @@ func initHandlers() { r.HandleFunc("/api/v1/files/detection/{fileId}/{action}", shuffle.HandleToggleRule).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/files/detection/{action}", shuffle.HandleFolderToggle).Methods("PUT", "OPTIONS") + r.HandleFunc("/api/v1/detection/siem/connect", shuffle.HandleConnectSiem).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/detection/siem/node_health", handleTenzirHealthUpdate).Methods("POST","OPTIONS") r.HandleFunc("/api/v1/detection/{triggerId}/selected_rules", shuffle.HandleGetSelectedRules).Methods("GET","OPTIONS") r.HandleFunc("/api/v1/detection/{triggerId}/selected_rules/save", shuffle.HandleSaveSelectedRules).Methods("POST","OPTIONS") From ccad70a2ee4cca6f1d862b7e9309b07ff85f86c3 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Wed, 3 Jul 2024 11:09:52 +0530 Subject: [PATCH 43/60] orborus can now start the tenzir node based on the request --- functions/onprem/orborus/orborus.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index d08f2bed..391a1620 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2072,7 +2072,16 @@ func main() { } toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - } else { + } else if incRequest.Type == "START_TENZIR" { + + err := deployTenzirNode() + if err != nil{ + log.Printf("[ERROR] failed to deploy the pipeline, reason: %s", err) + } + + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + + } else { newrequests = append(newrequests, incRequest) } } From d7741db32f8e19d21dbca7a25b5cf2ab6701d413 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Wed, 3 Jul 2024 11:10:13 +0530 Subject: [PATCH 44/60] adding connect to siem button --- frontend/src/views/Detection.jsx | 58 ++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/frontend/src/views/Detection.jsx b/frontend/src/views/Detection.jsx index f7e2d135..3f909758 100644 --- a/frontend/src/views/Detection.jsx +++ b/frontend/src/views/Detection.jsx @@ -10,12 +10,7 @@ import { import { Publish as PublishIcon } from "@mui/icons-material"; import { toast } from "react-toastify"; import RuleCard from "./RuleCard"; -import { styled } from "@mui/system"; - -const ConnectedButton = styled(Button)(({ theme, isConnected }) => ({ - backgroundColor: isConnected ? "green" : "red", - color: "white", -})); +import CircularProgress from "@material-ui/core/CircularProgress"; const handleDirectoryChange = (folderDisabled, setFolderDisabled, globalUrl) => { const action = folderDisabled ? "enable_folder" : "disable_folder"; @@ -54,6 +49,7 @@ const Detection = ({ }) => { const [searchQuery, setSearchQuery] = useState(""); const uploadRef = useRef(null); + const [loading, setLoading] = useState(false); const uploadFiles = (files) => { for (const key in files) { @@ -136,6 +132,41 @@ const Detection = ({ }); }; + const handleConnectClick = () => { + if (!isTenzirActive) { + setLoading(true); + const url = `${globalUrl}/api/v1/detection/siem/connect`; + + fetch(url, { + method: "GET", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => + response.json().then((responseJson) => { + if (responseJson["success"] === true) { + setTimeout(() => { + setLoading(false); + window.location.reload(); + }, 5000); + } else { + setLoading(false); + toast("Failed to connect to SIEM"); + } + }) + ) + .catch((error) => { + setLoading(false); + console.log(`Error in connecting to SIEM: `, error); + toast("An error occurred while connecting to SIEM"); + }); + } else { + console.log("Already connected to SIEM"); + } + }; + const filteredRules = ruleInfo?.filter((rule) => rule.title.toLowerCase().includes(searchQuery.toLowerCase()) || rule.description.toLowerCase().includes(searchQuery.toLowerCase()) @@ -155,9 +186,14 @@ const Detection = ({ Sigma Detection Rules - - {isTenzirActive ? "Connected to SIEM" : "Not Connected to SIEM"} - + setSearchQuery(e.target.value)} /> -
+ const defaultEnvironment = environments.find( + (env) => env.default && env.Name.toLowerCase() !== "cloud" + ); + + if (selectedTrigger.environment === "onprem" && defaultEnvironment !== undefined) { + selectedTrigger.environment = defaultEnvironment.Name + setSelectedTrigger(selectedTrigger) } + const PipelineSidebar = Object.getOwnPropertyNames(selectedTrigger).length === 0 || workflow.triggers[selectedTriggerIndex] === undefined && selectedTrigger.trigger_type !== "SCHEDULE" ? null :

From 295bcf07a450078c71f0fd72469a63659da4034e Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Tue, 16 Jul 2024 20:25:13 +0530 Subject: [PATCH 46/60] adding a new endpoint for downloading files from a repo --- backend/go-app/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 4ac495de..8f247fc8 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5188,6 +5188,7 @@ func initHandlers() { // PS: For cloud, this has to use cloud storage. // https://developer.box.com/reference/get-files-id-content/ r.HandleFunc("/api/v1/files/download_remote", shuffle.HandleDownloadRemoteFiles).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v2/files/download_remote", shuffle.HandleDownloadRemoteFiles2).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/files/namespaces/{namespace}", shuffle.HandleGetFileNamespace).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}/content", shuffle.HandleGetFileContent).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/create", shuffle.HandleCreateFile).Methods("POST", "OPTIONS") From 94749e679e66dca9421d20071068456d13094d93 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Tue, 16 Jul 2024 20:26:11 +0530 Subject: [PATCH 47/60] making the pipeline side bar to auto select the defualt env initially --- frontend/src/views/AngularWorkflow.jsx | 96 ++++++++++---------------- 1 file changed, 38 insertions(+), 58 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index b74a7afd..c3ce2581 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -8290,11 +8290,11 @@ const releaseToConnectLabel = "Release to Connect" toast("Pipeline deleted!") return } - + if (trigger.parameters){ trigger.parameters.push({ name: data.name, value: data.command, - }); + });} if (data.type === "stop") trigger.status = "stopped"; else trigger.status = "running"; @@ -8302,7 +8302,6 @@ const releaseToConnectLabel = "Release to Connect" setSelectedTrigger(trigger); setWorkflow(workflow); - console.log("Should set the status to running and save"); saveWorkflow(workflow); } }) @@ -15347,7 +15346,7 @@ const releaseToConnectLabel = "Release to Connect" (env) => env.default && env.Name.toLowerCase() !== "cloud" ); - if (selectedTrigger.environment === "onprem" && defaultEnvironment !== undefined) { + if (selectedTrigger.trigger_type === "PIPELINE" && selectedTrigger.environment === "onprem" && defaultEnvironment !== undefined) { selectedTrigger.environment = defaultEnvironment.Name setSelectedTrigger(selectedTrigger) } @@ -15450,11 +15449,25 @@ const releaseToConnectLabel = "Release to Connect" key="syslogListener" onClick={() => { if(selectedTrigger.status === "running"){ - toast("please stop the trigger to edit the configuration"); + //toast("please stop the trigger to edit the configuration"); return; } else { setSelectedOption("Syslog listener"); - setTenzirConfigModalOpen(true); + const url = `${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` + const command = `from tcp://192.168.1.100:5162 read syslog | import` + const pipelineConfig = { + command: command, + name: selectedTrigger.label, + type: "create", + environment: selectedTrigger.environment, + workflow_id: workflow.id, + trigger_id: selectedTrigger.id, + start_node: "", + url:url, + }; + submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig); + + }}} style={{ border: "1px solid rgba(255,255,255,0.3)", @@ -15470,12 +15483,15 @@ const releaseToConnectLabel = "Release to Connect" control={ setSelectedOption("Syslog listener")} + onChange={() => { + if (selectedTrigger.status !== "running"){ + setSelectedOption("Syslog listener")}} + } value={"Syslog listener"} name="option" /> } - label="Start Syslog listener" + label= {selectedOption === "Syslog listener" && selectedTrigger.status === "running" ? "listening at 192.168.1.100:5162" : "Start Syslog listener"} />

@@ -15483,11 +15499,12 @@ const releaseToConnectLabel = "Release to Connect" key="sigmaRulesearch" onClick={() => { if(selectedTrigger.status === "running"){ - toast("please stop the trigger to edit the configuration"); + // toast("please stop the trigger to edit the configuration"); return; } else { setSelectedOption("SigmaRule"); - const command = `export | sigma /var/lib/tenzir/sigma_rules | to ${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` + const url = `${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` + const command = `export | sigma /var/lib/tenzir/sigma_rules | to ${url}` const pipelineConfig = { command: command, name: selectedTrigger.label, @@ -15496,6 +15513,7 @@ const releaseToConnectLabel = "Release to Connect" workflow_id: workflow.id, trigger_id: selectedTrigger.id, start_node: "", + url:url, }; submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig); @@ -15514,7 +15532,9 @@ const releaseToConnectLabel = "Release to Connect" control={ setSelectedOption("SigmaRule")} + onChange={() => { + if (selectedTrigger.status !== "running"){ + setSelectedOption("SigmaRule")}}} value={"Sigma Rulesearch"} name="option" /> @@ -15547,7 +15567,9 @@ const releaseToConnectLabel = "Release to Connect" control={ setSelectedOption("Kafka Queue")} + onChange={() => { + if (selectedTrigger.status !== "running"){ + setSelectedOption("Kafka Queue")}}} value={"Kafka Queue"} name="option" /> @@ -15564,7 +15586,7 @@ const releaseToConnectLabel = "Release to Connect" onClick={() => { if (selectedOption === "Kafka Queue"){ - + const url = `${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` const topic = (selectedTrigger?.parameters?.find(param => param.name === "topic")?.value) || '' const bootstrapServers = (selectedTrigger?.parameters?.find(param => param.name === "bootstrap_servers")?.value) || '' const groupId = (selectedTrigger?.parameters?.find(param => param.name === "group_id")?.value) || '' @@ -15594,10 +15616,10 @@ const releaseToConnectLabel = "Release to Connect" } else { command = `${command},auto.offset.reset=earliest` - } + } command = `${command},auto.offset.reset=earliest` command = `${command},client.id=${selectedTrigger.id},enable.auto.commit=true,auto.commit.interval.ms=1` - command = `${command} read json | to ${globalUrl}/api/v1/pipelines/pipeline_${selectedTrigger.id}` + command = `${command} read json | to ${url}` const pipelineConfig = { command: command, @@ -15607,28 +15629,9 @@ const releaseToConnectLabel = "Release to Connect" workflow_id: workflow.id, trigger_id: selectedTrigger.id, start_node: "", + url: url, }; submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig); - } else if (selectedOption === "Syslog listener"){ - let command = "" - const endpoint = (selectedTrigger?.parameters?.find(param => param.name === "endpoint")?.value) || '' - if(endpoint) { - command = `from tcp://${endpoint} | read syslog | import` - } else { - toast("please enter the topic name") - return; - } - - const pipelineConfig = { - command: command, - name: selectedTrigger.label, - type: "create", - environment: selectedTrigger.environment, - workflow_id: workflow.id, - trigger_id: selectedTrigger.id, - start_node: "", - }; - submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig); } }} color="primary" @@ -21438,29 +21441,6 @@ const releaseToConnectLabel = "Release to Connect"
) : null}{" "} -{selectedOption === "Syslog listener" ? ( -
- End Point - param.name === "endpoint", - )?.value || "" - } - /> -
- ) : null}{" "} From d6b78612df49b9a415c640d71c53ee13775bb3bf Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Tue, 16 Jul 2024 20:27:09 +0530 Subject: [PATCH 48/60] making enable / disable button to work only when connected to the siem --- frontend/src/views/Detection.jsx | 100 ++----------- frontend/src/views/DetectionDashboard.jsx | 174 ++++++++++++++-------- frontend/src/views/RuleCard.jsx | 78 +++++----- 3 files changed, 164 insertions(+), 188 deletions(-) diff --git a/frontend/src/views/Detection.jsx b/frontend/src/views/Detection.jsx index 3f909758..c86eeba2 100644 --- a/frontend/src/views/Detection.jsx +++ b/frontend/src/views/Detection.jsx @@ -1,4 +1,4 @@ -import React, { useState, useRef } from "react"; +import React, { useState } from "react"; import { Container, Box, @@ -7,12 +7,16 @@ import { Typography, Button, } from "@mui/material"; -import { Publish as PublishIcon } from "@mui/icons-material"; import { toast } from "react-toastify"; import RuleCard from "./RuleCard"; import CircularProgress from "@material-ui/core/CircularProgress"; -const handleDirectoryChange = (folderDisabled, setFolderDisabled, globalUrl) => { +const handleDirectoryChange = (folderDisabled, setFolderDisabled, globalUrl, isTenzirActive) => { + + if (!isTenzirActive) { + toast("connect to siem first for global enable/disable to work"); + return; + } const action = folderDisabled ? "enable_folder" : "disable_folder"; const url = `${globalUrl}/api/v1/files/detection/${action}`; @@ -44,94 +48,11 @@ const Detection = ({ ruleInfo, folderDisabled, setFolderDisabled, - openEditBar, isTenzirActive, }) => { const [searchQuery, setSearchQuery] = useState(""); - const uploadRef = useRef(null); const [loading, setLoading] = useState(false); - const uploadFiles = (files) => { - for (const key in files) { - try { - const filename = files[key].name; - const filedata = new FormData(); - filedata.append("shuffle_file", files[key]); - - if (typeof files[key] === "object") { - handleCreateFile(filename, filedata); - } - } catch (e) { - console.log("Error in dropzone: ", e); - } - } - - setTimeout(() => { - // Additional logic if needed - }, 2500); - }; - - const handleCreateFile = (filename, file) => { - const data = { - filename: filename, - org_id: "default", - workflow_id: "global", - namespace: "sigma", - }; - - fetch(globalUrl + "/api/v1/files/create", { - method: "POST", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - credentials: "include", - body: JSON.stringify(data), - }) - .then((response) => { - if (response.status !== 200) { - console.log("Status not 200 for apps :O!"); - return; - } - - return response.json(); - }) - .then((responseJson) => { - if (responseJson.success === true) { - handleFileUpload(responseJson.id, file); - } else { - toast("Failed to upload file ", filename); - } - }) - .catch((error) => { - toast("Failed to upload file ", filename); - console.log(error.toString()); - }); - }; - - const handleFileUpload = (file_id, file) => { - fetch(`${globalUrl}/api/v1/files/${file_id}/upload`, { - method: "POST", - credentials: "include", - body: file, - }) - .then((response) => { - if (response.status !== 200 && response.status !== 201) { - console.log("Status not 200 for apps :O!"); - toast("File was created, but failed to upload."); - return; - } - - return response.json(); - }) - .then((responseJson) => { - // Handle the response as needed - }) - .catch((error) => { - toast(error.toString()); - }); - }; - const handleConnectClick = () => { if (!isTenzirActive) { setLoading(true); @@ -150,7 +71,7 @@ const Detection = ({ setTimeout(() => { setLoading(false); window.location.reload(); - }, 5000); + }, 15000); } else { setLoading(false); toast("Failed to connect to SIEM"); @@ -240,8 +161,9 @@ const Detection = ({ - handleDirectoryChange(folderDisabled, setFolderDisabled, globalUrl) + handleDirectoryChange(folderDisabled, setFolderDisabled, globalUrl, isTenzirActive) } + disabled={!isTenzirActive} /> @@ -263,7 +185,7 @@ const Detection = ({ file_id={card.file_id} globalUrl={globalUrl} folderDisabled={folderDisabled} - openEditBar={() => openEditBar(card)} + isTenzirActive={isTenzirActive} {...card} /> ))} diff --git a/frontend/src/views/DetectionDashboard.jsx b/frontend/src/views/DetectionDashboard.jsx index e09814e4..4b2b1b06 100644 --- a/frontend/src/views/DetectionDashboard.jsx +++ b/frontend/src/views/DetectionDashboard.jsx @@ -1,68 +1,45 @@ import React, { useState, useEffect } from "react"; -import { Container} from "@mui/material"; +import { Container, CircularProgress, Typography } from "@mui/material"; import { toast } from "react-toastify"; import Detection from "./Detection"; -import EditComponent from "./EditRules"; - -const getSigmaInfo = (globalUrl, setRuleInfo, setFolderDisabled, setIsTenzirActive) => { - const url = globalUrl + "/api/v1/files/detection/sigma_rules"; - - fetch(url, { - method: "GET", - credentials: "include", - headers: { - "Content-Type": "application/json", - }, - }) - .then((response) => - response.json().then((responseJson) => { - if (responseJson["success"] === false) { - toast("Failed to get sigma rules"); - } else { - setRuleInfo(responseJson.sigma_info); - setFolderDisabled(responseJson.folder_disabled); - setIsTenzirActive(responseJson.is_tenzir_active); - - } - }) - ) - .catch((error) => { - console.log("Error in getting sigma files: ", error); - toast("An error occurred while fetching sigma rules"); - }); -}; const DetectionDashBoard = (props) => { const { globalUrl } = props; - const [ruleInfo, setRuleInfo] = useState([]); - const [selectedRule, setSelectedRule] = useState(null); - const [fileData, setFileData] = React.useState(""); - const [isTenzirActive, setIsTenzirActive] = React.useState(false); - + const [ruleInfo, setRuleInfo] = useState(null); + const [, setSelectedRule] = useState(null); + const [, setFileData] = useState(""); + const [isTenzirActive, setIsTenzirActive] = useState(false); const [folderDisabled, setFolderDisabled] = useState(false); + const [isLoading, setIsLoading] = useState(false); + const [importAttempts, setImportAttempts] = useState(0); + const maxImportAttempts = 2; useEffect(() => { - getSigmaInfo(globalUrl, setRuleInfo, setFolderDisabled, setIsTenzirActive); - }, [folderDisabled]); + const fetchTimeout = setTimeout(() => { + fetchSigmaInfo(); + }, 1000); // Delay by 1 second + + return () => clearTimeout(fetchTimeout); + }, [globalUrl]); useEffect(() => { - if (ruleInfo?.length > 0) { - openEditBar(ruleInfo[0]); + if (ruleInfo && ruleInfo.length === 0 && importAttempts < maxImportAttempts) { + importSigmaFromUrl(); } }, [ruleInfo]); const openEditBar = (rule) => { setSelectedRule(rule); - getFileContent(rule.file_id) + fetchFileContent(rule.file_id); }; const handleSave = (updatedContent) => { - toast("this will be saved"); + toast("This will be saved"); }; - const getFileContent = (file_id) => { + const fetchFileContent = (file_id) => { setFileData(""); - fetch(globalUrl + "/api/v1/files/" + file_id + "/content", { + fetch(`${globalUrl}/api/v1/files/${file_id}/content`, { method: "GET", headers: { "Content-Type": "application/json", @@ -77,38 +54,109 @@ const DetectionDashBoard = (props) => { } return response.text(); }) - .then((respdata) => { + .then((respdata) => { if (respdata.length === 0) { toast("Failed getting file. Is it deleted?"); return; } - return respdata - }) - .then((responseData) => { - - setFileData(responseData); + setFileData(respdata); }) .catch((error) => { toast(error.toString()); }); }; + const fetchSigmaInfo = () => { + const url = `${globalUrl}/api/v1/files/detection/sigma_rules`; + setIsLoading(true); + + fetch(url, { + method: "GET", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => response.json()) + .then((responseJson) => { + if (responseJson["success"] === false) { + toast("Failed to get sigma rules"); + } else { + setRuleInfo(responseJson.sigma_info || []); + setFolderDisabled(responseJson.folder_disabled); + setIsTenzirActive(responseJson.is_tenzir_active); + } + setIsLoading(false); + }) + .catch((error) => { + setIsLoading(false); + console.log("Error in getting sigma files: ", error); + toast("An error occurred while fetching sigma rules"); + setRuleInfo([]); + }); + }; + + const importSigmaFromUrl = () => { + setIsLoading(true); + setImportAttempts((prevAttempts) => prevAttempts + 1); + + const url = "https://github.com/satti-hari-krishna-reddy/shuffle_sigma"; + const folder = "sigma"; + + const parsedData = { + url: url, + path: folder, + field_3: "main", + }; + + toast(`Getting files from url ${url}. This may take a while if the repository is large. Please wait...`); + fetch(`${globalUrl}/api/v2/files/download_remote`, { + method: "POST", + mode: "cors", + headers: { + Accept: "application/json", + }, + body: JSON.stringify(parsedData), + credentials: "include", + }) + .then((response) => response.json()) + .then((responseJson) => { + if (responseJson.success) { + toast("Successfully loaded files from " + url); + fetchSigmaInfo(); // Fetch again after successful import + } else { + toast(responseJson.reason ? `Failed loading: ${responseJson.reason}` : "Failed loading"); + } + setIsLoading(false); + }) + .catch((error) => { + toast(error.toString()); + setIsLoading(false); + }); + }; + + if (isLoading && (!ruleInfo || ruleInfo.length === 0)) { + return ( + +
+ + Downloading rules, please wait... +
+ + ); + } + return ( - - {/* {selectedRule ? ( - - ) : null} */} - + + - ); + ); }; export default DetectionDashBoard; diff --git a/frontend/src/views/RuleCard.jsx b/frontend/src/views/RuleCard.jsx index fe334f85..b35d993a 100644 --- a/frontend/src/views/RuleCard.jsx +++ b/frontend/src/views/RuleCard.jsx @@ -10,7 +10,7 @@ import EditIcon from "@mui/icons-material/Edit"; import { toast } from "react-toastify"; import ShuffleCodeEditor from "../components/ShuffleCodeEditor1.jsx"; -const RuleCard = ({ ruleName, description, file_id, globalUrl, folderDisabled, ...otherProps }) => { +const RuleCard = ({ ruleName, description, file_id, globalUrl, folderDisabled, isTenzirActive, ...otherProps }) => { const [openCodeEditor, setOpenCodeEditor] = React.useState(false); const [fileData, setFileData] = React.useState(""); const [isEnabled, setIsEnabled] = React.useState(otherProps.is_enabled); @@ -22,6 +22,10 @@ const RuleCard = ({ ruleName, description, file_id, globalUrl, folderDisabled, . toast("enable the directory to enable individual rules"); return; } + if (!isTenzirActive) { + toast("connect to the siem to enable/disable the rule"); + return; + } const newIsEnabled = event.target.checked; toggleRule(file_id, !newIsEnabled, globalUrl, () => { setIsEnabled(newIsEnabled); @@ -73,6 +77,7 @@ const RuleCard = ({ ruleName, description, file_id, globalUrl, folderDisabled, . @@ -121,42 +126,43 @@ const toggleRule = (fileId, isCurrentlyEnabled, globalUrl, callback) => { toast(`An error occurred while ${action}ing the rule`); }); }; - const openEditBar = (file_id, setOpenCodeEditor, setFileData, globalUrl) => { - getFileContent(file_id, setFileData, globalUrl); + +const openEditBar = (file_id, setOpenCodeEditor, setFileData, globalUrl) => { + getFileContent(file_id, setFileData, globalUrl) + setOpenCodeEditor(true); - }; +}; - const getFileContent = (file_id, setFileData, globalUrl) => { - setFileData(""); - fetch(globalUrl + "/api/v1/files/" + file_id + "/content", { - method: "GET", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - credentials: "include", +const getFileContent = (file_id, setFileData, globalUrl) => { + setFileData(""); + fetch(globalUrl + "/api/v1/files/" + file_id + "/content", { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for file :O!"); + return ""; + } + return response.text(); }) - .then((response) => { - if (response.status !== 200) { - console.log("Status not 200 for file :O!"); - return ""; - } - return response.text(); - }) - .then((respdata) => { - if (respdata.length === 0) { - toast("Failed getting file. Is it deleted?"); - return; - } - return respdata - }) - .then((responseData) => { - - setFileData(responseData); - }) - .catch((error) => { - toast(error.toString()); - }); - }; - + .then((respdata) => { + if (respdata.length === 0) { + toast("Failed getting file. Is it deleted?"); + return; + } + return respdata + }) + .then((responseData) => { + + setFileData(responseData); + }) + .catch((error) => { + toast(error.toString()); + }); +}; export default RuleCard; From 9cf3f2fd6333ea46ce6dcbd1a519fd8552cb4c91 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Tue, 16 Jul 2024 20:28:00 +0530 Subject: [PATCH 49/60] a lot of things --- functions/onprem/orborus/orborus.go | 316 ++++++++++++++++++++++------ 1 file changed, 246 insertions(+), 70 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 391a1620..411571bb 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2052,14 +2052,28 @@ func main() { log.Printf("[ERROR] failed to deploy the pipeline, reason: %s", err) } - err = removeFile(fileName) + err = disableRule(fileName) if err != nil { log.Printf("[ERROR] Failed to disable the sigma file %s, reason: %s", fileName, err) } - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - } else if incRequest.Type == "DISABLE_SIGMA_FOLDER" { + } else if incRequest.Type == "ENABLE_SIGMA_FILE" { + fileName := incRequest.ExecutionArgument + err := deployTenzirNode() + if err != nil{ + log.Printf("[ERROR] failed to deploy the pipeline, reason: %s", err) + } + + err = enableRule(fileName) + if err != nil { + log.Printf("[ERROR] Failed to disable the sigma file %s, reason: %s", fileName, err) + } + + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + + } else if incRequest.Type == "DISABLE_SIGMA_FOLDER" { err := deployTenzirNode() if err != nil{ @@ -2526,9 +2540,19 @@ func deployTenzirNode() error { return nil } - containerInfo, err := dockercli.ContainerInspect(ctx, containerName) - if err != nil { - if dockerclient.IsErrNotFound(err) { + containerInfo, err := dockercli.ContainerInspect(ctx, containerName) + if err != nil { + if dockerclient.IsErrNotFound(err) { + // Create network if it doesn't exist + networkName := "tenzir-network" + networkSubnet := "192.168.1.0/24" + networkGateway := "192.168.1.1" + + err = createNetworkIfNotExists(ctx, networkName, networkSubnet, networkGateway) + if err != nil { + log.Printf("[ERROR] Failed to create network: %s", err) + return err + } // Check if image exists _, _, err := dockercli.ImageInspectWithRaw(ctx, imageName) @@ -2589,30 +2613,6 @@ func deployTenzirNode() error { return nil } -func checkTenzirNode() error { - retries := 5 - retryInterval := 3 * time.Second - url := fmt.Sprintf("%s/api/v0/ping",tenzirUrl) - forwardMethod := "POST" - - client := http.Client{} - req, err := http.NewRequest(forwardMethod, url, nil) - if err != nil { - log.Printf("[ERROR] Failed to create HTTP request: %s", err) - return err - } - - for i := 0; i < retries; i++ { - resp, err := client.Do(req) - if err == nil && resp.StatusCode == http.StatusOK { - return nil - } - time.Sleep(retryInterval) - } - - return fmt.Errorf("tenzir node is not available") -} - func createAndStartTenzirNode(ctx context.Context, containerName, imageName string, containerStartOptions container.StartOptions) error { healthconfig := &container.HealthConfig{ Test: []string{"tenzir --connection-timeout=30s --connection-retry-delay=1s 'api /ping'"}, @@ -2628,23 +2628,34 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri Entrypoint: []string{containerName}, } - hostConfig := &container.HostConfig{ - PortBindings: nat.PortMap{ - "5160/tcp": []nat.PortBinding{{HostPort: "5160"}}, - }, - Mounts: []mount.Mount{ - { - Type: mount.TypeVolume, - Source: containerName, - Target: "/var/lib/tenzir/", - }, - }, - VolumeDriver: "local", - } - _, err := dockercli.ContainerCreate(ctx, config, hostConfig, nil, nil, containerName) - if err != nil { - return err - } + hostConfig := &container.HostConfig{ + PortBindings: nat.PortMap{ + "5160/tcp": []nat.PortBinding{{HostPort: "5160"}}, + }, + Mounts: []mount.Mount{ + { + Type: mount.TypeVolume, + Source: containerName, + Target: "/var/lib/tenzir/", + }, + }, + VolumeDriver: "local", + } + + networkingConfig := &network.NetworkingConfig{ + EndpointsConfig: map[string]*network.EndpointSettings{ + "tenzir-network": { + IPAMConfig: &network.EndpointIPAMConfig{ + IPv4Address: "192.168.1.100", + }, + }, + }, + } + + _, err := dockercli.ContainerCreate(ctx, config, hostConfig, networkingConfig, nil, containerName) + if err != nil { + return err + } err = dockercli.ContainerStart(ctx, containerName, containerStartOptions) if err != nil { @@ -2653,16 +2664,76 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri } log.Printf("[INFO] Tenzir Node container started successfully") - log.Printf("[INFO] Waiting for Tenzir to become available ...") - err = checkTenzirNode() - if err != nil { - return err - } - log.Printf("[INFO] Successfully deployed Tenzir Node !") + log.Printf("[INFO] Waiting for Tenzir to become available ...") + err = checkTenzirNode() + if err != nil { + return err + } + log.Printf("[INFO] Successfully deployed Tenzir Node!") return nil } +func createNetworkIfNotExists(ctx context.Context, networkName, subnet, gateway string) error { + networks, err := dockercli.NetworkList(ctx, types.NetworkListOptions{}) + if err != nil { + return err + } + + for _, network := range networks { + if network.Name == networkName { + // Network exists + return nil + } + } + + ipamConfig := &network.IPAM{ + Config: []network.IPAMConfig{ + { + Subnet: subnet, + Gateway: gateway, + }, + }, + } + + networkCreate := types.NetworkCreate{ + CheckDuplicate: true, + Driver: "bridge", + IPAM: ipamConfig, + } + + _, err = dockercli.NetworkCreate(ctx, networkName, networkCreate) + if err != nil { + return err + } + + return nil +} + +func checkTenzirNode() error { + retries := 5 + retryInterval := 3 * time.Second + url := fmt.Sprintf("%s/api/v0/ping",tenzirUrl) + forwardMethod := "POST" + + client := http.Client{} + req, err := http.NewRequest(forwardMethod, url, nil) + if err != nil { + log.Printf("[ERROR] Failed to create HTTP request: %s", err) + return err + } + + for i := 0; i < retries; i++ { + resp, err := client.Do(req) + if err == nil && resp.StatusCode == http.StatusOK { + return nil + } + time.Sleep(retryInterval) + } + + return fmt.Errorf("tenzir node is not available") +} + func createPipeline(command, identifier string) (string, error) { toBeDeleted := false @@ -2702,8 +2773,6 @@ func createPipeline(command, identifier string) (string, error) { //command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | sigma /var/lib/tenzir/rule.yaml" //command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | import" - //command = "export | to https://expert-acorn-v6vg4j4j5w7q2wg6g-5001.app.github.dev/api/v1/hooks/webhook_623eab3f-0af4-4d40-abb9-699d9a493411" - log.Printf("[HARI] this is the command %s", command) requestBody := map[string]interface{}{ "definition": command, @@ -2931,8 +3000,8 @@ func searchPipeline(identifier string) (string, error) { return "", errors.New("no existing pipeline found with name") } -func handleFileCategoryChange() error{ - apiEndpoint := baseUrl+"/api/v1/files/namespaces/sigma" +func handleFileCategoryChange() error { + apiEndpoint := baseUrl + "/api/v1/files/namespaces/sigma" req, err := http.NewRequest("GET", apiEndpoint, nil) if err != nil { return err @@ -2943,12 +3012,12 @@ func handleFileCategoryChange() error{ client := &http.Client{} resp, err := client.Do(req) if err != nil { - return err + return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return err + return fmt.Errorf("received non-200 response: %s", resp.Status) } out, err := os.Create("files.zip") @@ -2961,10 +3030,10 @@ func handleFileCategoryChange() error{ _, err = io.Copy(out, resp.Body) if err != nil { - return err + return err } - fmt.Println("ZIP file downloaded successfully.") + log.Println("ZIP file downloaded successfully.") err = extractZIP("files.zip", "sigma_rules") if err != nil { @@ -2978,7 +3047,45 @@ func handleFileCategoryChange() error{ return err } - fmt.Println("Files copied to container successfully.") + log.Println("Files copied to container successfully.") + + checkDisabledDirCmd := exec.Command("docker", "exec", "tenzir-node", "sh", "-c", "test -d /var/lib/tenzir/disabled_rules") + if err := checkDisabledDirCmd.Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { + // Directory does not exist, nothing to do + log.Println("[DEBUG] /var/lib/tenzir/disabled_rules does not exist.") + return nil + } + + return fmt.Errorf("error checking disabled rules directory: %v", err) + } + + // List files in /var/lib/tenzir/disabled_rules + listFilesCmd := exec.Command("docker", "exec", "tenzir-node", "sh", "-c", "ls /var/lib/tenzir/disabled_rules") + output, err := listFilesCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error listing files in disabled rules directory: %v, output: %s", err, output) + } + + files := strings.Split(strings.TrimSpace(string(output)), "\n") + for _, file := range files { + disabledFilePath := fmt.Sprintf("/var/lib/tenzir/sigma_rules/%s", file) + checkFileCmd := exec.Command("docker", "exec", "tenzir-node", "sh", "-c", fmt.Sprintf("test -f %s", disabledFilePath)) + if err := checkFileCmd.Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { + log.Printf("[ERROR] File does not exist: %s, moving on.\n", disabledFilePath) + continue + } + return fmt.Errorf("error checking file: %v", err) + } + + deleteFileCmd := exec.Command("docker", "exec", "-u", "root", "tenzir-node", "sh", "-c", fmt.Sprintf("rm -f %s", disabledFilePath)) + if err := deleteFileCmd.Run(); err != nil { + return fmt.Errorf("error deleting file: %v", err) + } + log.Printf("[INFO] Deleted file: %s\n", disabledFilePath) + } + return nil } @@ -3025,7 +3132,6 @@ func extractFile(f *zip.File, destDir string) error { func copyToTenzir(srcPath, destPath string) error { containerName := "tenzir-node" - // Check if the sigma_rules directory exists in the container checkCmd := exec.Command("docker", "exec", containerName, "test", "-d", destPath) if err := checkCmd.Run(); err == nil { rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", "-rf", destPath) @@ -3034,7 +3140,6 @@ func copyToTenzir(srcPath, destPath string) error { } } - // Copy the new directory to the container cpCmd := exec.Command("docker", "cp", srcPath, fmt.Sprintf("%s:%s", containerName, destPath)) var out bytes.Buffer cpCmd.Stdout = &out @@ -3049,10 +3154,19 @@ func copyToTenzir(srcPath, destPath string) error { } func removeAllFiles() error { - containerName := "tenzir-node" - sigmaPath := "/var/lib/tenzir/sigma_rules/*" + containerName := "tenzir-node" + sigmaPath := "/var/lib/tenzir/sigma_rules/*" - cmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("rm -rf %s", sigmaPath)) + checkCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("ls %s", sigmaPath)) + checkOutput, checkErr := checkCmd.CombinedOutput() + if checkErr != nil { + if strings.Contains(string(checkOutput), "No such file or directory") { + return nil // nothing to delete + } + return fmt.Errorf("error checking files: %v, output: %s", checkErr, checkOutput) + } + + cmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("rm -rf %s", sigmaPath)) output, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("error removing files: %v, output: %s", err, output) @@ -3064,16 +3178,21 @@ func removeFile(fileName string) error { containerName := "tenzir-node" srcPath := fmt.Sprintf("/var/lib/tenzir/sigma_rules/%s", fileName) - checkSrcCmd := exec.Command("docker", "exec", containerName, "test", "-f", srcPath) + checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) if err := checkSrcCmd.Run(); err != nil { - return fmt.Errorf("source file does not exist: %v", err) + // If the file does not exist, simply return nil + if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { + log.Printf("[ERROR] No such file: %s, nothing to delete\n", srcPath) + return nil + } + return fmt.Errorf("error checking source file: %v", err) } return removePath(containerName, srcPath) } func removePath(containerName, path string) error { - rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", "-rf", path) + rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("rm -rf %s", path)) output, err := rmCmd.CombinedOutput() if err != nil { return fmt.Errorf("error removing path: %v, output: %s", err, output) @@ -3127,6 +3246,63 @@ func sendTenzirHealthStatus() error { return nil } +func disableRule(fileName string) error { + containerName := "tenzir-node" + srcPath := fmt.Sprintf("/var/lib/tenzir/sigma_rules/%s", fileName) + destDir := "/var/lib/tenzir/disabled_rules" + destPath := fmt.Sprintf("%s/%s", destDir, fileName) + + checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) + if err := checkSrcCmd.Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { + fmt.Printf("File does not exist: %s\n", srcPath) + return nil // Nothing to disable + } + return fmt.Errorf("error checking source file: %v", err) + } + + checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mkdir -p %s", destDir)) + if err := checkDestDirCmd.Run(); err != nil { + return fmt.Errorf("error ensuring destination directory exists: %v", err) + } + + moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mv %s %s", srcPath, destPath)) + if err := moveCmd.Run(); err != nil { + return fmt.Errorf("error moving file: %v", err) + } + + fmt.Printf("File %s moved to %s successfully.\n", fileName, destDir) + return nil +} + +func enableRule(fileName string) error { + containerName := "tenzir-node" + srcPath := fmt.Sprintf("/var/lib/tenzir/disabled_rules/%s", fileName) + destDir := "/var/lib/tenzir/sigma_rules" + destPath := fmt.Sprintf("%s/%s", destDir, fileName) + + checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) + if err := checkSrcCmd.Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { + fmt.Printf("File does not exist: %s\n", srcPath) + return nil // Nothing to enable + } + return fmt.Errorf("error checking source file: %v", err) + } + + checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mkdir -p %s", destDir)) + if err := checkDestDirCmd.Run(); err != nil { + return fmt.Errorf("error ensuring destination directory exists: %v", err) + } + moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mv %s %s", srcPath, destPath)) + if err := moveCmd.Run(); err != nil { + return fmt.Errorf("error moving file: %v", err) + } + + fmt.Printf("File %s moved to %s successfully.\n", fileName, destDir) + return nil +} + // Is this ok to do with Docker? idk :) func getRunningWorkers(ctx context.Context, workerTimeout int) int { //log.Printf("[DEBUG] Getting running workers with API version %s", dockerApiVersion) From b33c45543ac1fa4454514f45a5b014997a390319 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Mon, 22 Jul 2024 11:35:55 +0530 Subject: [PATCH 50/60] pointing to the correct url --- backend/go-app/main.go | 2 +- frontend/src/views/DetectionDashboard.jsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 8f247fc8..860a16ef 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5188,7 +5188,7 @@ func initHandlers() { // PS: For cloud, this has to use cloud storage. // https://developer.box.com/reference/get-files-id-content/ r.HandleFunc("/api/v1/files/download_remote", shuffle.HandleDownloadRemoteFiles).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v2/files/download_remote", shuffle.HandleDownloadRemoteFiles2).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/files/download_remote_enhanced", shuffle.HandleEnhancedDownloadRemoteFiles).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/files/namespaces/{namespace}", shuffle.HandleGetFileNamespace).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}/content", shuffle.HandleGetFileContent).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/create", shuffle.HandleCreateFile).Methods("POST", "OPTIONS") diff --git a/frontend/src/views/DetectionDashboard.jsx b/frontend/src/views/DetectionDashboard.jsx index 4b2b1b06..3d4eb871 100644 --- a/frontend/src/views/DetectionDashboard.jsx +++ b/frontend/src/views/DetectionDashboard.jsx @@ -110,7 +110,7 @@ const DetectionDashBoard = (props) => { }; toast(`Getting files from url ${url}. This may take a while if the repository is large. Please wait...`); - fetch(`${globalUrl}/api/v2/files/download_remote`, { + fetch(`${globalUrl}/api/v1/files/download_remote_enhanced`, { method: "POST", mode: "cors", headers: { From 2b90a1b9afb55f5fff3b577c4fae604e7ea72b3c Mon Sep 17 00:00:00 2001 From: monilprajapati Date: Mon, 22 Jul 2024 16:08:46 +0530 Subject: [PATCH 51/60] Added the error message on the field about the variable --- frontend/src/components/ParsedAction.jsx | 119 +++++++---------------- 1 file changed, 35 insertions(+), 84 deletions(-) diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index c4cff82f..208b822e 100755 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -184,15 +184,8 @@ const ParsedAction = (props) => { const [hiddenDescription, setHiddenDescription] = React.useState(true); const [autoCompleting, setAutocompleting] = React.useState(false); const [selectedActionParameters, setSelectedActionParameters] = React.useState(selectedAction?.parameters || []); - const [selectedVariableParameter, setSelectedVariableParameter] = React.useState(""); - const [paramValues, setParamValues] = React.useState( - selectedAction?.parameters?.map((param) => { - return { - name: param.name, - value: param.value, - } - }) - ); + const [selectedVariableParameter, setSelectedVariableParameter] = React.useState(""); + const [paramUpdate, setParamUpdate] = React.useState(""); const [actionlist, setActionlist] = React.useState([]); const [jsonList, setJsonList] = React.useState([]); const [showDropdown, setShowDropdown] = React.useState(false); @@ -207,16 +200,6 @@ const ParsedAction = (props) => { } }, [expansionModalOpen]) -// useEffect(() => { -// setParamValues(selectedAction.parameters?.map((param) => { -// return { -// name: param.name, -// value: param.value, -// } -// })) -// },[ -// selectedAction, selectedApp,setNewSelectedAction, workflow, -// ]) useEffect(() => { if (selectedAction.parameters === null || selectedAction.parameters === undefined) { @@ -417,8 +400,8 @@ const ParsedAction = (props) => { } // Only set selected action parameters if they have changed - if (selectedAction.parameters && selectedAction.parameters.length > 0) { - setSelectedActionParameters(selectedAction.parameters); + if (selectedAction?.parameters && selectedAction?.parameters.length > 0) { + setSelectedActionParameters(selectedAction?.parameters); } // Only set selected variable parameter if it is null or undefined @@ -433,6 +416,7 @@ const ParsedAction = (props) => { useEffect(() => { const newActionList = []; + const parentActionList = []; // Process workflowExecutions if (workflowExecutions.length > 0) { @@ -560,89 +544,54 @@ const ParsedAction = (props) => { autocomplete: parentNode.label.split(" ").join("_"), example: exampleData, }); - } - } - } - } - // Update the actionlist state - setActionlist(newActionList); - }, [workflow.execution_variables, workflow.workflow_variables, workflowExecutions, workflow, selectedAction, listCache, getParents]); - - - const memoizedParam = useMemo(() => { - let appActions = []; - if (getParents) { - const parents = getParents(selectedAction); - if (parents.length > 1) { - const labels = []; - for (let parentNode of parents) { - if (parentNode.label !== "Execution Argument" && !labels.includes(parentNode.label)) { - labels.push(parentNode.label); - let exampleData = parentNode.example ?? ""; - if (!exampleData && workflowExecutions.length > 0) { - for (let exec of workflowExecutions) { - const foundResult = exec.results?.find(result => result.action.id === parentNode.id); - if (foundResult) { - const valid = validateJson(foundResult.result); - if (valid.valid && valid.result.success !== false) { - exampleData = valid.result; - break; - } - } - } - } - appActions.push({ + parentActionList.push({ type: "action", id: parentNode.id, name: parentNode.label, autocomplete: parentNode.label.split(" ").join("_"), example: exampleData, }); + + } } } } - let newParameters = selectedAction.parameters?.map((param) => { + let newParameters = selectedAction?.parameters?.map((param) => { let paramvalue = param.value; + let errorVars = []; if(paramvalue.includes("$")){ let actions = workflow.actions?.map((action) => { return "$"+action.label.toLowerCase(); }) - if(actionlist.length > 0){ - let appParentActions = appActions?.map(action => "$" + action.name.toLowerCase()); + if(newActionList.length > 0){ + let appParentActions = parentActionList?.map(action => "$" + action.name.toLowerCase()); let notPresentAction = actions?.filter((action) => !appParentActions?.includes(action)) - console.log("ACTIONS: ", actions) - console.log("APP ACTIONS: ", appParentActions) - console.log("NOT PRESENT: ", notPresentAction) notPresentAction?.forEach((action) => { - console.log("Not included Action: ", action) if(paramvalue.includes(action)){ - + errorVars.push(action); // paramvalue = paramvalue.replace(action, "") // paramvalue = paramvalue.replace(/^\s*[\r\n]/gm, ""); } }) } } - console.log("After removing param value: ", paramvalue) - return {...param, value: paramvalue} - }); - selectedAction.parameters = newParameters; - setSelectedActionParameters(newParameters); - setSelectedAction(selectedAction); - return newParameters; - },[actionlist,selectedAction,workflow.actions,workflow,selectedApp,setNewSelectedAction]) - useEffect(() => { - setParamValues(memoizedParam?.map((param) => { - return { - name: param.name, - value: param.value, + let message = ""; + if(errorVars.length > 0){ + if(errorVars.length === 1){ + message = errorVars[0] + " is not accessible in this action"; + }else{ + message = errorVars.join(", ") + " are not accessible in this action"; + } } - })) - },[memoizedParam]) + return {...param, value: paramvalue, error: message} + }); + setSelectedActionParameters(newParameters); + setActionlist(newActionList); + }, [workflow.execution_variables,paramUpdate, workflow.workflow_variables, workflowExecutions, workflow, selectedAction, listCache, getParents,setNewSelectedAction]); useEffect(() => { selectedNameChange(appActionName) @@ -653,13 +602,14 @@ const ParsedAction = (props) => { },[appActionName,delay]) const handleParamChange = (event, count,data) => { - const newParams = [...paramValues]; + const newParams = [...selectedActionParameters]; newParams.map((param) => { if (param.name === data.name) { param.value = event.target.value; } }) - setParamValues(newParams); + setSelectedActionParameters(newParams); + setParamUpdate(event.target.value); changeActionParameter(event, count, data) } const calculateHelpertext = (input_data) => { @@ -1248,7 +1198,7 @@ const ParsedAction = (props) => { } // FIXME: Issue #40 - selectedActionParameters not reset - if (Object.getOwnPropertyNames(selectedAction).length > 0 && selectedActionParameters.length > 0) { + if (Object.getOwnPropertyNames(selectedAction)?.length > 0 && selectedActionParameters?.length > 0) { var wrapperapp = { "id": "", "name": "noapp", @@ -2759,7 +2709,7 @@ const ParsedAction = (props) => { {suggestionInfo()} - {selectedActionParameters.map((data, count) => { + {selectedActionParameters?.map((data, count) => { if (data.variant === "") { data.variant = "STATIC_VALUE"; } @@ -3214,10 +3164,12 @@ const ParsedAction = (props) => { color="primary" // defaultValue={data.value} value={ - paramValues.find((param) => param.name === data.name) !== undefined - ? paramValues.find((param) => param.name === data.name).value - : "" + data?.value } + error={ + data?.error?.length > 0 ? true : false + } + helperText={data?.error?.length > 0 ? data.error : returnHelperText(data.name, data.value)} //options={{ // theme: 'gruvbox-dark', // keyMap: 'sublime', @@ -3240,7 +3192,6 @@ const ParsedAction = (props) => { // changeActionParameter(event, count, data); handleParamChange(event, count, data) }} - helperText={returnHelperText(data.name, data.value)} onBlur={(event) => { baseHelperText = calculateHelpertext(event.target.value) if (setLastSaved !== undefined) { From b26950546fe479d17d332ef0d2bd2b11c42f0a4f Mon Sep 17 00:00:00 2001 From: monilprajapati Date: Mon, 22 Jul 2024 19:21:23 +0530 Subject: [PATCH 52/60] Fixed the app crash Issue --- frontend/src/components/ParsedAction.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 034859e0..0c0398dd 100755 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -2696,7 +2696,7 @@ const ParsedAction = (props) => { // selectedAction.selectedAuthentication = e.target.value // selectedAction.authentication_id = e.target.value.id if ( - !selectedAction.auth_not_required && + // !selectedAction.auth_not_required && selectedAction.selectedAuthentication !== undefined && selectedAction.selectedAuthentication.fields !== undefined && selectedAction.selectedAuthentication.fields[data.name] !== From e22f6e470908c0025efa30a7ba4ab9b194dea7c1 Mon Sep 17 00:00:00 2001 From: monilprajapati Date: Wed, 24 Jul 2024 13:17:42 +0530 Subject: [PATCH 53/60] Added error in Auth field for unescaped dollar --- frontend/src/components/ParsedAction.jsx | 37 +++++++++++++++++++++--- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 208b822e..31378963 100755 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -566,10 +566,11 @@ const ParsedAction = (props) => { let actions = workflow.actions?.map((action) => { return "$"+action.label.toLowerCase(); }) - if(newActionList.length > 0){ + if(newActionList?.length > 0){ let appParentActions = parentActionList?.map(action => "$" + action.name.toLowerCase()); let notPresentAction = actions?.filter((action) => !appParentActions?.includes(action)) notPresentAction?.forEach((action) => { + action = action.replace(" ", "_"); if(paramvalue.includes(action)){ errorVars.push(action); // paramvalue = paramvalue.replace(action, "") @@ -582,9 +583,20 @@ const ParsedAction = (props) => { let message = ""; if(errorVars.length > 0){ if(errorVars.length === 1){ - message = errorVars[0] + " is not accessible in this action"; + message = errorVars[0] + " is not accessible in this action."; }else{ - message = errorVars.join(", ") + " are not accessible in this action"; + message = errorVars.join(", ") + " are not accessible in this action."; + } + } + + if (param?.configuration) { + let regex = /(^|[^\\])\$/; + if (regex.test(paramvalue)) { + if(message.length > 0){ + message += "\nUse \"\\$\" instead of \"$\"."; + }else{ + message = "Use \"\\$\" instead of \"$\"."; + } } } return {...param, value: paramvalue, error: message} @@ -1137,6 +1149,15 @@ const ParsedAction = (props) => { return helperText } + const errorHelperText = (name, value, error) => { + return ( +
+ {error} +
+ ); + } + + const analyzeFields = () => { if (selectedAction === undefined || selectedAction === null) { @@ -3169,7 +3190,7 @@ const ParsedAction = (props) => { error={ data?.error?.length > 0 ? true : false } - helperText={data?.error?.length > 0 ? data.error : returnHelperText(data.name, data.value)} + helperText={data?.error?.length > 0 ? errorHelperText(data?.name,data?.value,data?.error) : returnHelperText(data.name, data.value)} //options={{ // theme: 'gruvbox-dark', // keyMap: 'sublime', @@ -3946,6 +3967,14 @@ const ParsedAction = (props) => { - Description: {description} + { + data?.configuration ? + ( + + - Use "\$" instead of "$" + + ) : null + } ); From 56d96b331a69fac6890959086110542cbe5077c6 Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 26 Jul 2024 17:02:29 +0200 Subject: [PATCH 54/60] Force build new nginx without confd --- backend/app_sdk/app_base.py | 26 ++++++++++++++++++++++---- frontend/Dockerfile | 23 ++++++++++++++--------- frontend/entrypoint.sh | 5 +++-- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 415f1e94..a751319b 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -632,6 +632,7 @@ class AppBase: # I wonder if this actually works url = "%s%s" % (self.base_url, stream_path) + self.logger.info(f"[DEBUG][%s] Sending result to %s" % (self.current_execution_id, url)) try: log_contents = "disabled: add env SHUFFLE_LOGS_DISABLED=true to Orborus to re-enable logs for apps. Can not be enabled natively in Cloud except in Hybrid mode." @@ -656,6 +657,13 @@ class AppBase: except Exception as e: pass + # Check if type of headers is right + if not isinstance(headers, dict): + headers = {} + + if not "User-Agent" in headers: + headers["User-Agent"] = "Shuffle App" + try: finished = False ret = {} @@ -684,23 +692,23 @@ class AppBase: headerauth = headers["Authorization"] try: - self.logger.info(f"[ERROR] Bad resp ({ret.status_code}) in send_result for url '{url}'. Execution ID: %d, Authorization: %d, Header Auth: %d" % (len(action_result["execution_id"]), len(action_result["authorization"]), len(headerauth))) except Exception as e: self.logger.info(f"[ERROR] Bad resp ({ret.status_code}) in send_result for url '{url}' (no detail)") - pass time.sleep(sleeptime) # Proxyerrror except requests.exceptions.ProxyError as e: + self.logger.info(f"[ERROR][{self.current_execution_id}] Proxy error in send_result for url '{url}': {e}") + self.proxy_config = {} continue except requests.exceptions.RequestException as e: - time.sleep(sleeptime) + self.logger.info(f"[ERROR][{self.current_execution_id}] Request error in send_result for url '{url}': {e}") # Check if we have a read timeout. If we do, exit as we most likely sent the result without getting a good result if "Read timed out" in str(e): @@ -713,24 +721,34 @@ class AppBase: finished = True break + time.sleep(sleeptime) + #time.sleep(5) continue except TimeoutError as e: + self.logger.info(f"[ERROR][{self.current_execution_id}] Timeout error in send_result for url '{url}': {e}") + time.sleep(sleeptime) #time.sleep(5) continue except requests.exceptions.ConnectionError as e: + self.logger.info(f"[ERROR][{self.current_execution_id}] Connection error in send_result for url '{url}': {e}") + time.sleep(sleeptime) #time.sleep(5) continue except http.client.RemoteDisconnected as e: + self.logger.info(f"[ERROR][{self.current_execution_id}] RemoteDisconnected error in send_result for url '{url}': {e}") + time.sleep(sleeptime) #time.sleep(5) continue except urllib3.exceptions.ProtocolError as e: + self.logger.info(f"[ERROR][{self.current_execution_id}] ProtocolError error in send_result for url '{url}': {e}") + time.sleep(0.1) #time.sleep(5) @@ -3668,7 +3686,7 @@ class AppBase: #self.logger.info() if not multiexecution: - self.logger.info("NOT MULTI EXEC") + #self.logger.info("NOT MULTI EXEC") # Runs a single iteration here new_params = self.validate_unique_fields(params) if isinstance(new_params, list) and len(new_params) == 1: diff --git a/frontend/Dockerfile b/frontend/Dockerfile index e9d6d683..32b494c6 100755 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -25,29 +25,34 @@ COPY ./*.json /usr/src/app/ RUN npm run build --loglevel verbose 2>&1 # Production environment -FROM nginx:1.21.5 +FROM nginx:1.26.0 RUN mkdir -p /usr/share/nginx/html/build RUN mkdir -p /usr/share/nginx/html/css RUN mkdir -p /usr/share/nginx/html/js RUN mkdir -p /usr/share/nginx/html/img -COPY --from=builder /usr/src/app/build /usr/share/nginx/html -#Localhost certificate challenge: Y#XwrJ#DoZGz2w6x +# Localhost certificate challenge: Y#XwrJ#DoZGz2w6x +# Cert challenge doesn't matter to be here or not, as ALL production setups should be using their own certificates + reverse proxy: https://shuffler.io/docs/configuration#using-the-nginx-reverse-proxy-for-tls/ssl +COPY --from=builder /usr/src/app/build /usr/share/nginx/html COPY --from=builder /usr/src/app/certs/fullchain.pem /etc/nginx/fullchain.cert.pem COPY --from=builder /usr/src/app/certs/privkey.pem /etc/nginx/privkey.pem # install CONFD -ENV CONFD_VERSION 0.16.0 RUN apt-get update && apt-get install -y curl && apt-get clean -RUN curl -sSL https://github.com/kelseyhightower/confd/releases/download/v${CONFD_VERSION}/confd-${CONFD_VERSION}-linux-amd64 -o /usr/local/bin/confd && \ - chmod +x /usr/local/bin/confd -COPY ./confd /etc/confd +COPY ./confd/templates/nginx.conf /etc/nginx/nginx.conf +## OLD CONFD THINGS (not compatible with arm) +#ENV CONFD_VERSION 0.16.0 +#RUN curl -sSL https://github.com/kelseyhightower/confd/releases/download/v${CONFD_VERSION}/confd-${CONFD_VERSION}-linux-amd64 -o /usr/local/bin/confd && \ +# chmod +x /usr/local/bin/confd +#COPY ./confd /etc/confd # rewrite command & entrypoint with ours -COPY ./entrypoint.sh / -ENTRYPOINT [ "/entrypoint.sh" ] +#COPY ./entrypoint.sh / +#ENTRYPOINT [ "/entrypoint.sh" ] + + CMD ["nginx", "-g", "daemon off;"] EXPOSE 80 diff --git a/frontend/entrypoint.sh b/frontend/entrypoint.sh index 09be2558..f5542b05 100755 --- a/frontend/entrypoint.sh +++ b/frontend/entrypoint.sh @@ -1,7 +1,8 @@ #!/bin/bash -# generate configs -/usr/local/bin/confd -backend="env" -confdir="/etc/confd" -onetime +# generate configs - is this necessary? +# Removing confd if possible +#/usr/local/bin/confd -backend="env" -confdir="/etc/confd" -onetime # run main command exec "$@" From 94a2f4ce876345f6ae8185d8eb5dbd56bca9d119 Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 26 Jul 2024 17:35:33 +0200 Subject: [PATCH 55/60] Started using pure nginx conf syntax --- frontend/Dockerfile | 4 +++- frontend/confd/templates/nginx.conf | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 32b494c6..6602208d 100755 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,6 +1,8 @@ # Build environment FROM node:21 as builder +ENV NODE_OPTIONS="--max-old-space-size=4096" + RUN mkdir /usr/src/app WORKDIR /usr/src/app ENV PATH /usr/src/app/node_modules/.bin:$PATH @@ -52,7 +54,7 @@ COPY ./confd/templates/nginx.conf /etc/nginx/nginx.conf #COPY ./entrypoint.sh / #ENTRYPOINT [ "/entrypoint.sh" ] - +RUN export BACKEND_HOSTNAME=shuffle-backend CMD ["nginx", "-g", "daemon off;"] EXPOSE 80 diff --git a/frontend/confd/templates/nginx.conf b/frontend/confd/templates/nginx.conf index 3bb02c27..a119d69d 100755 --- a/frontend/confd/templates/nginx.conf +++ b/frontend/confd/templates/nginx.conf @@ -71,7 +71,8 @@ http { } location ~ /api/v(1|2) { - proxy_pass http://{{ getenv "BACKEND_HOSTNAME" "shuffle-backend" }}:5001; + #proxy_pass http://{{ getenv "BACKEND_HOSTNAME" "shuffle-backend" }}:5001; + proxy_pass http://$BACKEND_HOSTNAME:5001; proxy_buffering off; proxy_http_version 1.1; @@ -113,7 +114,7 @@ http { # Get the hostname from environment here? location ~ /api/v(1|2) { - proxy_pass http://{{ getenv "BACKEND_HOSTNAME" "shuffle-backend" }}:5001; + proxy_pass http://$BACKEND_HOSTNAME:5001; proxy_buffering off; proxy_http_version 1.1; From 4e7d03debbf83fa828ee1cfa919a848ef700eafd Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 26 Jul 2024 18:25:43 +0200 Subject: [PATCH 56/60] Another try with entrypoint rewrites --- frontend/Dockerfile | 8 ++++---- frontend/confd/templates/nginx.conf | 6 +++--- frontend/entrypoint.sh | 8 +++----- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 6602208d..fad87105 100755 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -43,7 +43,7 @@ COPY --from=builder /usr/src/app/certs/privkey.pem /etc/nginx/privkey.pem # install CONFD RUN apt-get update && apt-get install -y curl && apt-get clean -COPY ./confd/templates/nginx.conf /etc/nginx/nginx.conf +COPY ./confd/templates/nginx.conf /etc/nginx/nginx.conf.tmpl ## OLD CONFD THINGS (not compatible with arm) #ENV CONFD_VERSION 0.16.0 @@ -51,10 +51,10 @@ COPY ./confd/templates/nginx.conf /etc/nginx/nginx.conf # chmod +x /usr/local/bin/confd #COPY ./confd /etc/confd # rewrite command & entrypoint with ours -#COPY ./entrypoint.sh / -#ENTRYPOINT [ "/entrypoint.sh" ] -RUN export BACKEND_HOSTNAME=shuffle-backend +COPY ./entrypoint.sh / +ENV BACKEND_HOSTNAME="shuffle-backend" +ENTRYPOINT [ "/entrypoint.sh" ] CMD ["nginx", "-g", "daemon off;"] EXPOSE 80 diff --git a/frontend/confd/templates/nginx.conf b/frontend/confd/templates/nginx.conf index a119d69d..2c9df91e 100755 --- a/frontend/confd/templates/nginx.conf +++ b/frontend/confd/templates/nginx.conf @@ -71,8 +71,7 @@ http { } location ~ /api/v(1|2) { - #proxy_pass http://{{ getenv "BACKEND_HOSTNAME" "shuffle-backend" }}:5001; - proxy_pass http://$BACKEND_HOSTNAME:5001; + proxy_pass http://${BACKEND_HOSTNAME}:5001; proxy_buffering off; proxy_http_version 1.1; @@ -114,7 +113,8 @@ http { # Get the hostname from environment here? location ~ /api/v(1|2) { - proxy_pass http://$BACKEND_HOSTNAME:5001; + proxy_pass http://${BACKEND_HOSTNAME}:5001; + proxy_buffering off; proxy_http_version 1.1; diff --git a/frontend/entrypoint.sh b/frontend/entrypoint.sh index f5542b05..af1d0a43 100755 --- a/frontend/entrypoint.sh +++ b/frontend/entrypoint.sh @@ -1,8 +1,6 @@ -#!/bin/bash +#!/usr/bin/env sh +set -eu -# generate configs - is this necessary? -# Removing confd if possible -#/usr/local/bin/confd -backend="env" -confdir="/etc/confd" -onetime +envsubst '${BACKEND_HOSTNAME}' < /etc/nginx/nginx.conf.tmpl > /etc/nginx/nginx.conf -# run main command exec "$@" From d5274e9da1d69f350ce4ef63655adb69daff5b45 Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 26 Jul 2024 19:46:53 +0200 Subject: [PATCH 57/60] Optimized .env and docker-compose to work better with opensearch setups --- .env | 5 +++-- docker-compose.yml | 27 ++++++++++++++------------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.env b/.env index 298128cc..417e860f 100755 --- a/.env +++ b/.env @@ -97,14 +97,15 @@ SHUFFLE_MAX_EXECUTION_DEPTH= DATASTORE_EMULATOR_HOST=shuffle-database:8000 #SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200 SHUFFLE_OPENSEARCH_URL=https://shuffle-opensearch:9200 -SHUFFLE_OPENSEARCH_USERNAME="admin" -SHUFFLE_OPENSEARCH_PASSWORD="StrongShufflePassword321!" SHUFFLE_OPENSEARCH_CERTIFICATE_FILE= SHUFFLE_OPENSEARCH_APIKEY= SHUFFLE_OPENSEARCH_CLOUDID= SHUFFLE_OPENSEARCH_PROXY= SHUFFLE_OPENSEARCH_INDEX_PREFIX= SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY=true +SHUFFLE_OPENSEARCH_USERNAME="admin" +SHUFFLE_OPENSEARCH_PASSWORD="StrongShufflePassword321!" # In use for the first time setup of OpenSearch + backend of Shuffle +OPENSEARCH_INITIAL_ADMIN_PASSWORD="StrongShufflePassword321!" # In use for the first time setup of OpenSearch #Tenzir related SHUFFLE_TENZIR_URL= diff --git a/docker-compose.yml b/docker-compose.yml index 2f096df2..fcc7c0cb 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - image: ghcr.io/shuffle/shuffle-frontend:latest + image: ghcr.io/shuffle/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -15,7 +15,7 @@ services: depends_on: - backend backend: - image: ghcr.io/shuffle/shuffle-backend:latest + image: ghcr.io/shuffle/shuffle-backend:nightly container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -34,7 +34,7 @@ services: - SHUFFLE_FILE_LOCATION=/shuffle-files restart: unless-stopped orborus: - image: ghcr.io/shuffle/shuffle-orborus:latest + image: ghcr.io/shuffle/shuffle-orborus:nightly container_name: shuffle-orborus hostname: shuffle-orborus networks: @@ -48,9 +48,6 @@ services: - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:5001 - DOCKER_API_VERSION=1.40 - - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} - - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} - - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} - HTTP_PROXY=${HTTP_PROXY} - HTTPS_PROXY=${HTTPS_PROXY} - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} @@ -74,7 +71,6 @@ services: - node.name=shuffle-opensearch - node.store.allow_mmap=false - discovery.seed_hosts=shuffle-opensearch - - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${SHUFFLE_OPENSEARCH_PASSWORD} ulimits: memlock: soft: -1 @@ -83,7 +79,7 @@ services: soft: 65536 hard: 65536 volumes: - - ${DB_LOCATION}:/usr/share/opensearch/data:z + - shuffle-database:/usr/share/opensearch/data:z ports: - 9200:9200 networks: @@ -129,13 +125,18 @@ services: # networks: # - shuffle # + +volumes: + shuffle-database: + driver: local + driver_opts: + type: none + device: ${DB_LOCATION} + o: bind + networks: shuffle: driver: bridge - - # uncomment to set MTU for swarm mode. - # MTU should be whatever is your host's preferred MTU is. - # Refer to this doc to figure out what your host's MTU is: - # https://shuffler.io/docs/troubleshooting#TLS_timeout_error/Timeout_Errors/EOF_Errors # driver_opts: # com.docker.network.driver.mtu: 1460 + # uncomment to set MTU for swarm mode. MTU should be whatever is your host's preferred MTU is: https://shuffler.io/docs/troubleshooting#TLS_timeout_error/Timeout_Errors/EOF_Errors From 2fe2772e3bd2f1906e16446c970a04ad9d7fbe9f Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Mon, 29 Jul 2024 14:13:24 +0530 Subject: [PATCH 58/60] track the sigma rules --- backend/go-app/main.go | 47 +++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 860a16ef..23486930 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -2122,6 +2122,9 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { if err == nil { resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s"}`, workflowExecution.ExecutionId))) + + // Track Sigma rules + trackSigmaRules(ctx, pipeline.OrgId, jsonList) return } @@ -2130,23 +2133,39 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { } func parseConcatenatedJSONLogs(logs string) ([]map[string]interface{}, error) { - var jsonList []map[string]interface{} - decoder := json.NewDecoder(strings.NewReader(logs)) + var jsonList []map[string]interface{} + decoder := json.NewDecoder(strings.NewReader(logs)) - for decoder.More() { - var jsonObject map[string]interface{} - if err := decoder.Decode(&jsonObject); err != nil { - log.Printf("[WARNING] JSON decoding error: %s. Skipping this object.", err) - continue - } - jsonList = append(jsonList, jsonObject) - } + for decoder.More() { + var jsonObject map[string]interface{} + if err := decoder.Decode(&jsonObject); err != nil { + log.Printf("[WARNING] JSON decoding error: %s. Skipping this object.", err) + continue + } + jsonList = append(jsonList, jsonObject) + } - if err := decoder.Decode(&struct{}{}); err != io.EOF { - return nil, fmt.Errorf("error after decoding all JSON objects: %v", err) - } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + return nil, fmt.Errorf("error after decoding all JSON objects: %v", err) + } - return jsonList, nil + return jsonList, nil +} + +func trackSigmaRules(ctx context.Context, orgId string, jsonList []map[string]interface{}) { + ruleCount := make(map[string]int) + for _, logEntry := range jsonList { + if rule, ok := logEntry["rule"].(map[string]interface{}); ok { + if ruleName, ok := rule["title"].(string); ok { + ruleCount[ruleName]++ + } + } + } + + for ruleName, count := range ruleCount { + shuffle.IncrementCache(ctx, orgId, ruleName, count) + log.Printf("[INFO] Rule %s incremented by %d", ruleName, count) + } } func handleTenzirHealthUpdate(resp http.ResponseWriter, request *http.Request) { From 9734a014b943ede4521f286350b582c626f1a913 Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 31 Jul 2024 19:30:47 +0200 Subject: [PATCH 59/60] Updates from cloud pre merge --- frontend/src/components/Billing.jsx | 11 +- frontend/src/components/Branding.jsx | 72 +++++++++---- frontend/src/components/LicencePopup.jsx | 2 +- frontend/src/components/NewHeader.jsx | 24 +++-- frontend/src/components/ParsedAction.jsx | 129 ++++++++++++++++++++--- frontend/src/components/Searchfield.jsx | 2 +- frontend/src/views/Admin.jsx | 2 +- frontend/src/views/AngularWorkflow.jsx | 88 ++++++++++------ frontend/src/views/Apps.jsx | 7 +- 9 files changed, 253 insertions(+), 84 deletions(-) diff --git a/frontend/src/components/Billing.jsx b/frontend/src/components/Billing.jsx index 97b6cd16..d5b39411 100644 --- a/frontend/src/components/Billing.jsx +++ b/frontend/src/components/Billing.jsx @@ -54,7 +54,6 @@ const Billing = (props) => { const { globalUrl, userdata, serverside, billingInfo, stripeKey, selectedOrganization, handleGetOrg, clickedFromOrgTab } = props; //const alert = useAlert(); let navigate = useNavigate(); - const [selectedDealModalOpen, setSelectedDealModalOpen] = React.useState(false); const [dealList, setDealList] = React.useState([]); const [dealName, setDealName] = React.useState(""); @@ -1055,10 +1054,10 @@ const Billing = (props) => { Consultation & Management
- + You currently have a total of {inputHour} hours and {inputMinutes} minutes of professional services available by our experts. -
+
{editConsultation ? <> { {editConsultation && }
: null} - + Features
    @@ -1331,7 +1330,7 @@ const Billing = (props) => { Become a Shuffle Expert
    - + Public Training
      @@ -1346,7 +1345,7 @@ const Billing = (props) => {
    - + Private Training
      diff --git a/frontend/src/components/Branding.jsx b/frontend/src/components/Branding.jsx index 48466476..de38ef86 100644 --- a/frontend/src/components/Branding.jsx +++ b/frontend/src/components/Branding.jsx @@ -3,15 +3,25 @@ import ReactGA from 'react-ga4'; import theme from "../theme.jsx"; import { ToastContainer, toast } from "react-toastify" +import { + CheckCircle as CheckCircleIcon, +} from "@mui/icons-material"; + import { Paper, Typography, Divider, Button, + Tooltip, Grid, Card, } from "@mui/material"; +import { + red, + green, +} from "../views/AngularWorkflow.jsx" + //import { useAlert const Branding = (props) => { @@ -45,7 +55,7 @@ const Branding = (props) => { toast("Failed updating org: ", responseJson.reason); } else { if (joinStatus == "join") { - setPublishingInfo("Your organization is now part of the Creator Incentive Program. You can now create and publish content to your organization's page. You can also create a creator account to manage your organization's content.") + setPublishingInfo("Your organization is now part of the Partner Program. You can now create, publish and manage content for your organization's public page.") } else { setPublishingInfo("Your organization is no longer part of the Creator Incentive Program. You can still create a creator account to manage your organization's content.") } @@ -70,7 +80,15 @@ const Branding = (props) => { } const isOrganizationReady = () => { - console.log("Is organization ready?") + + // Check if it's a suborg + if (selectedOrganization.creator_org !== "") { + const comment = "Child orgs can't become creators" + if (!publishRequirements.includes(comment)) { + setPublishRequirements([...publishRequirements, comment]) + } + return false; + } // A simple checklist to ensure the button shows up properly if (selectedOrganization.name === selectedOrganization.org) { @@ -82,15 +100,6 @@ const Branding = (props) => { return false; } - // Check if it's a suborg - if (selectedOrganization.creator_org !== "") { - const comment = "Child orgs can't become creators" - if (!publishRequirements.includes(comment)) { - setPublishRequirements([...publishRequirements, comment]) - } - return false; - } - if (selectedOrganization.large_image === "" || selectedOrganization.large_image === theme.palette.defaultImage) { const comment = "Add a logo for your organization" if (!publishRequirements.includes(comment)) { @@ -102,6 +111,14 @@ const Branding = (props) => { return true } + const isPublished = selectedOrganization.creator_id === "" + const leadinfo = selectedOrganization.lead_info === undefined || selectedOrganization.lead_info === null || selectedOrganization.lead_info === "" ? "" : JSON.stringify(selectedOrganization.lead_info) + const isPartner = leadinfo.includes("partner") + + console.log("LEADINFO: ", leadinfo) + + console.log("SELECTEDORGANIZATION: ", selectedOrganization) + return (

      @@ -111,27 +128,46 @@ const Branding = (props) => { You can customize your organization's branding by uploading a logo, changing the color scheme and a lot more. + + {isPublished ? : } + {isPublished ? "Not Published" : "Published"} + + + + + {!isPartner ? : } + + {!isPartner? "Not Officially Partnered" : "Officially Partnered"} + + + + + + + +

      - Creator Incentive Program + Partner Program

      - By changing publishing settings, you agree to our Terms of Service, and acknowledge that your organization's non-sensitive data will be added as a creator account. None of your existing workflows, apps, or other stored data will be published. Any admin in your organization can manage the creator configuration. Becoming a creator organization is reversible.
      Support: support@shuffler.io + By changing publishing settings, you agree to our Terms of Service, and acknowledge that your organization's non-sensitive data will be added as a creator account. None of your existing workflows, apps, or other stored data will be published. Any admin in your organization can manage the creator configuration. Becoming a creator organization IS reversible.
      Support: support@shuffler.io {selectedOrganization.creator_id == "" ?   : - - - Modify your creator organization - + null } + diff --git a/frontend/src/components/LicencePopup.jsx b/frontend/src/components/LicencePopup.jsx index 7fa355e0..820b8253 100644 --- a/frontend/src/components/LicencePopup.jsx +++ b/frontend/src/components/LicencePopup.jsx @@ -690,7 +690,7 @@ const LicencePopup = (props) => { const priceItem = window.location.origin === "https://shuffler.io" ? shuffleVariant === 0 ? "app_executions" : "cores" : - shuffleVariant === 0 ? "price_1PbO0cEJjT17t98NsfEMUlMn" : "price_1PbNnaEJjT17t98NLadq6Lhq" + shuffleVariant === 0 ? "price_1PZPSSEJjT17t98NLJoTMYja" : "price_1PZPQuEJjT17t98N3yORUtd9" const successUrl = `${window.location.origin}/admin?admin_tab=billing&payment=success` const failUrl = `${window.location.origin}/pricing?admin_tab=billing&payment=failure` diff --git a/frontend/src/components/NewHeader.jsx b/frontend/src/components/NewHeader.jsx index ee486357..e196c2cf 100644 --- a/frontend/src/components/NewHeader.jsx +++ b/frontend/src/components/NewHeader.jsx @@ -143,7 +143,7 @@ const Header = (props) => { const [subAnchorEl, setSubAnchorEl] = React.useState(null); const [upgradeHovered, setUpgradeHovered] = React.useState(false); const [showTopbar, setShowTopbar] = useState(false) - const stripeKey = typeof window === 'undefined' || window.location === undefined ? "" : window.location.origin === "https://shuffler.io" ? "pk_live_XAxwE2Fp9DEbEcNYw4UKmyby00vIlIPPRp" : "pk_test_51PXYYMEJjT17t98NbDkojZ3DRvsFUQBs35LGMx3i436BXwEBVFKB9nCvHt0Q3M4MG3dz4mHheuWvfoYvpaL3GmsG00k1Rb2ksO" + const stripeKey = typeof window === 'undefined' || window.location === undefined ? "" : window.location.origin === "https://shuffler.io" ? "pk_live_51PXYYMEJjT17t98N20qEqItyt1fLQjrnn41lPeG2PjnSlZHTDNKHuisAbW00s4KAn86nGuqB9uSVU4ds8MutbnMU00DPXpZ8ZD" : "pk_test_51PXYYMEJjT17t98NbDkojZ3DRvsFUQBs35LGMx3i436BXwEBVFKB9nCvHt0Q3M4MG3dz4mHheuWvfoYvpaL3GmsG00k1Rb2ksO" let navigate = useNavigate(); const classes = useStyles(); @@ -171,8 +171,10 @@ const Header = (props) => { setTooltipOpen(true); }; + const topbar_var = "topbar_closed2" + useEffect(() => { - const topbar = localStorage.getItem("topbar_closed") + const topbar = localStorage.getItem(topbar_var) if (topbar === "true") { setShowTopbar(false) } else { @@ -919,8 +921,8 @@ const Header = (props) => { )}
      - handleMenuItemClick('/professional-support')}> - + handleMenuItemClick('/professional-services')}> + Professional Services @@ -929,9 +931,19 @@ const Header = (props) => { Training Courses + + + + handleMenuItemClick('/partners')}> + + Partner Program + + + + {/*
      ) : null} {workflow.execution_variables !== undefined && workflow.execution_variables !== null && @@ -2693,6 +2788,10 @@ const ParsedAction = (props) => { return null } + if (data.value === "authgroup controlled") { + return null + } + // selectedAction.selectedAuthentication = e.target.value // selectedAction.authentication_id = e.target.value.id if ( @@ -4086,11 +4185,7 @@ const ParsedAction = (props) => { onClose={() => { setShowAutocomplete(false); - if ( - !selectedActionParameters[count].value[ - selectedActionParameters[count].value.length - 1 - ] === "." - ) { + if (!selectedActionParameters[count].value[selectedActionParameters[count].value.length - 1] === ".") { setShowDropdown(false); } diff --git a/frontend/src/components/Searchfield.jsx b/frontend/src/components/Searchfield.jsx index c389f88d..6b3c50ae 100644 --- a/frontend/src/components/Searchfield.jsx +++ b/frontend/src/components/Searchfield.jsx @@ -124,7 +124,7 @@ const SearchField = props => { ); return ( -
      +
      {modalView} Limits & Cloud Sync /> Priorities /> Billing & Stats /> - Branding (Beta) /> + Partner /> action.category_label !== undefined && action.category_label !== null && action.category_label.length > 0) const actionIndex = startIndex < 0 ? 0 : startIndex + if (app.actions[actionIndex] === undefined || app.actions[actionIndex] === null) { + console.log("No actions found for app: ", app) + return + } + // Make the first action the most relevant one for them based on previous use if ( app.actions[actionIndex].parameters !== undefined && @@ -11605,7 +11614,7 @@ const releaseToConnectLabel = "Release to Connect"

      - Branch: Conditions - {selectedEdgeIndex} + Conditions

      0) ? "loading" : "success" var executionDelay = -75 const executionModal = ( @@ -18598,26 +18608,6 @@ const releaseToConnectLabel = "Release to Connect" - {executionData.status === "EXECUTING" ? ( - - - - - - ) : null} - {isCloud ? + {executionData.status === "EXECUTING" ? ( + + + ) : null} + + {isCloud ? + + + @@ -18722,7 +18735,18 @@ const releaseToConnectLabel = "Release to Connect" {executionData.workflow !== undefined && executionData.workflow !== null && executionData.workflow.actions !== undefined && executionData.workflow.actions !== null && executionData.workflow.actions.length > 0 ?
      - Env      + + {/*envStatus === "success" ? + + + + : envStatus === "failure" ? + + + + : null*/} + + Env      { @@ -19580,7 +19604,7 @@ const releaseToConnectLabel = "Release to Connect" return "The app's Docker Image is not available in the environment yet. Re-run the app to force a re-download of the app. If the problem persists, contact support" } - if (result.status !== 200 && result.url !== undefined && result.url !== null && (result.url.includes("192.168") || result.url.includes("172.16") || result.url.includes("10.0"))) { + if (result.status !== 200 && result.url !== undefined && result.url !== null && typeof result.url === "string" && (result.url.includes("192.168") || result.url.includes("172.16") || result.url.includes("10.0"))) { return "Consider whether your Orborus environment can connect to a local IP or not." } @@ -21737,7 +21761,7 @@ const releaseToConnectLabel = "Release to Connect"
      - {selectedVersion.name} + {selectedVersion?.name}
      {/* Cross icon to close it */} diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 4c816623..770e37c8 100755 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -2586,6 +2586,8 @@ const Apps = (props) => { if (parsedtext.indexOf("openapi") === -1 && parsedtext.indexOf("swagger") === -1) { setValidation(false); setOpenApiError("Error in generation: "+parsedtext); + + return; } } catch (e) { @@ -2684,10 +2686,11 @@ const Apps = (props) => { body: openApidata, credentials: "include", }) - .then((response) => { + .then((response) => { + setValidation(false); return response.json(); - }) + }) .then((responseJson) => { if (responseJson.success) { setAppValidation(responseJson.id); From eb13e975d3ac64f70c8cd2d00d3bad02abd5f542 Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 31 Jul 2024 20:12:13 +0200 Subject: [PATCH 60/60] No parsedaction parser - Monil suggestion :) --- frontend/src/components/ParsedAction.jsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 8e6e88f4..4f3052d8 100755 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -220,6 +220,7 @@ const ParsedAction = (props) => { } }, []) + /* useEffect(() => { setParamValues(selectedAction.parameters.map((param) => { return { @@ -230,6 +231,7 @@ const ParsedAction = (props) => { },[ selectedAction, selectedApp,setNewSelectedAction, workflow, ]) + */ useEffect(() => {