enhancing the ui for kafka pipeline

This commit is contained in:
satti-hari-krishna-reddy
2024-05-19 23:07:48 +05:30
parent 0708a51482
commit a3d43702d5
+464 -248
View File
@@ -69,6 +69,7 @@ import {
Badge, Badge,
AvatarGroup, AvatarGroup,
Autocomplete, Autocomplete,
Radio,
} from "@mui/material"; } from "@mui/material";
import { import {
@@ -250,7 +251,6 @@ export const triggers = [
id: "", id: "",
}, },
]; ];
// Adds specific text to items // Adds specific text to items
// https://stackoverflow.com/questions/19014250/rerender-view-on-browser-resize-with-react // https://stackoverflow.com/questions/19014250/rerender-view-on-browser-resize-with-react
@@ -551,6 +551,10 @@ const AngularWorkflow = (defaultprops) => {
const [highlightedApp, setHighlightedApp] = React.useState("") const [highlightedApp, setHighlightedApp] = React.useState("")
const [listCache, setListCache] = React.useState([]); const [listCache, setListCache] = React.useState([]);
const [selectedOption, setSelectedOption] = React.useState("");
const [tenzirConfigModalOpen, setTenzirConfigModalOpen] = React.useState(false);
const [suggestionBox, setSuggestionBox] = React.useState({ const [suggestionBox, setSuggestionBox] = React.useState({
"position": { "position": {
"top": 500, "top": 500,
@@ -1276,6 +1280,50 @@ const AngularWorkflow = (defaultprops) => {
}); });
}; };
const handleKafkaSubmit = (trigger) => {
if (trigger.trigger_type !== "PIPELINE") {
toast("Unable to save the configuration");
return;
}
trigger.parameters = [];
const topic = document.getElementById('topic')?.value;
const bootstrapServers = document.getElementById('bootstrap_servers')?.value;
const groupId = document.getElementById('group_id')?.value;
const autoOffsetReset = document.getElementById('auto_offset_reset')?.value;
if(topic) {
trigger.parameters.push({
name: "topic",
value: topic
});
}
if (bootstrapServers) {
trigger.parameters.push({
name: "bootstrap_servers",
value: bootstrapServers
});
}
if (groupId) {
trigger.parameters.push({
name: "group_id",
value: groupId
});
}
if (autoOffsetReset) {
trigger.parameters.push({
name: "auto_offset_reset",
value: autoOffsetReset
});
}
setTenzirConfigModalOpen(false);
};
const handleColoring = (actionId, status, label) => { const handleColoring = (actionId, status, label) => {
if (cy === undefined) { if (cy === undefined) {
return return
@@ -7232,32 +7280,31 @@ const AngularWorkflow = (defaultprops) => {
return; return;
} }
var mappedStartnode = "" var mappedStartnode = "";
const alledges = cy.edges().jsons() const alledges = cy.edges().jsons();
if (alledges !== undefined && alledges !== null && alledges.length > 0) { if (alledges !== undefined && alledges !== null && alledges.length > 0) {
for (let edgekey in alledges) { for (let edgekey in alledges) {
const tmp = alledges[edgekey] const tmp = alledges[edgekey];
console.log("TMP: ", tmp, tmp.data.source) console.log("TMP: ", tmp, tmp.data.source);
if (tmp.data.source === trigger.id) { if (tmp.data.source === trigger.id) {
mappedStartnode = tmp.data.target mappedStartnode = tmp.data.target;
break break;
} }
}
}
toast("Creating pipeline")
const data = usecase
const url = `${globalUrl}/api/v1/triggers/pipeline`
fetch(url, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json",
},
body: JSON.stringify(data),
credentials: "include",
} }
) }
const data = usecase;
if (data.type === "create") toast("Creating pipeline");
else toast("stopping pipeline");
const url = `${globalUrl}/api/v1/triggers/pipeline`;
fetch(url, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json",
},
body: JSON.stringify(data),
credentials: "include",
})
.then((response) => { .then((response) => {
if (response.status !== 200) { if (response.status !== 200) {
console.log("Status not 200 for stream results :O!"); console.log("Status not 200 for stream results :O!");
@@ -7269,9 +7316,18 @@ const AngularWorkflow = (defaultprops) => {
if (!responseJson.success) { if (!responseJson.success) {
toast("Failed to set pipeline: " + responseJson.reason); toast("Failed to set pipeline: " + responseJson.reason);
} else { } else {
toast("Successfully created pipeline"); if (data.type === "create") toast("Pipeline will be created!");
workflow.triggers[triggerindex].status = "running"; else toast("Pipeline will be stopped!");
trigger.status = "running";
trigger.parameters.push({
name: data.name,
value: data.command,
});
if (data.type === "stop") trigger.status = "stopped";
else trigger.status = "running";
workflow.triggers[triggerindex] = trigger;
setSelectedTrigger(trigger); setSelectedTrigger(trigger);
setWorkflow(workflow); setWorkflow(workflow);
console.log("Should set the status to running and save"); console.log("Should set the status to running and save");
@@ -7279,10 +7335,9 @@ const AngularWorkflow = (defaultprops) => {
} }
}) })
.catch((error) => { .catch((error) => {
//toast(error.toString()); console.log("Get pipeline error: ", error.toString());
console.log("Get schedule error: ", error.toString());
}); });
} };
const submitSchedule = (trigger, triggerindex) => { const submitSchedule = (trigger, triggerindex) => {
if (trigger.name.length <= 0) { if (trigger.name.length <= 0) {
@@ -7761,7 +7816,7 @@ const AngularWorkflow = (defaultprops) => {
if (trigger.trigger_type === "PIPELINE") { if (trigger.trigger_type === "PIPELINE") {
if (userdata.support !== true) { if (userdata.support !== true) {
return null // return null
} }
} }
@@ -7826,26 +7881,31 @@ const AngularWorkflow = (defaultprops) => {
></div> ></div>
<Grid <Grid
container container
style={{ margin: isMobile ? "10px 0px 0px 0px" : "10px 10px 10px 10px", flex: "10" }} style={{ margin: isMobile ? "10px 0px 0px 0px" : "10px 10px 10px 10px", flex: "10", alignItems: 'center' }}
wrap="nowrap"
> >
<Grid item> <Grid item>
<ButtonBase>{imageline}</ButtonBase> <ButtonBase>{imageline}</ButtonBase>
</Grid> </Grid>
{isMobile ? null : {isMobile ? null :
<Grid <Grid
style={{ item
display: "flex", container
flexDirection: "column", style={{
marginLeft: 20, display: "flex",
overflow: "hidden", flexDirection: "column",
}} marginLeft: 20,
> flexGrow: 1,
overflow: "hidden",
}}
>
<Grid item style={{ flex: "1", overflow: "hidden", }}> <Grid item style={{ flex: "1", overflow: "hidden", }}>
<h3 style={{ marginTop: 0, marginBottom: 0, overflow: "hidden" }}> <h3 style={{ marginTop: 0, marginBottom: 0, wordBreak: "break-word", overflowWrap: "break-word", flexBasis: 'auto' }}>
{trigger.name} {trigger.name}
</h3> </h3>
</Grid> </Grid>
<Grid item style={{ flex: "1" }}> <Grid item style={{ flex: "1", wordBreak: "break-word", overflowWrap: "break-word", whiteSpace: "normal", flexBasis: 'auto' }}>
{trigger.description} {trigger.description}
</Grid> </Grid>
</Grid> </Grid>
@@ -8784,11 +8844,11 @@ const AngularWorkflow = (defaultprops) => {
} }
if (app.trigger_type === "PIPELINE" && userdata.support !== true) { if (app.trigger_type === "PIPELINE" && userdata.support !== true) {
return null //return null
} }
if (app.id === "integration" && userdata.support !== true) { if (app.id === "integration" && userdata.support !== true) {
return null //return null
} }
var extraMessage = "" var extraMessage = ""
@@ -13692,226 +13752,241 @@ const AngularWorkflow = (defaultprops) => {
return null return null
} }
const PipelineSidebar = Object.getOwnPropertyNames(selectedTrigger).length === 0 || workflow.triggers[selectedTriggerIndex] === undefined && selectedTrigger.trigger_type !== "SCHEDULE" ? null : !userdata.support === true ? null : const PipelineSidebar = Object.getOwnPropertyNames(selectedTrigger).length === 0 || workflow.triggers[selectedTriggerIndex] === undefined && selectedTrigger.trigger_type !== "SCHEDULE" ? null :
<div style={appApiViewStyle}> <div style={appApiViewStyle}>
<h3 style={{ marginBottom: "5px" }}> <h3 style={{ marginBottom: "5px" }}>
{selectedTrigger.app_name}: {selectedTrigger.status} {selectedTrigger.app_name}: {selectedTrigger.status}
</h3> </h3>
<a <a
rel="noopener noreferrer" rel="noopener noreferrer"
target="_blank" target="_blank"
href="https://shuffler.io/docs/triggers#pipelines" href="https://shuffler.io/docs/triggers#pipelines"
style={{ textDecoration: "none", color: "#f85a3e" }} style={{ textDecoration: "none", color: "#f85a3e" }}
> >
What are pipelines? What are pipelines?
</a> </a>
<Divider <Divider
style={{ style={{
marginBottom: "10px", marginBottom: "10px",
marginTop: "10px", marginTop: "10px",
height: "1px", height: "1px",
width: "100%", width: "100%",
backgroundColor: "rgb(91, 96, 100)", backgroundColor: "rgb(91, 96, 100)",
}}
/>
<div>Name</div>
<TextField
style={{
backgroundColor: theme.palette.inputColor,
borderRadius: theme.palette.borderRadius,
}}
InputProps={{
style: {
},
}}
fullWidth
color="primary"
placeholder={selectedTrigger.label}
onChange={selectedTriggerChange}
/>
<div style={{ marginTop: "20px" }}>
<Typography>Environment</Typography>
<Select
MenuProps={{
disableScrollLock: true,
}}
value={selectedTrigger.environment}
disabled={selectedTrigger.status === "running"}
SelectDisplayProps={{}}
fullWidth
onChange={(e) => {
selectedTrigger.environment = e.target.value
setSelectedTrigger(selectedTrigger)
setWorkflow(workflow)
setUpdate(Math.random())
}} }}
/>
<div>Name</div>
<TextField
style={{ style={{
backgroundColor: theme.palette.inputColor, backgroundColor: theme.palette.inputColor,
color: "white", borderRadius: theme.palette.borderRadius,
height: 50,
}} }}
> InputProps={{
{environments.map((data) => { style: {},
if (data.archived) { }}
return null fullWidth
} color="primary"
placeholder={selectedTrigger.label}
onChange={selectedTriggerChange}
/>
if (data.Name.toLowerCase() === "cloud") { <div style={{ marginTop: "20px" }}>
return null <Typography>Environment</Typography>
} <Select
MenuProps={{
disableScrollLock: true,
}}
value={selectedTrigger.environment}
disabled={selectedTrigger.status === "running"}
SelectDisplayProps={{}}
fullWidth
onChange={(e) => {
selectedTrigger.environment = e.target.value;
setSelectedTrigger(selectedTrigger);
return ( setWorkflow(workflow);
<MenuItem setUpdate(Math.random());
key={data.id} }}
style={{ backgroundColor: theme.palette.inputColor, color: "white" }}
value={data.Name}
>
{data.Name}
</MenuItem>
);
})}
</Select>
</div>
<Divider
style={{
marginTop: "20px",
height: "1px",
width: "100%",
backgroundColor: "rgb(91, 96, 100)",
}}
/>
<div style={{ flex: "6", marginTop: 20, }}>
<div>
<b>Parameters</b>
{/*
<div
style={{
border: "1px solid rgba(255,255,255,0.3)",
borderRadius: theme.palette.borderRadius,
padding: 10,
cursor: "pointer",
}}
onClick={() => {
const pipelineConfig = {
"name": "HTTP Testing",
"type": "create",
"command": "from http://192.168.86.44:5002/api/v1/orgs/7e9b9007-5df2-4b47-bca5-c4d267ef2943/cache/CIDR%20ranges?type=text&authorization=cec9d01f-09b2-4419-8a0a-76c6046e3fef read lines | to http://192.168.86.44:5002/api/v1/hooks/webhook_665ace5f-f27b-496a-a365-6e07eb61078c write lines",
"environment": selectedTrigger.environment,
}
submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig)
}}
>
Run HTTP Request
</div>
*/}
<div
style={{
border: "1px solid rgba(255,255,255,0.3)",
borderRadius: theme.palette.borderRadius,
padding: 10,
cursor: "pointer",
marginTop: 5,
}}
onClick={() => {
const pipelineConfig = {
"name": selectedTrigger.label,
"type": "create",
"command": "load tcp://0.0.0.0:514 | read syslog | export",
"environment": selectedTrigger.environment,
}
submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig)
}}
>
Start Syslog listener
</div>
<div
style={{
border: "1px solid rgba(255,255,255,0.3)",
borderRadius: theme.palette.borderRadius,
padding: 10,
cursor: "pointer",
marginTop: 5,
}}
onClick={() => {
const pipelineConfig = {
"name": selectedTrigger.label,
"type": "create",
"command": "export --live | sigma /path/to/rules | to http://192.168.86.44:5002/api/v1/hooks/webhook_665ace5f-f27b-496a-a365-6e07eb61078c write lines",
"environment": selectedTrigger.environment,
}
submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig)
}}
>
Run Sigma Rulesearch
</div>
<div
style={{
border: "1px solid rgba(255,255,255,0.3)",
borderRadius: theme.palette.borderRadius,
padding: 10,
cursor: "pointer",
marginTop: 5,
}}
onClick={() => {
const pipelineConfig = {
"name": selectedTrigger.label,
"type": "create",
"command": "from kafka://1.2.3.4 --topic foo | to http://api.com X-Token:Secret",
"environment": selectedTrigger.environment,
}
submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig)
}}
>
Follow Kafka Queue
</div>
<div
style={{ style={{
marginTop: "20px", backgroundColor: theme.palette.inputColor,
marginBottom: "7px", color: "white",
display: "flex", height: 50,
}} }}
> >
<Button {environments.map((data) => {
style={{ flex: "1" }} if (data.archived) {
variant="contained" return null;
disabled={selectedTrigger.status === "running"} }
if (data.Name.toLowerCase() === "cloud") {
return null;
}
return (
<MenuItem
key={data.id}
style={{
backgroundColor: theme.palette.inputColor,
color: "white",
}}
value={data.Name}
>
{data.Name}
</MenuItem>
);
})}
</Select>
</div>
<Divider
style={{
marginTop: "20px",
height: "1px",
width: "100%",
backgroundColor: "rgb(91, 96, 100)",
}}
/>
<div style={{ flex: 6, marginTop: 20 }}>
<div>
<b>Parameters</b>
<div
key="syslogListener"
onClick={() => { onClick={() => {
toast("Should start. But it doesn't") // setSelectedOption("Syslog listener")
// setTenzirConfigModalOpen(true);
}}
style={{
border: "1px solid rgba(255,255,255,0.3)",
borderRadius: theme.palette.borderRadius,
padding: 10,
cursor: "not-allowed",
marginTop: 5,
display: "flex",
alignItems: "center",
}} }}
color="primary"
> >
Start <FormControlLabel
</Button> control={
<Button <Radio
style={{ flex: "1" }} checked={selectedOption === "Syslog listener"}
variant="contained" onChange={() => setSelectedOption("Syslog listener")}
disabled={selectedTrigger.status !== "running"} value={"Syslog listener"}
name="option"
disabled={true}
/>
}
label="Start Syslog listener"
/>
</div>
<div
key="sigmaRulesearch"
onClick={() => { onClick={() => {
toast("Should stop triggert") // setSelectedOption("Sigma Rulesearch")
// setTenzirConfigModalOpen(true);
}}
style={{
border: "1px solid rgba(255,255,255,0.3)",
borderRadius: theme.palette.borderRadius,
padding: 10,
cursor: "not-allowed",
marginTop: 5,
display: "flex",
alignItems: "center",
}} }}
color="primary"
> >
Stop <FormControlLabel
</Button> control={
<Radio
checked={selectedOption === "Sigma Rulesearch"}
onChange={() => setSelectedOption("Sigma Rulesearch")}
value={"Sigma Rulesearch"}
name="option"
disabled={true}
/>
}
label="Run Sigma Rulesearch"
/>
</div>
<div
key="kafkaQueue"
onClick={() => {
if(selectedTrigger.status === "running"){
toast("please stop the trigger to edit the configuration");
return;
} else {
setSelectedOption("Kafka Queue");
setTenzirConfigModalOpen(true);
}}}
style={{
border: "1px solid rgba(255,255,255,0.3)",
borderRadius: theme.palette.borderRadius,
padding: 10,
cursor: "pointer",
marginTop: 5,
display: "flex",
alignItems: "center",
}}
>
<FormControlLabel
control={
<Radio
checked={selectedOption === "Kafka Queue"}
onChange={() => setSelectedOption("Kafka Queue")}
value={"Kafka Queue"}
name="option"
/>
}
label="Follow Kafka Queue"
/>
</div>
<div style={{ marginTop: 20, marginBottom: 7, display: "flex" }}>
<Button
style={{ flex: 1, marginRight: 7 }}
variant="contained"
disabled={selectedTrigger.status === "running"}
onClick={() => {
const topic = document.getElementById('topic')?.value;
const bootstrapServers = document.getElementById('bootstrap_servers')?.value;
const groupId = document.getElementById('group_id')?.value;
const autoOffsetReset = document.getElementById('auto_offset_reset')?.value;
let command = "from kafka"
if(topic) command = `${command} --topic ${topic}`
if(bootstrapServers) command = `${command} --set bootstrap.servers=${bootstrapServers}`
if(groupId) command = `${command},group.id=${groupId}`
if(autoOffsetReset) command = `${command},auto.offset.reset=${autoOffsetReset}`
const pipelineConfig = {
command: command,
name: selectedTrigger.label,
type: "start",
environment: selectedTrigger.environment,
workflow_id: workflow.id,
trigger_id: selectedTrigger.id,
};
submitPipeline(selectedTrigger, selectedTriggerIndex, pipelineConfig);
}}
color="primary"
>
Start
</Button>
<Button
style={{ flex: 1 }}
variant="contained"
disabled={selectedTrigger.status !== "running"}
onClick={() => {
toast("Should stop trigger");
}}
color="primary"
>
Stop
</Button>
</div>
</div> </div>
</div> </div>
</div> </div>
</div>
const ScheduleSidebar = Object.getOwnPropertyNames(selectedTrigger).length === 0 || workflow.triggers[selectedTriggerIndex] === undefined && selectedTrigger.trigger_type !== "SCHEDULE" ? null : const ScheduleSidebar = Object.getOwnPropertyNames(selectedTrigger).length === 0 || workflow.triggers[selectedTriggerIndex] === undefined && selectedTrigger.trigger_type !== "SCHEDULE" ? null :
<div style={appApiViewStyle}> <div style={appApiViewStyle}>
@@ -14048,7 +14123,7 @@ const AngularWorkflow = (defaultprops) => {
selectedTrigger.status === "running" selectedTrigger.status === "running"
} }
defaultValue={ defaultValue={
selectedTrigger.parameters === undefined ? "" : selectedTrigger.parameters[0].value selectedTrigger.parameters === undefined ? "" : selectedTrigger.parameters[0]?.value
} }
color="primary" color="primary"
placeholder="" placeholder=""
@@ -18749,6 +18824,146 @@ const AngularWorkflow = (defaultprops) => {
</Dialog> </Dialog>
) : null; ) : null;
const tenzirConfigModal = tenzirConfigModalOpen ? (
<Dialog
PaperComponent={PaperComponent}
hideBackdrop={true}
disableEnforceFocus={true}
disableBackdropClick={true}
style={{ pointerEvents: "none" }}
open={tenzirConfigModalOpen}
PaperProps={{
style: {
pointerEvents: "auto",
color: "white",
minWidth: 600,
minHeight: 500,
maxHeight: 500,
padding: 15,
overflow: "hidden",
zIndex: 10012,
border: theme.palette.defaultBorder,
},
}}
>
<div
style={{
flex: 2,
padding: 0,
minHeight: isMobile ? "90%" : 700,
maxHeight: isMobile ? "90%" : 700,
overflowY: "auto",
overflowX: isMobile ? "auto" : "hidden",
}}
>
<DialogTitle id="tenzir-config-modal" style={{ cursor: "move" }}>
<div style={{ color: "white" }}>Configuration options for {selectedOption}</div>
</DialogTitle>
<DialogContent>
{selectedOption === "Kafka Queue" && (
<>
<b>Topic</b>
<TextField
id="topic"
style={{
backgroundColor: theme.palette.inputColor,
borderRadius: theme.palette.borderRadius,
}}
InputProps={{
style: {},
}}
fullWidth
color="primary"
placeholder={"topic name"}
defaultValue={(selectedTrigger?.parameters?.find(param => param.name === "topic")?.value) || ''}
/>
<b>bootstrap.servers</b>
<TextField
id="bootstrap_servers"
style={{
backgroundColor: theme.palette.inputColor,
borderRadius: theme.palette.borderRadius,
}}
InputProps={{
style: {},
}}
fullWidth
color="primary"
placeholder={"broker1.example.com:9092,192.168.1.100:9092"}
defaultValue={(selectedTrigger?.parameters?.find(param => param.name === "bootstrap_servers")?.value) || ''}
/>
<b>group.id</b>
<TextField
id="group_id"
style={{
backgroundColor: theme.palette.inputColor,
borderRadius: theme.palette.borderRadius,
}}
InputProps={{
style: {},
}}
fullWidth
color="primary"
placeholder={"tenzir"}
defaultValue={(selectedTrigger?.parameters?.find(param => param.name === "group_id")?.value) || ''}
/>
<b>auto.offest.reset</b>
<TextField
id="auto_offset_reset"
style={{
backgroundColor: theme.palette.inputColor,
borderRadius: theme.palette.borderRadius,
}}
InputProps={{
style: {},
}}
fullWidth
color="primary"
placeholder={"earliest"}
defaultValue={(selectedTrigger?.parameters?.find(param => param.name === "auto_offset_reset")?.value) || ''}
/>
</>
)}
</DialogContent>
<DialogActions>
<Button
style={{ borderRadius: "0px" }}
onClick={() => {
setTenzirConfigModalOpen(false);
}}
color="primary"
>
Cancel
</Button>
<Button
style={{ borderRadius: "0px" }}
onClick={() => {
handleKafkaSubmit(selectedTrigger);
}}
color="primary"
>
Submit
</Button>
</DialogActions>
</div>
<IconButton
style={{
zIndex: 5000,
position: "absolute",
top: 14,
right: 18,
color: "grey",
}}
onClick={() => {
setTenzirConfigModalOpen(false);
}}
>
<CloseIcon />
</IconButton>
</Dialog>
) : null;
// Should get AI autocompletes // Should get AI autocompletes
const aiSubmit = (value, setResponseMsg, setSuggestionLoading, inputAction) => { const aiSubmit = (value, setResponseMsg, setSuggestionLoading, inputAction) => {
if (setResponseMsg !== undefined) { if (setResponseMsg !== undefined) {
@@ -19565,6 +19780,7 @@ const AngularWorkflow = (defaultprops) => {
{codePopoutModal} {codePopoutModal}
{workflowRevisions} {workflowRevisions}
{authenticationModal} {authenticationModal}
{tenzirConfigModal}
{/*editWorkflowModal*/} {/*editWorkflowModal*/}
{executionArgumentModal} {executionArgumentModal}
{configureWorkflowModal} {configureWorkflowModal}