Merge branch 'nightly' of github.com:Shuffle/Shuffle into nightly
This commit is contained in:
Aditya
@@ -45,7 +45,7 @@ jobs:
|
||||
CHART_VERSION="${VERSION}"
|
||||
else
|
||||
APP_VERSION="latest"
|
||||
CHART_VERSION="2.1.1"
|
||||
CHART_VERSION="2.1.2"
|
||||
fi
|
||||
|
||||
echo "APP_VERSION set to ${APP_VERSION}"
|
||||
|
||||
@@ -403,6 +403,7 @@ func buildImage(tags []string, dockerfileLocation string) error {
|
||||
Image: "gcr.io/kaniko-project/executor:latest",
|
||||
Args: []string{
|
||||
"--verbosity=debug",
|
||||
"--log-format=text",
|
||||
"--dockerfile=Dockerfile",
|
||||
"--context=dir://" + contextDir,
|
||||
"--skip-tls-verify",
|
||||
@@ -413,6 +414,10 @@ func buildImage(tags []string, dockerfileLocation string) error {
|
||||
Name: "kaniko-workspace",
|
||||
MountPath: "/app/generated",
|
||||
},
|
||||
{
|
||||
Name: "docker-config",
|
||||
MountPath: "/kaniko/.docker/",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -427,6 +432,20 @@ func buildImage(tags []string, dockerfileLocation string) error {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "docker-config",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: os.Getenv("SHUFFLE_REGISTRY_SECRET"),
|
||||
Items: []corev1.KeyToPath{
|
||||
{
|
||||
Key: ".dockerconfigjson",
|
||||
Path: "config.json",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -26,7 +26,7 @@ require (
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/h2non/filetype v1.1.3
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
github.com/shuffle/shuffle-shared v0.9.58
|
||||
github.com/shuffle/shuffle-shared v0.9.61
|
||||
github.com/shuffle/singul v0.0.20
|
||||
golang.org/x/crypto v0.40.0
|
||||
google.golang.org/api v0.236.0
|
||||
@@ -96,7 +96,6 @@ require (
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.14.2 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
@@ -131,7 +130,6 @@ require (
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
github.com/skeema/knownhosts v1.3.1 // indirect
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
|
||||
github.com/spf13/cobra v1.9.1 // indirect
|
||||
github.com/spf13/pflag v1.0.6 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
|
||||
github.com/x448/float16 v0.8.4 // indirect
|
||||
|
||||
@@ -138,8 +138,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0=
|
||||
github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4=
|
||||
github.com/frikky/schemaless v0.0.24 h1:1bNg96gm82yLbRlDS0haQH/96X3GlCAEuTCSasXIJZE=
|
||||
github.com/frikky/schemaless v0.0.24/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
|
||||
github.com/frikky/schemaless v0.0.25 h1:qXjrKT54LWl3tzDFIUmevZ86F7FiusPpQKfa40xxNl8=
|
||||
github.com/frikky/schemaless v0.0.25/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
|
||||
github.com/fsouza/go-dockerclient v1.12.1 h1:FMoLq+Zhv9Oz/rFmu6JWkImfr6CBgZOPcL+bHW4gS0o=
|
||||
github.com/fsouza/go-dockerclient v1.12.1/go.mod h1:OqsgJJcpCwqyM3JED7TdfM9QVWS5O7jSYwXxYKmOooY=
|
||||
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
|
||||
@@ -344,8 +344,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN
|
||||
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
|
||||
github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU=
|
||||
github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0=
|
||||
github.com/shuffle/shuffle-shared v0.9.53 h1:ykziiRA82FZX0YiuErSmFLQMxvAFi6yUajgRjd3J5xE=
|
||||
github.com/shuffle/shuffle-shared v0.9.53/go.mod h1:g03P/ZM8pUwBbdxW0l9A1siMMuemEQshgDpO9hF23eE=
|
||||
github.com/shuffle/shuffle-shared v0.9.60 h1:8NiovcsSsVX8i0UHmbFBANjUQ/u+4PDDpJXg9JfMAUo=
|
||||
github.com/shuffle/shuffle-shared v0.9.60/go.mod h1:vK6t1WY5Nfg5vOAk6taT788jIGKs6/4iN1d8Argyn4o=
|
||||
github.com/shuffle/singul v0.0.20 h1:Lz+K4l2GJQ5W6o8ePr/2qlIcGuZuMC7W8Wj/JFw1XDI=
|
||||
github.com/shuffle/singul v0.0.20/go.mod h1:qY3ZmwaNwmRApLWMTqZaNEMXO6X8bjr/zY44xDU62hQ=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
|
||||
@@ -3393,6 +3393,8 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
|
||||
fmt.Sprintf("/apps"),
|
||||
orgId,
|
||||
false,
|
||||
"HIGH",
|
||||
"APP_BUID",
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -54,7 +54,6 @@ services:
|
||||
- SHUFFLE_STATS_DISABLED=true
|
||||
- SHUFFLE_LOGS_DISABLED=true
|
||||
- SHUFFLE_SWARM_CONFIG=run
|
||||
- SHUFFLE_APP_REPLICAS=3
|
||||
- CLEANUP=false
|
||||
- SHUFFLE_WORKER_IMAGE=ghcr.io/shuffle/shuffle-worker:latest
|
||||
env_file: .env
|
||||
|
||||
@@ -26,6 +26,7 @@
|
||||
"cpx": "^1.5.0",
|
||||
"create-react-app": "^5.0.1",
|
||||
"cytoscape": "^3.29.2",
|
||||
"cytoscape-bubblesets": "^4.1.0",
|
||||
"cytoscape-clipboard": "^2.2.1",
|
||||
"cytoscape-cxtmenu": "^3.4.0",
|
||||
"cytoscape-edgehandles": "^3.5.1",
|
||||
|
||||
@@ -0,0 +1,680 @@
|
||||
import React, { useMemo, useState, useEffect, useContext, memo } from "react";
|
||||
import { toast } from 'react-toastify';
|
||||
import { getTheme } from "../theme.jsx";
|
||||
import { Context } from "../context/ContextApi.jsx";
|
||||
import ReactJson from "react-json-view-ssr";
|
||||
import { GetIconInfo, collapseField, } from "../views/Workflows2.jsx";
|
||||
import defaultCytoscapeStyle from "../defaultCytoscapeStyle.jsx";
|
||||
import { v4 as uuidv4, } from "uuid";
|
||||
|
||||
import {
|
||||
Typography,
|
||||
Tooltip,
|
||||
IconButton,
|
||||
} from "@mui/material";
|
||||
|
||||
import {
|
||||
Refresh as RefreshIcon,
|
||||
} from "@mui/icons-material";
|
||||
|
||||
import cytoscape from "cytoscape";
|
||||
import CytoscapeComponent from "react-cytoscapejs";
|
||||
|
||||
//import spread from "cytoscape-spread";
|
||||
import BubbleSets from 'cytoscape-bubblesets';
|
||||
import coseBilkent from 'cytoscape-cose-bilkent';
|
||||
|
||||
cytoscape.use(coseBilkent)
|
||||
cytoscape.use(BubbleSets)
|
||||
const svgSize = 24;
|
||||
|
||||
|
||||
// Uses correlations between locations to build a graph around relationships
|
||||
// In the background, this is stored in the datastore_ngram index
|
||||
const CorrelationGraph = (props) => {
|
||||
const { globalUrl, userdata, datastoreData, } = props;
|
||||
|
||||
const {themeMode, supportEmail, brandColor} = useContext(Context)
|
||||
const theme = getTheme(themeMode, brandColor)
|
||||
|
||||
const [relations, setRelations] = useState(null);
|
||||
const [cy, setCy] = React.useState();
|
||||
const [correlations, setCorrelations] = useState([])
|
||||
const [visualisedData, setVisualisedData] = useState(undefined)
|
||||
const [selectedData, setselectedData] = useState(undefined)
|
||||
const [startNode, setStartNode] = useState(undefined)
|
||||
const [bb, setBb] = useState(undefined)
|
||||
const [handledKeys, setHandledKeys] = useState([])
|
||||
|
||||
const cystyle = useMemo(() => defaultCytoscapeStyle(theme), [themeMode]);
|
||||
const spreadLayout = {
|
||||
name: 'cose-bilkent',
|
||||
nodeRepulsion: 12500,
|
||||
idealEdgeLength: 200,
|
||||
gravity: 0.40,
|
||||
numIter: 10000,
|
||||
animationDuration: 500,
|
||||
fit: true,
|
||||
|
||||
animate: true,
|
||||
}
|
||||
|
||||
const loadCorrelations = (datastoreData) => {
|
||||
const searchData = {
|
||||
"type": "datastore",
|
||||
"key": datastoreData?.key,
|
||||
|
||||
"category": datastoreData?.category,
|
||||
}
|
||||
|
||||
if (datastoreData?.category === undefined || datastoreData?.category === null || datastoreData?.category.length < 1) {
|
||||
console.log("No category for datastore item, cannot load correlations.")
|
||||
return
|
||||
}
|
||||
|
||||
const checkKey = `${datastoreData?.category}|${datastoreData?.key}`
|
||||
if (handledKeys.includes(checkKey) === true) {
|
||||
//toast.warn("Correlations for this datastore item have already been loaded.")
|
||||
return
|
||||
}
|
||||
setHandledKeys([...handledKeys, checkKey])
|
||||
|
||||
const url = `${globalUrl}/api/v2/correlations`
|
||||
fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Accept: "application/json",
|
||||
},
|
||||
credentials: "include",
|
||||
body: JSON.stringify(searchData),
|
||||
})
|
||||
.then((response) => {
|
||||
if (response.status !== 200) {
|
||||
console.log("Status not 200 for workflows :O!");
|
||||
return;
|
||||
}
|
||||
|
||||
return response.json();
|
||||
})
|
||||
.then((responseJson) => {
|
||||
if (responseJson?.success === false) {
|
||||
toast.warn("Failed to load correlations. Please try again or contact support@shuffler if this persists.")
|
||||
} else {
|
||||
if (responseJson.length === 0) {
|
||||
toast.warn("No correlations found for this datastore item.")
|
||||
return
|
||||
}
|
||||
|
||||
const parsedInfo = {
|
||||
parentKey: datastoreData?.key,
|
||||
parentId: datastoreData?.id,
|
||||
relations: responseJson,
|
||||
}
|
||||
|
||||
setCorrelations(parsedInfo)
|
||||
}
|
||||
})
|
||||
.catch((error) => {
|
||||
toast(error.toString());
|
||||
});
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (cy === undefined) {
|
||||
return
|
||||
}
|
||||
|
||||
if (correlations === undefined || correlations === null || correlations?.relations === undefined) {
|
||||
console.log("No correlations to load into graph: ", correlations)
|
||||
return
|
||||
}
|
||||
|
||||
var startX = 250
|
||||
var startY = 400
|
||||
var added = false
|
||||
//var xyIncrement = 125
|
||||
for (var key in correlations?.relations) {
|
||||
var parentValue = correlations?.relations[key]
|
||||
|
||||
if (parentValue?.ref === undefined || parentValue?.ref === null || parentValue?.ref?.length < 2) {
|
||||
console.log("Skipping correlation with no ref: ", value)
|
||||
continue
|
||||
}
|
||||
|
||||
parentValue.id = `node_${parentValue.key}`
|
||||
const foundnode = cy.getElementById(parentValue.id)
|
||||
if (foundnode !== undefined && foundnode !== null && foundnode.length > 0) {
|
||||
console.log("Parent node already exists, skipping: ", parentValue)
|
||||
continue
|
||||
}
|
||||
|
||||
parentValue.type = "COMMENT"
|
||||
parentValue.height = 40
|
||||
|
||||
//parentValue.width = 200
|
||||
parentValue.width = Math.max(125, Math.min(300, parentValue.key.length * 10))
|
||||
|
||||
parentValue.is_valid = true
|
||||
parentValue.color = "white"
|
||||
parentValue.label = parentValue.key
|
||||
parentValue.clusters = [parentValue.id, correlations.parentId]
|
||||
|
||||
const parentNodeData = {
|
||||
group: "nodes",
|
||||
data: parentValue,
|
||||
position: {
|
||||
x: startX,
|
||||
y: startY,
|
||||
},
|
||||
}
|
||||
|
||||
cy.add(parentNodeData)
|
||||
|
||||
const newRootId = uuidv4()
|
||||
cy.add({
|
||||
group: "edges",
|
||||
data: {
|
||||
id: newRootId,
|
||||
source: correlations?.parentId,
|
||||
target: parentValue.id,
|
||||
correlation: true,
|
||||
},
|
||||
})
|
||||
|
||||
added = true
|
||||
for (var relKey in parentValue?.ref) {
|
||||
var refKey = parentValue?.ref[relKey]
|
||||
|
||||
|
||||
//const foundnode = cy.getElementById(`node_${key}`)
|
||||
//if (foundnode !== undefined && foundnode !== null && foundnode.length > 0) {
|
||||
// continue
|
||||
//}
|
||||
const original = refKey
|
||||
var category = ""
|
||||
if (refKey.includes("|")) {
|
||||
const splitRef = refKey.split("|")
|
||||
category = splitRef[0]
|
||||
|
||||
// Add the rest of them to key
|
||||
refKey = splitRef.slice(1).join("|")
|
||||
}
|
||||
|
||||
var value = {
|
||||
"key": refKey,
|
||||
"category": category,
|
||||
"relation": parentValue?.key,
|
||||
|
||||
"ref": original,
|
||||
}
|
||||
|
||||
value.id = `node_${value.ref}`
|
||||
const foundnode = cy.getElementById(value.id)
|
||||
if (foundnode !== undefined && foundnode !== null && foundnode.length > 0) {
|
||||
// Check if the node points to this one?
|
||||
console.log("Child node already exists, skipping: ", value.id, value)
|
||||
|
||||
// Verify if an edge exists
|
||||
const sourceExistingEdgeId = `edge_${parentValue.id}_${value.id}`
|
||||
const sourceCheck = cy.getElementById(sourceExistingEdgeId)
|
||||
if (sourceCheck === undefined || sourceCheck === null || sourceCheck.length < 1) {
|
||||
const destinationExistingEdgeId = `edge_${value.id}_${parentValue.id}`
|
||||
const destCheck = cy.getElementById(destinationExistingEdgeId)
|
||||
if (destCheck === undefined || destCheck === null || destCheck.length < 1) {
|
||||
// Get source and child - make sure their clusters align
|
||||
const sourceNode = cy.getElementById(parentValue.id)
|
||||
if (sourceNode === undefined || sourceNode === null || sourceNode.length < 1) {
|
||||
}
|
||||
|
||||
cy.add({
|
||||
group: "edges",
|
||||
data: {
|
||||
id: sourceExistingEdgeId,
|
||||
source: parentValue.id,
|
||||
target: value.id,
|
||||
|
||||
correlation: true,
|
||||
},
|
||||
})
|
||||
}
|
||||
} else {
|
||||
//console.log("Edge already exists between parent and child: ", existingEdgeId)
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
value.height = 40
|
||||
value.width = 40
|
||||
value.clusters = [value.id, parentValue.id]
|
||||
|
||||
const iconInfo = GetIconInfo({
|
||||
"app_name": category,
|
||||
"name": category,
|
||||
})
|
||||
|
||||
//console.log("Iconinfo: ", category, iconInfo)
|
||||
if (iconInfo?.originalIcon === undefined || iconInfo?.originalIcon === null || iconInfo?.originalIcon.length < 2) {
|
||||
value.type = "CORRELATION"
|
||||
|
||||
// Get text length from value.key
|
||||
//value.width = Math.max(125, Math.min(300, value.key.length * 10))
|
||||
value.backgroundcolor = randomColor()
|
||||
} else {
|
||||
|
||||
value.type = "CORRELATION"
|
||||
const svg_pin = `<svg width="${svgSize}" height="${svgSize}" viewBox="0 0 ${svgSize} ${svgSize}" version="1.1" xmlns="http://www.w3.org/2000/svg"><path d="${iconInfo.icon}" fill="${iconInfo.iconColor}"></path></svg>`;
|
||||
const svgpin_Url = encodeURI("data:image/svg+xml;utf-8," + svg_pin);
|
||||
value.large_image = svgpin_Url
|
||||
if (iconInfo.icon === undefined || iconInfo.icon === null || iconInfo.icon.length < 1) {
|
||||
console.log("Handle iconInfo.originalIcon instead")
|
||||
}
|
||||
|
||||
value.fillGradient = iconInfo.fillGradient
|
||||
value.fillstyle = "solid"
|
||||
value.app_name = "Shuffle Tools"
|
||||
|
||||
if (
|
||||
value.fillGradient !== undefined &&
|
||||
value.fillGradient !== null &&
|
||||
value.fillGradient.length > 0
|
||||
) {
|
||||
value.fillstyle = "linear-gradient";
|
||||
} else {
|
||||
value.iconBackground = iconInfo.iconBackgroundColor;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
value.is_valid = true
|
||||
value.color = "white"
|
||||
//value.backgroundcolor = "blue"
|
||||
value.label = value.key
|
||||
if (value.category !== undefined && value.category !== null && value.category.length > 0) {
|
||||
value.label = `${value.category}: ${value.key}`
|
||||
}
|
||||
|
||||
const nodeData = {
|
||||
group: "nodes",
|
||||
data: value,
|
||||
position: {
|
||||
x: startX,
|
||||
y: startY,
|
||||
},
|
||||
}
|
||||
|
||||
cy.add(nodeData)
|
||||
|
||||
const newChildRootId = `edge_${parentValue.id}_${value.id}`
|
||||
cy.add({
|
||||
group: "edges",
|
||||
data: {
|
||||
id: newChildRootId,
|
||||
source: parentValue.id,
|
||||
target: value.id,
|
||||
|
||||
correlation: true,
|
||||
},
|
||||
})
|
||||
|
||||
added = true
|
||||
}
|
||||
}
|
||||
|
||||
setTimeout(() => {
|
||||
if (added === false) {
|
||||
return
|
||||
}
|
||||
|
||||
cy.ready(() => {
|
||||
/*
|
||||
var layout = cy.makeLayout({
|
||||
name: 'circle',
|
||||
radius: 300,
|
||||
avoidOverlap: false,
|
||||
startAngle: 3.14,
|
||||
sweep: undefined,
|
||||
clockwise: true,
|
||||
|
||||
animate: true,
|
||||
})
|
||||
*/
|
||||
|
||||
var layout = cy.makeLayout(spreadLayout)
|
||||
layout.run()
|
||||
})
|
||||
}, 500)
|
||||
|
||||
}, [correlations])
|
||||
|
||||
useEffect(() => {
|
||||
var removeFields = ["type", "height", "width", "is_valid", "color", "label", "backgroundcolor", "success", "encrypted", "public_authorization", "suborg_distribution", "revision_id", "large_image", "fillstyle", "app_name", "fillGradient", ]
|
||||
if (visualisedData !== undefined) {
|
||||
var cleanedData = JSON.parse(JSON.stringify(visualisedData))
|
||||
for (var i = 0; i < removeFields.length; i++) {
|
||||
delete cleanedData[removeFields[i]]
|
||||
}
|
||||
|
||||
setVisualisedData(cleanedData)
|
||||
}
|
||||
|
||||
if (selectedData !== undefined) {
|
||||
var cleanedData2 = JSON.parse(JSON.stringify(selectedData))
|
||||
for (var j = 0; j < removeFields.length; j++) {
|
||||
delete cleanedData2[removeFields[j]]
|
||||
}
|
||||
|
||||
setselectedData(cleanedData2)
|
||||
}
|
||||
|
||||
}, [visualisedData, selectedData])
|
||||
|
||||
const onNodeHover = (event) => {
|
||||
const nodedata = JSON.parse(JSON.stringify(event.target.data()))
|
||||
if (nodedata === undefined || nodedata === null) {
|
||||
console.log("No node data on hover")
|
||||
return
|
||||
}
|
||||
|
||||
const cytoscapeElement = document.getElementById("cytoscape_view")
|
||||
if (cytoscapeElement !== undefined && cytoscapeElement !== null) {
|
||||
if (nodedata?.category === undefined || nodedata?.category === null || nodedata?.category.length < 1) {
|
||||
// Means it is a value
|
||||
} else {
|
||||
cytoscapeElement.style.cursor = "pointer"
|
||||
}
|
||||
}
|
||||
|
||||
// Only do it 1/20 times as it's super fast anyway
|
||||
// This is due to state management within events being weird
|
||||
if (Math.random() < 0.95) {
|
||||
setVisualisedData(nodedata)
|
||||
}
|
||||
|
||||
if (bb === undefined || bb === null) {
|
||||
console.log("BB undefined (2)")
|
||||
return
|
||||
}
|
||||
|
||||
if (nodedata?.clusters === undefined || nodedata?.clusters === null) {
|
||||
toast.warn("No cluster data for this node.")
|
||||
return
|
||||
}
|
||||
|
||||
for (var clusterKey in nodedata?.clusters) {
|
||||
const cluster = nodedata?.clusters[clusterKey]
|
||||
|
||||
var nodesInCluster = cy.nodes().filter((node) => {
|
||||
const nodeData = node.data()
|
||||
|
||||
// Check if it is an incoming or outgoing one
|
||||
if (nodeData?.clusters === undefined || nodeData?.clusters === null) {
|
||||
return false
|
||||
}
|
||||
|
||||
return nodeData?.clusters.includes(cluster)
|
||||
})
|
||||
|
||||
const edgesInCluster = cy.edges().filter((edge) => {
|
||||
const sourceNode = edge.source()
|
||||
const targetNode = edge.target()
|
||||
|
||||
const sourceData = sourceNode.data()
|
||||
const targetData = targetNode.data()
|
||||
if (sourceData?.clusters === undefined || sourceData?.clusters === null) {
|
||||
return false
|
||||
}
|
||||
|
||||
if (targetData?.clusters === undefined || targetData?.clusters === null) {
|
||||
return false
|
||||
}
|
||||
|
||||
return sourceData?.clusters.includes(cluster) && targetData?.clusters.includes(cluster)
|
||||
})
|
||||
|
||||
bb.addPath(nodesInCluster, edgesInCluster, null)
|
||||
}
|
||||
|
||||
const incomingNodes = event.target.incomers('node')
|
||||
bb.addPath(incomingNodes)
|
||||
const outgoingNodes = event.target.outgoers('node')
|
||||
bb.addPath(outgoingNodes)
|
||||
}
|
||||
|
||||
const onNodeHoverOut = (event) => {
|
||||
const nodedata = event.target.data();
|
||||
|
||||
const cytoscapeElement = document.getElementById("cytoscape_view")
|
||||
if (cytoscapeElement !== undefined && cytoscapeElement !== null) {
|
||||
cytoscapeElement.style.cursor = "default"
|
||||
}
|
||||
|
||||
setVisualisedData(undefined)
|
||||
|
||||
if (bb === undefined || bb === null) {
|
||||
console.log("BB undefined")
|
||||
return
|
||||
}
|
||||
|
||||
const activePaths = bb.getPaths()
|
||||
for (var pathKey in activePaths) {
|
||||
bb.removePath(activePaths[pathKey])
|
||||
}
|
||||
}
|
||||
|
||||
const onNodeSelect = (event) => {
|
||||
const nodedata = event.target.data();
|
||||
if (nodedata === undefined || nodedata === null) {
|
||||
return
|
||||
}
|
||||
|
||||
setselectedData(nodedata)
|
||||
|
||||
loadCorrelations(nodedata)
|
||||
}
|
||||
|
||||
const onUnselect = (event) => {
|
||||
const nodedata = event.target.data();
|
||||
if (nodedata === undefined || nodedata === null) {
|
||||
return
|
||||
}
|
||||
|
||||
setselectedData(undefined)
|
||||
}
|
||||
|
||||
const randomColor = () => {
|
||||
// Generate RGB values in a pleasant range (not too dark or oversaturated)
|
||||
const r = 80 + Math.floor(Math.random() * 150); // 80–230
|
||||
const g = 80 + Math.floor(Math.random() * 150);
|
||||
const b = 80 + Math.floor(Math.random() * 150);
|
||||
|
||||
// Slightly mix towards neutral gray to make tones modern and less primary
|
||||
const mix = 0.3;
|
||||
const gray = (r + g + b) / 3;
|
||||
const rr = Math.floor(r * (1 - mix) + gray * mix);
|
||||
const gg = Math.floor(g * (1 - mix) + gray * mix);
|
||||
const bb = Math.floor(b * (1 - mix) + gray * mix);
|
||||
|
||||
return `rgb(${rr}, ${gg}, ${bb})`;
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (cy === undefined) {
|
||||
console.log("No cy instance (yet)")
|
||||
return
|
||||
}
|
||||
|
||||
// If cytoscape has nodes, skip
|
||||
const existingNodes = cy.nodes()
|
||||
if (existingNodes !== undefined && existingNodes !== null && existingNodes.length > 0) {
|
||||
return
|
||||
}
|
||||
|
||||
if (datastoreData === undefined || datastoreData === null) {
|
||||
console.log("No data for correlation graph (2): ", datastoreData)
|
||||
return
|
||||
}
|
||||
|
||||
var parsedData = JSON.parse(JSON.stringify(datastoreData))
|
||||
|
||||
parsedData.id = `node_${datastoreData?.category}|${datastoreData?.key}`
|
||||
parsedData.large_image = "/images/singul_green.png"
|
||||
parsedData.type = "ACTION"
|
||||
parsedData.height = 125
|
||||
parsedData.width = 125
|
||||
parsedData.color = "white"
|
||||
parsedData.label = `start - ${parsedData?.category}: ${parsedData?.key}`
|
||||
parsedData.clusters = [parsedData.id]
|
||||
parsedData.is_valid = true
|
||||
|
||||
// Default node
|
||||
cy.add({
|
||||
group: "nodes",
|
||||
data: parsedData,
|
||||
position: {
|
||||
x: 500,
|
||||
y: 500,
|
||||
},
|
||||
})
|
||||
|
||||
loadCorrelations(parsedData)
|
||||
setStartNode(parsedData)
|
||||
|
||||
const bubbleSets = cy.bubbleSets({
|
||||
zIndex: 4,
|
||||
throttle: 10,
|
||||
interactive: true,
|
||||
|
||||
style: {
|
||||
"fill": randomColor(),
|
||||
},
|
||||
})
|
||||
|
||||
setBb(bubbleSets)
|
||||
}, [cy])
|
||||
|
||||
useEffect(() => {
|
||||
if (bb === undefined || bb === null) {
|
||||
return
|
||||
}
|
||||
|
||||
cy.on("mouseover", "node", (e) => onNodeHover(e))
|
||||
cy.on("mouseout", "node", (e) => onNodeHoverOut(e))
|
||||
cy.on("select", "node", (e) => onNodeSelect(e))
|
||||
cy.on("unselect", "node", (e) => onUnselect(e))
|
||||
}, [bb])
|
||||
|
||||
if (datastoreData === undefined || datastoreData === null) {
|
||||
console.log("No data for correlation graph: ", datastoreData)
|
||||
return null
|
||||
}
|
||||
|
||||
const getRelation = (startNode, targetNode) => {
|
||||
if (startNode?.id === targetNode?.id) {
|
||||
return 0
|
||||
}
|
||||
|
||||
const foundStartnode = cy.getElementById(startNode?.id)
|
||||
if (foundStartnode === undefined || foundStartnode === null || foundStartnode.length < 1) {
|
||||
return 0
|
||||
}
|
||||
|
||||
const foundTargetnode = cy.getElementById(targetNode?.id)
|
||||
if (foundTargetnode === undefined || foundTargetnode === null || foundTargetnode.length < 1) {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Check range between them
|
||||
const dijkstra = cy.elements().dijkstra(foundStartnode, (edge) => 1, false);
|
||||
const distance = dijkstra.distanceTo(foundTargetnode);
|
||||
if (distance === Infinity) {
|
||||
return -1
|
||||
}
|
||||
|
||||
if (distance <= 0) {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Divide by 2 due to skipping "values" -> focus on keys containing them
|
||||
return distance/2
|
||||
}
|
||||
|
||||
const foundValue = visualisedData !== undefined ? visualisedData : selectedData !== undefined ? selectedData : undefined
|
||||
const hoverName = foundValue !== undefined && foundValue?.key !== undefined ? foundValue?.key : ""
|
||||
const hoverCategory = foundValue !== undefined && foundValue?.category !== undefined ? foundValue?.category : ""
|
||||
const jumpsFromStart = hoverCategory === "" ? 0 : getRelation(startNode, foundValue)
|
||||
|
||||
return (
|
||||
<div style={{position: "relative", width: "95%", height: "95%", margin: "16px 16px 32px 32px", }}>
|
||||
<Tooltip title="Re-shuffle graph" placement="top">
|
||||
<IconButton
|
||||
variant="outlined"
|
||||
color="secondary"
|
||||
style={{position: "absolute", top: 8, left: 8, zIndex: 1000, backgroundColor: theme.palette.surfaceColor, }}
|
||||
onClick={() => {
|
||||
if (cy === undefined || cy === null) {
|
||||
return
|
||||
}
|
||||
|
||||
cy.ready(() => {
|
||||
var layout = cy.makeLayout(spreadLayout)
|
||||
layout.run()
|
||||
})
|
||||
}}
|
||||
>
|
||||
<RefreshIcon />
|
||||
</IconButton>
|
||||
</Tooltip>
|
||||
|
||||
{visualisedData === undefined && selectedData === undefined ? null :
|
||||
<div style={{position: "fixed", top: 100, right: 100, zIndex: 1000, maxWidth: 400, minWidth: 400, backgroundColor: theme.palette.surfaceColor, padding: "16px 16px 16px 16px", borderRadius: theme.palette.borderRadius, boxShadow: theme.palette.boxShadow, }}>
|
||||
<Typography variant="h6" style={{}}>
|
||||
{hoverCategory}{hoverCategory === "" ? "" : ":"} {hoverName}
|
||||
</Typography>
|
||||
{hoverCategory !== "" ?
|
||||
<Typography variant="body1" color="textSecondary" style={{marginBottom: 8, }}>
|
||||
Degrees from start: {jumpsFromStart}
|
||||
</Typography>
|
||||
: null}
|
||||
<div style={{marginTop: 8, }} />
|
||||
<ReactJson
|
||||
src={visualisedData !== undefined ? visualisedData : selectedData}
|
||||
theme={theme.palette.jsonTheme}
|
||||
style={theme.palette.reactJsonStyle}
|
||||
//shouldCollapse={(jsonField) => {
|
||||
// return collapseField(jsonField)
|
||||
//}}
|
||||
iconStyle={theme.palette.jsonIconStyle}
|
||||
collapseStringsAfterLength={theme.palette.jsonCollapseStringsAfterLength}
|
||||
displayArrayKey={false}
|
||||
displayDataTypes={false}
|
||||
name={false}
|
||||
/>
|
||||
</div>
|
||||
}
|
||||
|
||||
<CytoscapeComponent
|
||||
elements={[]}
|
||||
minZoom={0.1}
|
||||
maxZoom={2.0}
|
||||
wheelSensitivity={0.25}
|
||||
id="cytoscape_view"
|
||||
style={{
|
||||
width: 900,
|
||||
height: 700,
|
||||
backgroundColor: theme.palette.surfaceColor,
|
||||
|
||||
borderRadius: theme.palette.borderRadius,
|
||||
border: "1px solid rgba(255,255,255,0.1)",
|
||||
}}
|
||||
stylesheet={cystyle}
|
||||
cy={(incy) => {
|
||||
setCy(incy)
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default CorrelationGraph
|
||||
@@ -10,7 +10,7 @@ require (
|
||||
github.com/docker/docker v28.3.3+incompatible
|
||||
github.com/docker/go-connections v0.5.0
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
github.com/shuffle/shuffle-shared v0.9.58
|
||||
github.com/shuffle/shuffle-shared v0.9.61
|
||||
k8s.io/api v0.33.1
|
||||
k8s.io/apimachinery v0.33.1
|
||||
)
|
||||
|
||||
@@ -128,8 +128,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0=
|
||||
github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4=
|
||||
github.com/frikky/schemaless v0.0.24 h1:1bNg96gm82yLbRlDS0haQH/96X3GlCAEuTCSasXIJZE=
|
||||
github.com/frikky/schemaless v0.0.24/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
|
||||
github.com/frikky/schemaless v0.0.25 h1:qXjrKT54LWl3tzDFIUmevZ86F7FiusPpQKfa40xxNl8=
|
||||
github.com/frikky/schemaless v0.0.25/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
|
||||
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
|
||||
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
|
||||
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
|
||||
@@ -309,8 +309,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN
|
||||
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
|
||||
github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU=
|
||||
github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0=
|
||||
github.com/shuffle/shuffle-shared v0.9.53 h1:ykziiRA82FZX0YiuErSmFLQMxvAFi6yUajgRjd3J5xE=
|
||||
github.com/shuffle/shuffle-shared v0.9.53/go.mod h1:g03P/ZM8pUwBbdxW0l9A1siMMuemEQshgDpO9hF23eE=
|
||||
github.com/shuffle/shuffle-shared v0.9.60 h1:8NiovcsSsVX8i0UHmbFBANjUQ/u+4PDDpJXg9JfMAUo=
|
||||
github.com/shuffle/shuffle-shared v0.9.60/go.mod h1:vK6t1WY5Nfg5vOAk6taT788jIGKs6/4iN1d8Argyn4o=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
||||
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
|
||||
@@ -142,6 +142,28 @@ func init() {
|
||||
log.Printf("Unable to create docker client: %s", err)
|
||||
}
|
||||
|
||||
if os.Getenv("SHUFFLE_EC2_INSTANCE") == "true" {
|
||||
log.Printf("[INFO] Detected AWS EC2 instance. Setting up Docker Swarm with AWS optimizations.")
|
||||
containers, err := dockercli.ContainerList(context.Background(), container.ListOptions{})
|
||||
if err == nil {
|
||||
for _, container := range containers {
|
||||
if strings.Contains(container.Image, "shuffle-orborus") {
|
||||
if len(container.Names) != 0 {
|
||||
if strings.Contains(container.Names[0], "shuffle-orborus") {
|
||||
containerName = container.Names[0]
|
||||
containerName = strings.TrimPrefix(containerName, "/")
|
||||
os.Setenv("ORBORUS_CONTAINER_NAME", containerName)
|
||||
log.Printf("[DEBUG] Found orborus container name: %s", containerName)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.Printf("[ERROR] Failed to find orborus container: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
getThisContainerId()
|
||||
|
||||
if len(pipelineApikey) == 0 {
|
||||
@@ -2416,6 +2438,7 @@ func main() {
|
||||
}
|
||||
|
||||
if swarmConfig == "run" || swarmConfig == "swarm" || isKubernetes == "true" {
|
||||
|
||||
if isKubernetes != "true" {
|
||||
checkSwarmService(ctx)
|
||||
}
|
||||
@@ -2750,6 +2773,8 @@ func main() {
|
||||
fmt.Sprintf("/detections/Sigma"),
|
||||
org,
|
||||
true,
|
||||
"LOW",
|
||||
"TENZIR_START",
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
|
||||
@@ -11,7 +11,7 @@ require (
|
||||
github.com/docker/docker v28.3.3+incompatible
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
github.com/shuffle/shuffle-shared v0.9.58
|
||||
github.com/shuffle/shuffle-shared v0.9.61
|
||||
github.com/shuffle/singul v0.0.20
|
||||
k8s.io/api v0.33.1
|
||||
k8s.io/apimachinery v0.33.1
|
||||
|
||||
@@ -128,8 +128,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0=
|
||||
github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4=
|
||||
github.com/frikky/schemaless v0.0.24 h1:1bNg96gm82yLbRlDS0haQH/96X3GlCAEuTCSasXIJZE=
|
||||
github.com/frikky/schemaless v0.0.24/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
|
||||
github.com/frikky/schemaless v0.0.25 h1:qXjrKT54LWl3tzDFIUmevZ86F7FiusPpQKfa40xxNl8=
|
||||
github.com/frikky/schemaless v0.0.25/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
|
||||
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
|
||||
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
|
||||
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
|
||||
@@ -311,8 +311,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN
|
||||
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
|
||||
github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU=
|
||||
github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0=
|
||||
github.com/shuffle/shuffle-shared v0.9.53 h1:ykziiRA82FZX0YiuErSmFLQMxvAFi6yUajgRjd3J5xE=
|
||||
github.com/shuffle/shuffle-shared v0.9.53/go.mod h1:g03P/ZM8pUwBbdxW0l9A1siMMuemEQshgDpO9hF23eE=
|
||||
github.com/shuffle/shuffle-shared v0.9.60 h1:8NiovcsSsVX8i0UHmbFBANjUQ/u+4PDDpJXg9JfMAUo=
|
||||
github.com/shuffle/shuffle-shared v0.9.60/go.mod h1:vK6t1WY5Nfg5vOAk6taT788jIGKs6/4iN1d8Argyn4o=
|
||||
github.com/shuffle/singul v0.0.20 h1:Lz+K4l2GJQ5W6o8ePr/2qlIcGuZuMC7W8Wj/JFw1XDI=
|
||||
github.com/shuffle/singul v0.0.20/go.mod h1:qY3ZmwaNwmRApLWMTqZaNEMXO6X8bjr/zY44xDU62hQ=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
|
||||
Reference in New Issue
Block a user