Merge branch 'nightly' of github.com:Shuffle/Shuffle into nightly

.github/workflows/helm-release.yml

@@ -45,7 +45,7 @@ jobs:
             CHART_VERSION="${VERSION}"
           else
             APP_VERSION="latest"
-            CHART_VERSION="2.1.1"
+            CHART_VERSION="2.1.2"
           fi
 
           echo "APP_VERSION set to ${APP_VERSION}"

backend/go-app/docker.go

@@ -403,6 +403,7 @@ func buildImage(tags []string, dockerfileLocation string) error {
 								Image: "gcr.io/kaniko-project/executor:latest",
 								Args: []string{
 									"--verbosity=debug",
+									"--log-format=text",
 									"--dockerfile=Dockerfile",
 									"--context=dir://" + contextDir,
 									"--skip-tls-verify",
@@ -413,6 +414,10 @@ func buildImage(tags []string, dockerfileLocation string) error {
 										Name:      "kaniko-workspace",
 										MountPath: "/app/generated",
 									},
+									{
+										Name: "docker-config",
+										MountPath: "/kaniko/.docker/",
+									},
 								},
 							},
 						},
@@ -427,6 +432,20 @@ func buildImage(tags []string, dockerfileLocation string) error {
 									},
 								},
 							},
+							{
+								Name: "docker-config",
+								VolumeSource: corev1.VolumeSource{
+									Secret: &corev1.SecretVolumeSource{
+										SecretName: os.Getenv("SHUFFLE_REGISTRY_SECRET"),
+										Items: []corev1.KeyToPath{
+											{
+											Key: ".dockerconfigjson",
+											Path: "config.json",
+											},
+										},
+									},
+								},
+							},
 						},
 					},
 				},

backend/go-app/go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/gorilla/mux v1.8.1
 	github.com/h2non/filetype v1.1.3
 	github.com/satori/go.uuid v1.2.0
-	github.com/shuffle/shuffle-shared v0.9.58
+	github.com/shuffle/shuffle-shared v0.9.61
 	github.com/shuffle/singul v0.0.20
 	golang.org/x/crypto v0.40.0
 	google.golang.org/api v0.236.0
@@ -96,7 +96,6 @@ require (
 	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
 	github.com/googleapis/gax-go/v2 v2.14.2 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect
-	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -131,7 +130,6 @@ require (
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/skeema/knownhosts v1.3.1 // indirect
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
-	github.com/spf13/cobra v1.9.1 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect

backend/go-app/go.sum

@@ -138,8 +138,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0=
 github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4=
-github.com/frikky/schemaless v0.0.24 h1:1bNg96gm82yLbRlDS0haQH/96X3GlCAEuTCSasXIJZE=
+github.com/frikky/schemaless v0.0.25 h1:qXjrKT54LWl3tzDFIUmevZ86F7FiusPpQKfa40xxNl8=
-github.com/frikky/schemaless v0.0.24/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
+github.com/frikky/schemaless v0.0.25/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
 github.com/fsouza/go-dockerclient v1.12.1 h1:FMoLq+Zhv9Oz/rFmu6JWkImfr6CBgZOPcL+bHW4gS0o=
 github.com/fsouza/go-dockerclient v1.12.1/go.mod h1:OqsgJJcpCwqyM3JED7TdfM9QVWS5O7jSYwXxYKmOooY=
 github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
@@ -344,8 +344,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN
 github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
 github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU=
 github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0=
-github.com/shuffle/shuffle-shared v0.9.53 h1:ykziiRA82FZX0YiuErSmFLQMxvAFi6yUajgRjd3J5xE=
+github.com/shuffle/shuffle-shared v0.9.60 h1:8NiovcsSsVX8i0UHmbFBANjUQ/u+4PDDpJXg9JfMAUo=
-github.com/shuffle/shuffle-shared v0.9.53/go.mod h1:g03P/ZM8pUwBbdxW0l9A1siMMuemEQshgDpO9hF23eE=
+github.com/shuffle/shuffle-shared v0.9.60/go.mod h1:vK6t1WY5Nfg5vOAk6taT788jIGKs6/4iN1d8Argyn4o=
 github.com/shuffle/singul v0.0.20 h1:Lz+K4l2GJQ5W6o8ePr/2qlIcGuZuMC7W8Wj/JFw1XDI=
 github.com/shuffle/singul v0.0.20/go.mod h1:qY3ZmwaNwmRApLWMTqZaNEMXO6X8bjr/zY44xDU62hQ=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=

backend/go-app/walkoff.go

@@ -3393,6 +3393,8 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
 						fmt.Sprintf("/apps"),
 						orgId,
 						false,
+						"HIGH",
+						"APP_BUID",
 					)
 				}
 

docker-compose.yml

@@ -54,7 +54,6 @@ services:
       - SHUFFLE_STATS_DISABLED=true
       - SHUFFLE_LOGS_DISABLED=true
       - SHUFFLE_SWARM_CONFIG=run
-      - SHUFFLE_APP_REPLICAS=3
       - CLEANUP=false
       - SHUFFLE_WORKER_IMAGE=ghcr.io/shuffle/shuffle-worker:latest
     env_file: .env

frontend/package.json

@@ -26,6 +26,7 @@
     "cpx": "^1.5.0",
     "create-react-app": "^5.0.1",
     "cytoscape": "^3.29.2",
+    "cytoscape-bubblesets": "^4.1.0",
     "cytoscape-clipboard": "^2.2.1",
     "cytoscape-cxtmenu": "^3.4.0",
     "cytoscape-edgehandles": "^3.5.1",

frontend/src/components/CorrelationGraph.jsx

@@ -0,0 +1,680 @@
+import React, { useMemo, useState, useEffect, useContext, memo } from "react";
+import { toast } from 'react-toastify';
+import { getTheme } from "../theme.jsx";
+import { Context } from "../context/ContextApi.jsx";
+import ReactJson from "react-json-view-ssr";
+import { GetIconInfo, collapseField, } from "../views/Workflows2.jsx";
+import defaultCytoscapeStyle from "../defaultCytoscapeStyle.jsx";
+import { v4 as uuidv4, } from "uuid";
+
+import {
+  Typography,
+  Tooltip,
+  IconButton,
+} from "@mui/material";
+
+import {
+  Refresh as RefreshIcon,
+} from "@mui/icons-material";
+
+import cytoscape from "cytoscape";
+import CytoscapeComponent from "react-cytoscapejs";
+
+//import spread from "cytoscape-spread";
+import BubbleSets from 'cytoscape-bubblesets';
+import coseBilkent from 'cytoscape-cose-bilkent';
+
+cytoscape.use(coseBilkent)
+cytoscape.use(BubbleSets)
+const svgSize = 24;
+
+
+// Uses correlations between locations to build a graph around relationships
+// In the background, this is stored in the datastore_ngram index
+const CorrelationGraph = (props) => {
+	const { globalUrl, userdata, datastoreData, } = props;
+
+    const {themeMode, supportEmail, brandColor} = useContext(Context)
+    const theme = getTheme(themeMode, brandColor)
+
+	const [relations, setRelations] = useState(null);
+    const [cy, setCy] = React.useState();
+	const [correlations, setCorrelations] = useState([])
+	const [visualisedData, setVisualisedData] = useState(undefined)
+	const [selectedData, setselectedData] = useState(undefined)
+	const [startNode, setStartNode] = useState(undefined)
+	const [bb, setBb] = useState(undefined)
+	const [handledKeys, setHandledKeys] = useState([])
+
+  	const cystyle = useMemo(() => defaultCytoscapeStyle(theme), [themeMode]);
+	const spreadLayout = {
+		name: 'cose-bilkent',
+		nodeRepulsion: 12500,
+		idealEdgeLength: 200,
+		gravity: 0.40,
+		numIter: 10000,      
+		animationDuration: 500,
+		fit: true,
+
+		animate: true,
+	}
+
+	const loadCorrelations = (datastoreData) => {
+		const searchData = {
+			"type": "datastore",
+			"key": datastoreData?.key,
+
+			"category": datastoreData?.category,
+		}
+
+		if (datastoreData?.category === undefined || datastoreData?.category === null || datastoreData?.category.length < 1) {
+			console.log("No category for datastore item, cannot load correlations.")
+			return
+		}
+
+		const checkKey = `${datastoreData?.category}|${datastoreData?.key}`
+		if (handledKeys.includes(checkKey) === true) {
+			//toast.warn("Correlations for this datastore item have already been loaded.")
+			return
+		}
+		setHandledKeys([...handledKeys, checkKey])
+
+		const url = `${globalUrl}/api/v2/correlations`
+		fetch(url, {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+				Accept: "application/json",
+			},
+			credentials: "include",
+			body: JSON.stringify(searchData),
+		})
+		.then((response) => {
+			if (response.status !== 200) {
+				console.log("Status not 200 for workflows :O!");
+				return;
+			}
+
+			return response.json();
+		})
+		.then((responseJson) => {
+			if (responseJson?.success === false) {
+				toast.warn("Failed to load correlations. Please try again or contact support@shuffler if this persists.")
+			} else {
+				if (responseJson.length === 0) {
+					toast.warn("No correlations found for this datastore item.")
+					return
+				}
+
+				const parsedInfo = {
+					parentKey: datastoreData?.key,
+					parentId: datastoreData?.id,
+					relations: responseJson,
+				}
+
+				setCorrelations(parsedInfo)
+			}
+		})
+		.catch((error) => {
+			toast(error.toString());
+		});
+	}
+
+	useEffect(() => {
+		if (cy === undefined) {
+			return
+		}
+
+		if (correlations === undefined || correlations === null || correlations?.relations === undefined) {
+			console.log("No correlations to load into graph: ", correlations)
+			return
+		}
+
+		var startX = 250 
+		var startY = 400 
+		var added = false
+		//var xyIncrement = 125 
+		for (var key in correlations?.relations) {
+			var parentValue = correlations?.relations[key]
+
+			if (parentValue?.ref === undefined || parentValue?.ref === null || parentValue?.ref?.length < 2) {
+				console.log("Skipping correlation with no ref: ", value)
+				continue
+			}
+
+			parentValue.id = `node_${parentValue.key}`
+			const foundnode = cy.getElementById(parentValue.id)
+			if (foundnode !== undefined && foundnode !== null && foundnode.length > 0) {
+				console.log("Parent node already exists, skipping: ", parentValue)
+				continue
+			}
+
+			parentValue.type = "COMMENT"
+			parentValue.height = 40 
+
+			//parentValue.width = 200 
+			parentValue.width = Math.max(125, Math.min(300, parentValue.key.length * 10))
+
+			parentValue.is_valid = true
+			parentValue.color = "white"
+			parentValue.label = parentValue.key
+			parentValue.clusters = [parentValue.id, correlations.parentId]
+
+			const parentNodeData = {
+			  group: "nodes",
+			  data: parentValue,
+			  position: {
+				x: startX,
+				y: startY,
+			  },
+			}
+
+			cy.add(parentNodeData)
+
+		    const newRootId = uuidv4()
+			cy.add({
+				group: "edges",
+				data: {
+					id: newRootId,
+					source: correlations?.parentId,
+					target: parentValue.id,
+					correlation: true,
+				},
+			})
+
+			added = true 
+			for (var relKey in parentValue?.ref) {
+				var refKey = parentValue?.ref[relKey]
+
+
+				//const foundnode = cy.getElementById(`node_${key}`)
+				//if (foundnode !== undefined && foundnode !== null && foundnode.length > 0) {
+				//	continue
+				//}
+				const original = refKey
+				var category = ""
+				if (refKey.includes("|")) {
+					const splitRef = refKey.split("|")
+					category = splitRef[0]
+
+					// Add the rest of them to key
+					refKey = splitRef.slice(1).join("|")
+				}
+
+				var value = {
+					"key": refKey,
+					"category": category,
+					"relation": parentValue?.key,
+
+					"ref": original,
+				}
+
+				value.id = `node_${value.ref}`
+				const foundnode = cy.getElementById(value.id)
+				if (foundnode !== undefined && foundnode !== null && foundnode.length > 0) {
+					// Check if the node points to this one?
+					console.log("Child node already exists, skipping: ", value.id, value)
+
+					// Verify if an edge exists
+					const sourceExistingEdgeId = `edge_${parentValue.id}_${value.id}`
+					const sourceCheck = cy.getElementById(sourceExistingEdgeId)
+					if (sourceCheck === undefined || sourceCheck === null || sourceCheck.length < 1) {
+						const destinationExistingEdgeId = `edge_${value.id}_${parentValue.id}`
+						const destCheck = cy.getElementById(destinationExistingEdgeId)
+						if (destCheck === undefined || destCheck === null || destCheck.length < 1) {
+							// Get source and child - make sure their clusters align
+							const sourceNode = cy.getElementById(parentValue.id)
+							if (sourceNode === undefined || sourceNode === null || sourceNode.length < 1) {
+							}
+
+							cy.add({
+								group: "edges",
+								data: {
+									id: sourceExistingEdgeId,
+									source: parentValue.id,
+									target: value.id,
+
+									correlation: true,
+								},
+							})
+						}
+					} else {
+						//console.log("Edge already exists between parent and child: ", existingEdgeId)
+					}
+
+					continue
+				}
+
+				value.height = 40 
+				value.width = 40 
+				value.clusters = [value.id, parentValue.id]
+
+              	const iconInfo = GetIconInfo({
+					"app_name": category,
+					"name": category,
+				})
+
+				//console.log("Iconinfo: ", category, iconInfo)
+				if (iconInfo?.originalIcon === undefined || iconInfo?.originalIcon === null || iconInfo?.originalIcon.length < 2) {
+					value.type = "CORRELATION"
+
+					// Get text length from value.key
+					//value.width = Math.max(125, Math.min(300, value.key.length * 10))
+					value.backgroundcolor = randomColor()
+				} else {
+					
+					value.type = "CORRELATION"
+					const svg_pin = `<svg width="${svgSize}" height="${svgSize}" viewBox="0 0 ${svgSize} ${svgSize}" version="1.1" xmlns="http://www.w3.org/2000/svg"><path d="${iconInfo.icon}" fill="${iconInfo.iconColor}"></path></svg>`;
+					const svgpin_Url = encodeURI("data:image/svg+xml;utf-8," + svg_pin);
+					value.large_image = svgpin_Url
+					if (iconInfo.icon === undefined || iconInfo.icon === null || iconInfo.icon.length < 1) {
+						console.log("Handle iconInfo.originalIcon instead")
+					}
+
+					value.fillGradient = iconInfo.fillGradient
+					value.fillstyle = "solid"
+					value.app_name = "Shuffle Tools"
+
+					if (
+					  value.fillGradient !== undefined &&
+					  value.fillGradient !== null &&
+					  value.fillGradient.length > 0
+					) {
+					  value.fillstyle = "linear-gradient";
+					} else {
+					  value.iconBackground = iconInfo.iconBackgroundColor;
+					}
+					
+				}
+
+				value.is_valid = true
+				value.color = "white"
+				//value.backgroundcolor = "blue" 
+				value.label = value.key
+				if (value.category !== undefined && value.category !== null && value.category.length > 0) {
+					value.label = `${value.category}: ${value.key}`
+				}
+
+				const nodeData = {
+				  group: "nodes",
+				  data: value,
+				  position: {
+					x: startX,
+					y: startY,
+				  },
+				}
+
+				cy.add(nodeData)
+
+				const newChildRootId = `edge_${parentValue.id}_${value.id}`
+				cy.add({
+					group: "edges",
+					data: {
+						id: newChildRootId,
+						source: parentValue.id,
+						target: value.id,
+
+						correlation: true,
+					},
+				})
+			
+				added = true 
+			}
+		}
+
+		setTimeout(() => {
+			if (added === false) {
+				return
+			}
+
+			cy.ready(() => {
+				/*
+				var layout = cy.makeLayout({
+					name: 'circle',
+					radius: 300,
+					avoidOverlap: false,
+					startAngle: 3.14,
+					sweep: undefined,
+					clockwise: true,
+
+					animate: true,
+				})
+				*/
+
+				var layout = cy.makeLayout(spreadLayout)
+				layout.run()
+			})
+		}, 500)
+
+	}, [correlations])
+
+	useEffect(() => {
+		var removeFields = ["type", "height", "width", "is_valid", "color", "label", "backgroundcolor", "success", "encrypted", "public_authorization", "suborg_distribution", "revision_id", "large_image", "fillstyle", "app_name", "fillGradient", ]
+		if (visualisedData !== undefined) {
+			var cleanedData = JSON.parse(JSON.stringify(visualisedData))
+			for (var i = 0; i < removeFields.length; i++) {
+				delete cleanedData[removeFields[i]]
+			}
+
+			setVisualisedData(cleanedData)
+		}
+
+		if (selectedData !== undefined) {
+			var cleanedData2 = JSON.parse(JSON.stringify(selectedData))
+			for (var j = 0; j < removeFields.length; j++) {
+				delete cleanedData2[removeFields[j]]
+			}
+
+			setselectedData(cleanedData2)
+		}
+
+	}, [visualisedData, selectedData])
+
+	const onNodeHover = (event) => {
+		const nodedata = JSON.parse(JSON.stringify(event.target.data()))
+		if (nodedata === undefined || nodedata === null) {
+			console.log("No node data on hover")
+			return
+		}
+
+		const cytoscapeElement = document.getElementById("cytoscape_view")
+		if (cytoscapeElement !== undefined && cytoscapeElement !== null) {
+			if (nodedata?.category === undefined || nodedata?.category === null || nodedata?.category.length < 1) {
+				// Means it is a value
+			} else {
+		  		cytoscapeElement.style.cursor = "pointer"
+			}
+		}
+
+		// Only do it 1/20 times as it's super fast anyway
+		// This is due to state management within events being weird 
+		if (Math.random() < 0.95) {
+			setVisualisedData(nodedata)
+		}
+
+		if (bb === undefined || bb === null) {
+			console.log("BB undefined (2)")
+			return
+		}
+
+		if (nodedata?.clusters === undefined || nodedata?.clusters === null) {
+			toast.warn("No cluster data for this node.")
+			return
+		}
+
+		for (var clusterKey in nodedata?.clusters) {
+			const cluster = nodedata?.clusters[clusterKey]
+
+			var nodesInCluster = cy.nodes().filter((node) => {
+				const nodeData = node.data()
+
+				// Check if it is an incoming or outgoing one
+				if (nodeData?.clusters === undefined || nodeData?.clusters === null) {
+					return false
+				}
+
+				return nodeData?.clusters.includes(cluster)
+			})
+
+			const edgesInCluster = cy.edges().filter((edge) => {
+				const sourceNode = edge.source()
+				const targetNode = edge.target()
+
+				const sourceData = sourceNode.data()
+				const targetData = targetNode.data()
+				if (sourceData?.clusters === undefined || sourceData?.clusters === null) {
+					return false
+				}
+
+				if (targetData?.clusters === undefined || targetData?.clusters === null) {
+					return false
+				}
+
+				return sourceData?.clusters.includes(cluster) && targetData?.clusters.includes(cluster)
+			})
+
+			bb.addPath(nodesInCluster, edgesInCluster, null)
+		}
+
+		const incomingNodes = event.target.incomers('node')
+		bb.addPath(incomingNodes)
+		const outgoingNodes = event.target.outgoers('node')
+		bb.addPath(outgoingNodes)
+	}
+
+    const onNodeHoverOut = (event) => {
+		const nodedata = event.target.data();
+
+		const cytoscapeElement = document.getElementById("cytoscape_view")
+		if (cytoscapeElement !== undefined && cytoscapeElement !== null) {
+		  cytoscapeElement.style.cursor = "default"
+		}
+
+		setVisualisedData(undefined)
+
+		if (bb === undefined || bb === null) {
+			console.log("BB undefined")
+			return
+		}
+
+		const activePaths = bb.getPaths()
+		for (var pathKey in activePaths) {
+			bb.removePath(activePaths[pathKey])
+		}
+	}
+
+	const onNodeSelect = (event) => {
+		const nodedata = event.target.data();
+		if (nodedata === undefined || nodedata === null) {
+			return
+		}
+
+		setselectedData(nodedata)
+	
+		loadCorrelations(nodedata) 
+	}
+
+	const onUnselect = (event) => {
+		const nodedata = event.target.data();
+		if (nodedata === undefined || nodedata === null) {
+			return
+		}
+
+		setselectedData(undefined)
+	}
+
+	const randomColor = () => {
+	  // Generate RGB values in a pleasant range (not too dark or oversaturated)
+	  const r = 80 + Math.floor(Math.random() * 150);  // 80–230
+	  const g = 80 + Math.floor(Math.random() * 150);
+	  const b = 80 + Math.floor(Math.random() * 150);
+
+	  // Slightly mix towards neutral gray to make tones modern and less primary
+	  const mix = 0.3;
+	  const gray = (r + g + b) / 3;
+	  const rr = Math.floor(r * (1 - mix) + gray * mix);
+	  const gg = Math.floor(g * (1 - mix) + gray * mix);
+	  const bb = Math.floor(b * (1 - mix) + gray * mix);
+
+	  return `rgb(${rr}, ${gg}, ${bb})`;
+	}
+
+	useEffect(() => {
+		if (cy === undefined) {
+			console.log("No cy instance (yet)")
+			return
+		}
+
+		// If cytoscape has nodes, skip
+		const existingNodes = cy.nodes()
+		if (existingNodes !== undefined && existingNodes !== null && existingNodes.length > 0) {
+			return
+		}
+
+		if (datastoreData === undefined || datastoreData === null) {
+			console.log("No data for correlation graph (2): ", datastoreData)
+			return 
+		}
+
+		var parsedData = JSON.parse(JSON.stringify(datastoreData))
+
+		parsedData.id = `node_${datastoreData?.category}|${datastoreData?.key}`
+		parsedData.large_image = "/images/singul_green.png"
+	    parsedData.type = "ACTION"
+	    parsedData.height = 125 
+	    parsedData.width = 125 
+	    parsedData.color = "white"
+	    parsedData.label = `start - ${parsedData?.category}: ${parsedData?.key}`
+		parsedData.clusters = [parsedData.id]
+		parsedData.is_valid = true
+
+		// Default node
+		cy.add({
+		  group: "nodes",
+		  data: parsedData,
+		  position: {
+			x: 500,
+			y: 500,
+		  },
+		})
+
+		loadCorrelations(parsedData)
+		setStartNode(parsedData)
+
+		const bubbleSets = cy.bubbleSets({
+			zIndex: 4,
+			throttle: 10,
+			interactive: true,
+
+			style: {
+				"fill": randomColor(), 
+			},
+		})
+
+		setBb(bubbleSets)
+	}, [cy])
+
+	useEffect(() => {
+		if (bb === undefined || bb === null) {
+			return
+		}
+
+		cy.on("mouseover", "node", (e) => onNodeHover(e))
+		cy.on("mouseout", "node", (e) => onNodeHoverOut(e))
+		cy.on("select", "node", (e) => onNodeSelect(e))
+		cy.on("unselect", "node", (e) => onUnselect(e))
+	}, [bb])
+
+	if (datastoreData === undefined || datastoreData === null) {
+		console.log("No data for correlation graph: ", datastoreData)
+		return null
+	}
+
+	const getRelation = (startNode, targetNode) => {
+		if (startNode?.id === targetNode?.id) {
+			return 0
+		}
+
+		const foundStartnode = cy.getElementById(startNode?.id)
+		if (foundStartnode === undefined || foundStartnode === null || foundStartnode.length < 1) {
+			return 0
+		}
+
+		const foundTargetnode = cy.getElementById(targetNode?.id)
+		if (foundTargetnode === undefined || foundTargetnode === null || foundTargetnode.length < 1) {
+			return 0
+		} 
+
+		// Check range between them
+		const dijkstra = cy.elements().dijkstra(foundStartnode, (edge) => 1, false);
+		const distance = dijkstra.distanceTo(foundTargetnode);
+		if (distance === Infinity) {
+			return -1
+		}
+
+		if (distance <= 0) {
+			return 0
+		}
+
+		// Divide by 2 due to skipping "values" -> focus on keys containing them 
+		return distance/2
+	}
+
+	const foundValue = visualisedData !== undefined ? visualisedData : selectedData !== undefined ? selectedData : undefined
+	const hoverName = foundValue !== undefined && foundValue?.key !== undefined ? foundValue?.key : ""
+	const hoverCategory = foundValue !== undefined && foundValue?.category !== undefined ? foundValue?.category : ""
+	const jumpsFromStart = hoverCategory === "" ? 0 : getRelation(startNode, foundValue) 
+
+	return (
+		<div style={{position: "relative", width: "95%", height: "95%", margin: "16px 16px 32px 32px", }}>
+			<Tooltip title="Re-shuffle graph" placement="top">
+				<IconButton
+					variant="outlined"
+					color="secondary"
+					style={{position: "absolute", top: 8, left: 8, zIndex: 1000, backgroundColor: theme.palette.surfaceColor, }}
+					onClick={() => {
+						if (cy === undefined || cy === null) {
+							return
+						}
+
+						cy.ready(() => {
+							var layout = cy.makeLayout(spreadLayout)
+							layout.run()
+						})
+					}}
+				>
+					<RefreshIcon />
+				</IconButton>
+			</Tooltip>
+
+			{visualisedData === undefined && selectedData === undefined ? null :
+				<div style={{position: "fixed", top: 100, right: 100, zIndex: 1000, maxWidth: 400, minWidth: 400, backgroundColor: theme.palette.surfaceColor, padding: "16px 16px 16px 16px", borderRadius: theme.palette.borderRadius, boxShadow: theme.palette.boxShadow, }}>
+				  <Typography variant="h6" style={{}}>
+					{hoverCategory}{hoverCategory === "" ? "" : ":"} {hoverName}
+				  </Typography>
+				  {hoverCategory !== "" ?
+					  <Typography variant="body1" color="textSecondary" style={{marginBottom: 8, }}>
+						Degrees from start: {jumpsFromStart}
+					  </Typography>
+				  : null}
+				  <div style={{marginTop: 8, }} />
+				  <ReactJson
+					src={visualisedData !== undefined ? visualisedData : selectedData}
+					theme={theme.palette.jsonTheme}
+					style={theme.palette.reactJsonStyle}
+					//shouldCollapse={(jsonField) => {
+					//  return collapseField(jsonField)
+					//}}
+					iconStyle={theme.palette.jsonIconStyle}
+					collapseStringsAfterLength={theme.palette.jsonCollapseStringsAfterLength}
+					displayArrayKey={false}
+					displayDataTypes={false}
+					name={false}
+				  />
+				</div>
+			}
+
+            <CytoscapeComponent
+			  elements={[]}
+              minZoom={0.1}
+              maxZoom={2.0}
+              wheelSensitivity={0.25}
+              id="cytoscape_view"
+              style={{
+                width: 900,
+                height: 700,
+                backgroundColor: theme.palette.surfaceColor,
+
+				borderRadius: theme.palette.borderRadius,
+				border: "1px solid rgba(255,255,255,0.1)", 
+              }}
+              stylesheet={cystyle}
+              cy={(incy) => {
+                setCy(incy)
+			  }}
+            />
+		</div>
+	)
+}
+
+export default CorrelationGraph

functions/onprem/orborus/go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/docker/docker v28.3.3+incompatible
 	github.com/docker/go-connections v0.5.0
 	github.com/satori/go.uuid v1.2.0
-	github.com/shuffle/shuffle-shared v0.9.58
+	github.com/shuffle/shuffle-shared v0.9.61
 	k8s.io/api v0.33.1
 	k8s.io/apimachinery v0.33.1
 )

functions/onprem/orborus/go.sum

@@ -128,8 +128,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0=
 github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4=
-github.com/frikky/schemaless v0.0.24 h1:1bNg96gm82yLbRlDS0haQH/96X3GlCAEuTCSasXIJZE=
+github.com/frikky/schemaless v0.0.25 h1:qXjrKT54LWl3tzDFIUmevZ86F7FiusPpQKfa40xxNl8=
-github.com/frikky/schemaless v0.0.24/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
+github.com/frikky/schemaless v0.0.25/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
 github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
 github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
@@ -309,8 +309,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN
 github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
 github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU=
 github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0=
-github.com/shuffle/shuffle-shared v0.9.53 h1:ykziiRA82FZX0YiuErSmFLQMxvAFi6yUajgRjd3J5xE=
+github.com/shuffle/shuffle-shared v0.9.60 h1:8NiovcsSsVX8i0UHmbFBANjUQ/u+4PDDpJXg9JfMAUo=
-github.com/shuffle/shuffle-shared v0.9.53/go.mod h1:g03P/ZM8pUwBbdxW0l9A1siMMuemEQshgDpO9hF23eE=
+github.com/shuffle/shuffle-shared v0.9.60/go.mod h1:vK6t1WY5Nfg5vOAk6taT788jIGKs6/4iN1d8Argyn4o=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=

functions/onprem/orborus/orborus.go

@@ -142,6 +142,28 @@ func init() {
 		log.Printf("Unable to create docker client: %s", err)
 	}
 
+	if os.Getenv("SHUFFLE_EC2_INSTANCE") == "true" {
+		log.Printf("[INFO] Detected AWS EC2 instance. Setting up Docker Swarm with AWS optimizations.")
+		containers, err := dockercli.ContainerList(context.Background(), container.ListOptions{})
+		if err == nil {
+			for _, container := range containers {
+				if strings.Contains(container.Image, "shuffle-orborus") {
+					if len(container.Names) != 0 {
+						if strings.Contains(container.Names[0], "shuffle-orborus") {
+							containerName = container.Names[0]
+							containerName = strings.TrimPrefix(containerName, "/")
+							os.Setenv("ORBORUS_CONTAINER_NAME", containerName)
+							log.Printf("[DEBUG] Found orborus container name: %s", containerName)
+							break
+						}
+					}
+				}
+			}
+		} else {
+			log.Printf("[ERROR] Failed to find orborus container: %s", err)
+		}
+	}
+
 	getThisContainerId()
 
 	if len(pipelineApikey) == 0 {
@@ -2416,6 +2438,7 @@ func main() {
 	}
 
 	if swarmConfig == "run" || swarmConfig == "swarm" || isKubernetes == "true" {
+
 		if isKubernetes != "true" {
 			checkSwarmService(ctx)
 		}
@@ -2750,6 +2773,8 @@ func main() {
 								fmt.Sprintf("/detections/Sigma"),
 								org,
 								true,
+								"LOW",
+								"TENZIR_START",
 							)
 
 							if err != nil {

functions/onprem/worker/go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/docker/docker v28.3.3+incompatible
 	github.com/gorilla/mux v1.8.1
 	github.com/satori/go.uuid v1.2.0
-	github.com/shuffle/shuffle-shared v0.9.58
+	github.com/shuffle/shuffle-shared v0.9.61
 	github.com/shuffle/singul v0.0.20
 	k8s.io/api v0.33.1
 	k8s.io/apimachinery v0.33.1

functions/onprem/worker/go.sum

@@ -128,8 +128,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0=
 github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4=
-github.com/frikky/schemaless v0.0.24 h1:1bNg96gm82yLbRlDS0haQH/96X3GlCAEuTCSasXIJZE=
+github.com/frikky/schemaless v0.0.25 h1:qXjrKT54LWl3tzDFIUmevZ86F7FiusPpQKfa40xxNl8=
-github.com/frikky/schemaless v0.0.24/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
+github.com/frikky/schemaless v0.0.25/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
 github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
 github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
@@ -311,8 +311,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN
 github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
 github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU=
 github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0=
-github.com/shuffle/shuffle-shared v0.9.53 h1:ykziiRA82FZX0YiuErSmFLQMxvAFi6yUajgRjd3J5xE=
+github.com/shuffle/shuffle-shared v0.9.60 h1:8NiovcsSsVX8i0UHmbFBANjUQ/u+4PDDpJXg9JfMAUo=
-github.com/shuffle/shuffle-shared v0.9.53/go.mod h1:g03P/ZM8pUwBbdxW0l9A1siMMuemEQshgDpO9hF23eE=
+github.com/shuffle/shuffle-shared v0.9.60/go.mod h1:vK6t1WY5Nfg5vOAk6taT788jIGKs6/4iN1d8Argyn4o=
 github.com/shuffle/singul v0.0.20 h1:Lz+K4l2GJQ5W6o8ePr/2qlIcGuZuMC7W8Wj/JFw1XDI=
 github.com/shuffle/singul v0.0.20/go.mod h1:qY3ZmwaNwmRApLWMTqZaNEMXO6X8bjr/zY44xDU62hQ=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=