marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'nightly' of github.com:Shuffle/Shuffle into nightly

Browse Source
This commit is contained in:
Aditya
2025-10-29 21:06:35 +05:30
parent d8b0d5f5b2 dd052857e9
commit 0da6fa67af
62 changed files with 7297 additions and 1785 deletions
Download Patch File Download Diff File Expand all files Collapse all files
functions/onprem/orborus/go.mod
+1 -1
View File
@@ -10,7 +10,7 @@ require (
github.com/docker/docker v28.3.3+incompatible
github.com/docker/go-connections v0.5.0
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.9.27
github.com/shuffle/shuffle-shared v0.9.30
k8s.io/api v0.33.1
k8s.io/apimachinery v0.33.1
)
functions/onprem/orborus/go.sum
+2 -2
View File
@@ -328,8 +328,8 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G
github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8=
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
github.com/shuffle/shuffle-shared v0.9.27 h1:YwyWXsp4fCOAPmc1DD+NNf9sVa4RHzp26SvWKxH4ytc=
github.com/shuffle/shuffle-shared v0.9.27/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw=
github.com/shuffle/shuffle-shared v0.9.30 h1:3CYvNyD7sTxdxoZjTVrtaDqFvSWQRKAFGaga6rPGf8A=
github.com/shuffle/shuffle-shared v0.9.30/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
functions/onprem/orborus/orborus.go
+70 -45
View File
@@ -1148,10 +1148,18 @@ func deployK8sWorker(image string, identifier string, env []string) error {
env = append(env, fmt.Sprintf("SHUFFLE_APP_CONTAINER_SECURITY_CONTEXT=%s", appContainerSecurityContext))
}
if len(os.Getenv("SHUFFLE_APP_MOUNT_TMP_VOLUME")) > 0 {
env = append(env, fmt.Sprintf("SHUFFLE_APP_MOUNT_TMP_VOLUME=%s", os.Getenv("SHUFFLE_APP_MOUNT_TMP_VOLUME")))
}
if len(os.Getenv("SHUFFLE_LOGS_DISABLED")) > 0 {
env = append(env, fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED")))
}
if len(os.Getenv("SHUFFLE_APP_REPLICAS")) > 0 {
env = append(env, fmt.Sprintf("SHUFFLE_APP_REPLICAS=%s", os.Getenv("SHUFFLE_APP_REPLICAS")))
}
clientset, _, err := shuffle.GetKubernetesClient()
if err != nil {
log.Printf("[ERROR] Error getting kubernetes client:", err)
@@ -1397,6 +1405,7 @@ func deployK8sWorker(image string, identifier string, env []string) error {
return err
}
svcAppProtocol := "http"
service := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: identifier,
@@ -1406,9 +1415,10 @@ func deployK8sWorker(image string, identifier string, env []string) error {
Selector: matchLabels,
Ports: []corev1.ServicePort{
{
Protocol: "TCP",
Port: 33333,
TargetPort: intstr.FromInt(33333),
Protocol: "TCP",
AppProtocol: &svcAppProtocol,
Port: 33333,
TargetPort: intstr.FromInt(33333),
},
},
Type: corev1.ServiceTypeClusterIP,
@@ -2067,7 +2077,10 @@ func sendRemoveRequest(client *http.Client, toBeRemoved shuffle.ExecutionRequest
resultResp, err := client.Do(result)
if err != nil {
log.Printf("[ERROR] Failed making confirm request: %s", err)
if !strings.Contains(fmt.Sprintf("%s", err), "timeout") {
log.Printf("[ERROR] Failed making confirm request: %s", err)
}
time.Sleep(time.Duration(sleepTime) * time.Second)
return err
}
@@ -2196,12 +2209,23 @@ func main() {
os.Setenv("SHUFFLE_PIPELINE_ENABLED", "true")
}
log.Println("[INFO] Setting up execution environment")
if os.Getenv("SHUFFLE_SKIP_PIPELINES") != "true" && os.Getenv("SHUFFLE_PIPELINE_ENABLED") != "false" {
// Run in 15 seconds in a goroutine
go func() {
time.Sleep(15 * time.Second)
log.Printf("[INFO] Auto-downloading Sigma rules during startup")
ruleType := "sigma"
err := handleFileCategoryChange(ruleType)
if err != nil {
log.Printf("[WARNING] Failed downloading %s rules: %s", ruleType, err)
}
}()
}
log.Println("[INFO] Setting up execution environment for env '%s'", environment)
// //FIXME
if baseUrl == "" {
baseUrl = "https://shuffler.io"
//baseUrl = "http://localhost:5001"
}
if len(orborusUuid) == 0 {
@@ -2592,7 +2616,7 @@ func main() {
os.Setenv("SHUFFLE_SKIP_PIPELINES", "false")
tenzirDisabled = false
err = handleFileCategoryChange()
err = handleFileCategoryChange("sigma")
if err != nil {
log.Printf("[ERROR] Failed to download the file category: %s", err)
}
@@ -2602,7 +2626,7 @@ func main() {
} else if incRequest.Type == "DISABLE_SIGMA_FOLDER" {
log.Printf("[INFO] Got job to disable sigma rules")
err = removeFileCategory()
err = removeFileCategory("sigma")
if err != nil {
log.Printf("[ERROR] Failed to disable the sigma rules: %s", err)
}
@@ -3107,8 +3131,8 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri
Healthcheck: healthconfig,
ExposedPorts: nat.PortSet{
"5160/tcp": struct{}{},
"1514/udp": struct{}{},
"1514/tcp": struct{}{},
"1514/udp": struct{}{},
"1514/tcp": struct{}{},
},
Entrypoint: []string{containerName},
Env: []string{},
@@ -3154,8 +3178,8 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri
hostConfig := &container.HostConfig{
PortBindings: nat.PortMap{
"1514/tcp": []nat.PortBinding{{HostPort: "1514"}},
"1514/udp": []nat.PortBinding{{HostPort: "1514"}},
"1514/tcp": []nat.PortBinding{{HostPort: "1514"}},
"1514/udp": []nat.PortBinding{{HostPort: "1514"}},
"5160/tcp": []nat.PortBinding{{HostPort: "5160"}},
},
Mounts: []mount.Mount{
@@ -3165,16 +3189,16 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri
Target: "/tmp",
},
/*
{
Type: "bind",
Source: tenzirStorageFolder,
Target: "/var/log/tenzir/",
},
{
Type: "bind",
Source: tenzirStorageFolder,
Target: "/var/cache/tenzir/",
},
{
Type: "bind",
Source: tenzirStorageFolder,
Target: "/var/log/tenzir/",
},
{
Type: "bind",
Source: tenzirStorageFolder,
Target: "/var/cache/tenzir/",
},
*/
},
VolumeDriver: "local",
@@ -3392,7 +3416,7 @@ func createPipeline(command, identifier string) (string, error) {
"name": identifier,
"hidden": false,
"retry_delay": "500.0ms",
"unstoppable": true,
"unstoppable": true,
}
requestBodyJSON, err := json.Marshal(requestBody)
@@ -3464,20 +3488,20 @@ func updatePipelineState(command, pipelineId, action string) (string, error) {
url := fmt.Sprintf("%s/api/v0/pipeline/update", pipelineUrl)
forwardMethod := "POST"
requestBody := map[string]interface{}{
"id": pipelineId,
"action": action,
"id": pipelineId,
"action": action,
/*
"autostart": map[string]bool{
"created": true,
"completed": false,
"failed": false,
},
"autodelete": map[string]bool{
"completed": false,
"failed": false,
"stopped": false,
},
"autostart": map[string]bool{
"created": true,
"completed": false,
"failed": false,
},
"autodelete": map[string]bool{
"completed": false,
"failed": false,
"stopped": false,
},
*/
}
@@ -3637,8 +3661,8 @@ func searchPipeline(identifier string) (string, error) {
return "", errors.New("no existing pipeline found with name")
}
func handleFileCategoryChange() error {
apiEndpoint := baseUrl + "/api/v1/files/namespaces/sigma"
func handleFileCategoryChange(ruleType string) error {
apiEndpoint := fmt.Sprintf("%s/api/v1/files/namespaces/%s", baseUrl, ruleType)
req, err := http.NewRequest("GET", apiEndpoint, nil)
if err != nil {
return err
@@ -3685,14 +3709,13 @@ func handleFileCategoryChange() error {
}
//log.Println("[DEBUG] ZIP file downloaded successfully.")
tenzirStorageFolder := os.Getenv("SHUFFLE_STORAGE_FOLDER")
if len(tenzirStorageFolder) == 0 {
tenzirStorageFolder = "/tmp/"
}
tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/")
sigmaPath := fmt.Sprintf("%s/sigma_rules", tenzirStorageFolder)
sigmaPath := fmt.Sprintf("%s/%s_rules", tenzirStorageFolder, ruleType)
err = extractZIP("files.zip", sigmaPath)
if err != nil {
log.Printf("[ERROR] Failed to extract ZIP file: %s", err)
@@ -3780,7 +3803,7 @@ func copyToTenzir(srcPath, destPath string) error {
return nil
}
func removeFileCategory() error {
func removeFileCategory(ruleType string) error {
tenzirStorageFolder := os.Getenv("SHUFFLE_STORAGE_FOLDER")
if len(tenzirStorageFolder) == 0 {
tenzirStorageFolder = "/tmp/"
@@ -3789,14 +3812,14 @@ func removeFileCategory() error {
tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/")
//sigmaPath := "/var/lib/tenzir/sigma_rules/*"
sigmaPath := fmt.Sprintf("%s/sigma_rules", tenzirStorageFolder)
rulePath := fmt.Sprintf("%s/%s_rules", tenzirStorageFolder, ruleType)
err := os.RemoveAll(sigmaPath)
err := os.RemoveAll(rulePath)
if err != nil {
return fmt.Errorf("Error removing category files in %s: %v", sigmaPath, err)
return fmt.Errorf("Error removing category files in %s: %v", rulePath, err)
}
log.Printf("[INFO] Removed all local category data in %s", sigmaPath)
log.Printf("[INFO] Removed all local category data in %s", rulePath)
return nil
}
@@ -4216,7 +4239,9 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string,
newresp, err := client.Do(req)
if err != nil {
// Connection refused?
log.Printf("[ERROR][%s] Error running worker request to %s (1): %s", workflowExecution.ExecutionId, streamUrl, err)
if !strings.Contains(fmt.Sprintf("%s", err), "timeout") {
log.Printf("[ERROR][%s] Error running worker request to %s (1): %s", workflowExecution.ExecutionId, streamUrl, err)
}
if strings.Contains(fmt.Sprintf("%s", err), "connection refused") || strings.Contains(fmt.Sprintf("%s", err), "EOF") {
workerImage := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion)