marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed issue with app sdk not loading full execution properly for /api/v1/run actions

Browse Source
This commit is contained in:
Frikky
2024-09-26 16:15:33 +02:00
parent 91c0b2ca48
commit 21d481a189
4 changed files with 55 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app_sdk/app_base.py
+3 -18
View File
@@ -1767,27 +1767,12 @@ class AppBase:
self.send_result(self.action_result, headers, stream_path)
return
# Add async logger
# self.console_logger.handlers[0].stream.set_execution_id()
# FIXME: Shouldn't skip this, but it's good for minimzing API calls
#try:
# ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False)
# self.logger.info("Workflow: %d" % ret.status_code)
# if ret.status_code != 200:
# self.logger.info(ret.text)
#except requests.exceptions.ConnectionError as e:
# self.logger.info("Connectionerror: %s" % e)
# action_result["result"] = "Bad setup during startup: %s" % e
# self.send_result(action_result, headers, stream_path)
# return
# Verify whether there are any parameters with ACTION_RESULT required
# If found, we get the full results list from backend
# Forcing this to run due to potential self.full_execution loading issues in cloud run
fullexecution = {}
if isinstance(self.full_execution, str) and len(self.full_execution) == 0:
if True or (isinstance(self.full_execution, str) and len(self.full_execution) == 0):
#self.logger.info("[DEBUG] NO EXECUTION - LOADING!")
try:
failed = False
backend/go-app/main.go
+14
View File
@@ -497,6 +497,7 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini)
newUser.ActiveOrg = shuffle.OrgMini{
Id: org.Id,
Name: org.Name,
Role: newUser.Role,
}
if len(apikey) > 0 {
@@ -934,10 +935,22 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) {
//if err == nil {
if len(org.Id) > 0 {
if userInfo.Role == "" {
//err = shuffle.SetUser(ctx, &userInfo, false)
for _, user := range org.Users {
if user.Id != userInfo.Id {
continue
}
userInfo.ActiveOrg.Role = user.Role
}
}
userInfo.ActiveOrg = shuffle.OrgMini{
Id: org.Id,
Name: org.Name,
CreatorOrg: org.CreatorOrg,
ChildOrgs: org.ChildOrgs,
Role: userInfo.ActiveOrg.Role,
Image: org.Image,
}
@@ -5219,6 +5232,7 @@ func initHandlers() {
r.HandleFunc("/api/v1/orgs/{orgId}/delete_cache", shuffle.HandleDeleteCacheKeyPost).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/cache/{cache_key}", shuffle.HandleDeleteCacheKey).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/stats", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/stats", shuffle.HandleAppendStatistics).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/statistics", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/cache", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS")
functions/onprem/orborus/go.sum
+2
View File
@@ -271,6 +271,8 @@ github.com/shuffle/shuffle-shared v0.6.60 h1:8OaiNxNpzJmIbYIcXI3TIYZVrPJ1sSCa+u7
github.com/shuffle/shuffle-shared v0.6.60/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus=
github.com/shuffle/shuffle-shared v0.6.61 h1:+9CCLeZLiAVDgNRTkZxnIgz+FZ7UrEHez2BAGPS/axc=
github.com/shuffle/shuffle-shared v0.6.61/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus=
github.com/shuffle/shuffle-shared v0.6.63 h1:eNQMpVhe/mAMxl61W9Wj6/Z4PrtPeEnbjvMtDdT1mqw=
github.com/shuffle/shuffle-shared v0.6.63/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/skeema/knownhosts v1.2.1 h1:SHWdIUa82uGZz+F+47k8SY4QhhI291cXCpopT1lK2AQ=
github.com/skeema/knownhosts v1.2.1/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo=
functions/onprem/orborus/orborus.go
+36 -11
View File
@@ -30,6 +30,7 @@ import (
"strings"
"sync"
"time"
"math/rand"
//"os/signal"
//"syscall"
@@ -101,8 +102,9 @@ var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG")
var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME")
var orborusLabel = os.Getenv("SHUFFLE_ORBORUS_LABEL")
var memcached = os.Getenv("SHUFFLE_MEMCACHED")
var apiKey = os.Getenv("AUTH_FOR_ORBORUS")
// For it to download from Sigma?
var apiKey = os.Getenv("AUTH_FOR_ORBORUS")
var pipelineUrl = os.Getenv("SHUFFLE_PIPELINE_URL")
var executionIds = []string{}
@@ -1637,7 +1639,6 @@ func getOrborusStats(ctx context.Context) shuffle.OrborusStats {
func sendRemoveRequest(client *http.Client, toBeRemoved shuffle.ExecutionRequestWrapper, baseUrl, environment, auth, org string, sleepTime int) error {
confirmUrl := fmt.Sprintf("%s/api/v1/workflows/queue/confirm", baseUrl)
data, err := json.Marshal(toBeRemoved)
if err != nil {
log.Printf("[WARNING] Failed removal marshalling: %s", err)
@@ -1926,6 +1927,7 @@ func main() {
hasStarted := false
for {
_ = sendTenzirHealthStatus()
if req.Method == "POST" {
// Should find data to send (memory etc.)
@@ -2727,12 +2729,15 @@ func createNetworkIfNotExists(ctx context.Context, networkName, subnet, gateway
}
func checkTenzirNode() error {
retries := 5
retryInterval := 3 * time.Second
retries := 1
//retryInterval := 3 * time.Second
url := fmt.Sprintf("%s/api/v0/ping", pipelineUrl)
forwardMethod := "POST"
client := http.Client{}
client := http.Client{
Timeout: 1 * time.Second,
}
req, err := http.NewRequest(forwardMethod, url, nil)
if err != nil {
log.Printf("[ERROR] Failed to create HTTP request: %s", err)
@@ -2744,7 +2749,8 @@ func checkTenzirNode() error {
if err == nil && resp.StatusCode == http.StatusOK {
return nil
}
time.Sleep(retryInterval)
//time.Sleep(retryInterval)
}
return fmt.Errorf("tenzir node is not available")
@@ -3217,8 +3223,15 @@ func removePath(containerName, path string) error {
}
func sendTenzirHealthStatus() error {
// Check one in every 10 times only
randint := rand.Intn(10)
_ = randint
//if randint != 0 {
// return nil
//}
var status string
url := fmt.Sprintf("%s/api/v1/detection/siem/node_health", baseUrl)
url := fmt.Sprintf("%s/api/v1/detections/siem/health", baseUrl)
err := checkTenzirNode()
if err != nil {
return err
@@ -3226,12 +3239,24 @@ func sendTenzirHealthStatus() error {
status = "active"
}
log.Printf("[DEBUG] Sending health update to backend url %s", baseUrl)
log.Printf("[DEBUG] Sending Tenzir health update to backend url '%s'", baseUrl)
forwardMethod := "POST"
payload := map[string]interface{}{
"status": status,
"environment": environment,
"authorization": "",
"org_id": "",
}
if len(auth) > 0 {
payload["authorization"] = auth
}
if len(org) > 0 {
payload["org_id"] = org
}
payloadBytes, err := json.Marshal(payload)
if err != nil {
log.Printf("[ERROR] Failed to marshal payload: %s", err)
@@ -3247,8 +3272,8 @@ func sendTenzirHealthStatus() error {
log.Printf("[ERROR] Failed to create HTTP request: %s", err)
return err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Content-Type", "application/json")
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Do(req)
if err != nil {
@@ -3258,7 +3283,7 @@ func sendTenzirHealthStatus() error {
defer resp.Body.Close()
if resp.StatusCode != 200 {
log.Printf("[ERROR] Pipeline: Received non-successful HTTP status code: %d", resp.StatusCode)
log.Printf("[ERROR] Pipeline: status for URL %s: %d", url, resp.StatusCode)
return fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode)
}