fixed issue with app sdk not loading full execution properly for /api/v1/run actions

backend/app_sdk/app_base.py

@@ -1767,27 +1767,12 @@ class AppBase:
             self.send_result(self.action_result, headers, stream_path) 
             return
 
-
-        # Add async logger
-        # self.console_logger.handlers[0].stream.set_execution_id()
-
-        # FIXME: Shouldn't skip this, but it's good for minimzing API calls
-        #try:
-        #    ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False)
-        #    self.logger.info("Workflow: %d" % ret.status_code)
-        #    if ret.status_code != 200:
-        #        self.logger.info(ret.text)
-        #except requests.exceptions.ConnectionError as e:
-        #    self.logger.info("Connectionerror: %s" %  e)
-
-        #    action_result["result"] = "Bad setup during startup: %s" % e 
-        #    self.send_result(action_result, headers, stream_path) 
-        #    return
-
         # Verify whether there are any parameters with ACTION_RESULT required
         # If found, we get the full results list from backend
+
+        # Forcing this to run due to potential self.full_execution loading issues in cloud run
         fullexecution = {}
-        if isinstance(self.full_execution, str) and len(self.full_execution) == 0:
+        if True or (isinstance(self.full_execution, str) and len(self.full_execution) == 0):
             #self.logger.info("[DEBUG] NO EXECUTION - LOADING!")
             try:
                 failed = False

backend/go-app/main.go

@@ -497,6 +497,7 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini)
 	newUser.ActiveOrg = shuffle.OrgMini{
 		Id:   org.Id,
 		Name: org.Name,
+		Role: newUser.Role,
 	}
 
 	if len(apikey) > 0 {
@@ -934,10 +935,22 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) {
 
 	//if err == nil {
 	if len(org.Id) > 0 {
+		if userInfo.Role == "" {
+			//err = shuffle.SetUser(ctx, &userInfo, false)
+			for _, user := range org.Users {
+				if user.Id != userInfo.Id {
+					continue
+				}
+
+				userInfo.ActiveOrg.Role = user.Role
+			}
+		}
+
 		userInfo.ActiveOrg = shuffle.OrgMini{
 			Id:         org.Id,
 			Name:       org.Name,
 			CreatorOrg: org.CreatorOrg,
+			ChildOrgs:  org.ChildOrgs,
 			Role:       userInfo.ActiveOrg.Role,
 			Image:      org.Image,
 		}
@@ -5219,6 +5232,7 @@ func initHandlers() {
 	r.HandleFunc("/api/v1/orgs/{orgId}/delete_cache", shuffle.HandleDeleteCacheKeyPost).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/orgs/{orgId}/cache/{cache_key}", shuffle.HandleDeleteCacheKey).Methods("DELETE", "OPTIONS")
 	r.HandleFunc("/api/v1/orgs/{orgId}/stats", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS")
+	r.HandleFunc("/api/v1/orgs/{orgId}/stats", shuffle.HandleAppendStatistics).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/orgs/{orgId}/statistics", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS")
 
 	r.HandleFunc("/api/v1/orgs/{orgId}/cache", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS")

functions/onprem/orborus/go.sum

@@ -271,6 +271,8 @@ github.com/shuffle/shuffle-shared v0.6.60 h1:8OaiNxNpzJmIbYIcXI3TIYZVrPJ1sSCa+u7
 github.com/shuffle/shuffle-shared v0.6.60/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus=
 github.com/shuffle/shuffle-shared v0.6.61 h1:+9CCLeZLiAVDgNRTkZxnIgz+FZ7UrEHez2BAGPS/axc=
 github.com/shuffle/shuffle-shared v0.6.61/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus=
+github.com/shuffle/shuffle-shared v0.6.63 h1:eNQMpVhe/mAMxl61W9Wj6/Z4PrtPeEnbjvMtDdT1mqw=
+github.com/shuffle/shuffle-shared v0.6.63/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/skeema/knownhosts v1.2.1 h1:SHWdIUa82uGZz+F+47k8SY4QhhI291cXCpopT1lK2AQ=
 github.com/skeema/knownhosts v1.2.1/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo=

functions/onprem/orborus/orborus.go

@@ -30,6 +30,7 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"math/rand"
 
 	//"os/signal"
 	//"syscall"
@@ -101,8 +102,9 @@ var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG")
 var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME")
 var orborusLabel = os.Getenv("SHUFFLE_ORBORUS_LABEL")
 var memcached = os.Getenv("SHUFFLE_MEMCACHED")
-var apiKey = os.Getenv("AUTH_FOR_ORBORUS")
 
+// For it to download from Sigma? 
+var apiKey = os.Getenv("AUTH_FOR_ORBORUS") 
 var pipelineUrl = os.Getenv("SHUFFLE_PIPELINE_URL")
 
 var executionIds = []string{}
@@ -1637,7 +1639,6 @@ func getOrborusStats(ctx context.Context) shuffle.OrborusStats {
 
 func sendRemoveRequest(client *http.Client, toBeRemoved shuffle.ExecutionRequestWrapper, baseUrl, environment, auth, org string, sleepTime int) error {
 	confirmUrl := fmt.Sprintf("%s/api/v1/workflows/queue/confirm", baseUrl)
-
 	data, err := json.Marshal(toBeRemoved)
 	if err != nil {
 		log.Printf("[WARNING] Failed removal marshalling: %s", err)
@@ -1926,6 +1927,7 @@ func main() {
 	hasStarted := false
 	for {
 		_ = sendTenzirHealthStatus()
+
 		if req.Method == "POST" {
 			// Should find data to send (memory etc.)
 
@@ -2727,12 +2729,15 @@ func createNetworkIfNotExists(ctx context.Context, networkName, subnet, gateway
 }
 
 func checkTenzirNode() error {
-	retries := 5
+	retries := 1
-	retryInterval := 3 * time.Second
+
+	//retryInterval := 3 * time.Second
 	url := fmt.Sprintf("%s/api/v0/ping", pipelineUrl)
 	forwardMethod := "POST"
 
-	client := http.Client{}
+	client := http.Client{
+		Timeout: 1 * time.Second,
+	}
 	req, err := http.NewRequest(forwardMethod, url, nil)
 	if err != nil {
 		log.Printf("[ERROR] Failed to create HTTP request: %s", err)
@@ -2744,7 +2749,8 @@ func checkTenzirNode() error {
 		if err == nil && resp.StatusCode == http.StatusOK {
 			return nil
 		}
-		time.Sleep(retryInterval)
+
+		//time.Sleep(retryInterval)
 	}
 
 	return fmt.Errorf("tenzir node is not available")
@@ -3217,8 +3223,15 @@ func removePath(containerName, path string) error {
 }
 
 func sendTenzirHealthStatus() error {
+	// Check one in every 10 times only
+	randint := rand.Intn(10)
+	_ = randint
+	//if randint != 0 {
+	//	return nil
+	//}
+
 	var status string
-	url := fmt.Sprintf("%s/api/v1/detection/siem/node_health", baseUrl)
+	url := fmt.Sprintf("%s/api/v1/detections/siem/health", baseUrl)
 	err := checkTenzirNode()
 	if err != nil {
 		return err
@@ -3226,12 +3239,24 @@ func sendTenzirHealthStatus() error {
 		status = "active"
 	}
 
-	log.Printf("[DEBUG] Sending health update to backend url %s", baseUrl)
+	log.Printf("[DEBUG] Sending Tenzir health update to backend url '%s'", baseUrl)
-
 	forwardMethod := "POST"
 	payload := map[string]interface{}{
 		"status": status,
+		"environment": environment,
+		"authorization": "",
+		"org_id": "",
+
 	}
+
+	if len(auth) > 0 {
+		payload["authorization"] = auth
+	}
+
+	if len(org) > 0 {
+		payload["org_id"] = org
+	}
+
 	payloadBytes, err := json.Marshal(payload)
 	if err != nil {
 		log.Printf("[ERROR] Failed to marshal payload: %s", err)
@@ -3247,8 +3272,8 @@ func sendTenzirHealthStatus() error {
 		log.Printf("[ERROR] Failed to create HTTP request: %s", err)
 		return err
 	}
-	req.Header.Set("Content-Type", "application/json")
 
+	req.Header.Set("Content-Type", "application/json")
 	client := &http.Client{Timeout: 10 * time.Second}
 	resp, err := client.Do(req)
 	if err != nil {
@@ -3258,7 +3283,7 @@ func sendTenzirHealthStatus() error {
 
 	defer resp.Body.Close()
 	if resp.StatusCode != 200 {
-		log.Printf("[ERROR] Pipeline: Received non-successful HTTP status code: %d", resp.StatusCode)
+		log.Printf("[ERROR] Pipeline: status for URL %s: %d", url, resp.StatusCode)
 		return fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode)
 	}