diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 72a8570c..e1ae2378 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1767,27 +1767,12 @@ class AppBase: self.send_result(self.action_result, headers, stream_path) return - - # Add async logger - # self.console_logger.handlers[0].stream.set_execution_id() - - # FIXME: Shouldn't skip this, but it's good for minimzing API calls - #try: - # ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False) - # self.logger.info("Workflow: %d" % ret.status_code) - # if ret.status_code != 200: - # self.logger.info(ret.text) - #except requests.exceptions.ConnectionError as e: - # self.logger.info("Connectionerror: %s" % e) - - # action_result["result"] = "Bad setup during startup: %s" % e - # self.send_result(action_result, headers, stream_path) - # return - # Verify whether there are any parameters with ACTION_RESULT required # If found, we get the full results list from backend + + # Forcing this to run due to potential self.full_execution loading issues in cloud run fullexecution = {} - if isinstance(self.full_execution, str) and len(self.full_execution) == 0: + if True or (isinstance(self.full_execution, str) and len(self.full_execution) == 0): #self.logger.info("[DEBUG] NO EXECUTION - LOADING!") try: failed = False diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 694f28b8..acfde3c4 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -497,6 +497,7 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) newUser.ActiveOrg = shuffle.OrgMini{ Id: org.Id, Name: org.Name, + Role: newUser.Role, } if len(apikey) > 0 { @@ -934,10 +935,22 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { //if err == nil { if len(org.Id) > 0 { + if userInfo.Role == "" { + //err = shuffle.SetUser(ctx, &userInfo, false) + for _, user := range org.Users { + if user.Id != userInfo.Id { + continue + } + + userInfo.ActiveOrg.Role = user.Role + } + } + userInfo.ActiveOrg = shuffle.OrgMini{ Id: org.Id, Name: org.Name, CreatorOrg: org.CreatorOrg, + ChildOrgs: org.ChildOrgs, Role: userInfo.ActiveOrg.Role, Image: org.Image, } @@ -5219,6 +5232,7 @@ func initHandlers() { r.HandleFunc("/api/v1/orgs/{orgId}/delete_cache", shuffle.HandleDeleteCacheKeyPost).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/cache/{cache_key}", shuffle.HandleDeleteCacheKey).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/stats", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/orgs/{orgId}/stats", shuffle.HandleAppendStatistics).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/statistics", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/cache", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS") diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 8ca413eb..532623cd 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -271,6 +271,8 @@ github.com/shuffle/shuffle-shared v0.6.60 h1:8OaiNxNpzJmIbYIcXI3TIYZVrPJ1sSCa+u7 github.com/shuffle/shuffle-shared v0.6.60/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus= github.com/shuffle/shuffle-shared v0.6.61 h1:+9CCLeZLiAVDgNRTkZxnIgz+FZ7UrEHez2BAGPS/axc= github.com/shuffle/shuffle-shared v0.6.61/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus= +github.com/shuffle/shuffle-shared v0.6.63 h1:eNQMpVhe/mAMxl61W9Wj6/Z4PrtPeEnbjvMtDdT1mqw= +github.com/shuffle/shuffle-shared v0.6.63/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/skeema/knownhosts v1.2.1 h1:SHWdIUa82uGZz+F+47k8SY4QhhI291cXCpopT1lK2AQ= github.com/skeema/knownhosts v1.2.1/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index a99efa1b..d6be28d6 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -30,6 +30,7 @@ import ( "strings" "sync" "time" + "math/rand" //"os/signal" //"syscall" @@ -101,8 +102,9 @@ var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG") var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME") var orborusLabel = os.Getenv("SHUFFLE_ORBORUS_LABEL") var memcached = os.Getenv("SHUFFLE_MEMCACHED") -var apiKey = os.Getenv("AUTH_FOR_ORBORUS") +// For it to download from Sigma? +var apiKey = os.Getenv("AUTH_FOR_ORBORUS") var pipelineUrl = os.Getenv("SHUFFLE_PIPELINE_URL") var executionIds = []string{} @@ -1637,7 +1639,6 @@ func getOrborusStats(ctx context.Context) shuffle.OrborusStats { func sendRemoveRequest(client *http.Client, toBeRemoved shuffle.ExecutionRequestWrapper, baseUrl, environment, auth, org string, sleepTime int) error { confirmUrl := fmt.Sprintf("%s/api/v1/workflows/queue/confirm", baseUrl) - data, err := json.Marshal(toBeRemoved) if err != nil { log.Printf("[WARNING] Failed removal marshalling: %s", err) @@ -1926,6 +1927,7 @@ func main() { hasStarted := false for { _ = sendTenzirHealthStatus() + if req.Method == "POST" { // Should find data to send (memory etc.) @@ -2727,12 +2729,15 @@ func createNetworkIfNotExists(ctx context.Context, networkName, subnet, gateway } func checkTenzirNode() error { - retries := 5 - retryInterval := 3 * time.Second + retries := 1 + + //retryInterval := 3 * time.Second url := fmt.Sprintf("%s/api/v0/ping", pipelineUrl) forwardMethod := "POST" - client := http.Client{} + client := http.Client{ + Timeout: 1 * time.Second, + } req, err := http.NewRequest(forwardMethod, url, nil) if err != nil { log.Printf("[ERROR] Failed to create HTTP request: %s", err) @@ -2744,7 +2749,8 @@ func checkTenzirNode() error { if err == nil && resp.StatusCode == http.StatusOK { return nil } - time.Sleep(retryInterval) + + //time.Sleep(retryInterval) } return fmt.Errorf("tenzir node is not available") @@ -3217,8 +3223,15 @@ func removePath(containerName, path string) error { } func sendTenzirHealthStatus() error { + // Check one in every 10 times only + randint := rand.Intn(10) + _ = randint + //if randint != 0 { + // return nil + //} + var status string - url := fmt.Sprintf("%s/api/v1/detection/siem/node_health", baseUrl) + url := fmt.Sprintf("%s/api/v1/detections/siem/health", baseUrl) err := checkTenzirNode() if err != nil { return err @@ -3226,12 +3239,24 @@ func sendTenzirHealthStatus() error { status = "active" } - log.Printf("[DEBUG] Sending health update to backend url %s", baseUrl) - + log.Printf("[DEBUG] Sending Tenzir health update to backend url '%s'", baseUrl) forwardMethod := "POST" payload := map[string]interface{}{ "status": status, + "environment": environment, + "authorization": "", + "org_id": "", + } + + if len(auth) > 0 { + payload["authorization"] = auth + } + + if len(org) > 0 { + payload["org_id"] = org + } + payloadBytes, err := json.Marshal(payload) if err != nil { log.Printf("[ERROR] Failed to marshal payload: %s", err) @@ -3247,8 +3272,8 @@ func sendTenzirHealthStatus() error { log.Printf("[ERROR] Failed to create HTTP request: %s", err) return err } - req.Header.Set("Content-Type", "application/json") + req.Header.Set("Content-Type", "application/json") client := &http.Client{Timeout: 10 * time.Second} resp, err := client.Do(req) if err != nil { @@ -3258,7 +3283,7 @@ func sendTenzirHealthStatus() error { defer resp.Body.Close() if resp.StatusCode != 200 { - log.Printf("[ERROR] Pipeline: Received non-successful HTTP status code: %d", resp.StatusCode) + log.Printf("[ERROR] Pipeline: status for URL %s: %d", url, resp.StatusCode) return fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode) }