diff --git a/.env b/.env index 298128cc..48bd17bc 100755 --- a/.env +++ b/.env @@ -40,6 +40,7 @@ BACKEND_HOSTNAME=shuffle-backend BACKEND_PORT=5001 FRONTEND_PORT=3001 FRONTEND_PORT_HTTPS=3443 +AUTH_FOR_ORBORUS = # CHANGE THIS IF YOU WANT GOOD LOCAL EXECUTIONS: OUTER_HOSTNAME=shuffle-backend @@ -97,14 +98,15 @@ SHUFFLE_MAX_EXECUTION_DEPTH= DATASTORE_EMULATOR_HOST=shuffle-database:8000 #SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200 SHUFFLE_OPENSEARCH_URL=https://shuffle-opensearch:9200 -SHUFFLE_OPENSEARCH_USERNAME="admin" -SHUFFLE_OPENSEARCH_PASSWORD="StrongShufflePassword321!" SHUFFLE_OPENSEARCH_CERTIFICATE_FILE= SHUFFLE_OPENSEARCH_APIKEY= SHUFFLE_OPENSEARCH_CLOUDID= SHUFFLE_OPENSEARCH_PROXY= SHUFFLE_OPENSEARCH_INDEX_PREFIX= SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY=true +SHUFFLE_OPENSEARCH_USERNAME="admin" +SHUFFLE_OPENSEARCH_PASSWORD="StrongShufflePassword321!" # In use for the first time setup of OpenSearch + backend of Shuffle +OPENSEARCH_INITIAL_ADMIN_PASSWORD="StrongShufflePassword321!" # In use for the first time setup of OpenSearch #Tenzir related SHUFFLE_TENZIR_URL= diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 415f1e94..a751319b 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -632,6 +632,7 @@ class AppBase: # I wonder if this actually works url = "%s%s" % (self.base_url, stream_path) + self.logger.info(f"[DEBUG][%s] Sending result to %s" % (self.current_execution_id, url)) try: log_contents = "disabled: add env SHUFFLE_LOGS_DISABLED=true to Orborus to re-enable logs for apps. Can not be enabled natively in Cloud except in Hybrid mode." @@ -656,6 +657,13 @@ class AppBase: except Exception as e: pass + # Check if type of headers is right + if not isinstance(headers, dict): + headers = {} + + if not "User-Agent" in headers: + headers["User-Agent"] = "Shuffle App" + try: finished = False ret = {} @@ -684,23 +692,23 @@ class AppBase: headerauth = headers["Authorization"] try: - self.logger.info(f"[ERROR] Bad resp ({ret.status_code}) in send_result for url '{url}'. Execution ID: %d, Authorization: %d, Header Auth: %d" % (len(action_result["execution_id"]), len(action_result["authorization"]), len(headerauth))) except Exception as e: self.logger.info(f"[ERROR] Bad resp ({ret.status_code}) in send_result for url '{url}' (no detail)") - pass time.sleep(sleeptime) # Proxyerrror except requests.exceptions.ProxyError as e: + self.logger.info(f"[ERROR][{self.current_execution_id}] Proxy error in send_result for url '{url}': {e}") + self.proxy_config = {} continue except requests.exceptions.RequestException as e: - time.sleep(sleeptime) + self.logger.info(f"[ERROR][{self.current_execution_id}] Request error in send_result for url '{url}': {e}") # Check if we have a read timeout. If we do, exit as we most likely sent the result without getting a good result if "Read timed out" in str(e): @@ -713,24 +721,34 @@ class AppBase: finished = True break + time.sleep(sleeptime) + #time.sleep(5) continue except TimeoutError as e: + self.logger.info(f"[ERROR][{self.current_execution_id}] Timeout error in send_result for url '{url}': {e}") + time.sleep(sleeptime) #time.sleep(5) continue except requests.exceptions.ConnectionError as e: + self.logger.info(f"[ERROR][{self.current_execution_id}] Connection error in send_result for url '{url}': {e}") + time.sleep(sleeptime) #time.sleep(5) continue except http.client.RemoteDisconnected as e: + self.logger.info(f"[ERROR][{self.current_execution_id}] RemoteDisconnected error in send_result for url '{url}': {e}") + time.sleep(sleeptime) #time.sleep(5) continue except urllib3.exceptions.ProtocolError as e: + self.logger.info(f"[ERROR][{self.current_execution_id}] ProtocolError error in send_result for url '{url}': {e}") + time.sleep(0.1) #time.sleep(5) @@ -3668,7 +3686,7 @@ class AppBase: #self.logger.info() if not multiexecution: - self.logger.info("NOT MULTI EXEC") + #self.logger.info("NOT MULTI EXEC") # Runs a single iteration here new_params = self.validate_unique_fields(params) if isinstance(new_params, list) and len(new_params) == 1: diff --git a/backend/go-app/main.go b/backend/go-app/main.go index d24cbc64..23486930 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1978,7 +1978,6 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { } func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { - if request.Method != "POST" { request.Method = "POST" } @@ -1999,7 +1998,7 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { location := strings.Split(request.URL.String(), "/") var pipelineId string - + if location[1] == "api" { if len(location) <= 4 { log.Printf("[INFO] Couldn't handle location. Too short in pipeline: %d", len(location)) @@ -2013,7 +2012,7 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { userAgent := request.Header.Get("User-Agent") if strings.Contains(strings.ToLower(userAgent), "microsoftpreview") || strings.Contains(strings.ToLower(userAgent), "googlebot") { - log.Printf("[AUDIT] Blocking googlebot and microsoftbot for pielines. UA: '%s'", userAgent) + log.Printf("[AUDIT] Blocking googlebot and microsoftbot for pipelines. UA: '%s'", userAgent) resp.WriteHeader(400) resp.Write([]byte(`{"success": false, "reason": "Google/Microsoft preview bots not allowed. Please change the useragent."}`)) return @@ -2058,11 +2057,27 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { return } - parsedBody := shuffle.GetExecutionbody(body) + // Parse concatenated JSON logs + jsonList, err := parseConcatenatedJSONLogs(string(body)) + if err != nil { + log.Printf("[DEBUG] JSON parsing error: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + parsedBody, err := json.Marshal(jsonList) + if err != nil { + log.Printf("[ERROR] Failed to marshal jsonList: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false}`)) + return + } + newBody := shuffle.ExecutionStruct{ Start: pipeline.StartNode, ExecutionSource: "pipeline", - ExecutionArgument: parsedBody, + ExecutionArgument: string(parsedBody), } workflow, err := shuffle.GetWorkflow(ctx, pipeline.WorkflowId) @@ -2093,8 +2108,7 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { } if len(pipeline.StartNode) == 0 { - log.Printf("[WARNING] No start node for pipeline %s - running with workflow default.", pipeline.TriggerId) - + log.Printf("[WARNING] No start node for pipeline %s - running with workflow default.") } newRequest := &http.Request{ @@ -2108,6 +2122,9 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { if err == nil { resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s"}`, workflowExecution.ExecutionId))) + + // Track Sigma rules + trackSigmaRules(ctx, pipeline.OrgId, jsonList) return } @@ -2115,6 +2132,82 @@ func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, executionResp))) } +func parseConcatenatedJSONLogs(logs string) ([]map[string]interface{}, error) { + var jsonList []map[string]interface{} + decoder := json.NewDecoder(strings.NewReader(logs)) + + for decoder.More() { + var jsonObject map[string]interface{} + if err := decoder.Decode(&jsonObject); err != nil { + log.Printf("[WARNING] JSON decoding error: %s. Skipping this object.", err) + continue + } + jsonList = append(jsonList, jsonObject) + } + + if err := decoder.Decode(&struct{}{}); err != io.EOF { + return nil, fmt.Errorf("error after decoding all JSON objects: %v", err) + } + + return jsonList, nil +} + +func trackSigmaRules(ctx context.Context, orgId string, jsonList []map[string]interface{}) { + ruleCount := make(map[string]int) + for _, logEntry := range jsonList { + if rule, ok := logEntry["rule"].(map[string]interface{}); ok { + if ruleName, ok := rule["title"].(string); ok { + ruleCount[ruleName]++ + } + } + } + + for ruleName, count := range ruleCount { + shuffle.IncrementCache(ctx, orgId, ruleName, count) + log.Printf("[INFO] Rule %s incremented by %d", ruleName, count) + } +} + +func handleTenzirHealthUpdate(resp http.ResponseWriter, request *http.Request) { + if request.Method != "POST" { + request.Method = "POST" + } + + type HealthUpdate struct { + Status string `json:"status"` + } + + var healthUpdate HealthUpdate + err := json.NewDecoder(request.Body).Decode(&healthUpdate) + if err != nil { + resp.WriteHeader(http.StatusBadRequest) + fmt.Fprintf(resp, "Failed to decode JSON: %v", err) + return + } + ctx := context.Background() + status := healthUpdate.Status + + result, err := shuffle.GetDisabledRules(ctx) + if (err != nil && err.Error() == "rules doesn't exist") || err == nil { + result.IsTenzirActive = status + result.LastActive = time.Now().Unix() + + err = shuffle.StoreDisabledRules(ctx, *result) + if err != nil { + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false}`)) + return + } + + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) + return + } + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false}`)) + return +} + func executeCloudAction(action shuffle.CloudSyncJob, apikey string) error { data, err := json.Marshal(action) if err != nil { @@ -5114,6 +5207,7 @@ func initHandlers() { // PS: For cloud, this has to use cloud storage. // https://developer.box.com/reference/get-files-id-content/ r.HandleFunc("/api/v1/files/download_remote", shuffle.HandleDownloadRemoteFiles).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/files/download_remote_enhanced", shuffle.HandleEnhancedDownloadRemoteFiles).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/files/namespaces/{namespace}", shuffle.HandleGetFileNamespace).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}/content", shuffle.HandleGetFileContent).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/create", shuffle.HandleCreateFile).Methods("POST", "OPTIONS") @@ -5122,6 +5216,15 @@ func initHandlers() { r.HandleFunc("/api/v1/files/{fileId}", shuffle.HandleGetFileMeta).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}", shuffle.HandleDeleteFile).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/files", shuffle.HandleGetFiles).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/files/detection/sigma_rules", shuffle.HandleGetSigmaRules).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/files/detection/{fileId}/{action}", shuffle.HandleToggleRule).Methods("PUT", "OPTIONS") + r.HandleFunc("/api/v1/files/detection/{action}", shuffle.HandleFolderToggle).Methods("PUT", "OPTIONS") + + r.HandleFunc("/api/v1/detection/siem/connect", shuffle.HandleConnectSiem).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/detection/siem/node_health", handleTenzirHealthUpdate).Methods("POST","OPTIONS") + r.HandleFunc("/api/v1/detection/{triggerId}/selected_rules", shuffle.HandleGetSelectedRules).Methods("GET","OPTIONS") + r.HandleFunc("/api/v1/detection/{triggerId}/selected_rules/save", shuffle.HandleSaveSelectedRules).Methods("POST","OPTIONS") + // Introduced in 0.9.21 to handle notifications for e.g. failed Workflow r.HandleFunc("/api/v1/notifications", shuffle.HandleCreateNotification).Methods("POST", "OPTIONS") diff --git a/docker-compose.yml b/docker-compose.yml index 2f096df2..fcc7c0cb 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - image: ghcr.io/shuffle/shuffle-frontend:latest + image: ghcr.io/shuffle/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -15,7 +15,7 @@ services: depends_on: - backend backend: - image: ghcr.io/shuffle/shuffle-backend:latest + image: ghcr.io/shuffle/shuffle-backend:nightly container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -34,7 +34,7 @@ services: - SHUFFLE_FILE_LOCATION=/shuffle-files restart: unless-stopped orborus: - image: ghcr.io/shuffle/shuffle-orborus:latest + image: ghcr.io/shuffle/shuffle-orborus:nightly container_name: shuffle-orborus hostname: shuffle-orborus networks: @@ -48,9 +48,6 @@ services: - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:5001 - DOCKER_API_VERSION=1.40 - - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} - - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} - - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} - HTTP_PROXY=${HTTP_PROXY} - HTTPS_PROXY=${HTTPS_PROXY} - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} @@ -74,7 +71,6 @@ services: - node.name=shuffle-opensearch - node.store.allow_mmap=false - discovery.seed_hosts=shuffle-opensearch - - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${SHUFFLE_OPENSEARCH_PASSWORD} ulimits: memlock: soft: -1 @@ -83,7 +79,7 @@ services: soft: 65536 hard: 65536 volumes: - - ${DB_LOCATION}:/usr/share/opensearch/data:z + - shuffle-database:/usr/share/opensearch/data:z ports: - 9200:9200 networks: @@ -129,13 +125,18 @@ services: # networks: # - shuffle # + +volumes: + shuffle-database: + driver: local + driver_opts: + type: none + device: ${DB_LOCATION} + o: bind + networks: shuffle: driver: bridge - - # uncomment to set MTU for swarm mode. - # MTU should be whatever is your host's preferred MTU is. - # Refer to this doc to figure out what your host's MTU is: - # https://shuffler.io/docs/troubleshooting#TLS_timeout_error/Timeout_Errors/EOF_Errors # driver_opts: # com.docker.network.driver.mtu: 1460 + # uncomment to set MTU for swarm mode. MTU should be whatever is your host's preferred MTU is: https://shuffler.io/docs/troubleshooting#TLS_timeout_error/Timeout_Errors/EOF_Errors diff --git a/frontend/Dockerfile b/frontend/Dockerfile index e9d6d683..fad87105 100755 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,6 +1,8 @@ # Build environment FROM node:21 as builder +ENV NODE_OPTIONS="--max-old-space-size=4096" + RUN mkdir /usr/src/app WORKDIR /usr/src/app ENV PATH /usr/src/app/node_modules/.bin:$PATH @@ -25,28 +27,33 @@ COPY ./*.json /usr/src/app/ RUN npm run build --loglevel verbose 2>&1 # Production environment -FROM nginx:1.21.5 +FROM nginx:1.26.0 RUN mkdir -p /usr/share/nginx/html/build RUN mkdir -p /usr/share/nginx/html/css RUN mkdir -p /usr/share/nginx/html/js RUN mkdir -p /usr/share/nginx/html/img -COPY --from=builder /usr/src/app/build /usr/share/nginx/html -#Localhost certificate challenge: Y#XwrJ#DoZGz2w6x +# Localhost certificate challenge: Y#XwrJ#DoZGz2w6x +# Cert challenge doesn't matter to be here or not, as ALL production setups should be using their own certificates + reverse proxy: https://shuffler.io/docs/configuration#using-the-nginx-reverse-proxy-for-tls/ssl +COPY --from=builder /usr/src/app/build /usr/share/nginx/html COPY --from=builder /usr/src/app/certs/fullchain.pem /etc/nginx/fullchain.cert.pem COPY --from=builder /usr/src/app/certs/privkey.pem /etc/nginx/privkey.pem # install CONFD -ENV CONFD_VERSION 0.16.0 RUN apt-get update && apt-get install -y curl && apt-get clean -RUN curl -sSL https://github.com/kelseyhightower/confd/releases/download/v${CONFD_VERSION}/confd-${CONFD_VERSION}-linux-amd64 -o /usr/local/bin/confd && \ - chmod +x /usr/local/bin/confd -COPY ./confd /etc/confd +COPY ./confd/templates/nginx.conf /etc/nginx/nginx.conf.tmpl +## OLD CONFD THINGS (not compatible with arm) +#ENV CONFD_VERSION 0.16.0 +#RUN curl -sSL https://github.com/kelseyhightower/confd/releases/download/v${CONFD_VERSION}/confd-${CONFD_VERSION}-linux-amd64 -o /usr/local/bin/confd && \ +# chmod +x /usr/local/bin/confd +#COPY ./confd /etc/confd # rewrite command & entrypoint with ours + COPY ./entrypoint.sh / +ENV BACKEND_HOSTNAME="shuffle-backend" ENTRYPOINT [ "/entrypoint.sh" ] CMD ["nginx", "-g", "daemon off;"] diff --git a/frontend/confd/templates/nginx.conf b/frontend/confd/templates/nginx.conf index 3bb02c27..2c9df91e 100755 --- a/frontend/confd/templates/nginx.conf +++ b/frontend/confd/templates/nginx.conf @@ -71,7 +71,7 @@ http { } location ~ /api/v(1|2) { - proxy_pass http://{{ getenv "BACKEND_HOSTNAME" "shuffle-backend" }}:5001; + proxy_pass http://${BACKEND_HOSTNAME}:5001; proxy_buffering off; proxy_http_version 1.1; @@ -113,7 +113,8 @@ http { # Get the hostname from environment here? location ~ /api/v(1|2) { - proxy_pass http://{{ getenv "BACKEND_HOSTNAME" "shuffle-backend" }}:5001; + proxy_pass http://${BACKEND_HOSTNAME}:5001; + proxy_buffering off; proxy_http_version 1.1; diff --git a/frontend/entrypoint.sh b/frontend/entrypoint.sh index 09be2558..af1d0a43 100755 --- a/frontend/entrypoint.sh +++ b/frontend/entrypoint.sh @@ -1,7 +1,6 @@ -#!/bin/bash +#!/usr/bin/env sh +set -eu -# generate configs -/usr/local/bin/confd -backend="env" -confdir="/etc/confd" -onetime +envsubst '${BACKEND_HOSTNAME}' < /etc/nginx/nginx.conf.tmpl > /etc/nginx/nginx.conf -# run main command exec "$@" diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 639b80bd..ef77004a 100755 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -15,6 +15,7 @@ import HealthPage from "./components/HealthPage.jsx"; import theme from "./theme"; import Apps from "./views/Apps"; import AppCreator from "./views/AppCreator"; +import DetectionDashBoard from "./views/DetectionDashboard.jsx"; import Welcome from "./views/Welcome.jsx"; import Dashboard from "./views/Dashboard.jsx"; @@ -414,6 +415,11 @@ const App = (message, props) => { /> } /> + } + /> { const { globalUrl, userdata, serverside, billingInfo, stripeKey, selectedOrganization, handleGetOrg, clickedFromOrgTab } = props; //const alert = useAlert(); let navigate = useNavigate(); - const [selectedDealModalOpen, setSelectedDealModalOpen] = React.useState(false); const [dealList, setDealList] = React.useState([]); const [dealName, setDealName] = React.useState(""); @@ -1055,10 +1054,10 @@ const Billing = (props) => { Consultation & Management
- + You currently have a total of {inputHour} hours and {inputMinutes} minutes of professional services available by our experts. -
+
{editConsultation ? <> { {editConsultation && }
: null} - + Features