From 79b24f76d8d1615279e440b0deff95e3160b6445 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 25 Mar 2022 00:43:16 +0100 Subject: [PATCH] Added basic zip upload reader sample with files in-memory based on @Dhaval's work. --- .env | 2 +- backend/app_sdk/app_base.py | 35 ++-- backend/app_sdk/build.sh | 2 +- backend/go-app/go.mod | 4 +- backend/go-app/main.go | 59 +++++++ backend/tests/files.sh | 2 +- docker-compose.yml | 4 +- .../src/components/DetectionFramework.jsx | 4 +- frontend/src/views/Admin.jsx | 15 +- frontend/src/views/AngularWorkflow.jsx | 153 +++++++++--------- frontend/src/views/Dashboard.jsx | 125 ++++++++------ functions/extensions/wazuh/ossec.conf | 2 +- functions/onprem/orborus/build.sh | 2 +- functions/onprem/orborus/orborus.go | 13 ++ functions/onprem/worker/build.sh | 2 +- 15 files changed, 273 insertions(+), 151 deletions(-) diff --git a/.env b/.env index b0e85331..75c593d5 100644 --- a/.env +++ b/.env @@ -9,7 +9,7 @@ SHUFFLE_DOWNLOAD_WORKFLOW_USERNAME= SHUFFLE_DOWNLOAD_WORKFLOW_PASSWORD= SHUFFLE_DOWNLOAD_WORKFLOW_BRANCH= -SHUFFLE_APP_DOWNLOAD_LOCATION=https://github.com/frikky/shuffle-apps +SHUFFLE_APP_DOWNLOAD_LOCATION=https://github.com/shuffle/python-apps SHUFFLE_DOWNLOAD_AUTH_USERNAME= SHUFFLE_DOWNLOAD_AUTH_PASSWORD= SHUFFLE_DOWNLOAD_AUTH_BRANCH= diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 6d60db98..2d8e4b02 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1,4 +1,5 @@ import os +import ast import copy import sys import re @@ -273,9 +274,10 @@ class AppBase: action_result["result"] = self.run_magic_parser(action_result["result"]) else: self.logger.warning(f"[ERROR] Magic output not defined.") + except KeyError as e: + self.logger.warning(f"[ERROR] Failed to run magic autoparser (send result) - keyerror: {e}") except Exception as e: self.logger.warning(f"[ERROR] Failed to run magic autoparser (send result): {e}") - pass # Try it with some magic @@ -383,8 +385,9 @@ class AppBase: if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true": try: - self.log_capture_string.flush() - self.log_capture_string.close() + pass + #self.log_capture_string.flush() + #self.log_capture_string.close() except Exception as e: print(f"[WARNING] Failed to flush logs: {e}") pass @@ -973,6 +976,20 @@ class AppBase: filebytes = BytesIO(ret1.content) myzipfile = zipfile.ZipFile(filebytes) + + # Unzip and build here! + #for member in files.namelist(): + # filename = os.path.basename(member) + # if not filename: + # continue + + # self.logger.info("File: %s" % member) + # source = files.open(member) + # with open("%s/%s" % (basedir, source.name), "wb+") as tmp: + # filedata = source.read() + # self.logger.info("Filedata (%s): %s" % (source.name, filedata)) + # tmp.write(filedata) + return myzipfile # Things to consider for files: @@ -2594,8 +2611,6 @@ class AppBase: params[item["key"]] = item["value"] except KeyError: self.logger.info("[DEBUG] No authentication specified!") - pass - #action["authentication"] # Fixes OpenAPI body parameters for later. newparams = [] @@ -3038,7 +3053,7 @@ class AppBase: if isinstance(value, str): params[key] = ast.literal_eval(value) except Exception as e: - self.logger.info("[DEBUG] Failed parsing value with ast: {e}") + self.logger.info(f"[DEBUG] Failed parsing value with ast: {e}") continue except Exception as e: self.logger.info("[DEBUG] Failed looping objects. Non critical: {e}") @@ -3206,8 +3221,8 @@ class AppBase: # https://ptb.discord.com/channels/747075026288902237/882017498550112286/882043773138382890 except (requests.exceptions.RequestException, TimeoutError) as e: - self.logger.info(f"Failed to execute request (requests): {e}") - self.logger.exception(f"Failed to execute {e}-{action['id']}") + self.logger.info(f"[ERROR] Failed to execute request (requests): {e}") + self.logger.exception(f"[ERROR] Failed to execute {e}-{action['id']}") self.action_result["status"] = "SUCCESS" try: self.action_result["result"] = json.dumps({ @@ -3219,8 +3234,8 @@ class AppBase: self.action_result["result"] = f"Request error: {e}" except Exception as e: - self.logger.info(f"Failed to execute: {e}") - self.logger.exception(f"Failed to execute {e}-{action['id']}") + self.logger.info(f"[ERROR] Failed to execute: {e}") + self.logger.exception(f"[ERROR] Failed to execute {e}-{action['id']}") self.action_result["status"] = "FAILURE" #self.action_result["result"] = f"General exception: {e}" self.action_result["result"] = json.dumps({ diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index c889c20b..0a6d6a54 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -3,7 +3,7 @@ ### DEFAULT NAME=shuffle-app_sdk -VERSION=0.9.64 +VERSION=0.9.65 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index c2060f41..755324f0 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module main go 1.16 -replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi //replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch @@ -24,7 +24,7 @@ require ( github.com/h2non/filetype v1.1.3 github.com/nirasan/go-oauth-pkce-code-verifier v0.0.0-20170819232839-0fbfe93532da // indirect github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.2.11 + github.com/shuffle/shuffle-shared v0.2.15 go4.org v0.0.0-20201209231011-d4a079459e60 // indirect golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce google.golang.org/api v0.65.0 diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 36ccd48c..accb4f44 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -4,6 +4,7 @@ import ( uuid "github.com/satori/go.uuid" "github.com/shuffle/shuffle-shared" + "archive/zip" "bufio" "bytes" "context" @@ -5759,6 +5760,63 @@ func makeWorkflowPublic(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } +func handleAppZipUpload(resp http.ResponseWriter, request *http.Request) { + cors := shuffle.HandleCors(resp, request) + if cors { + return + } + + //https://stackoverflow.com/questions/22964950/http-request-formfile-handle-zip-files + request.ParseMultipartForm(32 << 20) + f, _, err := request.FormFile("shuffle_file") + if err != nil { + log.Printf("[ERROR] Couldn't upload file: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Failed uploading file. Correct usage is: shuffle_file=@filepath"}`)) + return + } + + fileSize, err := f.Seek(0, 2) //2 = from end + if err != nil { + panic(err) + } + _, err = f.Seek(0, 0) + if err != nil { + panic(err) + } + + buf := new(bytes.Buffer) + fileSize, err = io.Copy(buf, f) + if err != nil { + panic(err) + } + + zipdata, err := zip.NewReader(bytes.NewReader(buf.Bytes()), fileSize) + if err != nil { + panic(err) + } + + for _, item := range zipdata.File { + log.Printf("\n\nName: %s\n\n", item.FileHeader.Name) + log.Printf("item: %#v", item) + + rr, err := item.Open() + if err != nil { + log.Fatal(err) + } + + _, err = io.Copy(os.Stdout, rr) + if err != nil { + log.Fatal(err) + } + rr.Close() + + } + + resp.WriteHeader(200) + resp.Write([]byte("OK")) +} + func initHandlers() { var err error ctx := context.Background() @@ -5857,6 +5915,7 @@ func initHandlers() { // App specific // From here down isnt checked for org specific + r.HandleFunc("/api/v1/apps/upload", handleAppZipUpload).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}/activate", activateWorkflowAppDocker).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps/frameworkConfiguration", shuffle.GetFrameworkConfiguration).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps/frameworkConfiguration", shuffle.SetFrameworkConfiguration).Methods("POST", "OPTIONS") diff --git a/backend/tests/files.sh b/backend/tests/files.sh index 91171924..71776148 100755 --- a/backend/tests/files.sh +++ b/backend/tests/files.sh @@ -19,4 +19,4 @@ curl http://localhost:5001/api/v1/apps/upload -H "Authorization: Bearer db0373c6 #curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" -d '{"filename": "rule2.yar", "org_id": "b4e88fe9-352b-47b4-b280-960181670acf", "workflow_id": "global", "namespace": "yara"}' #curl http://localhost:5001/api/v1/files/5cb941ad-fa1c-4444-a685-92024b1fa31c/upload -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" -F 'shuffle_file=@upload.sh' -curl http://localhost:5001/api/v1/files/namespaces/yara -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" --output rules.zip +#curl http://localhost:5001/api/v1/files/namespaces/yara -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" --output rules.zip diff --git a/docker-compose.yml b/docker-compose.yml index 266676d2..223e38fb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,8 +27,8 @@ services: - shuffle volumes: - /var/run/docker.sock:/var/run/docker.sock - - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps - - ${SHUFFLE_FILE_LOCATION}:/shuffle-files + - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z + - ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z #- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate env_file: .env environment: diff --git a/frontend/src/components/DetectionFramework.jsx b/frontend/src/components/DetectionFramework.jsx index bb74e843..6dba209a 100644 --- a/frontend/src/components/DetectionFramework.jsx +++ b/frontend/src/components/DetectionFramework.jsx @@ -922,7 +922,9 @@ const Framework = (props) => { const baselocationX = 285*scale const baselocationY = 50*scale const shiftmodifier = 3*scale - const svgSize = `${40*scale}px` + //const svgSize = `${40*scale}px` + const svgSize = `${40}px` + console.log("Size: ", svgSize) console.log("Framework: ", parsedFrameworkData) diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 3e369f4b..0bc5bf45 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -1010,8 +1010,9 @@ const Admin = (props) => { body: file, }) .then((response) => { - if (response.status !== 200) { + if (response.status !== 200 && response.status !== 201) { console.log("Status not 200 for apps :O!"); + alert.error("File was created, but failed to upload.") return; } @@ -1022,17 +1023,21 @@ const Admin = (props) => { //setFiles(responseJson) }) .catch((error) => { - //alert.error(error.toString()) + alert.error(error.toString()) }); }; const handleCreateFile = (filename, file) => { - const data = { + var data = { filename: filename, org_id: selectedOrganization.id, workflow_id: "global", }; + if (selectedNamespace !== undefined && selectedNamespace !== null && selectedNamespace.length > 0 && selectedNamespace !== "default") { + data.namespace = selectedNamespace + } + fetch(globalUrl + "/api/v1/files/create", { method: "POST", headers: { @@ -3991,7 +3996,7 @@ const Admin = (props) => { > {environment.archived ? "Activate" : "Disable"} - @@ -4240,7 +4245,7 @@ const Admin = (props) => { /> Environments diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 1349bbff..ac492fd9 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1795,103 +1795,103 @@ const AngularWorkflow = (defaultprops) => { // // Wait for new node to possibly be selected - setTimeout(() => { - const typeIds = cy.elements('node:selected').jsons(); - console.log("Found: ", typeIds) - for (var idkey in typeIds) { - const item = typeIds[idkey] - console.log("items: ", item) - if (item.data.isButton === true) { - console.log("Reselect old node & return - or just return?") - - if (item.data.buttonType === "delete" && item.data.attachedTo === nodedata.id) { - console.log("delete of same node!") - } - return + //setTimeout(() => { + const typeIds = cy.elements('node:selected').jsons(); + console.log("Found: ", typeIds) + for (var idkey in typeIds) { + const item = typeIds[idkey] + console.log("items: ", item) + if (item.data.isButton === true) { + console.log("Reselect old node & return - or just return?") + + if (item.data.buttonType === "delete" && item.data.attachedTo === nodedata.id) { + console.log("delete of same node!") } + return } + } - //if (nodedata.app_name === undefined && nodedata.source === undefined) { - // return; - //} - //event.target.removeClass("selected"); - // + //if (nodedata.app_name === undefined && nodedata.source === undefined) { + // return; + //} + //event.target.removeClass("selected"); + // - //// If button is clicked, select current node + //// If button is clicked, select current node - // Attempt at rewrite of name in other actions in following nodes. - // Should probably be done in the onBlur for the textfield instead - /* - if (event.target.data().type === "ACTION") { - const nodeaction = event.target.data() - const curaction = workflow.actions.find(a => a.id === nodeaction.id) - console.log("workflowaction: ", curaction) - console.log("nodeaction: ", nodeaction) - if (nodeaction.label !== curaction.label) { - console.log("BEACH!") + // Attempt at rewrite of name in other actions in following nodes. + // Should probably be done in the onBlur for the textfield instead + /* + if (event.target.data().type === "ACTION") { + const nodeaction = event.target.data() + const curaction = workflow.actions.find(a => a.id === nodeaction.id) + console.log("workflowaction: ", curaction) + console.log("nodeaction: ", nodeaction) + if (nodeaction.label !== curaction.label) { + console.log("BEACH!") - var params = [] - const fixedName = "$"+curaction.label.toLowerCase().replace(" ", "_") - for (var actionkey in workflow.actions) { - if (workflow.actions[actionkey].id === curaction.id) { + var params = [] + const fixedName = "$"+curaction.label.toLowerCase().replace(" ", "_") + for (var actionkey in workflow.actions) { + if (workflow.actions[actionkey].id === curaction.id) { + continue + } + + for (var paramkey in workflow.actions[actionkey].parameters) { + const param = workflow.actions[actionkey].parameters[paramkey] + if (param.value === null || param.value === undefined || !param.value.includes("$")) { continue } - for (var paramkey in workflow.actions[actionkey].parameters) { - const param = workflow.actions[actionkey].parameters[paramkey] - if (param.value === null || param.value === undefined || !param.value.includes("$")) { - continue - } - - const innername = param.value.toLowerCase().replace(" ", "_") - if (innername.includes(fixedName)) { - //workflow.actions[actionkey].parameters[paramkey].replace( - //console.log("FOUND!: ", innername) - } + const innername = param.value.toLowerCase().replace(" ", "_") + if (innername.includes(fixedName)) { + //workflow.actions[actionkey].parameters[paramkey].replace( + //console.log("FOUND!: ", innername) } } } } - */ + } + */ - //cy.removeListener("select"); - //cy.on("select", "node", (e) => onNodeSelect(e, appAuthentication)); - //cy.on("select", "edge", (e) => onEdgeSelect(e)); + //cy.removeListener("select"); + //cy.on("select", "node", (e) => onNodeSelect(e, appAuthentication)); + //cy.on("select", "edge", (e) => onEdgeSelect(e)); - // FIXME - check if they have value before overriding like this for no reason. - // Would save a lot of time (400~ ms -> 30ms) - //console.log("ACTION: ", selectedAction) - //console.log("APP: ", selectedApp) + // FIXME - check if they have value before overriding like this for no reason. + // Would save a lot of time (400~ ms -> 30ms) + //console.log("ACTION: ", selectedAction) + //console.log("APP: ", selectedApp) - ReactDOM.unstable_batchedUpdates(() => { - setSelectedAction({}); - setSelectedApp({}); - setSelectedTrigger({}); - setSelectedComment({}) - setSelectedEdge({}); + ReactDOM.unstable_batchedUpdates(() => { + setSelectedAction({}); + setSelectedApp({}); + setSelectedTrigger({}); + setSelectedComment({}) + setSelectedEdge({}); - setSelectedEdge({}) - setSelectedActionEnvironment({}) - setTriggerAuthentication({}) - setSelectedTriggerIndex(-1) - setTriggerFolders([]) - setSubworkflow({}) + setSelectedEdge({}) + setSelectedActionEnvironment({}) + setTriggerAuthentication({}) + setSelectedTriggerIndex(-1) + setTriggerFolders([]) + setSubworkflow({}) - // Can be used for right side view - setRightSideBarOpen(false); - setScrollConfig({ - top: 0, - left: 0, - selected: "", - }); + // Can be used for right side view + setRightSideBarOpen(false); + setScrollConfig({ + top: 0, + left: 0, + selected: "", + }); //console.timeEnd("UNSELECT"); - }) - }, 150) + }) + //}, 150) }; const onEdgeSelect = (event) => { @@ -3600,7 +3600,12 @@ const AngularWorkflow = (defaultprops) => { // FIXME: Don't allow multiple in cloud yet. Cloud -> Onprem isn't stable. if (isCloud) { - setEnvironments([{ Name: "Cloud", Type: "cloud" }]); + console.log("Envs: ", responseJson) + if (responseJson.length > 0) { + setEnvironments(responseJson); + } else { + setEnvironments([{ Name: "Cloud", Type: "cloud" }]); + } } else { setEnvironments(responseJson); } diff --git a/frontend/src/views/Dashboard.jsx b/frontend/src/views/Dashboard.jsx index e81f6196..ffe6cea3 100644 --- a/frontend/src/views/Dashboard.jsx +++ b/frontend/src/views/Dashboard.jsx @@ -74,7 +74,7 @@ const UsecaseListComponent = ({keys, isCloud, globalUrl, frameworkData, isLogged } const getUsecase = (name, index, subindex) => { - fetch(globalUrl + "/api/v1/workflows/usecases/"+name.replaceAll(" ", "_"), { + fetch(`${globalUrl}/api/v1/workflows/usecases/${escape(name.replaceAll(" ", "_"))}`, { method: "GET", headers: { "Content-Type": "application/json", @@ -100,7 +100,20 @@ const UsecaseListComponent = ({keys, isCloud, globalUrl, frameworkData, isLogged setExpandedIndex(index) setExpandedItem(subindex) - }) + + setTimeout(() => { + //console.log("Scroll!") + const found = document.getElementById("selected_box"); + console.log("Found to scroll: ", found) + if (found !== undefined && found !== null) { + //console.log("FOUND!!") + found.scrollTo({ + top: 100, + behavior: "smooth", + }) + } + }, 100); +}) .catch((error) => { //alert.error(error.toString()); setInputUsecase({}) @@ -224,67 +237,77 @@ const UsecaseListComponent = ({keys, isCloud, globalUrl, frameworkData, isLogged {subcase.name} -
- {finished ? - - { - }} - > - - - - : null} - {subcase.blogpost !== null && subcase.blogpost !== undefined && subcase.blogpost.length > 0 ? - + {finished ? { }} > - + - - : null} - {subcase.video !== null && subcase.video !== undefined && subcase.video.length > 0 ? - - 0 ? + - { - }} + - - - - - : null} -
+ { + }} + > + + + + + : null} + {subcase.video !== null && subcase.video !== undefined && subcase.video.length > 0 ? + + + { + }} + > + + + + + : null} : -
+
{subcase.name} @@ -482,7 +505,7 @@ const UsecaseListComponent = ({keys, isCloud, globalUrl, frameworkData, isLogged
: -
+
{subcase.description} diff --git a/functions/extensions/wazuh/ossec.conf b/functions/extensions/wazuh/ossec.conf index 45674065..e87bc53d 100644 --- a/functions/extensions/wazuh/ossec.conf +++ b/functions/extensions/wazuh/ossec.conf @@ -1,6 +1,6 @@ custom-shuffle 9 - http://:/api/v1/hooks/webhook_ + http://:/api/v1/hooks/webhook_hookid json diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 7e4d42fb..0f10630e 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.9.64 +VERSION=0.9.65 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 4eb9c9fe..622927bc 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -65,6 +65,10 @@ var baseimagename = os.Getenv("SHUFFLE_BASE_IMAGE_NAME") var baseimageregistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY") var baseimagetagsuffix = os.Getenv("SHUFFLE_BASE_IMAGE_TAG_SUFFIX") +// Used for cloud with auth +var auth = os.Getenv("AUTH") +var org = os.Getenv("ORG") + //var orgId = os.Getenv("ORG_ID") var baseUrl = os.Getenv("BASE_URL") var environment = os.Getenv("ENVIRONMENT_NAME") @@ -901,6 +905,15 @@ func main() { zombiecounter := 0 req.Header.Add("Content-Type", "application/json") req.Header.Add("Org-Id", environment) + + if len(auth) > 0 { + req.Header.Add("Authorization", auth) + } + + if len(org) > 0 { + req.Header.Add("Org", org) + } + log.Printf("[INFO] Waiting for executions at %s with Environment %s", fullUrl, environment) hasStarted := false for { diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 88fcbc9e..abeed075 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.9.64 +VERSION=0.9.65 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin .