marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added basic zip upload reader sample with files in-memory based on @Dhaval's work.

Browse Source
This commit is contained in:
frikky
2022-03-25 00:43:16 +01:00
parent 9628b1c533
commit 79b24f76d8
15 changed files with 273 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+1 -1
View File
@@ -9,7 +9,7 @@ SHUFFLE_DOWNLOAD_WORKFLOW_USERNAME=
SHUFFLE_DOWNLOAD_WORKFLOW_PASSWORD=
SHUFFLE_DOWNLOAD_WORKFLOW_BRANCH=
SHUFFLE_APP_DOWNLOAD_LOCATION=https://github.com/frikky/shuffle-apps
SHUFFLE_APP_DOWNLOAD_LOCATION=https://github.com/shuffle/python-apps
SHUFFLE_DOWNLOAD_AUTH_USERNAME=
SHUFFLE_DOWNLOAD_AUTH_PASSWORD=
SHUFFLE_DOWNLOAD_AUTH_BRANCH=
backend/app_sdk/app_base.py
+25 -10
View File
@@ -1,4 +1,5 @@
import os
import ast
import copy
import sys
import re
@@ -273,9 +274,10 @@ class AppBase:
action_result["result"] = self.run_magic_parser(action_result["result"])
else:
self.logger.warning(f"[ERROR] Magic output not defined.")
except KeyError as e:
self.logger.warning(f"[ERROR] Failed to run magic autoparser (send result) - keyerror: {e}")
except Exception as e:
self.logger.warning(f"[ERROR] Failed to run magic autoparser (send result): {e}")
pass
# Try it with some magic
@@ -383,8 +385,9 @@ class AppBase:
if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true":
try:
self.log_capture_string.flush()
self.log_capture_string.close()
pass
#self.log_capture_string.flush()
#self.log_capture_string.close()
except Exception as e:
print(f"[WARNING] Failed to flush logs: {e}")
pass
@@ -973,6 +976,20 @@ class AppBase:
filebytes = BytesIO(ret1.content)
myzipfile = zipfile.ZipFile(filebytes)
# Unzip and build here!
#for member in files.namelist():
# filename = os.path.basename(member)
# if not filename:
# continue
# self.logger.info("File: %s" % member)
# source = files.open(member)
# with open("%s/%s" % (basedir, source.name), "wb+") as tmp:
# filedata = source.read()
# self.logger.info("Filedata (%s): %s" % (source.name, filedata))
# tmp.write(filedata)
return myzipfile
# Things to consider for files:
@@ -2594,8 +2611,6 @@ class AppBase:
params[item["key"]] = item["value"]
except KeyError:
self.logger.info("[DEBUG] No authentication specified!")
pass
#action["authentication"]
# Fixes OpenAPI body parameters for later.
newparams = []
@@ -3038,7 +3053,7 @@ class AppBase:
if isinstance(value, str):
params[key] = ast.literal_eval(value)
except Exception as e:
self.logger.info("[DEBUG] Failed parsing value with ast: {e}")
self.logger.info(f"[DEBUG] Failed parsing value with ast: {e}")
continue
except Exception as e:
self.logger.info("[DEBUG] Failed looping objects. Non critical: {e}")
@@ -3206,8 +3221,8 @@ class AppBase:
# https://ptb.discord.com/channels/747075026288902237/882017498550112286/882043773138382890
except (requests.exceptions.RequestException, TimeoutError) as e:
self.logger.info(f"Failed to execute request (requests): {e}")
self.logger.exception(f"Failed to execute {e}-{action['id']}")
self.logger.info(f"[ERROR] Failed to execute request (requests): {e}")
self.logger.exception(f"[ERROR] Failed to execute {e}-{action['id']}")
self.action_result["status"] = "SUCCESS"
try:
self.action_result["result"] = json.dumps({
@@ -3219,8 +3234,8 @@ class AppBase:
self.action_result["result"] = f"Request error: {e}"
except Exception as e:
self.logger.info(f"Failed to execute: {e}")
self.logger.exception(f"Failed to execute {e}-{action['id']}")
self.logger.info(f"[ERROR] Failed to execute: {e}")
self.logger.exception(f"[ERROR] Failed to execute {e}-{action['id']}")
self.action_result["status"] = "FAILURE"
#self.action_result["result"] = f"General exception: {e}"
self.action_result["result"] = json.dumps({
backend/app_sdk/build.sh
+1 -1
View File
@@ -3,7 +3,7 @@
### DEFAULT
NAME=shuffle-app_sdk
VERSION=0.9.64
VERSION=0.9.65
docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly
backend/go-app/go.mod
+2 -2
View File
@@ -2,7 +2,7 @@ module main
go 1.16
replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared
//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared
//replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi
//replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch
@@ -24,7 +24,7 @@ require (
github.com/h2non/filetype v1.1.3
github.com/nirasan/go-oauth-pkce-code-verifier v0.0.0-20170819232839-0fbfe93532da // indirect
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.2.11
github.com/shuffle/shuffle-shared v0.2.15
go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce
google.golang.org/api v0.65.0
backend/go-app/main.go
+59
View File
@@ -4,6 +4,7 @@ import (
uuid "github.com/satori/go.uuid"
"github.com/shuffle/shuffle-shared"
"archive/zip"
"bufio"
"bytes"
"context"
@@ -5759,6 +5760,63 @@ func makeWorkflowPublic(resp http.ResponseWriter, request *http.Request) {
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func handleAppZipUpload(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
//https://stackoverflow.com/questions/22964950/http-request-formfile-handle-zip-files
request.ParseMultipartForm(32 << 20)
f, _, err := request.FormFile("shuffle_file")
if err != nil {
log.Printf("[ERROR] Couldn't upload file: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed uploading file. Correct usage is: shuffle_file=@filepath"}`))
return
}
fileSize, err := f.Seek(0, 2) //2 = from end
if err != nil {
panic(err)
}
_, err = f.Seek(0, 0)
if err != nil {
panic(err)
}
buf := new(bytes.Buffer)
fileSize, err = io.Copy(buf, f)
if err != nil {
panic(err)
}
zipdata, err := zip.NewReader(bytes.NewReader(buf.Bytes()), fileSize)
if err != nil {
panic(err)
}
for _, item := range zipdata.File {
log.Printf("\n\nName: %s\n\n", item.FileHeader.Name)
log.Printf("item: %#v", item)
rr, err := item.Open()
if err != nil {
log.Fatal(err)
}
_, err = io.Copy(os.Stdout, rr)
if err != nil {
log.Fatal(err)
}
rr.Close()
}
resp.WriteHeader(200)
resp.Write([]byte("OK"))
}
func initHandlers() {
var err error
ctx := context.Background()
@@ -5857,6 +5915,7 @@ func initHandlers() {
// App specific
// From here down isnt checked for org specific
r.HandleFunc("/api/v1/apps/upload", handleAppZipUpload).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appId}/activate", activateWorkflowAppDocker).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/frameworkConfiguration", shuffle.GetFrameworkConfiguration).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/frameworkConfiguration", shuffle.SetFrameworkConfiguration).Methods("POST", "OPTIONS")
backend/tests/files.sh
+1 -1
View File
@@ -19,4 +19,4 @@ curl http://localhost:5001/api/v1/apps/upload -H "Authorization: Bearer db0373c6
#curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" -d '{"filename": "rule2.yar", "org_id": "b4e88fe9-352b-47b4-b280-960181670acf", "workflow_id": "global", "namespace": "yara"}'
#curl http://localhost:5001/api/v1/files/5cb941ad-fa1c-4444-a685-92024b1fa31c/upload -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" -F 'shuffle_file=@upload.sh'
curl http://localhost:5001/api/v1/files/namespaces/yara -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" --output rules.zip
#curl http://localhost:5001/api/v1/files/namespaces/yara -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" --output rules.zip
docker-compose.yml
+2 -2
View File
@@ -27,8 +27,8 @@ services:
- shuffle
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps
- ${SHUFFLE_FILE_LOCATION}:/shuffle-files
- ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z
- ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z
#- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate
env_file: .env
environment:
frontend/src/components/DetectionFramework.jsx
+3 -1
View File
@@ -922,7 +922,9 @@ const Framework = (props) => {
const baselocationX = 285*scale
const baselocationY = 50*scale
const shiftmodifier = 3*scale
const svgSize = `${40*scale}px`
//const svgSize = `${40*scale}px`
const svgSize = `${40}px`
console.log("Size: ", svgSize)
console.log("Framework: ", parsedFrameworkData)
frontend/src/views/Admin.jsx
+10 -5
View File
@@ -1010,8 +1010,9 @@ const Admin = (props) => {
body: file,
})
.then((response) => {
if (response.status !== 200) {
if (response.status !== 200 && response.status !== 201) {
console.log("Status not 200 for apps :O!");
alert.error("File was created, but failed to upload.")
return;
}
@@ -1022,17 +1023,21 @@ const Admin = (props) => {
//setFiles(responseJson)
})
.catch((error) => {
//alert.error(error.toString())
alert.error(error.toString())
});
};
const handleCreateFile = (filename, file) => {
const data = {
var data = {
filename: filename,
org_id: selectedOrganization.id,
workflow_id: "global",
};
if (selectedNamespace !== undefined && selectedNamespace !== null && selectedNamespace.length > 0 && selectedNamespace !== "default") {
data.namespace = selectedNamespace
}
fetch(globalUrl + "/api/v1/files/create", {
method: "POST",
headers: {
@@ -3991,7 +3996,7 @@ const Admin = (props) => {
>
{environment.archived ? "Activate" : "Disable"}
</Button>
<Button variant={environment.archived ? "contained" : "outlined"} style={{borderRadius: "0px"}} onClick={() => {
<Button variant={"outlined"} style={{borderRadius: "0px"}} onClick={() => {
console.log("Should clear executions for: ", environment)
abortEnvironmentWorkflows(environment)
}} color="primary">Clear executions</Button>
@@ -4240,7 +4245,7 @@ const Admin = (props) => {
/>
<Tab
index={5}
disabled={isCloud || userdata.admin !== "true"}
disabled={userdata.admin !== "true"}
label=<span>
<EcoIcon style={iconStyle} />
Environments
frontend/src/views/AngularWorkflow.jsx
+7 -2
View File
@@ -1795,7 +1795,7 @@ const AngularWorkflow = (defaultprops) => {
//
// Wait for new node to possibly be selected
setTimeout(() => {
//setTimeout(() => {
const typeIds = cy.elements('node:selected').jsons();
console.log("Found: ", typeIds)
for (var idkey in typeIds) {
@@ -1891,7 +1891,7 @@ const AngularWorkflow = (defaultprops) => {
});
//console.timeEnd("UNSELECT");
})
}, 150)
//}, 150)
};
const onEdgeSelect = (event) => {
@@ -3600,7 +3600,12 @@ const AngularWorkflow = (defaultprops) => {
// FIXME: Don't allow multiple in cloud yet. Cloud -> Onprem isn't stable.
if (isCloud) {
console.log("Envs: ", responseJson)
if (responseJson.length > 0) {
setEnvironments(responseJson);
} else {
setEnvironments([{ Name: "Cloud", Type: "cloud" }]);
}
} else {
setEnvironments(responseJson);
}
frontend/src/views/Dashboard.jsx
+31 -8
View File
@@ -74,7 +74,7 @@ const UsecaseListComponent = ({keys, isCloud, globalUrl, frameworkData, isLogged
}
const getUsecase = (name, index, subindex) => {
fetch(globalUrl + "/api/v1/workflows/usecases/"+name.replaceAll(" ", "_"), {
fetch(`${globalUrl}/api/v1/workflows/usecases/${escape(name.replaceAll(" ", "_"))}`, {
method: "GET",
headers: {
"Content-Type": "application/json",
@@ -100,6 +100,19 @@ const UsecaseListComponent = ({keys, isCloud, globalUrl, frameworkData, isLogged
setExpandedIndex(index)
setExpandedItem(subindex)
setTimeout(() => {
//console.log("Scroll!")
const found = document.getElementById("selected_box");
console.log("Found to scroll: ", found)
if (found !== undefined && found !== null) {
//console.log("FOUND!!")
found.scrollTo({
top: 100,
behavior: "smooth",
})
}
}, 100);
})
.catch((error) => {
//alert.error(error.toString());
@@ -224,14 +237,17 @@ const UsecaseListComponent = ({keys, isCloud, globalUrl, frameworkData, isLogged
<Typography variant="h6" style={{maxWidth: 215}}>
<b>{subcase.name}</b>
</Typography>
<div style={{position: "absolute", top: -26, right: -20, width: 50, }}>
{finished ?
<Tooltip
title="A workflow has been assigned for this use case"
placement="top"
>
<IconButton
style={{}}
style={{
position: "absolute",
bottom: 15,
right: -15,
}}
onClick={(e) => {
}}
>
@@ -251,7 +267,11 @@ const UsecaseListComponent = ({keys, isCloud, globalUrl, frameworkData, isLogged
placement="top"
>
<IconButton
style={{marginTop: finished ? 5 : 40, }}
style={{
position: "absolute",
bottom: -25,
right: -15,
}}
onClick={(e) => {
}}
>
@@ -272,7 +292,11 @@ const UsecaseListComponent = ({keys, isCloud, globalUrl, frameworkData, isLogged
placement="top"
>
<IconButton
style={{paddingTop: 5,}}
style={{
position: "absolute",
bottom: -65,
right: -15,
}}
onClick={(e) => {
}}
>
@@ -282,9 +306,8 @@ const UsecaseListComponent = ({keys, isCloud, globalUrl, frameworkData, isLogged
</a>
: null}
</div>
</div>
:
<div style={{textAlign: "left", position: "relative",}}>
<div style={{textAlign: "left", position: "relative",}} id="selected_box">
<Typography variant="h6">
<b>{subcase.name}</b>
</Typography>
@@ -482,7 +505,7 @@ const UsecaseListComponent = ({keys, isCloud, globalUrl, frameworkData, isLogged
</div>
</div>
:
<div style={{flex: 1, textAlign: "left",}}>
<div style={{flex: 1, textAlign: "left", marginRight: 10, }}>
<Typography variant="body1">
{subcase.description}
</Typography>
functions/extensions/wazuh/ossec.conf
+1 -1
View File
@@ -1,6 +1,6 @@
<integration>
<name>custom-shuffle</name>
<level>9</level>
<hook_url>http://<IP>:<PORT>/api/v1/hooks/webhook_<HOOK_ID></hook_url>
<hook_url>http://<IP>:<PORT>/api/v1/hooks/webhook_hookid</hook_url>
<alert_format>json</alert_format>
</integration>
functions/onprem/orborus/build.sh
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=shuffle-orborus
VERSION=0.9.64
VERSION=0.9.65
echo "Running docker build with $NAME:$VERSION"
#docker rmi frikky/shuffle:$NAME --force
functions/onprem/orborus/orborus.go
+13
View File
@@ -65,6 +65,10 @@ var baseimagename = os.Getenv("SHUFFLE_BASE_IMAGE_NAME")
var baseimageregistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")
var baseimagetagsuffix = os.Getenv("SHUFFLE_BASE_IMAGE_TAG_SUFFIX")
// Used for cloud with auth
var auth = os.Getenv("AUTH")
var org = os.Getenv("ORG")
//var orgId = os.Getenv("ORG_ID")
var baseUrl = os.Getenv("BASE_URL")
var environment = os.Getenv("ENVIRONMENT_NAME")
@@ -901,6 +905,15 @@ func main() {
zombiecounter := 0
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Org-Id", environment)
if len(auth) > 0 {
req.Header.Add("Authorization", auth)
}
if len(org) > 0 {
req.Header.Add("Org", org)
}
log.Printf("[INFO] Waiting for executions at %s with Environment %s", fullUrl, environment)
hasStarted := false
for {
functions/onprem/worker/build.sh
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=shuffle-worker
VERSION=0.9.64
VERSION=0.9.65
echo "Running docker build with $NAME:$VERSION"
#CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin .