Added basic zip upload reader sample with files in-memory based on @Dhaval's work.

This commit is contained in:
frikky
2022-03-25 00:43:16 +01:00
parent 9628b1c533
commit 79b24f76d8
15 changed files with 273 additions and 151 deletions
+1 -1
View File
@@ -9,7 +9,7 @@ SHUFFLE_DOWNLOAD_WORKFLOW_USERNAME=
SHUFFLE_DOWNLOAD_WORKFLOW_PASSWORD=
SHUFFLE_DOWNLOAD_WORKFLOW_BRANCH=
SHUFFLE_APP_DOWNLOAD_LOCATION=https://github.com/frikky/shuffle-apps
SHUFFLE_APP_DOWNLOAD_LOCATION=https://github.com/shuffle/python-apps
SHUFFLE_DOWNLOAD_AUTH_USERNAME=
SHUFFLE_DOWNLOAD_AUTH_PASSWORD=
SHUFFLE_DOWNLOAD_AUTH_BRANCH=
+25 -10
View File
@@ -1,4 +1,5 @@
import os
import ast
import copy
import sys
import re
@@ -273,9 +274,10 @@ class AppBase:
action_result["result"] = self.run_magic_parser(action_result["result"])
else:
self.logger.warning(f"[ERROR] Magic output not defined.")
except KeyError as e:
self.logger.warning(f"[ERROR] Failed to run magic autoparser (send result) - keyerror: {e}")
except Exception as e:
self.logger.warning(f"[ERROR] Failed to run magic autoparser (send result): {e}")
pass
# Try it with some magic
@@ -383,8 +385,9 @@ class AppBase:
if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true":
try:
self.log_capture_string.flush()
self.log_capture_string.close()
pass
#self.log_capture_string.flush()
#self.log_capture_string.close()
except Exception as e:
print(f"[WARNING] Failed to flush logs: {e}")
pass
@@ -973,6 +976,20 @@ class AppBase:
filebytes = BytesIO(ret1.content)
myzipfile = zipfile.ZipFile(filebytes)
# Unzip and build here!
#for member in files.namelist():
# filename = os.path.basename(member)
# if not filename:
# continue
# self.logger.info("File: %s" % member)
# source = files.open(member)
# with open("%s/%s" % (basedir, source.name), "wb+") as tmp:
# filedata = source.read()
# self.logger.info("Filedata (%s): %s" % (source.name, filedata))
# tmp.write(filedata)
return myzipfile
# Things to consider for files:
@@ -2594,8 +2611,6 @@ class AppBase:
params[item["key"]] = item["value"]
except KeyError:
self.logger.info("[DEBUG] No authentication specified!")
pass
#action["authentication"]
# Fixes OpenAPI body parameters for later.
newparams = []
@@ -3038,7 +3053,7 @@ class AppBase:
if isinstance(value, str):
params[key] = ast.literal_eval(value)
except Exception as e:
self.logger.info("[DEBUG] Failed parsing value with ast: {e}")
self.logger.info(f"[DEBUG] Failed parsing value with ast: {e}")
continue
except Exception as e:
self.logger.info("[DEBUG] Failed looping objects. Non critical: {e}")
@@ -3206,8 +3221,8 @@ class AppBase:
# https://ptb.discord.com/channels/747075026288902237/882017498550112286/882043773138382890
except (requests.exceptions.RequestException, TimeoutError) as e:
self.logger.info(f"Failed to execute request (requests): {e}")
self.logger.exception(f"Failed to execute {e}-{action['id']}")
self.logger.info(f"[ERROR] Failed to execute request (requests): {e}")
self.logger.exception(f"[ERROR] Failed to execute {e}-{action['id']}")
self.action_result["status"] = "SUCCESS"
try:
self.action_result["result"] = json.dumps({
@@ -3219,8 +3234,8 @@ class AppBase:
self.action_result["result"] = f"Request error: {e}"
except Exception as e:
self.logger.info(f"Failed to execute: {e}")
self.logger.exception(f"Failed to execute {e}-{action['id']}")
self.logger.info(f"[ERROR] Failed to execute: {e}")
self.logger.exception(f"[ERROR] Failed to execute {e}-{action['id']}")
self.action_result["status"] = "FAILURE"
#self.action_result["result"] = f"General exception: {e}"
self.action_result["result"] = json.dumps({
+1 -1
View File
@@ -3,7 +3,7 @@
### DEFAULT
NAME=shuffle-app_sdk
VERSION=0.9.64
VERSION=0.9.65
docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly
+2 -2
View File
@@ -2,7 +2,7 @@ module main
go 1.16
replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared
//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared
//replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi
//replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch
@@ -24,7 +24,7 @@ require (
github.com/h2non/filetype v1.1.3
github.com/nirasan/go-oauth-pkce-code-verifier v0.0.0-20170819232839-0fbfe93532da // indirect
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.2.11
github.com/shuffle/shuffle-shared v0.2.15
go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce
google.golang.org/api v0.65.0
+59
View File
@@ -4,6 +4,7 @@ import (
uuid "github.com/satori/go.uuid"
"github.com/shuffle/shuffle-shared"
"archive/zip"
"bufio"
"bytes"
"context"
@@ -5759,6 +5760,63 @@ func makeWorkflowPublic(resp http.ResponseWriter, request *http.Request) {
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func handleAppZipUpload(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
//https://stackoverflow.com/questions/22964950/http-request-formfile-handle-zip-files
request.ParseMultipartForm(32 << 20)
f, _, err := request.FormFile("shuffle_file")
if err != nil {
log.Printf("[ERROR] Couldn't upload file: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed uploading file. Correct usage is: shuffle_file=@filepath"}`))
return
}
fileSize, err := f.Seek(0, 2) //2 = from end
if err != nil {
panic(err)
}
_, err = f.Seek(0, 0)
if err != nil {
panic(err)
}
buf := new(bytes.Buffer)
fileSize, err = io.Copy(buf, f)
if err != nil {
panic(err)
}
zipdata, err := zip.NewReader(bytes.NewReader(buf.Bytes()), fileSize)
if err != nil {
panic(err)
}
for _, item := range zipdata.File {
log.Printf("\n\nName: %s\n\n", item.FileHeader.Name)
log.Printf("item: %#v", item)
rr, err := item.Open()
if err != nil {
log.Fatal(err)
}
_, err = io.Copy(os.Stdout, rr)
if err != nil {
log.Fatal(err)
}
rr.Close()
}
resp.WriteHeader(200)
resp.Write([]byte("OK"))
}
func initHandlers() {
var err error
ctx := context.Background()
@@ -5857,6 +5915,7 @@ func initHandlers() {
// App specific
// From here down isnt checked for org specific
r.HandleFunc("/api/v1/apps/upload", handleAppZipUpload).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appId}/activate", activateWorkflowAppDocker).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/frameworkConfiguration", shuffle.GetFrameworkConfiguration).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/frameworkConfiguration", shuffle.SetFrameworkConfiguration).Methods("POST", "OPTIONS")
+1 -1
View File
@@ -19,4 +19,4 @@ curl http://localhost:5001/api/v1/apps/upload -H "Authorization: Bearer db0373c6
#curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" -d '{"filename": "rule2.yar", "org_id": "b4e88fe9-352b-47b4-b280-960181670acf", "workflow_id": "global", "namespace": "yara"}'
#curl http://localhost:5001/api/v1/files/5cb941ad-fa1c-4444-a685-92024b1fa31c/upload -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" -F 'shuffle_file=@upload.sh'
curl http://localhost:5001/api/v1/files/namespaces/yara -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" --output rules.zip
#curl http://localhost:5001/api/v1/files/namespaces/yara -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" --output rules.zip
+2 -2
View File
@@ -27,8 +27,8 @@ services:
- shuffle
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps
- ${SHUFFLE_FILE_LOCATION}:/shuffle-files
- ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z
- ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z
#- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate
env_file: .env
environment:
@@ -922,7 +922,9 @@ const Framework = (props) => {
const baselocationX = 285*scale
const baselocationY = 50*scale
const shiftmodifier = 3*scale
const svgSize = `${40*scale}px`
//const svgSize = `${40*scale}px`
const svgSize = `${40}px`
console.log("Size: ", svgSize)
console.log("Framework: ", parsedFrameworkData)
+10 -5
View File
@@ -1010,8 +1010,9 @@ const Admin = (props) => {
body: file,
})
.then((response) => {
if (response.status !== 200) {
if (response.status !== 200 && response.status !== 201) {
console.log("Status not 200 for apps :O!");
alert.error("File was created, but failed to upload.")
return;
}
@@ -1022,17 +1023,21 @@ const Admin = (props) => {
//setFiles(responseJson)
})
.catch((error) => {
//alert.error(error.toString())
alert.error(error.toString())
});
};
const handleCreateFile = (filename, file) => {
const data = {
var data = {
filename: filename,
org_id: selectedOrganization.id,
workflow_id: "global",
};
if (selectedNamespace !== undefined && selectedNamespace !== null && selectedNamespace.length > 0 && selectedNamespace !== "default") {
data.namespace = selectedNamespace
}
fetch(globalUrl + "/api/v1/files/create", {
method: "POST",
headers: {
@@ -3991,7 +3996,7 @@ const Admin = (props) => {
>
{environment.archived ? "Activate" : "Disable"}
</Button>
<Button variant={environment.archived ? "contained" : "outlined"} style={{borderRadius: "0px"}} onClick={() => {
<Button variant={"outlined"} style={{borderRadius: "0px"}} onClick={() => {
console.log("Should clear executions for: ", environment)
abortEnvironmentWorkflows(environment)
}} color="primary">Clear executions</Button>
@@ -4240,7 +4245,7 @@ const Admin = (props) => {
/>
<Tab
index={5}
disabled={isCloud || userdata.admin !== "true"}
disabled={userdata.admin !== "true"}
label=<span>
<EcoIcon style={iconStyle} />
Environments
+79 -74
View File
@@ -1795,103 +1795,103 @@ const AngularWorkflow = (defaultprops) => {
//
// Wait for new node to possibly be selected
setTimeout(() => {
const typeIds = cy.elements('node:selected').jsons();
console.log("Found: ", typeIds)
for (var idkey in typeIds) {
const item = typeIds[idkey]
console.log("items: ", item)
if (item.data.isButton === true) {
console.log("Reselect old node & return - or just return?")
if (item.data.buttonType === "delete" && item.data.attachedTo === nodedata.id) {
console.log("delete of same node!")
}
return
//setTimeout(() => {
const typeIds = cy.elements('node:selected').jsons();
console.log("Found: ", typeIds)
for (var idkey in typeIds) {
const item = typeIds[idkey]
console.log("items: ", item)
if (item.data.isButton === true) {
console.log("Reselect old node & return - or just return?")
if (item.data.buttonType === "delete" && item.data.attachedTo === nodedata.id) {
console.log("delete of same node!")
}
return
}
}
//if (nodedata.app_name === undefined && nodedata.source === undefined) {
// return;
//}
//event.target.removeClass("selected");
//
//if (nodedata.app_name === undefined && nodedata.source === undefined) {
// return;
//}
//event.target.removeClass("selected");
//
//// If button is clicked, select current node
//// If button is clicked, select current node
// Attempt at rewrite of name in other actions in following nodes.
// Should probably be done in the onBlur for the textfield instead
/*
if (event.target.data().type === "ACTION") {
const nodeaction = event.target.data()
const curaction = workflow.actions.find(a => a.id === nodeaction.id)
console.log("workflowaction: ", curaction)
console.log("nodeaction: ", nodeaction)
if (nodeaction.label !== curaction.label) {
console.log("BEACH!")
// Attempt at rewrite of name in other actions in following nodes.
// Should probably be done in the onBlur for the textfield instead
/*
if (event.target.data().type === "ACTION") {
const nodeaction = event.target.data()
const curaction = workflow.actions.find(a => a.id === nodeaction.id)
console.log("workflowaction: ", curaction)
console.log("nodeaction: ", nodeaction)
if (nodeaction.label !== curaction.label) {
console.log("BEACH!")
var params = []
const fixedName = "$"+curaction.label.toLowerCase().replace(" ", "_")
for (var actionkey in workflow.actions) {
if (workflow.actions[actionkey].id === curaction.id) {
var params = []
const fixedName = "$"+curaction.label.toLowerCase().replace(" ", "_")
for (var actionkey in workflow.actions) {
if (workflow.actions[actionkey].id === curaction.id) {
continue
}
for (var paramkey in workflow.actions[actionkey].parameters) {
const param = workflow.actions[actionkey].parameters[paramkey]
if (param.value === null || param.value === undefined || !param.value.includes("$")) {
continue
}
for (var paramkey in workflow.actions[actionkey].parameters) {
const param = workflow.actions[actionkey].parameters[paramkey]
if (param.value === null || param.value === undefined || !param.value.includes("$")) {
continue
}
const innername = param.value.toLowerCase().replace(" ", "_")
if (innername.includes(fixedName)) {
//workflow.actions[actionkey].parameters[paramkey].replace(
//console.log("FOUND!: ", innername)
}
const innername = param.value.toLowerCase().replace(" ", "_")
if (innername.includes(fixedName)) {
//workflow.actions[actionkey].parameters[paramkey].replace(
//console.log("FOUND!: ", innername)
}
}
}
}
*/
}
*/
//cy.removeListener("select");
//cy.on("select", "node", (e) => onNodeSelect(e, appAuthentication));
//cy.on("select", "edge", (e) => onEdgeSelect(e));
//cy.removeListener("select");
//cy.on("select", "node", (e) => onNodeSelect(e, appAuthentication));
//cy.on("select", "edge", (e) => onEdgeSelect(e));
// FIXME - check if they have value before overriding like this for no reason.
// Would save a lot of time (400~ ms -> 30ms)
//console.log("ACTION: ", selectedAction)
//console.log("APP: ", selectedApp)
// FIXME - check if they have value before overriding like this for no reason.
// Would save a lot of time (400~ ms -> 30ms)
//console.log("ACTION: ", selectedAction)
//console.log("APP: ", selectedApp)
ReactDOM.unstable_batchedUpdates(() => {
setSelectedAction({});
setSelectedApp({});
setSelectedTrigger({});
setSelectedComment({})
setSelectedEdge({});
ReactDOM.unstable_batchedUpdates(() => {
setSelectedAction({});
setSelectedApp({});
setSelectedTrigger({});
setSelectedComment({})
setSelectedEdge({});
setSelectedEdge({})
setSelectedActionEnvironment({})
setTriggerAuthentication({})
setSelectedTriggerIndex(-1)
setTriggerFolders([])
setSubworkflow({})
setSelectedEdge({})
setSelectedActionEnvironment({})
setTriggerAuthentication({})
setSelectedTriggerIndex(-1)
setTriggerFolders([])
setSubworkflow({})
// Can be used for right side view
setRightSideBarOpen(false);
setScrollConfig({
top: 0,
left: 0,
selected: "",
});
// Can be used for right side view
setRightSideBarOpen(false);
setScrollConfig({
top: 0,
left: 0,
selected: "",
});
//console.timeEnd("UNSELECT");
})
}, 150)
})
//}, 150)
};
const onEdgeSelect = (event) => {
@@ -3600,7 +3600,12 @@ const AngularWorkflow = (defaultprops) => {
// FIXME: Don't allow multiple in cloud yet. Cloud -> Onprem isn't stable.
if (isCloud) {
setEnvironments([{ Name: "Cloud", Type: "cloud" }]);
console.log("Envs: ", responseJson)
if (responseJson.length > 0) {
setEnvironments(responseJson);
} else {
setEnvironments([{ Name: "Cloud", Type: "cloud" }]);
}
} else {
setEnvironments(responseJson);
}
+74 -51
View File
@@ -74,7 +74,7 @@ const UsecaseListComponent = ({keys, isCloud, globalUrl, frameworkData, isLogged
}
const getUsecase = (name, index, subindex) => {
fetch(globalUrl + "/api/v1/workflows/usecases/"+name.replaceAll(" ", "_"), {
fetch(`${globalUrl}/api/v1/workflows/usecases/${escape(name.replaceAll(" ", "_"))}`, {
method: "GET",
headers: {
"Content-Type": "application/json",
@@ -100,7 +100,20 @@ const UsecaseListComponent = ({keys, isCloud, globalUrl, frameworkData, isLogged
setExpandedIndex(index)
setExpandedItem(subindex)
})
setTimeout(() => {
//console.log("Scroll!")
const found = document.getElementById("selected_box");
console.log("Found to scroll: ", found)
if (found !== undefined && found !== null) {
//console.log("FOUND!!")
found.scrollTo({
top: 100,
behavior: "smooth",
})
}
}, 100);
})
.catch((error) => {
//alert.error(error.toString());
setInputUsecase({})
@@ -224,67 +237,77 @@ const UsecaseListComponent = ({keys, isCloud, globalUrl, frameworkData, isLogged
<Typography variant="h6" style={{maxWidth: 215}}>
<b>{subcase.name}</b>
</Typography>
<div style={{position: "absolute", top: -26, right: -20, width: 50, }}>
{finished ?
<Tooltip
title="A workflow has been assigned for this use case"
placement="top"
>
<IconButton
style={{}}
onClick={(e) => {
}}
>
<DoneAllIcon style={{ color: usecase.color }} />
</IconButton>
</Tooltip>
: null}
{subcase.blogpost !== null && subcase.blogpost !== undefined && subcase.blogpost.length > 0 ?
<a
href={subcase.blogpost}
rel="noopener noreferrer"
target="_blank"
style={{ textDecoration: "none", color: "#f85a3e" }}
>
{finished ?
<Tooltip
title="Click to visit the blogpost"
title="A workflow has been assigned for this use case"
placement="top"
>
<IconButton
style={{marginTop: finished ? 5 : 40, }}
style={{
position: "absolute",
bottom: 15,
right: -15,
}}
onClick={(e) => {
}}
>
<DescriptionIcon style={{ color: usecase.color }} />
<DoneAllIcon style={{ color: usecase.color }} />
</IconButton>
</Tooltip>
</a>
: null}
{subcase.video !== null && subcase.video !== undefined && subcase.video.length > 0 ?
<a
href={subcase.video}
rel="noopener noreferrer"
target="_blank"
style={{ textDecoration: "none", color: "#f85a3e" }}
>
<Tooltip
title="Click to see a video for this usecase"
placement="top"
: null}
{subcase.blogpost !== null && subcase.blogpost !== undefined && subcase.blogpost.length > 0 ?
<a
href={subcase.blogpost}
rel="noopener noreferrer"
target="_blank"
style={{ textDecoration: "none", color: "#f85a3e" }}
>
<IconButton
style={{paddingTop: 5,}}
onClick={(e) => {
}}
<Tooltip
title="Click to visit the blogpost"
placement="top"
>
<PlayArrowIcon style={{ color: usecase.color }} />
</IconButton>
</Tooltip>
</a>
: null}
</div>
<IconButton
style={{
position: "absolute",
bottom: -25,
right: -15,
}}
onClick={(e) => {
}}
>
<DescriptionIcon style={{ color: usecase.color }} />
</IconButton>
</Tooltip>
</a>
: null}
{subcase.video !== null && subcase.video !== undefined && subcase.video.length > 0 ?
<a
href={subcase.video}
rel="noopener noreferrer"
target="_blank"
style={{ textDecoration: "none", color: "#f85a3e" }}
>
<Tooltip
title="Click to see a video for this usecase"
placement="top"
>
<IconButton
style={{
position: "absolute",
bottom: -65,
right: -15,
}}
onClick={(e) => {
}}
>
<PlayArrowIcon style={{ color: usecase.color }} />
</IconButton>
</Tooltip>
</a>
: null}
</div>
:
<div style={{textAlign: "left", position: "relative",}}>
<div style={{textAlign: "left", position: "relative",}} id="selected_box">
<Typography variant="h6">
<b>{subcase.name}</b>
</Typography>
@@ -482,7 +505,7 @@ const UsecaseListComponent = ({keys, isCloud, globalUrl, frameworkData, isLogged
</div>
</div>
:
<div style={{flex: 1, textAlign: "left",}}>
<div style={{flex: 1, textAlign: "left", marginRight: 10, }}>
<Typography variant="body1">
{subcase.description}
</Typography>
+1 -1
View File
@@ -1,6 +1,6 @@
<integration>
<name>custom-shuffle</name>
<level>9</level>
<hook_url>http://<IP>:<PORT>/api/v1/hooks/webhook_<HOOK_ID></hook_url>
<hook_url>http://<IP>:<PORT>/api/v1/hooks/webhook_hookid</hook_url>
<alert_format>json</alert_format>
</integration>
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=shuffle-orborus
VERSION=0.9.64
VERSION=0.9.65
echo "Running docker build with $NAME:$VERSION"
#docker rmi frikky/shuffle:$NAME --force
+13
View File
@@ -65,6 +65,10 @@ var baseimagename = os.Getenv("SHUFFLE_BASE_IMAGE_NAME")
var baseimageregistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")
var baseimagetagsuffix = os.Getenv("SHUFFLE_BASE_IMAGE_TAG_SUFFIX")
// Used for cloud with auth
var auth = os.Getenv("AUTH")
var org = os.Getenv("ORG")
//var orgId = os.Getenv("ORG_ID")
var baseUrl = os.Getenv("BASE_URL")
var environment = os.Getenv("ENVIRONMENT_NAME")
@@ -901,6 +905,15 @@ func main() {
zombiecounter := 0
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Org-Id", environment)
if len(auth) > 0 {
req.Header.Add("Authorization", auth)
}
if len(org) > 0 {
req.Header.Add("Org", org)
}
log.Printf("[INFO] Waiting for executions at %s with Environment %s", fullUrl, environment)
hasStarted := false
for {
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=shuffle-worker
VERSION=0.9.64
VERSION=0.9.65
echo "Running docker build with $NAME:$VERSION"
#CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin .