marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added basic zip upload reader sample with files in-memory based on @Dhaval's work.

Browse Source
This commit is contained in:
frikky
2022-03-25 00:43:16 +01:00
parent 9628b1c533
commit 79b24f76d8
15 changed files with 273 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app_sdk/app_base.py
+25 -10
View File
@@ -1,4 +1,5 @@
import os
import ast
import copy
import sys
import re
@@ -273,9 +274,10 @@ class AppBase:
action_result["result"] = self.run_magic_parser(action_result["result"])
else:
self.logger.warning(f"[ERROR] Magic output not defined.")
except KeyError as e:
self.logger.warning(f"[ERROR] Failed to run magic autoparser (send result) - keyerror: {e}")
except Exception as e:
self.logger.warning(f"[ERROR] Failed to run magic autoparser (send result): {e}")
pass
# Try it with some magic
@@ -383,8 +385,9 @@ class AppBase:
if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true":
try:
self.log_capture_string.flush()
self.log_capture_string.close()
pass
#self.log_capture_string.flush()
#self.log_capture_string.close()
except Exception as e:
print(f"[WARNING] Failed to flush logs: {e}")
pass
@@ -973,6 +976,20 @@ class AppBase:
filebytes = BytesIO(ret1.content)
myzipfile = zipfile.ZipFile(filebytes)
# Unzip and build here!
#for member in files.namelist():
# filename = os.path.basename(member)
# if not filename:
# continue
# self.logger.info("File: %s" % member)
# source = files.open(member)
# with open("%s/%s" % (basedir, source.name), "wb+") as tmp:
# filedata = source.read()
# self.logger.info("Filedata (%s): %s" % (source.name, filedata))
# tmp.write(filedata)
return myzipfile
# Things to consider for files:
@@ -2594,8 +2611,6 @@ class AppBase:
params[item["key"]] = item["value"]
except KeyError:
self.logger.info("[DEBUG] No authentication specified!")
pass
#action["authentication"]
# Fixes OpenAPI body parameters for later.
newparams = []
@@ -3038,7 +3053,7 @@ class AppBase:
if isinstance(value, str):
params[key] = ast.literal_eval(value)
except Exception as e:
self.logger.info("[DEBUG] Failed parsing value with ast: {e}")
self.logger.info(f"[DEBUG] Failed parsing value with ast: {e}")
continue
except Exception as e:
self.logger.info("[DEBUG] Failed looping objects. Non critical: {e}")
@@ -3206,8 +3221,8 @@ class AppBase:
# https://ptb.discord.com/channels/747075026288902237/882017498550112286/882043773138382890
except (requests.exceptions.RequestException, TimeoutError) as e:
self.logger.info(f"Failed to execute request (requests): {e}")
self.logger.exception(f"Failed to execute {e}-{action['id']}")
self.logger.info(f"[ERROR] Failed to execute request (requests): {e}")
self.logger.exception(f"[ERROR] Failed to execute {e}-{action['id']}")
self.action_result["status"] = "SUCCESS"
try:
self.action_result["result"] = json.dumps({
@@ -3219,8 +3234,8 @@ class AppBase:
self.action_result["result"] = f"Request error: {e}"
except Exception as e:
self.logger.info(f"Failed to execute: {e}")
self.logger.exception(f"Failed to execute {e}-{action['id']}")
self.logger.info(f"[ERROR] Failed to execute: {e}")
self.logger.exception(f"[ERROR] Failed to execute {e}-{action['id']}")
self.action_result["status"] = "FAILURE"
#self.action_result["result"] = f"General exception: {e}"
self.action_result["result"] = json.dumps({
backend/app_sdk/build.sh
+1 -1
View File
@@ -3,7 +3,7 @@
### DEFAULT
NAME=shuffle-app_sdk
VERSION=0.9.64
VERSION=0.9.65
docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly
backend/go-app/go.mod
+2 -2
View File
@@ -2,7 +2,7 @@ module main
go 1.16
replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared
//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared
//replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi
//replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch
@@ -24,7 +24,7 @@ require (
github.com/h2non/filetype v1.1.3
github.com/nirasan/go-oauth-pkce-code-verifier v0.0.0-20170819232839-0fbfe93532da // indirect
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.2.11
github.com/shuffle/shuffle-shared v0.2.15
go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce
google.golang.org/api v0.65.0
backend/go-app/main.go
+59
View File
@@ -4,6 +4,7 @@ import (
uuid "github.com/satori/go.uuid"
"github.com/shuffle/shuffle-shared"
"archive/zip"
"bufio"
"bytes"
"context"
@@ -5759,6 +5760,63 @@ func makeWorkflowPublic(resp http.ResponseWriter, request *http.Request) {
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func handleAppZipUpload(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
//https://stackoverflow.com/questions/22964950/http-request-formfile-handle-zip-files
request.ParseMultipartForm(32 << 20)
f, _, err := request.FormFile("shuffle_file")
if err != nil {
log.Printf("[ERROR] Couldn't upload file: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed uploading file. Correct usage is: shuffle_file=@filepath"}`))
return
}
fileSize, err := f.Seek(0, 2) //2 = from end
if err != nil {
panic(err)
}
_, err = f.Seek(0, 0)
if err != nil {
panic(err)
}
buf := new(bytes.Buffer)
fileSize, err = io.Copy(buf, f)
if err != nil {
panic(err)
}
zipdata, err := zip.NewReader(bytes.NewReader(buf.Bytes()), fileSize)
if err != nil {
panic(err)
}
for _, item := range zipdata.File {
log.Printf("\n\nName: %s\n\n", item.FileHeader.Name)
log.Printf("item: %#v", item)
rr, err := item.Open()
if err != nil {
log.Fatal(err)
}
_, err = io.Copy(os.Stdout, rr)
if err != nil {
log.Fatal(err)
}
rr.Close()
}
resp.WriteHeader(200)
resp.Write([]byte("OK"))
}
func initHandlers() {
var err error
ctx := context.Background()
@@ -5857,6 +5915,7 @@ func initHandlers() {
// App specific
// From here down isnt checked for org specific
r.HandleFunc("/api/v1/apps/upload", handleAppZipUpload).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appId}/activate", activateWorkflowAppDocker).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/frameworkConfiguration", shuffle.GetFrameworkConfiguration).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/frameworkConfiguration", shuffle.SetFrameworkConfiguration).Methods("POST", "OPTIONS")
backend/tests/files.sh
+1 -1
View File
@@ -19,4 +19,4 @@ curl http://localhost:5001/api/v1/apps/upload -H "Authorization: Bearer db0373c6
#curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" -d '{"filename": "rule2.yar", "org_id": "b4e88fe9-352b-47b4-b280-960181670acf", "workflow_id": "global", "namespace": "yara"}'
#curl http://localhost:5001/api/v1/files/5cb941ad-fa1c-4444-a685-92024b1fa31c/upload -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" -F 'shuffle_file=@upload.sh'
curl http://localhost:5001/api/v1/files/namespaces/yara -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" --output rules.zip
#curl http://localhost:5001/api/v1/files/namespaces/yara -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" --output rules.zip