Added magic parser configuration to frontend with basic app sdk parsing

This commit is contained in:
frikky
2021-12-10 23:11:58 +01:00
parent 92a280bf2c
commit 5532296f74
16 changed files with 300 additions and 52 deletions
+1 -1
View File
@@ -50,8 +50,8 @@ SHUFFLE_PASS_APP_PROXY=FALSE
TZ=Europe/Amsterdam # Timezone-handler in Orborus, Worker and Apps
ORBORUS_CONTAINER_NAME= # Used to FIND the containername. cgroup v2: issue 501
SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io
SHUFFLE_BASE_IMAGE_NAME=frikky
SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io
SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.80"
# Used for auto-cleanup of containers. REALLY important at scale.
+4 -3
View File
@@ -5,13 +5,14 @@
Shuffle Automation
</h1><h4 align="center">
[Shuffle](https://shuffler.io) is an automation platform for and by the community, focusing on accessibility for anyone to automate. Security operations is complex, but it doesn't have to be.
![Shuffle](https://shuffler.io) is an automation platform for and by the community, focusing on accessibility for anyone to automate. Security operations is complex, but it doesn't have to be.
[_Key Features_](https://shuffler.io/docs/features) —
[_Community & Support_](https://discord.gg/B2CBzUm)
[_Community & Support_](https://discord.gg/B2CBzUm)
[_Documentation_](https://shuffler.io/docs) —
[_Getting Started_](https://shuffler.io/docs/getting_started) —
[_Development_](https://github.com/frikky/Shuffle/blob/master/.github/CONTRIBUTING.md)
[_Development_](https://github.com/frikky/Shuffle/blob/master/.github/CONTRIBUTING.md)
Follow us on Twitter at [@shuffleio](https://twitter.com/shuffleio).
+10
View File
@@ -8,5 +8,15 @@ This is the SDK used for apps to behave like they should.
4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...)
5. Rebuild the Docker image (click load in GUI?)
## Cloud updates
1. Go to shuffle cloud on GCP
2. Go to Cloud Storage
3. Find shuffler.appspot.com
4. Navigate to generated_apps/baseline
5. Update SDK there. This will make all new apps run with the new SDK
## Cloud app force-updates
1. Run the "stitcher.go" program in the public shuffle-shared repository.
# LICENSE
Everything in here is MIT, not AGPLv3 as indicated by the license.
+99 -3
View File
@@ -62,12 +62,100 @@ class AppBase:
if len(self.base_url) == 0:
self.base_url = self.url
# Checks output for whether it should be automatically parsed or not
def run_magic_parser(self, input_data):
if not isinstance(input_data, str):
return input_data
# Don't touch existing JSON/lists
if (input_data.startswith("[") and input_data.endswith("]")) or (input_data.startswith("{") and input_data.endswith("}")):
return input_data
# Don't touch large data.
if len(input_data) > 100000:
return input_data
new_input = input_data
try:
#new_input.strip()
new_input = input_data.split()
new_return = []
index = 0
for item in new_input:
splititem = ","
if ", " in item:
splititem = ", "
elif "," in item:
splititem = ","
else:
new_return.append(item)
index += 1
continue
#print("FIX ITEM %s" % item)
for subitem in item.split(splititem):
new_return.insert(index, subitem)
index += 1
# Prevent large data or infinite loops
if index > 10000:
return input_data
fixed_return = []
for item in new_return:
if not item:
continue
if not isinstance(item, str):
fixed_return.append(item)
continue
if item.endswith(","):
item = item[0:-1]
fixed_return.append(item)
new_input = fixed_return
except Exception as e:
self.logger.info(f"[DEBUG] Failed to run magic parser (2): {e}")
return input_data
try:
new_input = input_data.split()
except Exception as e:
self.logger.info(f"[DEBUG] Failed to run magic parser (1): {e}")
return input_data
# Won't ever touch this one?
if isinstance(input_data, list) or isinstance(input_data, object):
try:
return json.dumps(new_input)
except Exception as e:
self.logger.info(f"[DEBUG] Failed to run magic parser: {e}")
return new_input
# FIXME: Add more info like logs in here.
# Docker logs: https://forums.docker.com/t/docker-logs-inside-the-docker-container/68190/2
def send_result(self, action_result, headers, stream_path):
if action_result["status"] == "EXECUTING":
action_result["status"] = "FAILURE"
try:
if self.original_action["run_magic_output"] == True:
self.logger.warning("[INFO] Action result ran with Magic parser output.")
action_result["result"] = self.run_magic_parser(action_result["result"])
else:
self.logger.warning("[ERROR] Magic output not defined.")
except Exception as e:
self.logger.warning("[ERROR] Failed to run magic autoparser: {e}")
pass
# Try it with some magic
self.logger.info(f"""[DEBUG] Inside Send result with status {action_result["status"]}""")
# FIXME: Add cleanup of parameters to not send to frontend here
@@ -981,12 +1069,20 @@ class AppBase:
except json.decoder.JSONDecodeError:
pass
self.action_result["result"] = "Bad result from backend: %d" % ret.status_code
self.action_result["result"] = json.dumps({
"success": False,
"reason": f"Bad result from backend during startup of app: {ret.status_code}",
"extended_reason": f"{ret.text}"
})
self.send_result(self.action_result, headers, stream_path)
return
except requests.exceptions.ConnectionError as e:
self.logger.info("[DEBUG] FullExec Connectionerror: %s" % e)
self.action_result["result"] = "Connection error during startup: %s" % e
self.action_result["result"] = json.dumps({
"success": False,
"reason": f"Connection error during startup: {e}"
})
self.send_result(self.action_result, headers, stream_path)
return
else:
@@ -2805,7 +2901,7 @@ class AppBase:
# Dump the result as a string of a list
#self.logger.info("RESULTS: %s" % results)
if isinstance(results, list) or isinstance(results, dict):
self.logger.info("JSON OBJECT? ", json_object)
self.logger.info(f"JSON OBJECT? {json_object}")
# This part is weird lol
if json_object:
+1 -1
View File
@@ -3,7 +3,7 @@
### DEFAULT
NAME=shuffle-app_sdk
VERSION=0.9.35
VERSION=0.9.40
docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly
+1 -1
View File
@@ -1,7 +1,7 @@
urllib3==1.26.5
requests==2.25.1
MarkupSafe==2.0.1
liquidpy==0.7.2
liquidpy==0.7.3
flask[async]==2.0.2
#waitress==2.0.0
#flask==1.1.2
+1
View File
@@ -299,6 +299,7 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
}
if !downloaded {
return errors.New(fmt.Sprintf("Failed to build / download images %s", strings.Join(tags, ",")))
}
//baseDockerName
+1 -1
View File
@@ -2,7 +2,7 @@ module main
go 1.15
//replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared
replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared
//replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi
//replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch
+21 -2
View File
@@ -2467,6 +2467,7 @@ func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId
log.Printf("[INFO] Should send email to %s during execution.", email)
}
if strings.Contains(triggerType, "sms") {
action := shuffle.CloudSyncJob{
Type: "user_input",
@@ -2491,7 +2492,11 @@ func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId
return err
}
log.Printf("Should send SMS to %s during execution.", sms)
log.Printf("[DEBUG] Should send SMS to %s during execution.", sms)
}
if strings.Contains(triggerType, "subflow") {
log.Printf("[DEBUG] Should run a subflow with the result for user input.")
}
return nil
@@ -2927,7 +2932,21 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
for _, item := range buildLaterFirst {
err = buildImageMemory(fs, item.Tags, item.Extra, true)
if err != nil {
log.Printf("Failed image build memory: %s", err)
orgId := ""
log.Printf("[DEBUG] Failed image build memory. Creating notification with org %#v: %s", orgId, err)
if len(item.Tags) > 0 {
err = shuffle.CreateOrgNotification(
ctx,
fmt.Sprintf("App failed to build"),
fmt.Sprintf("The app %s with image %s failed to build. Check backend logs with docker! docker logs shuffle-backend", item.Tags[0], item.Extra),
fmt.Sprintf("/apps"),
orgId,
false,
)
}
} else {
if len(item.Tags) > 0 {
log.Printf("[INFO] Successfully built image %s", item.Tags[0])
+1 -1
View File
@@ -64,7 +64,7 @@ services:
- SHUFFLE_SWARM_NETWORK_NAME=shuffle-executions
restart: unless-stopped
opensearch:
image: opensearchproject/opensearch:1.1.0
image: opensearchproject/opensearch:1.2.0
hostname: shuffle-opensearch
container_name: shuffle-opensearch
environment:
+4
View File
@@ -5,6 +5,8 @@
"private": true,
"dependencies": {
"@babel/core": "^7.15.8",
"@emotion/react": "^11.7.0",
"@emotion/styled": "^11.6.0",
"@material-ui/core": "^4.5.2",
"@material-ui/data-grid": "^4.0.0-alpha.22",
"@material-ui/icons": "^4.5.1",
@@ -12,6 +14,8 @@
"@material-ui/styles": "^4.5.2",
"@material-ui/utils": "^4.11.2",
"@metamask/detect-provider": "^1.2.0",
"@mui/icons-material": "^5.2.1",
"@mui/material": "^5.2.3",
"@uiw/react-codemirror": "^3.2.1",
"@use-it/interval": "^1.0.0",
"babel-eslint": "^10.1.0",
@@ -164,7 +164,10 @@ const ConfigureWorkflow = (props) => {
newaction.must_authenticate = true;
newaction.action_ids.push(action.id);
}
}
} else if (action.authentication_id !== "" && app.authentication.required === true) {
console.log("Should verify authentication ID ", action.authentication_id)
}
newaction.app = app;
}
+40 -3
View File
@@ -80,7 +80,10 @@ import {
LockOpen as LockOpenIcon,
ExpandMore as ExpandMoreIcon,
VpnKey as VpnKeyIcon,
} from "@material-ui/icons";
AutoFixHigh as AutoFixHighIcon,
} from '@mui/icons-material';
//} from "@material-ui/icons";
import Autocomplete from "@material-ui/lab/Autocomplete";
import CodeMirror from "@uiw/react-codemirror";
@@ -2169,7 +2172,7 @@ const ParsedAction = (props) => {
marginTop: "auto",
marginBottom: "auto",
height: 30,
paddingLeft: 25,
marginLeft: 15,
paddingRight: 0,
}}
onClick={() => {
@@ -2189,7 +2192,7 @@ const ParsedAction = (props) => {
marginTop: "auto",
marginBottom: "auto",
height: 30,
paddingLeft: 25,
marginLeft: 15,
paddingRight: 0,
}}
onClick={() => {}}
@@ -2209,6 +2212,40 @@ const ParsedAction = (props) => {
</Tooltip>
</a>
</IconButton>
<IconButton
style={{
marginTop: "auto",
marginBottom: "auto",
height: 30,
marginLeft: 15,
paddingRight: 0,
}}
onClick={() => {
//setAuthenticationModalOpen(true);
console.log("Should enable/disable magic!")
console.log("Action: ", selectedAction)
if (selectedAction.run_magic_output === undefined) {
selectedAction.run_magic_output = true
} else {
if (selectedAction.run_magic_output === true) {
selectedAction.run_magic_output = false
} else {
selectedAction.run_magic_output = true
}
}
setSelectedAction(selectedAction)
setUpdate(Math.random());
}}
>
<Tooltip
color="primary"
title={selectedAction.run_magic_output === undefined || selectedAction.run_magic_output === null || selectedAction.run_magic_output === false ? "Click to enable magic parsing" : "Click to disable magic parsing"}
placement="top"
>
<AutoFixHighIcon style={{ color: selectedAction.run_magic_output === undefined || selectedAction.run_magic_output === null || selectedAction.run_magic_output === false ? "white" : "#f86a3e"}} />
</Tooltip>
</IconButton>
</div>
</div>
<div style={{ display: "flex", flexDirection: "column" }}>
+71 -32
View File
@@ -1754,13 +1754,17 @@ const AngularWorkflow = (props) => {
var idnumber = -1;
if (curElement.id.startsWith("rightside_field_")) {
console.log("FOUND FIELD WITH NUMBER: ", curElement.id);
// Find exact position to put the text
const idsplit = curElement.id.split("_");
console.log(idsplit);
if (idsplit.length === 3 && !isNaN(idsplit[2])) {
console.log("ADDING TO PARAM ", idsplit[2]);
console.log("PARAM: ", selectedAction);
selectedAction.parameters[idsplit[2]].value = newValue;
selectedAction.parameters[idsplit[2]].value += newValue;
paramname = selectedAction.parameters[idsplit[2]].name;
idnumber = idsplit[2];
}
@@ -1849,7 +1853,7 @@ const AngularWorkflow = (props) => {
const onNodeDrag = (event, selectedAction) => {
const nodedata = event.target.data();
if (nodedata.finished === false) {
return;
console.log("NOT FINISHED - ADD EXAMPLE BRANCHES TO CLOSEST!!")
}
if (nodedata.app_name !== undefined) {
@@ -1859,6 +1863,15 @@ const AngularWorkflow = (props) => {
if (currentNode.data.attachedTo === nodedata.id) {
cy.getElementById(currentNode.data.id).remove();
}
// Calculate location
//currentNode.position.x >
//if (nodedata.position.x > 0 && nodedata.position.y > 0) {
// console.log("Positive both")
//}
//console.log(currentNode.position)
//console.log(nodedata.position)
}
} else {
//console.log("No appid? ", nodedata)
@@ -2389,6 +2402,9 @@ const AngularWorkflow = (props) => {
return "";
}
console.log("NOT REPLACING ON PURPOSE!!")
return ""
// Basically just a stupid if-else :)
const synonyms = {
id: [
@@ -2491,6 +2507,7 @@ const AngularWorkflow = (props) => {
// Takes an action as input, then runs through and updates the relevant fields
// based on previous actions'
// Uses lots of synonyms
const RunAutocompleter = (dstdata) => {
// **PS: The right action should already be set here**
// 1. Check execution argument
@@ -2617,10 +2634,13 @@ const AngularWorkflow = (props) => {
(data) => data.id === edge.source
);
if (targetnode === -1) {
alert.error("Can't make arrow to starting node");
event.target.remove();
found = true;
break;
if (targetnode.type !== "TRIGGER") {
alert.error("Can't make arrow to starting node");
event.target.remove();
break;
}
found = true;
}
} else if (edge.source === workflow.branches[key].source_id) {
// FIXME: Verify multi-target for triggers
@@ -2953,16 +2973,16 @@ const AngularWorkflow = (props) => {
console.log("DELETE");
break;
case 38:
console.log("UP");
//console.log("UP");
break;
case 37:
console.log("LEFT");
//console.log("LEFT");
break;
case 40:
console.log("DOWN");
//console.log("DOWN");
break;
case 39:
console.log("RIGHT");
//console.log("RIGHT");
break;
case 90:
if (event.ctrlKey) {
@@ -3169,7 +3189,12 @@ const AngularWorkflow = (props) => {
}
}
setEnvironments(responseJson);
// FIXME: Don't allow multiple in cloud yet. Cloud -> Onprem isn't stable.
if (isCloud) {
setEnvironments({ name: "Cloud", type: "cloud" });
} else {
setEnvironments(responseJson);
}
})
.catch((error) => {
alert.error(error.toString());
@@ -3307,7 +3332,7 @@ const AngularWorkflow = (props) => {
document.title = "Workflow - " + workflow.name;
registerKeys();
}
});
})
const animationDuration = 150;
const onNodeHoverOut = (event) => {
@@ -3480,11 +3505,9 @@ const AngularWorkflow = (props) => {
const onNodeHover = (event) => {
const nodedata = event.target.data();
/*
if (nodedata.finished === false) {
return;
}
*/
//var parentNode = cy.$("#" + event.target.data("id"));
//if (parentNode.data("isButton") || parentNode.data("buttonId")) return;
@@ -4794,6 +4817,7 @@ const AngularWorkflow = (props) => {
parameters: parameters,
isStartNode: false,
large_image: app.large_image,
run_magic_output: true,
authentication: [],
execution_variable: undefined,
example: example,
@@ -6701,6 +6725,9 @@ const AngularWorkflow = (props) => {
</div>
{outlookButton}
{gmailButton}
<Typography variant="body2" color="textSecondary">
If you have trouble using the triggers, please <a href="https://shuffler.io/contact" rel="noopener noreferrer" target="_blank">contact us</a> to get access
</Typography>
</span>
)}
@@ -10075,7 +10102,10 @@ const AngularWorkflow = (props) => {
}
console.log("COPY: ", copy);
var newitem = JSON.parse(base);
//var newitem = JSON.parse(base);
var newitem = validateJson(base).result
to_be_copied = "$" + base_node_name.toLowerCase().replaceAll(" ", "_");
for (var key in copy.namespace) {
if (copy.namespace[key].includes("Results for")) {
@@ -10292,7 +10322,7 @@ const AngularWorkflow = (props) => {
{workflowExecutions.length > 0 ? (
<div>
{workflowExecutions.map((data, index) => {
executionDelay += 75
executionDelay += 50
const statusColor =
data.status === "FINISHED"
@@ -10441,7 +10471,7 @@ const AngularWorkflow = (props) => {
)}
</div>
) : (
<div style={{ padding: 25, maxWidth: 365, overflowX: "hidden" }}>
<div style={{ padding: "25px 15px 25px 15px", maxWidth: 365, overflowX: "hidden" }}>
<Breadcrumbs
aria-label="breadcrumb"
separator=""
@@ -10466,13 +10496,13 @@ const AngularWorkflow = (props) => {
onClick={() => {}}
>
<ArrowBackIcon style={{ color: "rgba(255,255,255,0.5)" }} />
</IconButton>
<h2
style={{ color: "rgba(255,255,255,0.5)", cursor: "pointer" }}
onClick={() => {}}
>
See other Executions
</h2>
</IconButton>
<h2
style={{ color: "rgba(255,255,255,0.5)", cursor: "pointer" }}
onClick={() => {}}
>
See other Executions
</h2>
</span>
</Breadcrumbs>
<Divider
@@ -10482,7 +10512,7 @@ const AngularWorkflow = (props) => {
marginBottom: 10,
}}
/>
<div style={{ display: "flex" }}>
<div style={{ display: "flex", marginLeft: 10, }}>
<h2>Execution info</h2>
<Tooltip
color="primary"
@@ -10531,7 +10561,7 @@ const AngularWorkflow = (props) => {
</div>
{executionData.status !== undefined &&
executionData.status.length > 0 ? (
<div style={{ display: "flex" }}>
<div style={{ display: "flex", marginLeft: 10, }}>
<Typography variant="body1">
<b>Status &nbsp;&nbsp;</b>
</Typography>
@@ -10544,7 +10574,7 @@ const AngularWorkflow = (props) => {
executionData.execution_source !== null &&
executionData.execution_source.length > 0 &&
executionData.execution_source !== "default" ? (
<div style={{ display: "flex" }}>
<div style={{ display: "flex", marginLeft: 10, }}>
<Typography variant="body1">
<b>Source &nbsp;&nbsp;</b>
</Typography>
@@ -10581,7 +10611,7 @@ const AngularWorkflow = (props) => {
</div>
) : null}
{executionData.started_at !== undefined ? (
<div style={{ display: "flex" }}>
<div style={{ display: "flex", marginLeft: 10, }}>
<Typography variant="body1">
<b>Started &nbsp;&nbsp;</b>
</Typography>
@@ -10593,7 +10623,7 @@ const AngularWorkflow = (props) => {
{executionData.completed_at !== undefined &&
executionData.completed_at !== null &&
executionData.completed_at > 0 ? (
<div style={{ display: "flex" }}>
<div style={{ display: "flex", marginLeft: 10, }}>
<Typography variant="body1" onClick={() => {
console.log(executionData)
}}>
@@ -10809,18 +10839,27 @@ const AngularWorkflow = (props) => {
<div
key={index}
style={{
marginBottom: 40,
marginBottom: 20,
border:
data.action.sub_action === true
? "1px solid rgba(255,255,255,0.3)"
: null,
? "1px solid rgba(255,255,255,0.3)"
: "1px solid rgba(255,255,255, 0.3)",
borderRadius: theme.palette.borderRadius,
backgroundColor: theme.palette.inputColor,
padding: "15px 10px 10px 10px",
overflow: "hidden",
}}
onMouseOver={() => {
var currentnode = cy.getElementById(data.action.id);
if (currentnode.length !== 0) {
currentnode.addClass("shuffle-hover-highlight");
}
// Add a hover highlight
//var copyText = document.getElementById(
// "copy_element_shuffle"
//)
}}
onMouseOut={() => {
var currentnode = cy.getElementById(data.action.id);
+3
View File
@@ -4676,12 +4676,15 @@ const AppCreator = (props) => {
for (var key in invalid) {
if (e.target.value.includes(invalid[key])) {
alert.error("Can't use " + invalid[key] + " in name");
setName(e.target.value.replaceAll(".", "").replaceAll("#", "").replaceAll(":", "").replaceAll(",", ""))
return;
}
}
if (e.target.value.length > 29) {
alert.error("Choose a shorter name (max 29).");
setName(e.target.value.slice(0,28))
return;
}
+38 -3
View File
@@ -398,12 +398,31 @@ export const validateJson = (showResult) => {
var result = showResult;
try {
const result = jsonvalid ? JSON.parse(showResult) : showResult;
result = jsonvalid ? JSON.parse(showResult) : showResult;
} catch (e) {
//console.log("Failed parsing JSON even though its valid: ", e)
jsonvalid = false;
}
if (jsonvalid === false) {
if (typeof showResult === 'string') {
showResult = showResult.trim()
}
try {
var newstr = showResult.replaceAll("'", '"')
console.log("Try replacements and trimming with new value: ", newstr)
result = JSON.parse(newstr)
jsonvalid = true
} catch (e) {
console.log("Failed parsing JSON even though its valid (2): ", e)
jsonvalid = false
}
}
//console.log("VALID: ", jsonvalid, result)
return {
valid: jsonvalid,
@@ -458,6 +477,8 @@ const Workflows = (props) => {
const [submitLoading, setSubmitLoading] = React.useState(false);
const [actionImageList, setActionImageList] = React.useState([]);
const [firstLoad, setFirstLoad] = React.useState(true);
const isCloud =
window.location.host === "localhost:3002" ||
window.location.host === "shuffler.io";
@@ -839,6 +860,11 @@ const Workflows = (props) => {
setFilteredWorkflows(responseJson);
setWorkflowDone(true);
// Ensures the zooming happens only once per load
setTimeout(() => {
setFirstLoad(false)
}, 100)
} else {
if (isLoggedIn) {
alert.error("An error occurred while loading workflows");
@@ -1764,6 +1790,7 @@ const Workflows = (props) => {
if (file.type !== "application/json") {
if (file.type !== undefined) {
alert.error("File has to contain valid json");
setImportLoading(false);
}
continue;
@@ -2581,7 +2608,11 @@ const Workflows = (props) => {
data.large_image = theme.palette.defaultImage;
}
appDelay += 75
if (firstLoad) {
appDelay += 75
} else {
appDelay = 0
}
return (
<Zoom key={index} in={true} style={{ transitionDelay: `${appDelay}ms` }}>
@@ -2650,7 +2681,11 @@ const Workflows = (props) => {
<NewWorkflowPaper />
</Zoom>
{filteredWorkflows.map((data, index) => {
workflowDelay += 75
if (firstLoad) {
workflowDelay += 75
} else {
workflowDelay = 0
}
return (
<Zoom key={index} in={true} style={{ transitionDelay: `${workflowDelay}ms` }}>