marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added major JSON change to action

Browse Source
This commit is contained in:
frikky
2020-05-14 18:53:52 +02:00
parent c6d055e8d1
commit a8b133ffa2
7 changed files with 174 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -87,8 +87,8 @@ There will be a major overhaul to the backend specifically. I'm currently moving
- * Full OpenAPI support with authentication schemes in App creator (not Oauth2 yet)
- * Change workflow name
- * User run statistics
- Extended result data usage, build json with answers, not just "from previous action"
- Workflows - IMPORT DEFAULT WORKFLOWS - Create some towards e.g. TheHive & MISP.
- Extended data usage - build json with answers
- Documentation - General documentation /docs rewrite
- API doc - 1. In Shuffle. 2. In e.g. python
- Fix scheduler
frontend/src/AngularWorkflow.js
+33 -14
View File
@@ -1028,8 +1028,6 @@ const AngularWorkflow = (props) => {
}, {
duration: animationDuration,
})
console.log(previousnodecolor)
}
const onNodeHover = (event) => {
@@ -2134,11 +2132,15 @@ const AngularWorkflow = (props) => {
var staticcolor = "inherit"
var actioncolor = "inherit"
var varcolor = "inherit"
var multiline = false
var multiline
if (data.multiline !== undefined && data.multiline !== null && data.multiline === true) {
multiline = true
}
if (data.value.startsWith("{") && data.value.endsWith("}")) {
multiline = true
}
var placeholder = "Static value"
if (data.example !== undefined && data.example !== null && data.example.length > 0) {
placeholder = data.example
@@ -2165,12 +2167,24 @@ const AngularWorkflow = (props) => {
onChange={(event) => {
changeActionParameter(event, count)
}}
onBlur={(event) => {
// Super basic check
if (event.target.value.startsWith("{")) {
console.log("VALIDATING JSON")
try {
JSON.parse(event.target.value)
} catch (e) {
alert.error("Failed to parse json")
}
}
}}
/>
// Remap data based on variant
if (data.variant === "STATIC_VALUE") {
staticcolor = "#f85a3e"
console.log("DATA IS STATIC")
} else if (data.variant === "ACTION_RESULT") {
// Gets the parents of the current node
var parents = getParents(selectedAction)
@@ -2390,7 +2404,7 @@ const AngularWorkflow = (props) => {
<div style={{marginTop: "20px"}}>
Environment:
<Select
value={selectedActionEnvironment.Name === undefined ? "" : selectedActionEnvironment.Name}
value={selectedActionEnvironment === undefined || selectedActionEnvironment.Name === undefined ? "" : selectedActionEnvironment.Name}
PaperProps={{
style: {
backgroundColor: inputColor,
@@ -2597,12 +2611,13 @@ const AngularWorkflow = (props) => {
}
const AppConditionHandler = (props) => {
const { tmpdata, type } = props;
const { tmpdata, type } = props;
if (tmpdata === undefined) {
return tmpdata
}
const [data, ] = useState(tmpdata)
const [multiline, setMultiline] = useState(false)
if (data.variant === "") {
data.variant = "STATIC_VALUE"
@@ -2611,9 +2626,8 @@ const AngularWorkflow = (props) => {
var staticcolor = "inherit"
var actioncolor = "inherit"
var varcolor = "inherit"
var multiline = false
if (data.multiline !== undefined && data.multiline !== null && data.multiline === true) {
multiline = true
setMultiline(true)
}
var placeholder = "Static value"
@@ -2635,10 +2649,14 @@ const AngularWorkflow = (props) => {
}}
fullWidth
multiline={multiline}
rows="5"
rows={5}
color="primary"
defaultValue={data.value}
placeholder={placeholder}
onClick={() => {
console.log("CHANGE FIELD")
setMultiline(!multiline)
}}
onBlur={(e) => {
changeActionVariable(data.action_field, e.target.value)
}}
@@ -2701,12 +2719,12 @@ const AngularWorkflow = (props) => {
setCurrentView("variables")
datafield =
<div>
<div>
Looks like you don't have any variables yet.
</div>
<div style={{width: "100%", margin: "auto"}}>
<Button style={{margin: "auto", marginTop: "10px",}} color="primary" variant="outlined" onClick={() => setVariablesModalOpen(true)}>New workflow variable</Button>
</div>
<div>
Looks like you don't have any variables yet.
</div>
<div style={{width: "100%", margin: "auto"}}>
<Button style={{margin: "auto", marginTop: "10px",}} color="primary" variant="outlined" onClick={() => setVariablesModalOpen(true)}>New workflow variable</Button>
</div>
</div>
} else {
// FIXME - this is a shitty solution that needs re-renders all the time
@@ -4022,6 +4040,7 @@ const AngularWorkflow = (props) => {
}}
color="secondary"
placeholder={"Execution Argument"}
defaultValue={executionText}
onBlur={(e) => {
setExecutionText(e.target.value)
}}
frontend/src/App.js
+2 -2
View File
@@ -38,12 +38,12 @@ import AlertTemplate from "react-alert-template-basic";
import { positions, Provider } from "react-alert";
// Testing - localhost
//const globalUrl = "http://192.168.3.6:5001"
const globalUrl = "http://192.168.3.6:5001"
//console.log("HOST: ", process.env)
// Production - backend proxy forwarding in nginx
const globalUrl = window.location.origin
//const globalUrl = window.location.origin
const surfaceColor = "#27292D"
const inputColor = "#383B40"
frontend/src/Workflows.js
+35 -28
View File
@@ -86,7 +86,7 @@ const Workflows = (props) => {
if (isLoggedIn) {
alert.error("An error occurred while loading workflows")
} else {
window.location.pathname = "/login"
window.location = "/login"
}
return
@@ -493,15 +493,17 @@ const Workflows = (props) => {
}
var t = new Date(data.started_at*1000)
var jsonvalid = true
var showResult = data.result.trim()
if ((showResult.startsWith("{") && showResult.endsWith("}")) || (showResult.startsWith("[{") && showResult.endsWith("}]"))) {
//showResult = <JSONPretty
// id="json-pretty"
// theme={JSONPrettyMon}
// data={showResult}/>
showResult = replaceAll(showResult, " None", " \"None\"");
console.log(showResult)
showResult = replaceAll(showResult, " None", " \"None\"");
try {
JSON.parse(showResult)
} catch (e) {
jsonvalid = false
}
console.log("VALID: ", jsonvalid)
if (jsonvalid) {
showResult = <ReactJson
src={JSON.parse(showResult)}
theme="solarized"
@@ -531,13 +533,13 @@ const Workflows = (props) => {
<Grid container style={{margin: "10px 10px 10px 10px", flex: "1"}}>
<Grid style={{display: "flex", flexDirection: "column", width: "100%"}}>
<Grid item style={{flex: "1"}}>
<h4 style={{marginBottom: "0px", marginTop: "10px"}}><b>Status</b>: {data.status}</h4>
<h4 style={{marginBottom: "0px", marginTop: "10px"}}><b>Name</b>: {data.action.label}</h4>
</Grid>
<Grid item style={{flex: "1", justifyContent: "center"}}>
App: {data.action.app_name}, Version: {data.action.app_version}
</Grid>
<Grid item style={{flex: "1", justifyContent: "center"}}>
Action: {data.action.name}, Environment: {data.action.environment}
Action: {data.action.name}, Environment: {data.action.environment}, Status: {data.status}
</Grid>
<div style={{display: "flex", flex: "1"}}>
<Grid item style={{flex: "10", justifyContent: "center"}}>
@@ -578,25 +580,30 @@ const Workflows = (props) => {
var parsedArgument = selectedExecution.execution_argument
if (selectedExecution.execution_argument !== undefined && selectedExecution.execution_argument.length > 0) {
parsedArgument = replaceAll(parsedArgument, " None", " \"None\"");
}
var arg = null
if (selectedExecution.execution_argument !== undefined && selectedExecution.execution_argument.length > 0) {
var jsonvalid = false
var showResult = selectedExecution.execution_argument.trim()
showResult = replaceAll(showResult, " None", " \"None\"");
try {
JSON.parse(showResult)
} catch (e) {
jsonvalid = false
}
arg = jsonvalid ?
<ReactJson
src={JSON.parse(showResult)}
theme="solarized"
collapsed={true}
displayDataTypes={false}
name={"Execution argument"}
/>
: showResult
}
const arg = selectedExecution.execution_argument !== undefined && selectedExecution.execution_argument.length > 0 ?
<div>
Execution argument: {(parsedArgument.startsWith("{") && parsedArgument.endsWith("}")) || (parsedArgument.startsWith("[{") && parsedArgument.endsWith("}]")) ?
<div>
<ReactJson
src={JSON.parse(parsedArgument)}
theme="solarized"
collapsed={true}
displayDataTypes={false}
name={"Execution argument"}
/>
</div>
:
selectedExecution.execution_argument
}
</div>
: null
/*
<div>
ID: {selectedExecution.execution_id}
functions/onprem/app_sdk/README.md
+7
View File
@@ -1,3 +1,10 @@
# app_sdk
This is the SDK used for apps to behave like they should.
To change it in the backend, upload it to Buckets/shuffler.appspot.com/generated_apps/baseline.
## If you want to update apps.. Why doesn't this work every time..?:
1. Write your code & check if runtime works
2. Build app_base image
3. docker rm $(docker ps -aq) # Remove all stopped containers
4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...)
5. Rebuild the Docker image (click load in GUI?)
functions/onprem/app_sdk/app_base.py
+95 -3
View File
@@ -1,5 +1,6 @@
import os
import sys
import re
import time
import json
import logging
@@ -63,7 +64,7 @@ class AppBase:
# Add async logger
# self.console_logger.handlers[0].stream.set_execution_id()
self.logger.info("Before initial stream result")
#self.logger.info("Before initial stream result")
try:
ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result)
self.logger.info("Workflow: %d" % ret.status_code)
@@ -72,7 +73,8 @@ class AppBase:
except requests.exceptions.ConnectionError as e:
print("Connectionerror: %s" % e)
return
self.logger.info("AFTER initial stream result")
#self.logger.info("AFTER initial stream result")
self.logger.info("THIS IS THE NEW UPDATE")
# Verify whether there are any parameters with ACTION_RESULT required
# If found, we get the full results list from backend
@@ -105,13 +107,102 @@ class AppBase:
self.logger.info("AFTER FULLEXEC stream result")
# Takes a workflow execution as argument
# Returns a string if the result is single, or a list if it's a list
# Not implemented: lists
def get_json_value(execution_data, input_data):
parsersplit = input_data.split(".")
actionname = parsersplit[0][1:]
print(f"Actionname: {actionname}")
# 1. Find the action
baseresult = ""
try:
if actionname.lower() == "exec":
baseresult = execution_data["execution_argument"]
else:
for result in execution_data["results"]:
if result["action"]["label"].lower() == actionname.lower():
baseresult = result["result"]
break
except KeyError as error:
print(f"Error: {error}")
print(f"After first trycatch")
# 2. Find the JSON data
if len(baseresult) == 0:
return ""
if len(parsersplit) == 1:
return baseresult
baseresult = baseresult.replace("\'", "\"")
basejson = {}
try:
basejson = json.loads(baseresult)
except json.decoder.JSONDecodeError as e:
return baseresult
try:
for value in parsersplit[1:]:
if value == "#":
print("HANDLE RECURSIVE LOOP")
pass
else:
if isinstance(basejson[value], str):
print(f"LOADING STRING {basejson[value]} AS JSON?")
try:
parsedjson = json.loads(basejson[value])
except json.decoder.JSONDecodeError as e:
print("RETURNING BECAUSE {basejson[value]} is a normal string")
return basejson
else:
basejson = basejson[value]
except KeyError as e:
print(f"Keyerror: {e}")
return basejson
except IndexError as e:
print(f"Indexerror: {e}")
return basejson
return basejson
def parse_params(action, fullexecution, parameter):
jsonparsevalue = "$."
# Regex to find all the things
if parameter["variant"] == "STATIC_VALUE":
data = parameter["value"]
self.logger.debug(f"\n\nHandle static data with JSON: {data}\n\n")
match = ".*([$]{1}(\w+\.?){1,})"
actualitem = re.findall(match, data, re.MULTILINE)
self.logger.info("PARSED: %s" % actualitem)
if len(actualitem) > 0:
for replace in actualitem:
try:
to_be_replaced = replace[0]
except IndexError:
continue
value = get_json_value(fullexecution, to_be_replaced)
# Check if json inside string
if isinstance(value, str) > 0:
print(f"VALUE: {value}")
self.logger.info(f"CONVERT DATA FROM {parameter['value']} to {data}")
parameter["value"] = data
if parameter["variant"] == "WORKFLOW_VARIABLE":
for item in fullexecution["workflow"]["workflow_variables"]:
if parameter["action_field"] == item["name"]:
parameter["value"] = item["value"]
break
elif parameter["variant"] == "ACTION_RESULT":
# FIXME - calculate value based on action_field and $if prominent
# FIND THE RIGHT LABEL
@@ -258,6 +349,7 @@ class AppBase:
print(sourcevalue)
destinationvalue = condition["destination"]["value"]
if condition["destination"]["variant"]== "" or condition["destination"]["variant"]== "STATIC_VALUE":
condition["destination"]["variant"] = "STATIC_VALUE"
else:
@@ -398,7 +490,7 @@ class AppBase:
action_result["completed_at"] = int(time.time())
# I wonder if this actually works
self.logger.info("Before last stream result")
#self.logger.info("Before last stream result")
try:
ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result)
self.logger.info("Result: %d" % ret.status_code)
functions/onprem/app_sdk/update_dockerhub
+1 -1
View File
@@ -1,4 +1,4 @@
#!/bin/bash
docker rmi frikky/shuffle:app_sdk
docker build . -t frikky/shuffle:app_sdk
docker build . -t frikky/shuffle:app_sdk --no-cache
docker push frikky/shuffle:app_sdk