marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Continuing forparser for app_sdk

Browse Source
This commit is contained in:
frikky
2020-05-20 11:06:22 +02:00
parent 5d092ddd41
commit c557d232c9
9 changed files with 333 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/go-app/walkoff.go
+81 -65
View File
@@ -1338,86 +1338,102 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
// Check every app action and param to see whether they exist
newActions = []Action{}
for _, action := range workflow.Actions {
curapp := WorkflowApp{}
// FIXME - can this work with ONLY AppID?
for _, app := range workflowApps {
if app.ID == action.AppID {
curapp = app
break
}
reservedApps := []string{
"0ca8887e-b4af-4e3e-887c-87e9d3bc3d3e",
}
if app.Name == action.AppName && app.AppVersion == action.AppVersion {
curapp = app
builtin := false
for _, id := range reservedApps {
if id == action.AppID {
builtin = true
break
}
}
// Check to see if the whole app is valid
if curapp.Name != action.AppName {
log.Printf("App %s doesn't exist.", action.AppName)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App %s doesn't exist"}`, action.AppName)))
return
}
if builtin {
newActions = append(newActions, action)
} else {
curapp := WorkflowApp{}
// FIXME - can this work with ONLY AppID?
for _, app := range workflowApps {
if app.ID == action.AppID {
curapp = app
break
}
// Check tosee if the appaction is valid
curappaction := WorkflowAppAction{}
for _, curAction := range curapp.Actions {
if action.Name == curAction.Name {
curappaction = curAction
break
}
log.Println(action.Name, curAction.Name)
}
// Check to see if the action is valid
if curappaction.Name != action.Name {
log.Printf("Appaction %s doesn't exist.", action.Name)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// FIXME - check all parameters to see if they're valid
// Includes checking required fields
newParams := []WorkflowAppActionParameter{}
for _, param := range curappaction.Parameters {
found := false
// Handles check for parameter exists + value not empty in used fields
for _, actionParam := range action.Parameters {
if actionParam.Name == param.Name {
found = true
if actionParam.Value == "" && actionParam.Variant == "STATIC_VALUE" && actionParam.Required == true {
log.Printf("Appaction %s with required param '%s' is empty.", action.Name, param.Name)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name)))
return
}
if actionParam.Variant == "" {
actionParam.Variant = "STATIC_VALUE"
}
newParams = append(newParams, actionParam)
if app.Name == action.AppName && app.AppVersion == action.AppVersion {
curapp = app
break
}
}
// Handles check for required params
if !found && param.Required {
log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name)
// Check to see if the whole app is valid
if curapp.Name != action.AppName {
log.Printf("App %s doesn't exist.", action.AppName)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App %s doesn't exist"}`, action.AppName)))
return
}
// Check tosee if the appaction is valid
curappaction := WorkflowAppAction{}
for _, curAction := range curapp.Actions {
if action.Name == curAction.Name {
curappaction = curAction
break
}
log.Println(action.Name, curAction.Name)
}
// Check to see if the action is valid
if curappaction.Name != action.Name {
log.Printf("Appaction %s doesn't exist.", action.Name)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
// FIXME - check all parameters to see if they're valid
// Includes checking required fields
action.Parameters = newParams
newActions = append(newActions, action)
newParams := []WorkflowAppActionParameter{}
for _, param := range curappaction.Parameters {
found := false
// Handles check for parameter exists + value not empty in used fields
for _, actionParam := range action.Parameters {
if actionParam.Name == param.Name {
found = true
if actionParam.Value == "" && actionParam.Variant == "STATIC_VALUE" && actionParam.Required == true {
log.Printf("Appaction %s with required param '%s' is empty.", action.Name, param.Name)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name)))
return
}
if actionParam.Variant == "" {
actionParam.Variant = "STATIC_VALUE"
}
newParams = append(newParams, actionParam)
}
}
// Handles check for required params
if !found && param.Required {
log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
action.Parameters = newParams
newActions = append(newActions, action)
}
}
workflow.Actions = newActions
backend/tests/forparser.py
+138
View File
File diff suppressed because one or more lines are too long
frontend/src/AngularWorkflow.js
+64 -8
View File
@@ -285,7 +285,6 @@ const AngularWorkflow = (props) => {
currentnode = currentnode[0]
const outgoingEdges = currentnode.outgoers('edge')
const incomingEdges = currentnode.incomers('edge')
console.log("NODE: ", currentnode)
//currentnode.removeClass('success-highlight failure-highlight executing-highlight')
switch (item.status) {
@@ -668,6 +667,56 @@ const AngularWorkflow = (props) => {
// // FIXME - handle this, as we can't have more than one of each :)
// //setVersionedApps(newapps)
//}
// Builtin actions that should ran in Worker and not apps
const getExtraApps = () => {
const data = [{
name: "Filter",
is_valid: true,
id: "0ca8887e-b4af-4e3e-887c-87e9d3bc3d3e",
link: "https://shuffler.io",
app_version: "1.0.0",
generated: true,
downloaded: false,
sharing: false,
verified: false,
tested: false,
owner: "",
private_id: "",
description: "Filter",
environment: "Shuffle",
small_image: "",
large_image: "",
contact_info: {name: "", url: ""},
authentication: {required: false, parameters: [],},
actions: [{
description: "Filter cases",
id: "",
name: "filter_cases",
node_type: "action",
environment: "Shuffle",
sharing: false,
private_id: "",
app_id: "",
authentication: null,
tested: true,
parameters: [{
description: "",
id: "",
name: "Field to look for",
example: "$testing_1.#.id",
value: "",
multiline: true,
action_field: "",
variant: "",
required: true,
schema: {type: "string"},
}]
}],
}]
return data
}
const getApps = () => {
fetch(globalUrl+"/api/v1/workflows/apps", {
@@ -688,9 +737,12 @@ const AngularWorkflow = (props) => {
.then((responseJson) => {
// FIXME - handle versions on left bar
//handleAppVersioning(responseJson)
setApps(responseJson)
setFilteredApps(responseJson)
})
var tmpapps = []
tmpapps = tmpapps.concat(getExtraApps())
tmpapps = tmpapps.concat(responseJson)
setApps(tmpapps)
setFilteredApps(tmpapps)
})
.catch(error => {
alert.error(error.toString())
});
@@ -780,6 +832,7 @@ const AngularWorkflow = (props) => {
}
console.log("Selected: ", data.id)
console.log(curaction)
setRequiresAuthentication(curapp.authentication.required)
setSelectedApp(curapp)
@@ -1848,7 +1901,7 @@ const AngularWorkflow = (props) => {
authentication: [],
}
console.log(newAppData)
// const image = "url("+app.large_image+")"
// FIXME - find the cytoscape offset position
// Can this be done with zoom calculations?
@@ -2358,7 +2411,8 @@ const AngularWorkflow = (props) => {
<b>{data.name}: </b>
</div>
<Tooltip color="primary" title="Static data" placement="top">
<div style={{cursor: "pointer", color: staticcolor}} onClick={() => {
<div style={{cursor: "pointer", color: staticcolor}} onClick={(e) => {
e.preventDefault()
changeActionParameterVariant("STATIC_VALUE", count)
}}>
<CreateIcon />
@@ -2366,7 +2420,8 @@ const AngularWorkflow = (props) => {
</Tooltip>
&nbsp;|&nbsp;
<Tooltip color="primary" title="Data from previous action" placement="top">
<div style={{cursor: "pointer", color: actioncolor}} onClick={() => {
<div style={{cursor: "pointer", color: actioncolor}} onClick={(e) => {
e.preventDefault()
changeActionParameterVariant("ACTION_RESULT", count)
}}>
<AppsIcon />
@@ -2374,7 +2429,8 @@ const AngularWorkflow = (props) => {
</Tooltip>
&nbsp;|&nbsp;
<Tooltip color="primary" title="Use local variable" placement="top">
<div style={{cursor: "pointer", color: varcolor}} onClick={() => {
<div style={{cursor: "pointer", color: varcolor}} onClick={(e) => {
e.preventDefault()
changeActionParameterVariant("WORKFLOW_VARIABLE", count)
}}>
<FavoriteBorderIcon />
frontend/src/App.js
+2 -2
View File
@@ -38,12 +38,12 @@ import AlertTemplate from "react-alert-template-basic";
import { positions, Provider } from "react-alert";
// Testing - localhost
//const globalUrl = "http://192.168.3.6:5001"
const globalUrl = "http://192.168.3.6:5001"
//console.log("HOST: ", process.env)
// Production - backend proxy forwarding in nginx
const globalUrl = window.location.origin
//const globalUrl = window.location.origin
const surfaceColor = "#27292D"
const inputColor = "#383B40"
frontend/src/Apps.js
+8 -11
View File
@@ -55,6 +55,11 @@ const Apps = (props) => {
useEffect(() => {
if (apps.length <= 0 && firstrequest) {
document.title = "Shuffle - Apps"
if (!isLoggedIn && isLoaded) {
window.location = "/login"
}
setFirstrequest(false)
getApps()
}
@@ -79,7 +84,7 @@ const Apps = (props) => {
const getApps = () => {
fetch(globalUrl+"/api/v1/workflows/apps", {
method: 'GET',
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
@@ -93,7 +98,7 @@ const Apps = (props) => {
return response.json()
})
.then((responseJson) => {
.then((responseJson) => {
setApps(responseJson)
setFilteredApps(responseJson)
if (responseJson.length > 0) {
@@ -452,15 +457,7 @@ const Apps = (props) => {
</div>
</div>
:
<div style={{width: "600px", margin: "auto", color: "white", paddingBottom: "50px"}}>
<h2>Available integrations</h2>
<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
{apps.map(data => {
return (
appPaper(data)
)
})}
</div>
null
// Gets the URL itself (hopefully this works in most cases?
// Will then forward the data to an internal endpoint to validate the api
frontend/src/Workflows.js
+4 -3
View File
@@ -22,6 +22,7 @@ import PlayArrowIcon from '@material-ui/icons/PlayArrow';
//import JSONPrettyMon from 'react-json-pretty/dist/monikai'
import ReactJson from 'react-json-view'
import {Link} from 'react-router-dom';
import { useAlert } from "react-alert";
import Dialog from '@material-ui/core/Dialog';
@@ -398,13 +399,13 @@ const Workflows = (props) => {
}
}}>
<Grid item style={{flex: "1", justifyContent: "center"}}>
<a href={"/workflows/"+data.id}>
<Link to={"/workflows/"+data.id}>
<Tooltip color="primary" title="Edit workflow" placement="bottom">
<Button style={{}} color="primary" variant="text" style={{marginRight: 10}} onClick={() => {}}>
<EditIcon style={{marginRight: 10}}/> Edit
</Button>
</Tooltip>
</a>
</Link>
<Tooltip color="primary" title="Execute workflow" placement="bottom">
<Button style={{}} color="secondary" variant="text" onClick={() => executeWorkflow(data.id)}>
<PlayArrowIcon />
@@ -889,7 +890,7 @@ const Workflows = (props) => {
</div>
<div>
<p>
<b>Shuffle</b> is a flexible, easy to use, automation framework allowing users to integrate their services and devices to reduce the amount of manual labor required for those tasks. <a href="/docs/workflows" style={{textDecoration: "none", color: "#f85a3e"}}>Click here for more information.</a>
<b>Shuffle</b> is a flexible, easy to use, automation framework allowing users to integrate their services and devices to reduce the amount of manual labor required for those tasks. <Link to="/docs/workflows" style={{textDecoration: "none", color: "#f85a3e"}}>Click here for more information.</Link>
</p>
</div>
<div>
functions/onprem/app_sdk/app_base.py
+12 -7
View File
@@ -122,9 +122,11 @@ class AppBase:
baseresult = execution_data["execution_argument"]
else:
for result in execution_data["results"]:
if result["action"]["label"].lower() == actionname.lower():
resultlabel = result["action"]["label"].replace(" ", "_", -1)
if resultlabel.lower() == actionname.lower():
baseresult = result["result"]
break
except KeyError as error:
print(f"Error: {error}")
@@ -143,9 +145,14 @@ class AppBase:
basejson = json.loads(baseresult)
except json.decoder.JSONDecodeError as e:
return baseresult
def loop_recursion(data):
for value in data:
pass
try:
for value in parsersplit[1:]:
print("VALUE: %s", value)
if value == "#":
print("HANDLE RECURSIVE LOOP")
pass
@@ -177,7 +184,7 @@ class AppBase:
data = parameter["value"]
self.logger.debug(f"\n\nHandle static data with JSON: {data}\n\n")
match = ".*([$]{1}(\w+\.?){1,})"
match = ".*([$]{1}([a-zA-Z0-9()# _-]+\.?){1,})"
actualitem = re.findall(match, data, re.MULTILINE)
self.logger.info("PARSED: %s" % actualitem)
if len(actualitem) > 0:
@@ -450,17 +457,15 @@ class AppBase:
calltimes = 1
result = ""
paramiter = []
for parameter in action["parameters"]:
#self.logger.info(parameter)
#print(fullexecution)
all_executions = []
for parameter in action["parameters"]:
check, value = parse_params(action, fullexecution, parameter)
if check:
raise Exception(check)
if isinstance(value, list):
params[parameter["name"]] = value
# p["value"]
# FIXME - this is horrible, but works for now
#for i in range(calltimes):
functions/onprem/worker/worker.bin
BIN
View File
Binary file not shown.
functions/onprem/worker/worker.go
+24 -2
View File
@@ -78,6 +78,7 @@ type WorkflowExecution struct {
type Action struct {
AppName string `json:"app_name" datastore:"app_name"`
AppVersion string `json:"app_version" datastore:"app_version"`
AppID string `json:"app_id" datastore:"app_id"`
Errors []string `json:"errors" datastore:"errors"`
ID string `json:"id" datastore:"id"`
IsValid bool `json:"is_valid" datastore:"is_valid"`
@@ -258,7 +259,7 @@ func shutdown(executionId, workflowId string) {
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", authorization)
//req.Header.Add("Authorization", authorization)
client := &http.Client{}
_, err = client.Do(req)
if err != nil {
@@ -336,6 +337,21 @@ func removeContainer(containername string) error {
return nil
}
func runFilter(workflowExecution WorkflowExecution, action Action) {
// 1. Get the parameter $.#.id
if action.Label == "filter_cases" && len(action.Parameters) > 0 {
if action.Parameters[0].Variant == "ACTION_RESULT" {
param := action.Parameters[0]
value := param.Value
// Loop cases.. Hmm, that's tricky
}
} else {
log.Printf("No handler for filter %s with %d params", action.Label, len(action.Parameters))
}
}
func handleExecution(client *http.Client, req *http.Request, workflowExecution WorkflowExecution) error {
// if no onprem runs (shouldn't happen, but extra check), exit
// if there are some, load the images ASAP for the app
@@ -364,7 +380,6 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W
}
toExecuteOnprem = append(toExecuteOnprem, action.ID)
actionName := fmt.Sprintf("%s:%s_%s", baseimagename, action.AppName, action.AppVersion)
found := false
for _, app := range onpremApps {
@@ -555,12 +570,19 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W
// marshal action and put it in there rofl
log.Printf("Time to execute %s with app %s:%s, function %s, env %s with %d parameters.", action.ID, action.AppName, action.AppVersion, action.Name, action.Environment, len(action.Parameters))
actionData, err := json.Marshal(action)
if err != nil {
log.Printf("Failed unmarshalling action: %s", err)
continue
}
if action.AppID == "0ca8887e-b4af-4e3e-887c-87e9d3bc3d3e" {
log.Printf("\nShould run filter: %#v\n\n", action)
runFilter(workflowExecution, action)
continue
}
//log.Println(string(actionData))
// FIXME - add proper FUNCTION_APIKEY from user definition
env := []string{