marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #4 from frikky/dev

Browse Source 
Dev merge for for-loops
This commit is contained in:
Frikkylikeme
2020-05-21 09:36:13 +02:00
committed by GitHub
parent eb9b752d25 b4d94e803f
commit b32aedf629
12 changed files with 519 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/go-app/main.go
+4 -41
View File
@@ -170,6 +170,7 @@ type User struct {
CreationTime int64 `datastore:"creation_time" json:"creation_time"`
}
// timeout maybe? idk
type session struct {
Username string `datastore:"Username,noindex"`
Session string `datastore:"session,noindex"`
@@ -1399,19 +1400,6 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) {
// FIXME - check memcache here
// Get the item from the memcache
ctx := context.Background()
//if item, err := memcache.Get(ctx, c.Value); err == memcache.ErrCacheMiss {
// // Not in cache
//} else if err != nil {
// log.Printf("Error getting item: %v", err)
//} else {
// var Userdata User
// err = json.Unmarshal(item.Value, &Userdata)
// if err == nil {
// resp.WriteHeader(200)
// resp.Write([]byte(`{"success": true, "reason": "OK"}`))
// return
// }
//}
sessionToken := c.Value
session, err := getSession(ctx, sessionToken)
@@ -1440,7 +1428,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) {
return
}
expiration := time.Now().Add(1200 * time.Second)
expiration := time.Now().Add(3600 * time.Second)
http.SetCookie(resp, &http.Cookie{
Name: "session_token",
Value: UserInfo.Session,
@@ -1449,31 +1437,6 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) {
returnData := fmt.Sprintf(`{"success": true, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}]}`, UserInfo.Session, expiration.Unix())
//b, err := json.Marshal(UserInfo)
//if err != nil {
// log.Printf("Failed marshalling: %s", err)
// resp.WriteHeader(401)
// resp.Write([]byte(`{"success": false}`))
// return
//}
// Adding to cache here
// Only keeping it in for 24 hours
//item := &memcache.Item{
// Key: c.Value,
// Value: b,
// Expiration: time.Hour * 24,
//}
//if err := memcache.Add(ctx, item); err == memcache.ErrNotStored {
// if err := memcache.Set(ctx, item); err != nil {
// log.Printf("Error setting item: %v", err)
// }
//} else if err != nil {
// log.Printf("error adding item: %v", err)
//} else {
// log.Printf("Set cache for %s", item.Key)
//}
resp.WriteHeader(200)
resp.Write([]byte(returnData))
}
@@ -2008,7 +1971,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) {
// FIXME - have timeout here
if len(Userdata.Session) != 0 {
//log.Println("Nonexisting session")
expiration := time.Now().Add(1200 * time.Second)
expiration := time.Now().Add(3600 * time.Second)
http.SetCookie(resp, &http.Cookie{
Name: "session_token",
@@ -2034,7 +1997,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) {
http.SetCookie(resp, &http.Cookie{
Name: "session_token",
Value: sessionToken.String(),
Expires: time.Now().Add(1200 * time.Second),
Expires: time.Now().Add(3600 * time.Second),
})
// ADD TO DATABASE
backend/go-app/walkoff.go
+81 -65
View File
@@ -1338,86 +1338,102 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
// Check every app action and param to see whether they exist
newActions = []Action{}
for _, action := range workflow.Actions {
curapp := WorkflowApp{}
// FIXME - can this work with ONLY AppID?
for _, app := range workflowApps {
if app.ID == action.AppID {
curapp = app
break
}
reservedApps := []string{
"0ca8887e-b4af-4e3e-887c-87e9d3bc3d3e",
}
if app.Name == action.AppName && app.AppVersion == action.AppVersion {
curapp = app
builtin := false
for _, id := range reservedApps {
if id == action.AppID {
builtin = true
break
}
}
// Check to see if the whole app is valid
if curapp.Name != action.AppName {
log.Printf("App %s doesn't exist.", action.AppName)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App %s doesn't exist"}`, action.AppName)))
return
}
if builtin {
newActions = append(newActions, action)
} else {
curapp := WorkflowApp{}
// FIXME - can this work with ONLY AppID?
for _, app := range workflowApps {
if app.ID == action.AppID {
curapp = app
break
}
// Check tosee if the appaction is valid
curappaction := WorkflowAppAction{}
for _, curAction := range curapp.Actions {
if action.Name == curAction.Name {
curappaction = curAction
break
}
log.Println(action.Name, curAction.Name)
}
// Check to see if the action is valid
if curappaction.Name != action.Name {
log.Printf("Appaction %s doesn't exist.", action.Name)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// FIXME - check all parameters to see if they're valid
// Includes checking required fields
newParams := []WorkflowAppActionParameter{}
for _, param := range curappaction.Parameters {
found := false
// Handles check for parameter exists + value not empty in used fields
for _, actionParam := range action.Parameters {
if actionParam.Name == param.Name {
found = true
if actionParam.Value == "" && actionParam.Variant == "STATIC_VALUE" && actionParam.Required == true {
log.Printf("Appaction %s with required param '%s' is empty.", action.Name, param.Name)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name)))
return
}
if actionParam.Variant == "" {
actionParam.Variant = "STATIC_VALUE"
}
newParams = append(newParams, actionParam)
if app.Name == action.AppName && app.AppVersion == action.AppVersion {
curapp = app
break
}
}
// Handles check for required params
if !found && param.Required {
log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name)
// Check to see if the whole app is valid
if curapp.Name != action.AppName {
log.Printf("App %s doesn't exist.", action.AppName)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App %s doesn't exist"}`, action.AppName)))
return
}
// Check tosee if the appaction is valid
curappaction := WorkflowAppAction{}
for _, curAction := range curapp.Actions {
if action.Name == curAction.Name {
curappaction = curAction
break
}
log.Println(action.Name, curAction.Name)
}
// Check to see if the action is valid
if curappaction.Name != action.Name {
log.Printf("Appaction %s doesn't exist.", action.Name)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
// FIXME - check all parameters to see if they're valid
// Includes checking required fields
action.Parameters = newParams
newActions = append(newActions, action)
newParams := []WorkflowAppActionParameter{}
for _, param := range curappaction.Parameters {
found := false
// Handles check for parameter exists + value not empty in used fields
for _, actionParam := range action.Parameters {
if actionParam.Name == param.Name {
found = true
if actionParam.Value == "" && actionParam.Variant == "STATIC_VALUE" && actionParam.Required == true {
log.Printf("Appaction %s with required param '%s' is empty.", action.Name, param.Name)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name)))
return
}
if actionParam.Variant == "" {
actionParam.Variant = "STATIC_VALUE"
}
newParams = append(newParams, actionParam)
}
}
// Handles check for required params
if !found && param.Required {
log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
action.Parameters = newParams
newActions = append(newActions, action)
}
}
workflow.Actions = newActions
backend/tests/forparser.py
+158
View File
File diff suppressed because one or more lines are too long
backend/tests/run_function.py
+1 -1
View File
@@ -53,7 +53,7 @@ if __name__ == "__main__":
"authorization": "hey",
}
apikey = "eyJhbGciOiJSUzI1NiIsImtpZCI6IjYwZjQwNjBlNThkNzVmZDNmNzBiZWZmODhjNzk0YTc3NTMyN2FhMzEiLCJ0eXAiOiJKV1QifQ.eyJhdWQiOiJodHRwczovL3NodWZmbGVyLmlvL2FwaS92MS93b3JrZmxvd3MvMWQ5ZDhjZTItNTY2ZS00YzNmLThhMzctNWQ2YzdkMjAwMGI1L2V4ZWN1dGUiLCJhenAiOiIxMDMwNzY3ODIwNjE0MjQ2MTg0MjIiLCJlbWFpbCI6InNjaGVkdWxlckBzaHVmZmxlLTI0MTUxNy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJleHAiOjE1NjU1Mjc1NTEsImlhdCI6MTU2NTUyMzk1MSwiaXNzIjoiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tIiwic3ViIjoiMTAzMDc2NzgyMDYxNDI0NjE4NDIyIn0.r0EDq9fjhf_5CPTiltyfk_L3uYJp577Uy0yYPcCAl2nv50_z_oUtbWGBpQLL8gcj-NGd3g4E52Qur8k6hCMIQweLS6WAb1279vGffEoCNDfkWb3Oy-yJGP1kzwLvqFJqnHLkSWYXNWvSyWnEimW8Rryx_m1BXS5wcA8l4NIr83kS7fPZrTwjnwFSeGSThwk91DVARzapQb8r0GEgOUyHZ1aBXnV98mikzSUt-5xFKe9eMdD22YJAj0Ru-DxAxs5nOqghX4PMRysWjshjOMrlR1piPWxqAmewp8YKZDCQ5gXskpeAFBDoULT971Wsx_NCohnJsFqx1JfPS9ZYMTW2oQ"
apikey = ""
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {apikey}"
frontend/src/AngularWorkflow.js
+62 -6
View File
@@ -285,7 +285,6 @@ const AngularWorkflow = (props) => {
currentnode = currentnode[0]
const outgoingEdges = currentnode.outgoers('edge')
const incomingEdges = currentnode.incomers('edge')
console.log("NODE: ", currentnode)
//currentnode.removeClass('success-highlight failure-highlight executing-highlight')
switch (item.status) {
@@ -668,6 +667,56 @@ const AngularWorkflow = (props) => {
// // FIXME - handle this, as we can't have more than one of each :)
// //setVersionedApps(newapps)
//}
// Builtin actions that should ran in Worker and not apps
const getExtraApps = () => {
const data = [{
name: "Filter",
is_valid: true,
id: "0ca8887e-b4af-4e3e-887c-87e9d3bc3d3e",
link: "https://shuffler.io",
app_version: "1.0.0",
generated: true,
downloaded: false,
sharing: false,
verified: false,
tested: false,
owner: "",
private_id: "",
description: "Filter",
environment: "Shuffle",
small_image: "",
large_image: "",
contact_info: {name: "", url: ""},
authentication: {required: false, parameters: [],},
actions: [{
description: "Filter cases",
id: "",
name: "filter_cases",
node_type: "action",
environment: "Shuffle",
sharing: false,
private_id: "",
app_id: "",
authentication: null,
tested: true,
parameters: [{
description: "",
id: "",
name: "Field to look for",
example: "$testing_1.#.id",
value: "",
multiline: true,
action_field: "",
variant: "",
required: true,
schema: {type: "string"},
}]
}],
}]
return data
}
const getApps = () => {
fetch(globalUrl+"/api/v1/workflows/apps", {
@@ -688,9 +737,12 @@ const AngularWorkflow = (props) => {
.then((responseJson) => {
// FIXME - handle versions on left bar
//handleAppVersioning(responseJson)
//var tmpapps = []
//tmpapps = tmpapps.concat(getExtraApps())
//tmpapps = tmpapps.concat(responseJson)
setApps(responseJson)
setFilteredApps(responseJson)
})
})
.catch(error => {
alert.error(error.toString())
});
@@ -780,6 +832,7 @@ const AngularWorkflow = (props) => {
}
console.log("Selected: ", data.id)
console.log(curaction)
setRequiresAuthentication(curapp.authentication.required)
setSelectedApp(curapp)
@@ -1848,7 +1901,7 @@ const AngularWorkflow = (props) => {
authentication: [],
}
console.log(newAppData)
// const image = "url("+app.large_image+")"
// FIXME - find the cytoscape offset position
// Can this be done with zoom calculations?
@@ -2358,7 +2411,8 @@ const AngularWorkflow = (props) => {
<b>{data.name}: </b>
</div>
<Tooltip color="primary" title="Static data" placement="top">
<div style={{cursor: "pointer", color: staticcolor}} onClick={() => {
<div style={{cursor: "pointer", color: staticcolor}} onClick={(e) => {
e.preventDefault()
changeActionParameterVariant("STATIC_VALUE", count)
}}>
<CreateIcon />
@@ -2366,7 +2420,8 @@ const AngularWorkflow = (props) => {
</Tooltip>
&nbsp;|&nbsp;
<Tooltip color="primary" title="Data from previous action" placement="top">
<div style={{cursor: "pointer", color: actioncolor}} onClick={() => {
<div style={{cursor: "pointer", color: actioncolor}} onClick={(e) => {
e.preventDefault()
changeActionParameterVariant("ACTION_RESULT", count)
}}>
<AppsIcon />
@@ -2374,7 +2429,8 @@ const AngularWorkflow = (props) => {
</Tooltip>
&nbsp;|&nbsp;
<Tooltip color="primary" title="Use local variable" placement="top">
<div style={{cursor: "pointer", color: varcolor}} onClick={() => {
<div style={{cursor: "pointer", color: varcolor}} onClick={(e) => {
e.preventDefault()
changeActionParameterVariant("WORKFLOW_VARIABLE", count)
}}>
<FavoriteBorderIcon />
frontend/src/App.js
+2
View File
@@ -92,11 +92,13 @@ const App = (message, props) => {
})
.then(response => response.json())
.then(responseJson => {
console.log(responseJson)
if (responseJson.success === true) {
setUserData(responseJson)
setIsLoggedIn(true)
// Updating cookie every request
console.log("COOKIES: ", cookies)
for (var key in responseJson["cookies"]) {
setCookie(responseJson["cookies"][key].key, responseJson["cookies"][key].value, {path: "/"})
}
frontend/src/Apps.js
+8 -11
View File
@@ -55,6 +55,11 @@ const Apps = (props) => {
useEffect(() => {
if (apps.length <= 0 && firstrequest) {
document.title = "Shuffle - Apps"
if (!isLoggedIn && isLoaded) {
window.location = "/login"
}
setFirstrequest(false)
getApps()
}
@@ -79,7 +84,7 @@ const Apps = (props) => {
const getApps = () => {
fetch(globalUrl+"/api/v1/workflows/apps", {
method: 'GET',
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
@@ -93,7 +98,7 @@ const Apps = (props) => {
return response.json()
})
.then((responseJson) => {
.then((responseJson) => {
setApps(responseJson)
setFilteredApps(responseJson)
if (responseJson.length > 0) {
@@ -452,15 +457,7 @@ const Apps = (props) => {
</div>
</div>
:
<div style={{width: "600px", margin: "auto", color: "white", paddingBottom: "50px"}}>
<h2>Available integrations</h2>
<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
{apps.map(data => {
return (
appPaper(data)
)
})}
</div>
null
// Gets the URL itself (hopefully this works in most cases?
// Will then forward the data to an internal endpoint to validate the api
frontend/src/Workflows.js
+5 -4
View File
@@ -22,6 +22,7 @@ import PlayArrowIcon from '@material-ui/icons/PlayArrow';
//import JSONPrettyMon from 'react-json-pretty/dist/monikai'
import ReactJson from 'react-json-view'
import {Link} from 'react-router-dom';
import { useAlert } from "react-alert";
import Dialog from '@material-ui/core/Dialog';
@@ -173,7 +174,7 @@ const Workflows = (props) => {
})
.then((responseJson) => {
setWorkflowExecutions(responseJson)
if (responseJson.length > 0 && Object.getOwnPropertyNames(selectedExecution).length === 0) {
if (responseJson.length > 0) {
setSelectedExecution(responseJson[0])
}
})
@@ -398,13 +399,13 @@ const Workflows = (props) => {
}
}}>
<Grid item style={{flex: "1", justifyContent: "center"}}>
<a href={"/workflows/"+data.id}>
<Link to={"/workflows/"+data.id}>
<Tooltip color="primary" title="Edit workflow" placement="bottom">
<Button style={{}} color="primary" variant="text" style={{marginRight: 10}} onClick={() => {}}>
<EditIcon style={{marginRight: 10}}/> Edit
</Button>
</Tooltip>
</a>
</Link>
<Tooltip color="primary" title="Execute workflow" placement="bottom">
<Button style={{}} color="secondary" variant="text" onClick={() => executeWorkflow(data.id)}>
<PlayArrowIcon />
@@ -889,7 +890,7 @@ const Workflows = (props) => {
</div>
<div>
<p>
<b>Shuffle</b> is a flexible, easy to use, automation framework allowing users to integrate their services and devices to reduce the amount of manual labor required for those tasks. <a href="/docs/workflows" style={{textDecoration: "none", color: "#f85a3e"}}>Click here for more information.</a>
<b>Shuffle</b> is a flexible, easy to use, automation framework allowing users to integrate their services and devices to reduce the amount of manual labor required for those tasks. <Link to="/docs/workflows" style={{textDecoration: "none", color: "#f85a3e"}}>Click here for more information.</Link>
</p>
</div>
<div>
functions/onprem/app_sdk/app_base.py
+161 -95
View File
@@ -86,8 +86,6 @@ class AppBase:
"execution_id": self.current_execution_id
}
self.logger.info("Auth: %s", tmpdata)
self.logger.info("Before FULLEXEC stream result")
ret = requests.post(
"%s/api/v1/streams/results" % (self.url),
@@ -109,10 +107,9 @@ class AppBase:
# Takes a workflow execution as argument
# Returns a string if the result is single, or a list if it's a list
# Not implemented: lists
def get_json_value(execution_data, input_data):
parsersplit = input_data.split(".")
actionname = parsersplit[0][1:]
actionname = parsersplit[0][1:].replace(" ", "_", -1)
print(f"Actionname: {actionname}")
# 1. Find the action
@@ -122,12 +119,14 @@ class AppBase:
baseresult = execution_data["execution_argument"]
else:
for result in execution_data["results"]:
if result["action"]["label"].lower() == actionname.lower():
resultlabel = result["action"]["label"].replace(" ", "_", -1).lower()
if resultlabel.lower() == actionname.lower():
baseresult = result["result"]
break
except KeyError as error:
print(f"Error: {error}")
print(f"After first trycatch")
# 2. Find the JSON data
@@ -145,12 +144,22 @@ class AppBase:
return baseresult
try:
cnt = 0
for value in parsersplit[1:]:
cnt += 1
if value == "#":
print("HANDLE RECURSIVE LOOP")
pass
# FIXME - not recursive - should go deeper if there are more #
print("HANDLE RECURSIVE LOOP ")
returnlist = []
for innervalue in basejson:
#print("Value: %s" % value[parsersplit[cnt+1]])
returnlist.append(innervalue[parsersplit[cnt+1]])
# Example format: ${[]}$
return "${%s%s}$" % (parsersplit[cnt+1], json.dumps(returnlist))
else:
print("BASE: ", basejson)
if isinstance(basejson[value], str):
print(f"LOADING STRING '%s' AS JSON" % basejson[value])
try:
@@ -160,24 +169,23 @@ class AppBase:
return basejson[value]
else:
basejson = basejson[value]
except KeyError as e:
print(f"Keyerror: {e}")
return basejson
return "KeyError: %s" % e
except IndexError as e:
print(f"Indexerror: {e}")
return basejson
return "IndexError: %s" % e
return basejson
def parse_params(action, fullexecution, parameter):
jsonparsevalue = "$."
match = ".*([$]{1}([a-zA-Z0-9()# _-]+\.?){1,})"
# Regex to find all the things
if parameter["variant"] == "STATIC_VALUE":
data = parameter["value"]
self.logger.debug(f"\n\nHandle static data with JSON: {data}\n\n")
match = ".*([$]{1}(\w+\.?){1,})"
actualitem = re.findall(match, data, re.MULTILINE)
self.logger.info("PARSED: %s" % actualitem)
if len(actualitem) > 0:
@@ -192,11 +200,13 @@ class AppBase:
parameter["value"] = parameter["value"].replace(to_be_replaced, value)
elif isinstance(value, dict):
parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value))
else:
print("Unknown type %s" % type(value))
try:
parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value))
except json.decoder.JSONDecodeError as e:
parameter["value"] = parameter["value"].replace(to_be_replaced, value)
# Check if json inside string
#self.logger.info(f"CONVERT DATA FROM {parameter['value']} to {data}")
#parameter["value"] = data
if parameter["variant"] == "WORKFLOW_VARIABLE":
for item in fullexecution["workflow"]["workflow_variables"]:
@@ -210,81 +220,47 @@ class AppBase:
# GET THE LABEL'S RESULT
tmpvalue = ""
print(parameter["action_field"])
self.logger.info("ACTION FIELD: %s" % parameter["action_field"])
#"$%s%s" %
fullname = "$"
if parameter["action_field"] == "Execution Argument":
tmpvalue = fullexecution["execution_argument"]
fullname += "exec"
else:
self.logger.info("WORKFLOW EXEC BELOW")
self.logger.info(fullexecution)
self.logger.info(fullexecution["results"])
self.logger.info(fullexecution["workflow"]["actions"])
self.logger.info("ACTIONS ABOVE")
# redundancy..
fullname += parameter["action_field"]
tmpid = ""
for item in fullexecution["workflow"]["actions"]:
if item["label"] == parameter["action_field"]:
tmpid = item["id"]
if not tmpid:
self.logger.error("Value not found for that id: %s. Exiting" % parameter["action_field"])
raise Exception("Value for %s was not found in workflow actions" % parameter["action_field"])
for subresult in fullexecution["results"]:
if subresult["action"]["id"] == tmpid:
tmpvalue = subresult["result"]
break
if not tmpvalue:
self.logger.error("Value not found for label %s. Exiting" % parameter["action_field"])
raise Exception("Value for %s was not found" % parameter["action_field"])
# Override locally with JSON data
if parameter["value"].startswith(jsonparsevalue):
parsersplit = parameter["value"].split(".")
# Convert to json here
self.logger.info("JSON HANDLING: %s" % tmpvalue)
tmpvalue = tmpvalue.replace("\'", "\"")
try:
if isinstance(tmpvalue, str):
newtmp = json.loads(tmpvalue)
except json.decoder.JSONDecodeError as e:
raise Exception("JSON error: %s" % e)
try:
#previousvalue = parsersplit[1]
for value in parsersplit[1:]:
# Might need to be recursive here, because it can go
# multiple layers ($.result.#.test.users.#.name)
# That would give executions of:
# 1 + result.length + users.length
# This is also just for one param
#if parsersplit[1:][count] == "#":
if value == "#":
# This means we already have an array
# for item in newtmp:
self.logger.info("THERE SHOULD BE A LOOP HERE")
# This works, but it needs to be split into multiples hurr
# Whenever there is a loop, there is a need to
# check whether there are more loops, then do
# recursion to all the bottom leaves
#paramnamevalue.append(newtmp
newtmp = newtmp[0]
# Choose numero uno which will then be handled by the next again
# params[parameter["name"]].append(value.nextitem)
else:
newtmp = newtmp[value]
except KeyError as e:
return "KeyError: %s" % e, ""
except IndexError as e:
return "IndexError: %s" % e, ""
parameter["value"] = str(newtmp)
fullname += parameter["value"][2:]
else:
parameter["value"] = tmpvalue
fullname = "$%s" % parameter["action_field"]
self.logger.info("Fullname: %s" % fullname)
actualitem = re.findall(match, fullname, re.MULTILINE)
self.logger.info("PARSED: %s" % actualitem)
if len(actualitem) > 0:
for replace in actualitem:
try:
to_be_replaced = replace[0]
except IndexError:
print("Nothing to replace?: " % e)
continue
# This will never be a loop aka multi argument
parameter["value"] = to_be_replaced
value = get_json_value(fullexecution, to_be_replaced)
if isinstance(value, str):
parameter["value"] = parameter["value"].replace(to_be_replaced, value)
elif isinstance(value, dict):
parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value))
else:
print("Unknown type %s" % type(value))
try:
parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value))
except json.decoder.JSONDecodeError as e:
parameter["value"] = parameter["value"].replace(to_be_replaced, value)
return "", parameter["value"]
@@ -449,22 +425,112 @@ class AppBase:
# which is super fast, but has a bad overview (potentially good tho)
calltimes = 1
result = ""
paramiter = []
for parameter in action["parameters"]:
#self.logger.info(parameter)
#print(fullexecution)
all_executions = []
# Multi_parameter has the data for each. variable
minlength = 0
multi_parameters = json.loads(json.dumps(params))
multiexecution = False
for parameter in action["parameters"]:
check, value = parse_params(action, fullexecution, parameter)
if check:
raise Exception(check)
params[parameter["name"]] = value
# p["value"]
# Custom format for ${name[0,1,2,...]}$
submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})"
actualitem = re.findall(submatch, value, re.MULTILINE)
if len(actualitem) > 0:
multiexecution = True
# This is here to handle for loops within variables.. kindof
# 1. Find the length of the longest array
# 2. Build an array with the base values based on parameter["value"]
# 3. Get the n'th value of the generated list from values
# 4. Execute all n answers
replacements = {}
for replace in actualitem:
try:
to_be_replaced = replace[0]
actualitem = replace[2]
except IndexError:
continue
itemlist = json.loads(actualitem)
if len(itemlist) > minlength:
minlength = len(itemlist)
replacements[to_be_replaced] = actualitem
# This is a result array for JUST this value..
# What if there are more?
resultarray = []
for i in range(0, minlength):
tmpitem = json.loads(json.dumps(parameter["value"]))
for key, value in replacements.items():
replacement = json.loads(value)[i]
tmpitem = tmpitem.replace(key, replacement, -1)
resultarray.append(tmpitem)
# With this parameter ready, add it to... a greater list of parameters. Rofl
multi_parameters[parameter["name"]] = resultarray
else:
print("Hello, in here?: %s" % value)
params[parameter["name"]] = value
multi_parameters[parameter["name"]] = value
# FIXME - this is horrible, but works for now
#for i in range(calltimes):
result += await func(**params)
if not multiexecution:
print("Params: %s" % params)
print("RUNNING NORMAL EXECUTION")
result += await func(**params)
else:
print("MULTI EXECUTION: ", multi_parameters)
# 1. Use number of executions based on longest array
# 2. Find the right value from the parsed multi_params
results = []
json_object = False
for i in range(0, minlength):
# To be able to use the results as a list:
baseparams = json.loads(json.dumps(multi_parameters))
try:
for key, value in baseparams.items():
if isinstance(value, list):
baseparams[key] = value[i]
except IndexError as e:
print("IndexError: %s" % e)
baseparams[key] = "IndexError: %s" % e
except KeyError as e:
print("KeyError: %s" % e)
baseparams[key] = "KeyError: %s" % e
#print("Running with params %s" % baseparams)
ret = await func(**baseparams)
print("Inner ret: %s" % ret)
try:
results.append(json.loads(ret))
json_object = True
except json.decoder.JSONDecodeError as e:
results.append(ret)
# Dump the result as a string of a list
print("RESULTS: %s" % results)
if isinstance(results, list):
print("JSON OBJECT? ", json_object)
if json_object:
result = json.dumps(results)
else:
result = "[\""+"\", \"".join(results)+"\"]"
else:
print("Normal result?")
result = results
print("RESULT: %s" % result)
action_result["status"] = "SUCCESS"
action_result["result"] = str(result)
@@ -491,7 +557,7 @@ class AppBase:
action_result["completed_at"] = int(time.time())
# I wonder if this actually works
#self.logger.info("Before last stream result")
self.logger.info("Before last stream result")
try:
ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result)
self.logger.info("Result: %d" % ret.status_code)
functions/onprem/app_sdk/update_dockerhub → functions/onprem/app_sdk/build.sh
View File
functions/onprem/worker/worker.bin
BIN
View File
Binary file not shown.
functions/onprem/worker/worker.go
+37 -10
View File
@@ -78,6 +78,7 @@ type WorkflowExecution struct {
type Action struct {
AppName string `json:"app_name" datastore:"app_name"`
AppVersion string `json:"app_version" datastore:"app_version"`
AppID string `json:"app_id" datastore:"app_id"`
Errors []string `json:"errors" datastore:"errors"`
ID string `json:"id" datastore:"id"`
IsValid bool `json:"is_valid" datastore:"is_valid"`
@@ -258,7 +259,7 @@ func shutdown(executionId, workflowId string) {
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", authorization)
//req.Header.Add("Authorization", authorization)
client := &http.Client{}
_, err = client.Do(req)
if err != nil {
@@ -336,6 +337,22 @@ func removeContainer(containername string) error {
return nil
}
func runFilter(workflowExecution WorkflowExecution, action Action) {
// 1. Get the parameter $.#.id
if action.Label == "filter_cases" && len(action.Parameters) > 0 {
if action.Parameters[0].Variant == "ACTION_RESULT" {
param := action.Parameters[0]
value := param.Value
_ = value
// Loop cases.. Hmm, that's tricky
}
} else {
log.Printf("No handler for filter %s with %d params", action.Label, len(action.Parameters))
}
}
func handleExecution(client *http.Client, req *http.Request, workflowExecution WorkflowExecution) error {
// if no onprem runs (shouldn't happen, but extra check), exit
// if there are some, load the images ASAP for the app
@@ -364,7 +381,6 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W
}
toExecuteOnprem = append(toExecuteOnprem, action.ID)
actionName := fmt.Sprintf("%s:%s_%s", baseimagename, action.AppName, action.AppVersion)
found := false
for _, app := range onpremApps {
@@ -383,21 +399,25 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W
}
pullOptions := types.ImagePullOptions{}
_ = pullOptions
for _, image := range onpremApps {
log.Printf("Image: %s", image)
// Kind of gambling that the image exists.
if strings.Contains(image, " ") {
image = strings.ReplaceAll(image, " ", "-")
}
reader, err := dockercli.ImagePull(context.Background(), image, pullOptions)
if err != nil {
log.Printf("Failed getting %s. The app is missing or some other issue", image)
//shutdown(workflowExecution.ExecutionId)
}
// FIXME: Reimplement for speed later
// Skip to make it faster
//reader, err := dockercli.ImagePull(context.Background(), image, pullOptions)
//if err != nil {
// log.Printf("Failed getting %s. The app is missing or some other issue", image)
// shutdown(workflowExecution.ExecutionId)
//}
//io.Copy(os.Stdout, reader)
_ = reader
log.Printf("Successfully downloaded and built %s", image)
////io.Copy(os.Stdout, reader)
//_ = reader
//log.Printf("Successfully downloaded and built %s", image)
}
// Process the parents etc. How?
@@ -555,12 +575,19 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W
// marshal action and put it in there rofl
log.Printf("Time to execute %s with app %s:%s, function %s, env %s with %d parameters.", action.ID, action.AppName, action.AppVersion, action.Name, action.Environment, len(action.Parameters))
actionData, err := json.Marshal(action)
if err != nil {
log.Printf("Failed unmarshalling action: %s", err)
continue
}
if action.AppID == "0ca8887e-b4af-4e3e-887c-87e9d3bc3d3e" {
log.Printf("\nShould run filter: %#v\n\n", action)
runFilter(workflowExecution, action)
continue
}
//log.Println(string(actionData))
// FIXME - add proper FUNCTION_APIKEY from user definition
env := []string{