Major changes to multi-file in Shuffle. Not finished yet.

This commit is contained in:
frikky
2020-12-06 05:30:55 +01:00
parent 0ef55b2f03
commit a20b848aba
7 changed files with 214 additions and 154 deletions
+65 -36
View File
@@ -57,47 +57,68 @@ class AppBase:
# - How can you download / stream a file?
# - Can you decide if you want a stream or the files directly?
def get_file(self, value):
print("INSIDE GET_FILE")
full_execution = self.full_execution
org_id = full_execution["workflow"]["execution_org"]["id"]
print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value))
get_path = "/api/v1/files/%s?execution_id=%s" % (value, full_execution["execution_id"])
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer %s" % self.authorization
}
if isinstance(value, list):
print("IS LIST!")
#if len(value) == 1:
# value = value[0]
else:
value = [value]
ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers)
print("RET1: %s" % ret1.text)
if ret1.status_code != 200:
returns = []
for item in value:
print("VALUE: %s" % item)
if len(item) != 36:
print("Bad length for value")
continue
#return {
# "filename": "",
# "data": "",
# "success": False,
#}
get_path = "/api/v1/files/%s?execution_id=%s" % (item, full_execution["execution_id"])
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer %s" % self.authorization
}
ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers)
print("RET1: %s" % ret1.text)
if ret1.status_code != 200:
returns.append({
"filename": "",
"data": "",
"success": False,
})
continue
content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"])
ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers)
print("Ret2: %s" % ret2.text)
if ret2.status_code == 200:
tmpdata = ret1.json()
returndata = {
"success": True,
"filename": tmpdata["filename"],
"data": ret2.content,
}
returns.append(returndata)
if len(returns) == 0:
return {
"filename": "",
"data": "",
"success": False,
"filename": "",
"data": b"",
}
content_path = "/api/v1/files/%s/content?execution_id=%s" % (value, full_execution["execution_id"])
ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers)
print("Ret2: %s" % ret2.text)
if ret2.status_code == 200:
tmpdata = ret1.json()
returndata = {
"success": True,
"filename": tmpdata["filename"],
"data": ret2.content,
}
# open('facebook.ico', 'wb').write(r.content)
return returndata
return {
"success": False,
"filename": "",
"data": b"",
}
elif len(returns) == 1:
return returns[0]
else:
return returns
# Sets files in the backend
def set_files(self, infiles):
@@ -522,6 +543,7 @@ class AppBase:
# Magical way of returning which makes app sdk identify
# it as multi execution
return newvalue, True
elif len(actualitem) > 0:
# FIXME: This is absolutely not perfect.
print("In recursion v2: ", actualitem)
@@ -704,7 +726,7 @@ class AppBase:
print("SET DATA WRAPPER TO %s!" % parsersplit[-1])
parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data))
print("Before last return")
print("Before last return with %s" % appendresult)
return str(parseditem)+str(appendresult), is_loop
# Parses parameters sent to it and returns whether it did it successfully with the values found
@@ -1205,15 +1227,22 @@ class AppBase:
if listitem in filteredlist:
continue
filteredlist.append(listitem)
# FIXME: Subsub required?. Recursion!
# Basically multiply what we have with the outer loop?
#
if isinstance(listitem, list):
for subitem in listitem:
filteredlist.append(subitem)
else:
filteredlist.append(listitem)
#print("New list length: %d" % len(filteredlist))
if len(filteredlist) > 1:
print("Calculating new multi-loop length with %d lists" % len(filteredlist))
tmplength = 1
for innerlist in filteredlist:
print("List length: %d. %d*%d" % (len(innerlist), len(innerlist), tmplength))
tmplength = len(innerlist)*tmplength
print("List length: %d. %d*%d" % (tmplength, len(innerlist), tmplength))
minlength = tmplength
@@ -1257,6 +1286,7 @@ class AppBase:
except ValueError:
result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres)
print("Can't handle type %s value from function" % (type(newres)))
print("POST NEWRES RESULT: ", result)
else:
print("APP_SDK DONE: Starting MULTI execution (length: %d) with values %s" % (minlength, multi_parameters))
@@ -1286,7 +1316,6 @@ class AppBase:
if isinstance(value, list):
try:
newvalue = value[i]
print("NEWVALUE: %s" % newvalue)
except IndexError:
pass
+1 -1
View File
@@ -1,7 +1,7 @@
package main
/*
Handles files within Workflows.
Handles files within Workflows.of Shuffle
*/
import (
+2 -2
View File
@@ -6813,7 +6813,7 @@ func remoteOrgJobHandler(org Org, interval int) error {
respBody, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("Failed body read in job sync: %s", err)
log.Printf("[ERROR] Failed body read in job sync: %s", err)
return err
}
@@ -6821,7 +6821,7 @@ func remoteOrgJobHandler(org Org, interval int) error {
err = remoteOrgJobController(org, respBody)
if err != nil {
log.Printf("Failed job controller run: %s", err)
log.Printf("[ERROR] Failed job controller run for %s: %s", respBody, err)
return err
}
return nil
+1 -1
View File
@@ -5765,7 +5765,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) {
}
// Query for the specifci workflowId
q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(20)
q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(30)
var workflowExecutions []WorkflowExecution
_, err = dbclient.GetAll(ctx, q, &workflowExecutions)
if err != nil {
+2 -2
View File
@@ -2,7 +2,7 @@ version: '3'
services:
frontend:
#build: ./frontend
image: ghcr.io/frikky/shuffle-frontend:0.8.0
image: ghcr.io/frikky/shuffle-frontend:0.8.3
container_name: shuffle-frontend
hostname: shuffle-frontend
ports:
@@ -17,7 +17,7 @@ services:
- backend
backend:
build: ./backend
image: ghcr.io/frikky/shuffle-backend:0.8.11
image: ghcr.io/frikky/shuffle-backend:0.8.3
container_name: shuffle-backend
hostname: ${BACKEND_HOSTNAME}
# Here for debugging:
+12 -7
View File
@@ -116,7 +116,7 @@ const Apps = (props) => {
const [apps, setApps] = React.useState([])
const [filteredApps, setFilteredApps] = React.useState([])
const [validation, setValidation] = React.useState(false)
const [isLoading, setIsLoading] = React.useState(false)
const [isLoading, setIsLoading] = React.useState(true)
const [appSearchLoading, setAppSearchLoading] = React.useState(false)
const [selectedAction, setSelectedAction] = React.useState({})
const [searchBackend, setSearchBackend] = React.useState(false)
@@ -207,6 +207,7 @@ const Apps = (props) => {
credentials: "include",
})
.then((response) => {
setIsLoading(false)
if (response.status !== 200) {
console.log("Status not 200 for apps :O!")
}
@@ -231,6 +232,7 @@ const Apps = (props) => {
})
.catch(error => {
alert.error(error.toString())
setIsLoading(false)
});
}
@@ -742,7 +744,7 @@ const Apps = (props) => {
<Divider style={{height: 1, backgroundColor: dividerColor, marginTop: 20, marginBottom: 20}} />
<div style={{}}>
<Button
variant="text"
variant="contained"
component="label"
color="primary"
style={{marginRight: 10, }}
@@ -932,11 +934,14 @@ const Apps = (props) => {
}
</Paper>
:
<Paper square style={uploadViewPaperStyle}>
<h4 style={{margin: 10}}>
No apps have been created, uploaded or downloaded yet. Click "Load existing apps" above to get the baseline. This may take a while as its building docker images.
</h4>
</Paper>
isLoading ?
<CircularProgress style={{width: 40, height: 40, margin: "auto"}}/>
:
<Paper square style={uploadViewPaperStyle}>
<h4 style={{margin: 10}}>
No apps have been created, uploaded or downloaded yet. Click "Load existing apps" above to get the baseline. This may take a while as its building docker images.
</h4>
</Paper>
}
</div>
</div>
+131 -105
View File
@@ -74,6 +74,7 @@ const Workflows = (props) => {
const [update, setUpdate] = React.useState("test");
const [deleteModalOpen, setDeleteModalOpen] = React.useState(false);
const [editingWorkflow, setEditingWorkflow] = React.useState({})
const [executionLoading, setExecutionLoading] = React.useState(false)
const { start, stop } = useInterval({
duration: 5000,
startImmediate: false,
@@ -245,6 +246,7 @@ const Workflows = (props) => {
}
const getWorkflowExecution = (id) => {
setExecutionLoading(true)
fetch(globalUrl+"/api/v1/workflows/"+id+"/executions", {
method: 'GET',
headers: {
@@ -254,6 +256,7 @@ const Workflows = (props) => {
credentials: "include",
})
.then((response) => {
setExecutionLoading(false)
if (response.status !== 200) {
console.log("Status not 200 for WORKFLOW EXECUTION :O!")
}
@@ -275,6 +278,7 @@ const Workflows = (props) => {
}
})
.catch(error => {
setExecutionLoading(false)
alert.error(error.toString())
});
}
@@ -858,10 +862,17 @@ const Workflows = (props) => {
</div>
)
}
return (
<h4>
There are no executiondetails yet. Click "execute" to run your first one.
</h4>
executionLoading ?
<div style={{marginTop: 25, textAlign: "center"}}>
<CircularProgress />
</div>
:
<h4>
There are no executiondetails yet. Click "execute" to run your first one.
</h4>
)
}
@@ -880,9 +891,14 @@ const Workflows = (props) => {
)
}
return (
<h4>
There are no executions for this workflow yet
</h4>
executionLoading ?
<div style={{marginTop: 25, textAlign: "center"}}>
<CircularProgress />
</div>
:
<h4>
There are no executions for this workflow yet
</h4>
)
}
@@ -1111,107 +1127,113 @@ const Workflows = (props) => {
workflowViewStyle.display = "none"
}
const workflowView = workflows.length > 0 ?
<div style={viewStyle}>
<div style={workflowViewStyle}>
<div style={{display: "flex"}}>
<div style={{flex: "4"}}>
<h2>Workflows</h2>
const WorkflowView = () => {
if (workflows.length === 0) {
return (
<div style={emptyWorkflowStyle}>
<Paper style={boxStyle}>
<div>
<h2>Welcome to Shuffle</h2>
</div>
<div>
<p>
<b>Shuffle</b> is a flexible, easy to use, automation platform allowing users to integrate their services and devices freely. It's made to significantly reduce the amount of manual labor, and is focused on security applications. <a href="/docs/about" style={{textDecoration: "none", color: "#f85a3e"}}>Click here to learn more.</a>
</p>
</div>
<div>
If you want to jump straight into it, click here to create your first workflow:
</div>
<div>
<Button color="primary" style={{marginTop: "20px",}} variant="outlined" onClick={() => setModalOpen(true)}>New workflow</Button>
</div>
</Paper>
</div>
<div style={{marginTop: 20}}>
<Tooltip color="primary" title={"Create new workflow"} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => setModalOpen(true)}><AddIcon /></Button>
</Tooltip>
{/*
<Tooltip color="primary" title={"Import workflows"} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
<PublishIcon />
</Button>
</Tooltip>
*/}
<Tooltip color="primary" title={`Download ALL workflows (${workflows.length})`} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => {
exportAllWorkflows()
}}>
<GetAppIcon />
</Button>
</Tooltip>
<Tooltip color="primary" title={"Import workflows"} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => setLoadWorkflowsModalOpen(true)}>
<CloudDownloadIcon />
</Button>
</Tooltip>
<input hidden type="file" multiple="multiple" ref={(ref) => upload = ref} onChange={importFiles} />
</div>
</div>
<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
)
}
<div style={scrollStyle}>
{workflows.map((data, index) => {
return (
<WorkflowPaper key={index} data={data} />
)
})}
</div>
</div>
<div style={{flex: viewSize.executionsView, marginLeft: "10px", marginRight: "10px"}}>
<div style={{display: "flex"}}>
<div style={{flex: "10"}}>
<h2>Executions: {selectedWorkflow.name}</h2>
return (
<div style={viewStyle}>
<div style={workflowViewStyle}>
<div style={{display: "flex"}}>
<div style={{flex: "4"}}>
<h2>Workflows</h2>
</div>
<div style={{marginTop: 20}}>
<Tooltip color="primary" title={"Create new workflow"} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => setModalOpen(true)}><AddIcon /></Button>
</Tooltip>
{/*
<Tooltip color="primary" title={"Import workflows"} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
<PublishIcon />
</Button>
</Tooltip>
*/}
<Tooltip color="primary" title={`Download ALL workflows (${workflows.length})`} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => {
exportAllWorkflows()
}}>
<GetAppIcon />
</Button>
</Tooltip>
<Tooltip color="primary" title={"Import workflows"} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => setLoadWorkflowsModalOpen(true)}>
<CloudDownloadIcon />
</Button>
</Tooltip>
<input hidden type="file" multiple="multiple" ref={(ref) => upload = ref} onChange={importFiles} />
</div>
</div>
<div style={{flex: "1"}}>
<Button color="primary" style={{marginTop: "20px"}} variant="text" onClick={() => {
alert.info("Refreshing executions");
getWorkflowExecution(selectedWorkflow.id)
}}>
<CachedIcon />
</Button>
</div>
</div>
<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
<div style={scrollStyle}>
<ExecutionsView />
</div>
</div>
<div style={{flex: viewSize.executionResults, marginLeft: "10px", marginRight: "10px", minWidth: "33%"}}>
<div style={{display: "flex"}}>
<div style={{flex: "1"}}>
<h2>Execution Timeline</h2>
</div>
<div style={{flex: 1}}>
<FormControlLabel
style={{color: "white", marginBottom: "0px", marginTop: "10px"}}
label={<div style={{color: "white"}}>Collapse results</div>}
control={<Switch checked={collapseJson} onChange={() => {setCollapseJson(!collapseJson)}} />}
/>
</div>
</div>
<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
<div style={scrollStyle}>
<ExecutionDetails />
</div>
</div>
</div>
:
<div style={emptyWorkflowStyle}>
<Paper style={boxStyle}>
<div>
<h2>Welcome to Shuffle</h2>
</div>
<div>
<p>
<b>Shuffle</b> is a flexible, easy to use, automation platform allowing users to integrate their services and devices freely. It's made to significantly reduce the amount of manual labor, and is focused on security applications. <a href="/docs/about" style={{textDecoration: "none", color: "#f85a3e"}}>Click here to learn more.</a>
</p>
</div>
<div>
If you want to jump straight into it, click here to create your first workflow:
</div>
<div>
<Button color="primary" style={{marginTop: "20px",}} variant="outlined" onClick={() => setModalOpen(true)}>New workflow</Button>
</div>
</Paper>
</div>
<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
<div style={scrollStyle}>
{workflows.map((data, index) => {
return (
<WorkflowPaper key={index} data={data} />
)
})}
</div>
</div>
<div style={{flex: viewSize.executionsView, marginLeft: "10px", marginRight: "10px"}}>
<div style={{display: "flex"}}>
<div style={{flex: "10"}}>
<h2>Executions: {selectedWorkflow.name}</h2>
</div>
<div style={{flex: "1"}}>
<Button color="primary" style={{marginTop: "20px"}} variant="text" onClick={() => {
alert.info("Refreshing executions");
getWorkflowExecution(selectedWorkflow.id)
}}>
<CachedIcon />
</Button>
</div>
</div>
<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
<div style={scrollStyle}>
<ExecutionsView />
</div>
</div>
<div style={{flex: viewSize.executionResults, marginLeft: "10px", marginRight: "10px", minWidth: "33%"}}>
<div style={{display: "flex"}}>
<div style={{flex: "1"}}>
<h2>Execution Timeline</h2>
</div>
<div style={{flex: 1}}>
<FormControlLabel
style={{color: "white", marginBottom: "0px", marginTop: "10px"}}
label={<div style={{color: "white"}}>Collapse results</div>}
control={<Switch checked={collapseJson} onChange={() => {setCollapseJson(!collapseJson)}} />}
/>
</div>
</div>
<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
<div style={scrollStyle}>
<ExecutionDetails />
</div>
</div>
</div>
)
}
const importWorkflowsFromUrl = (url) => {
console.log("IMPORT WORKFLOWS FROM ", downloadUrl)
@@ -1384,13 +1406,17 @@ const Workflows = (props) => {
const loadedCheck = isLoaded && isLoggedIn && workflowDone ?
<div>
{workflowView}
<WorkflowView />
{modalView}
{deleteModal}
{workflowDownloadModalOpen}
</div>
:
<div>
<div style={{paddingTop: 250, width: 250, margin: "auto", textAlign: "center"}}>
<CircularProgress />
<Typography>
Loading Workflows
</Typography>
</div>