marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed execution speed with too many executions

Browse Source
This commit is contained in:
frikky
2020-06-19 19:30:14 +02:00
parent 46bc9ef37f
commit a773cdeea8
11 changed files with 432 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app_sdk/app_base.py
+56 -10
View File
@@ -327,13 +327,36 @@ class AppBase:
print("BEFORE VARIABLES!")
if len(baseresult) == 0:
print("Variables: %s" % execution_data["workflow"]["workflow_variables"])
for variable in execution_data["workflow"]["workflow_variables"]:
variablename = variable["name"].replace(" ", "_", -1).lower()
try:
print("WF Variables: %s" % execution_data["workflow"]["workflow_variables"])
for variable in execution_data["workflow"]["workflow_variables"]:
variablename = variable["name"].replace(" ", "_", -1).lower()
if variablename.lower() == actionname_lower:
baseresult = variable["value"]
break
if variablename.lower() == actionname_lower:
baseresult = variable["value"]
break
except KeyError as e:
print("KeyError wf variables: %s" % e)
pass
except TypeError as e:
print("TypeError wf variables: %s" % e)
pass
print("BEFORE EXECUTION VAR")
if len(baseresult) == 0:
try:
print("Execution Variables: %s" % execution_data["execution_variables"])
for variable in execution_data["execution_variables"]:
variablename = variable["name"].replace(" ", "_", -1).lower()
if variablename.lower() == actionname_lower:
baseresult = variable["value"]
break
except KeyError as e:
print("KeyError exec variables: %s" % e)
pass
except TypeError as e:
print("TypeError exec variables: %s" % e)
pass
except KeyError as error:
print(f"KeyError in JSON: {error}")
@@ -421,10 +444,33 @@ class AppBase:
if parameter["variant"] == "WORKFLOW_VARIABLE":
for item in fullexecution["workflow"]["workflow_variables"]:
if parameter["action_field"] == item["name"]:
parameter["value"] = item["value"]
break
print("Handling workflow variable")
found = False
try:
for item in fullexecution["workflow"]["workflow_variables"]:
if parameter["action_field"] == item["name"]:
found = True
parameter["value"] = item["value"]
break
except KeyError as e:
print("KeyError WF variable 1: %s" % e)
pass
except TypeError as e:
print("TypeError WF variables 1: %s" % e)
pass
if not found:
try:
for item in fullexecution["execution_variables"]:
if parameter["action_field"] == item["name"]:
parameter["value"] = item["value"]
break
except KeyError as e:
print("KeyError WF variable 2: %s" % e)
pass
except TypeError as e:
print("TypeError WF variables 2: %s" % e)
pass
elif parameter["variant"] == "ACTION_RESULT":
# FIXME - calculate value based on action_field and $if prominent
backend/go-app/docker.go
+6 -5
View File
@@ -191,13 +191,14 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
)
//log.Printf("IMAGERESPONSE: %#v", imageBuildResponse.Body)
defer imageBuildResponse.Body.Close()
_, newerr := io.Copy(os.Stdout, imageBuildResponse.Body)
if newerr != nil {
log.Printf("Failed reading Docker build STDOUT: %s", newerr)
}
if err != nil {
// Read the STDOUT from the build process
defer imageBuildResponse.Body.Close()
_, newerr := io.Copy(os.Stdout, imageBuildResponse.Body)
if newerr != nil {
log.Printf("Failed reading Docker build STDOUT: %s", newerr)
}
return err
}
backend/go-app/main.go
+3 -2
View File
@@ -2148,7 +2148,7 @@ func getApikey(ctx context.Context, apikey string) (User, error) {
var users []User
_, err := dbclient.GetAll(ctx, q, &users)
if err != nil {
log.Printf("Error getting users apikey: %s", err)
log.Printf("Error getting users apikey (getapikey): %s", err)
return User{}, err
}
@@ -5969,7 +5969,7 @@ func runInit(ctx context.Context) {
var users []User
_, err = dbclient.GetAll(ctx, q, &users)
if err != nil {
log.Printf("Error getting users apikey: %s", err)
log.Printf("Error getting users apikey (runinit): %s", err)
} else {
if len(users) == 0 {
log.Printf("No active users found - setting ALL to active")
@@ -6030,6 +6030,7 @@ func runInit(ctx context.Context) {
} else {
log.Printf("Setting up %d schedule(s)", len(schedules))
for _, schedule := range schedules {
//log.Printf("Schedule: %#v", schedule)
job := func() {
request := &http.Request{
Method: "POST",
backend/go-app/walkoff.go
+78 -57
View File
@@ -138,8 +138,7 @@ type WorkflowAppAction struct {
ID string `json:"id" datastore:"id"`
Name string `json:"name" datastore:"name"`
Value string `json:"value" datastore:"value"`
} `json:"execution_variables" datastore:"execution_variables"`
} `json:"execution_variable" datastore:"execution_variables"`
Returns struct {
Description string `json:"description" datastore:"returns" yaml:"description,omitempty"`
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
@@ -149,41 +148,53 @@ type WorkflowAppAction struct {
// FIXME: Generate a callback authentication ID?
type WorkflowExecution struct {
Type string `json:"type" datastore:"type"`
Status string `json:"status" datastore:"status"`
Start string `json:"start" datastore:"start"`
ExecutionArgument string `json:"execution_argument" datastore:"execution_argument"`
ExecutionId string `json:"execution_id" datastore:"execution_id"`
WorkflowId string `json:"workflow_id" datastore:"workflow_id"`
LastNode string `json:"last_node" datastore:"last_node"`
Authorization string `json:"authorization" datastore:"authorization"`
Result string `json:"result" datastore:"result,noindex"`
StartedAt int64 `json:"started_at" datastore:"started_at"`
CompletedAt int64 `json:"completed_at" datastore:"completed_at"`
ProjectId string `json:"project_id" datastore:"project_id"`
Locations []string `json:"locations" datastore:"locations"`
Workflow Workflow `json:"workflow" datastore:"workflow,noindex"`
Results []ActionResult `json:"results" datastore:"results,noindex"`
Type string `json:"type" datastore:"type"`
Status string `json:"status" datastore:"status"`
Start string `json:"start" datastore:"start"`
ExecutionArgument string `json:"execution_argument" datastore:"execution_argument"`
ExecutionId string `json:"execution_id" datastore:"execution_id"`
WorkflowId string `json:"workflow_id" datastore:"workflow_id"`
LastNode string `json:"last_node" datastore:"last_node"`
Authorization string `json:"authorization" datastore:"authorization"`
Result string `json:"result" datastore:"result,noindex"`
StartedAt int64 `json:"started_at" datastore:"started_at"`
CompletedAt int64 `json:"completed_at" datastore:"completed_at"`
ProjectId string `json:"project_id" datastore:"project_id"`
Locations []string `json:"locations" datastore:"locations"`
Workflow Workflow `json:"workflow" datastore:"workflow,noindex"`
Results []ActionResult `json:"results" datastore:"results,noindex"`
ExecutionVariables []struct {
Description string `json:"description" datastore:"description"`
ID string `json:"id" datastore:"id"`
Name string `json:"name" datastore:"name"`
Value string `json:"value" datastore:"value"`
} `json:"execution_variables,omitempty" datastore:"execution_variables,omitempty"`
}
// Added environment for location to execute
type Action struct {
AppName string `json:"app_name" datastore:"app_name"`
AppVersion string `json:"app_version" datastore:"app_version"`
AppID string `json:"app_id" datastore:"app_id"`
Errors []string `json:"errors" datastore:"errors"`
ID string `json:"id" datastore:"id"`
IsValid bool `json:"is_valid" datastore:"is_valid"`
IsStartNode bool `json:"isStartNode" datastore:"isStartNode"`
Sharing bool `json:"sharing" datastore:"sharing"`
PrivateID string `json:"private_id" datastore:"private_id"`
Label string `json:"label" datastore:"label"`
SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"`
LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false`
Environment string `json:"environment" datastore:"environment"`
Name string `json:"name" datastore:"name"`
Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"`
Position struct {
AppName string `json:"app_name" datastore:"app_name"`
AppVersion string `json:"app_version" datastore:"app_version"`
AppID string `json:"app_id" datastore:"app_id"`
Errors []string `json:"errors" datastore:"errors"`
ID string `json:"id" datastore:"id"`
IsValid bool `json:"is_valid" datastore:"is_valid"`
IsStartNode bool `json:"isStartNode" datastore:"isStartNode"`
Sharing bool `json:"sharing" datastore:"sharing"`
PrivateID string `json:"private_id" datastore:"private_id"`
Label string `json:"label" datastore:"label"`
SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"`
LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false`
Environment string `json:"environment" datastore:"environment"`
Name string `json:"name" datastore:"name"`
Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"`
ExecutionVariable struct {
Description string `json:"description" datastore:"description"`
ID string `json:"id" datastore:"id"`
Name string `json:"name" datastore:"name"`
Value string `json:"value" datastore:"value"`
} `json:"execution_variable,omitempty" datastore:"execution_variable,omitempty"`
Position struct {
X float64 `json:"x" datastore:"x"`
Y float64 `json:"y" datastore:"y"`
} `json:"position"`
@@ -265,7 +276,7 @@ type Workflow struct {
ID string `json:"id" datastore:"id"`
Name string `json:"name" datastore:"name"`
Value string `json:"value" datastore:"value"`
} `json:"execution_variables" datastore:"execution_variables"`
} `json:"execution_variables,omitempty" datastore:"execution_variables,omitempty"`
}
type ActionResult struct {
@@ -796,13 +807,20 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
}
if found {
// FIXME - this is broken, but why
//if workflowExecution.Results[outerindex].Status == actionResult.Status {
// log.Printf("Status of %s is already %s", actionResult.Action.ID, actionResult.Status)
// resp.WriteHeader(401)
// resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Status of %s is already %s"}`, actionResult.Action.ID, actionResult.Status)))
// return
//}
// If result exists and execution variable exists, update execution value
//log.Printf("Exec var backend: %s", workflowExecution.Results[outerindex].Action.ExecutionVariable.Name)
actionVarName := workflowExecution.Results[outerindex].Action.ExecutionVariable.Name
// Finds potential execution arguments
if len(actionVarName) > 0 {
log.Printf("EXECUTION VARIABLE LOCAL: %s", actionVarName)
for index, execvar := range workflowExecution.ExecutionVariables {
if execvar.Name == actionVarName {
// Sets the value for the variable
workflowExecution.ExecutionVariables[index].Value = actionResult.Result
break
}
}
}
log.Printf("Updating %s in %s from %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, workflowExecution.Results[outerindex].Status, actionResult.Status)
workflowExecution.Results[outerindex] = actionResult
@@ -1013,6 +1031,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) {
if len(newActions) == 0 {
log.Printf("APPENDING NEW APP FOR NEW WORKFLOW")
// Adds the Testing app if it's a new workflow
workflowapps, err := getAllWorkflowApps(ctx)
if err == nil {
for _, item := range workflowapps {
@@ -1046,7 +1065,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) {
}
}
} else {
log.Printf("Has actions already?")
log.Printf("Has %d actions already", len(newActions))
}
workflow.Actions = newActions
@@ -1412,6 +1431,8 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
"0ca8887e-b4af-4e3e-887c-87e9d3bc3d3e",
}
log.Printf("%s Action execution var: %s", action.Label, action.ExecutionVariable.Name)
builtin := false
for _, id := range reservedApps {
if id == action.AppID {
@@ -1935,6 +1956,8 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
// Status for the entire workflow.
workflowExecution.Status = "EXECUTING"
}
workflowExecution.ExecutionVariables = workflow.ExecutionVariables
// Local authorization for this single workflow used in workers.
// FIXME: Used for cloud
@@ -2095,7 +2118,6 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) {
}
func stopSchedule(resp http.ResponseWriter, request *http.Request) {
log.Printf("Delete?")
cors := handleCors(resp, request)
if cors {
return
@@ -2262,20 +2284,19 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) {
func deleteSchedule(ctx context.Context, id string) error {
log.Printf("Should stop schedule %s!", id)
//newscheduler "github.com/carlescere/scheduler"
log.Printf("Schedules: %#v", scheduledJobs)
if value, exists := scheduledJobs[id]; exists {
log.Printf("STOP THIS ONE: %s", value)
// Looks like this does the trick? Hurr
value.Lock()
err := DeleteKey(ctx, "schedules", id)
if err != nil {
log.Printf("Failed to delete schedule: %s", err)
return err
}
err := DeleteKey(ctx, "schedules", id)
if err != nil {
log.Printf("Failed to delete schedule: %s", err)
return err
} else {
// FIXME - allow it to kind of stop anyway?
return errors.New("Can't find the schedule.")
if value, exists := scheduledJobs[id]; exists {
log.Printf("STOP THIS ONE: %s", value)
// Looks like this does the trick? Hurr
value.Lock()
} else {
// FIXME - allow it to kind of stop anyway?
return errors.New("Can't find the schedule.")
}
}
return nil
@@ -2717,7 +2738,7 @@ func deleteUser(resp http.ResponseWriter, request *http.Request) {
var users []User
_, err := dbclient.GetAll(ctx, q, &users)
if err != nil {
log.Printf("Error getting users apikey: %s", err)
log.Printf("Error getting users apikey (deleteuser): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed getting users for verification"}`))
return
docker-compose.yml
+2 -2
View File
@@ -1,7 +1,7 @@
version: '3'
services:
frontend:
build: ./frontend
#build: ./frontend
image: frikky/shuffle:frontend
container_name: shuffle-frontend
hostname: shuffle-frontend
@@ -14,7 +14,7 @@ services:
depends_on:
- backend
backend:
build: ./backend
#build: ./backend
image: frikky/shuffle:backend
container_name: shuffle-backend
hostname: ${BACKEND_HOSTNAME}
frontend/src/Admin.js
+2 -1
View File
@@ -43,7 +43,7 @@ const Admin = (props) => {
console.log("INPUT: ", data)
// Just use this one?
const url = globalUrl+'/api/v1/workflows/'+data["workflow_id datastore:"]+"/schedule/"+data.id
const url = globalUrl+'/api/v1/workflows/'+data["workflow_id"]+"/schedule/"+data.id
console.log("URL: ", url)
fetch(url, {
method: 'DELETE',
@@ -54,6 +54,7 @@ const Admin = (props) => {
})
.then(response =>
response.json().then(responseJson => {
console.log("RESP: ", responseJson)
if (responseJson["success"] === false) {
alert.error("Failed stopping schedule")
} else {
frontend/src/AngularWorkflow.js
+221 -29
View File
@@ -112,6 +112,7 @@ const AngularWorkflow = (props) => {
const [appAuthentication, setAppAuthentication] = React.useState({});
const [variablesModalOpen, setVariablesModalOpen] = React.useState(false);
const [executionVariablesModalOpen, setExecutionVariablesModalOpen] = React.useState(false);
const [authenticationModalOpen, setAuthenticationModalOpen] = React.useState(false);
const [conditionsModalOpen, setConditionsModalOpen] = React.useState(false);
const [newVariableName, setNewVariableName] = React.useState("");
@@ -791,7 +792,6 @@ const AngularWorkflow = (props) => {
const onNodeSelect = (event) => {
const data = event.target.data()
console.log("NODE: ", data)
setLastSaved(false)
//console.log(data)
@@ -1412,22 +1412,6 @@ const AngularWorkflow = (props) => {
const [open, setOpen] = React.useState(false);
const [anchorEl, setAnchorEl] = React.useState(null);
if (workflow.workflow_variables === undefined || workflow.workflow_variables === null || workflow.workflow_variables.length === 0) {
return (
<div style={appViewStyle}>
<div style={appScrollStyle}>
<div style={{margin: 10}}>
Looks like you don't have any variables yet.
<div/>
<div style={{width: "100%", margin: "auto"}}>
<Button fullWidth style={{margin: "auto", marginTop: "10px", borderRadius: 0}} color="primary" variant="outlined" onClick={() => setVariablesModalOpen(true)}>Make a new workflow variable</Button>
</div>
</div>
</div>
</div>
)
}
const menuClick = (event) => {
setOpen(!open)
setAnchorEl(event.currentTarget);
@@ -1438,8 +1422,13 @@ const AngularWorkflow = (props) => {
setWorkflow(workflow)
}
const deleteExecutionVariable = (variableName) => {
workflow.execution_variables = workflow.execution_variables.filter(data => data.name !== variableName)
setWorkflow(workflow)
}
const variableScrollStyle = {
marginTop: "10px",
margin: 15,
overflow: "scroll",
height: "66vh",
overflowX: "auto",
@@ -1450,7 +1439,9 @@ const AngularWorkflow = (props) => {
return (
<div style={appViewStyle}>
<div style={variableScrollStyle}>
{workflow.workflow_variables.map(variable=> {
<a href="https://shuffler.io/docs/workflows#variables" target="_blank" style={{textDecoration: "none", color: "#f85a3e"}}>What are WORKFLOW variables?</a>
{workflow.workflow_variables === null ?
null : workflow.workflow_variables.map(variable=> {
return (
<div>
<Paper square style={paperVariableStyle} onClick={() => {
@@ -1508,10 +1499,69 @@ const AngularWorkflow = (props) => {
</div>
)
})}
<div style={{flex: "1"}}>
<Button fullWidth style={{margin: "auto", marginTop: "10px",}} color="primary" variant="outlined" onClick={() => setVariablesModalOpen(true)}>New workflow variable</Button>
</div>
<Divider style={{marginBottom: 20, marginTop: 20, height: 1, width: "100%", backgroundColor: "rgb(91, 96, 100)"}}/>
<a href="https://shuffler.io/docs/workflows#execution_variables" target="_blank" style={{textDecoration: "none", color: "#f85a3e"}}>What are EXECUTION variables?</a>
{workflow.execution_variables === null ?
null : workflow.execution_variables.map(variable=> {
return (
<div>
<Paper square style={paperVariableStyle} onClick={() => {
}}>
<div style={{marginLeft: "10px", marginTop: "5px", marginBottom: "5px", width: "2px", backgroundColor: "orange", marginRight: "5px"}} />
<div style={{display: "flex", width: "100%"}}>
<div style={{flex: "10", marginTop: "15px", marginLeft: "10px", overflow: "hidden"}} onClick={() => {
setNewVariableName(variable.name)
setExecutionVariablesModalOpen(true)}}>
Name: {variable.name}
</div>
<div style={{flex: "1", marginLeft: "0px"}}>
<IconButton
aria-label="more"
aria-controls="long-menu"
aria-haspopup="true"
onClick={menuClick}
style={{color: "white"}}
>
<MoreVertIcon />
</IconButton>
<Menu
id="long-menu"
anchorEl={anchorEl}
keepMounted
open={open}
PaperProps={{
style: {
backgroundColor: surfaceColor,
}
}}
onClose={() => {
setOpen(false)
setAnchorEl(null)
}}
>
</div>
<div style={{flex: "1"}}>
<Button fullWidth style={{margin: "auto", marginTop: "10px",}} color="primary" variant="outlined" onClick={() => setVariablesModalOpen(true)}>New workflow variable</Button>
<MenuItem style={{backgroundColor: surfaceColor, color: "white"}} onClick={() => {
setOpen(false)
setNewVariableName(variable.name)
setExecutionVariablesModalOpen(true)
}} key={"Edit"}>{"Edit"}</MenuItem>
<MenuItem style={{backgroundColor: surfaceColor, color: "white"}} onClick={() => {
deleteExecutionVariable(variable.name)
setOpen(false)
}} key={"Delete"}>{"Delete"}</MenuItem>
</Menu>
</div>
</div>
</Paper>
</div>
)
})}
<div style={{flex: "1"}}>
<Button fullWidth style={{margin: "auto", marginTop: "10px",}} color="primary" variant="outlined" onClick={() => setExecutionVariablesModalOpen(true)}>New execution variable</Button>
</div>
</div>
</div>
)
@@ -1862,6 +1912,7 @@ const AngularWorkflow = (props) => {
isStartNode: false,
large_image: app.large_image,
authentication: [],
execution_variable: undefined,
}
// const image = "url("+app.large_image+")"
@@ -2361,7 +2412,7 @@ const AngularWorkflow = (props) => {
} else if (data.variant === "WORKFLOW_VARIABLE") {
varcolor = "#f85a3e"
if (workflow.workflow_variables === null || workflow.workflow_variables === undefined || workflow.workflow_variables.length === 0) {
if ((workflow.workflow_variables === null || workflow.workflow_variables === undefined || workflow.workflow_variables.length === 0) && (workflow.execution_variables === null || workflow.execution_variables === undefined || workflow.execution_variables.length === 0)) {
setCurrentView(2)
datafield =
<div>
@@ -2389,15 +2440,22 @@ const AngularWorkflow = (props) => {
fullWidth
value={selectedAction.parameters[count].action_field}
onChange={(e) => {
console.log(e.target.value)
changeActionParameterVariable(e.target.value, count)
}}
style={{backgroundColor: inputColor, color: "white", height: "50px"}}
>
{workflow.workflow_variables.map(data => (
{workflow.workflow_variables !== null ? workflow.workflow_variables.map(data => (
<MenuItem style={{backgroundColor: inputColor, color: "white"}} value={data.name}>
{data.name}
</MenuItem>
))}
)) : null}
<Divider />
{workflow.execution_variables !== null ? workflow.execution_variables.map(data => (
<MenuItem style={{backgroundColor: inputColor, color: "white"}} value={data.name}>
{data.name}
</MenuItem>
)) : null}
</Select>
}
@@ -2544,7 +2602,7 @@ const AngularWorkflow = (props) => {
placeholder={selectedAction.label}
onChange={selectedNameChange}
/>
{environments !== undefined && environments.length > 1 ?
{environments !== undefined && environments !== null && environments.length > 1 ?
<div style={{marginTop: "20px"}}>
Environment
<Select
@@ -2576,6 +2634,42 @@ const AngularWorkflow = (props) => {
</Select>
</div>
: null}
{workflow.execution_variables !== undefined && workflow.execution_variables !== null && workflow.execution_variables.length > 0 ?
<div style={{marginTop: "20px"}}>
Execution variables
<Select
value={selectedAction.execution_variable !== undefined ? selectedAction.execution_variable.name : {"name": "Select a variable"}}
PaperProps={{
style: {
backgroundColor: inputColor,
}
}}
SelectDisplayProps={{
style: {
marginLeft: 10,
}
}}
fullWidth
onChange={(e) => {
console.log("Variable", e.target.value)
//selectedAction.
const value = workflow.execution_variables.find(a => a.name === e.target.value.name)
console.log("FOUND: ", value)
selectedAction.execution_variable = value
setSelectedAction(selectedAction)
//setSelectedActionEnvironment(env)
//selectedAction.environment = env.Name
}}
style={{backgroundColor: inputColor, color: "white", height: "50px"}}
>
{workflow.execution_variables.map(data => (
<MenuItem style={{backgroundColor: inputColor, color: "white"}} value={data}>
{data.name}
</MenuItem>
))}
</Select>
</div>
: null}
{/*requiresAuthentication ?
<div style={{marginTop: "20px"}}>
<Button fullWidth style={{margin: "auto", marginTop: "10px",}} color="primary" variant="contained" onClick={() => setAuthenticationModalOpen(true)}>
@@ -4309,7 +4403,7 @@ const AngularWorkflow = (props) => {
)
} else if (Object.getOwnPropertyNames(selectedTrigger).length > 0) {
if (selectedTrigger.trigger_type === "SCHEDULE") {
console.log("SCHEDULE")
//console.log("SCHEDULE")
return(
<div style={rightsidebarStyle}>
<ScheduleSidebar />
@@ -4492,7 +4586,6 @@ const AngularWorkflow = (props) => {
executionData.results.map(data => {
var showResult = data.result.trim()
showResult.split(" None").join(" \"None\"")
console.log("RESULT: ", showResult)
// showResult = replaceAll(showResult, " None", " \"None\"")
// Super basic check.
@@ -4555,7 +4648,7 @@ const AngularWorkflow = (props) => {
cy={(incy) => {
// FIXME: There's something specific loading when
// you do the first hover of a node. Why is this different?
console.log("CY: ", incy)
//console.log("CY: ", incy)
setCy(incy)
}}
/>
@@ -4569,6 +4662,104 @@ const AngularWorkflow = (props) => {
<div style={{color: "white"}}>
TMP FOR NOT LOGGED IN
</div>
const executionVariableModal = executionVariablesModalOpen ?
<Dialog modal
open={executionVariablesModalOpen}
onClose={() => {
setNewVariableName("")
setExecutionVariablesModalOpen(false)
}}
PaperProps={{
style: {
backgroundColor: surfaceColor,
color: "white",
},
}}
>
<FormControl>
<DialogTitle><span style={{color: "white"}}>Execution Variable</span></DialogTitle>
<DialogContent>
Execution Variables are TEMPORARY variables that you can ony be set and used during execution. Learn more <a href="https://shuffler.io/docs/workflow#execution_variables" target="_blank" style={{textDecoration: "none", color: "#f85a3e"}}>here</a>
<TextField
onBlur={(event) => setNewVariableName(event.target.value)}
color="primary"
placeholder="Name"
style={{marginTop: 25}}
InputProps={{
style:{
color: "white"
}
}}
margin="dense"
fullWidth
defaultValue={newVariableName}
/>
</DialogContent>
<DialogActions>
<Button
style={{borderRadius: "0px"}}
onClick={() => {
setNewVariableName("")
setExecutionVariablesModalOpen(false)
}} color="primary">
Cancel
</Button>
<Button style={{borderRadius: "0px"}} disabled={newVariableName.length === 0} onClick={() => {
console.log("VARIABLES! ", newVariableName)
if (workflow.execution_variables === undefined || workflow.execution_variables === null) {
workflow.execution_variables = []
}
// try to find one with the same name
const found = workflow.execution_variables.findIndex(data => data.name === newVariableName)
//console.log(found)
if (found !== -1) {
if (newVariableName.length > 0) {
workflow.execution_variables[found].name = newVariableName
}
} else {
workflow.execution_variables.push({
"name": newVariableName,
"description": "An execution variable",
"value": "",
"id": uuid.v4(),
})
}
setExecutionVariablesModalOpen(false)
setNewVariableName("")
setWorkflow(workflow)
}} color="primary">
Submit
</Button>
</DialogActions>
{workflowExecutions.length > 0 ?
<DialogContent>
<Divider style={{backgroundColor: "white", marginTop: 15, marginBottom: 15,}}/>
<b style={{marginBottom: 10}}>Values from last 3 executions</b>
{workflowExecutions.slice(0,3).map((execution, index) => {
if (execution.execution_variables === undefined || execution.execution_variables === null || execution.execution_variables === 0) {
return null
}
const variable = execution.execution_variables.find(data => data.name === newVariableName)
if (variable === undefined || variable.value === undefined) {
return null
}
return (
<div>
{index+1}: {variable.value}
</div>
)
})}
</DialogContent>
: null
}
</FormControl>
</Dialog>
: null
const variablesModal = variablesModalOpen ?
<Dialog modal
@@ -4808,6 +4999,7 @@ const AngularWorkflow = (props) => {
<div>
{newView}
{variablesModal}
{executionVariableModal}
{conditionsModal}
{authenticationModal}
</div>
frontend/src/App.js
-2
View File
@@ -90,13 +90,11 @@ const App = (message, props) => {
})
.then(response => response.json())
.then(responseJson => {
console.log(responseJson)
if (responseJson.success === true) {
setUserData(responseJson)
setIsLoggedIn(true)
// Updating cookie every request
console.log("COOKIES: ", cookies)
for (var key in responseJson["cookies"]) {
setCookie(responseJson["cookies"][key].key, responseJson["cookies"][key].value, {path: "/"})
}
frontend/src/Apps.js
+1 -1
View File
@@ -905,6 +905,7 @@ const Apps = (props) => {
</Dialog>
: null
const circularLoader = validation ? <CircularProgress color="primary" /> : null
const appsModalLoad = loadAppsModalOpen ?
<Dialog modal
open={loadAppsModalOpen}
@@ -998,7 +999,6 @@ const Apps = (props) => {
: null
const errorText = openApiError.length > 0 ? <div>Error: {openApiError}</div> : null
const circularLoader = validation ? <CircularProgress color="primary" /> : null
const modalView = openApiModal ?
<Dialog modal
open={openApiModal}
functions/onprem/orborus/orborus.go
+63 -47
View File
@@ -36,7 +36,8 @@ var orgId = os.Getenv("ORG_ID")
var sleepTime = 3
// Timeout if somethinc rashes
var workerTimeout = 600
//var workerTimeout = 600
var workerTimeout = 300
type ExecutionRequestWrapper struct {
Data []ExecutionRequest `json:"data"`
@@ -52,7 +53,7 @@ type ExecutionRequest struct {
}
// Deploys the internal worker whenever something happens
func deployWorker(cli *dockerclient.Client, image string, identifier string, env []string) error {
func deployWorker(cli *dockerclient.Client, image string, identifier string, env []string) {
// Binds is the actual "-v" volume.
hostConfig := &container.HostConfig{
LogConfig: container.LogConfig{
@@ -101,16 +102,40 @@ func deployWorker(cli *dockerclient.Client, image string, identifier string, env
if err != nil {
log.Println(err)
return err
return
}
err = cli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{})
if err != nil {
log.Printf("Failed to start container in environment %s: %s", environment, err)
return
//stats, err := cli.ContainerInspect(context.Background(), containerName)
//if err != nil {
// log.Printf("Failed checking worker %s", containerName)
// return
//}
//containerStatus := stats.ContainerJSONBase.State.Status
//if containerStatus != "running" {
// log.Printf("Status of %s is %s. Should be running. Will reset", containerName, containerStatus)
// err = stopWorker(containerName)
// if err != nil {
// log.Printf("Failed stopping worker %s", execution.ExecutionId)
// return
// }
// err = deployWorker(cli, workerImage, containerName, env)
// if err != nil {
// log.Printf("Failed executing worker %s in state %s", execution.ExecutionId, containerStatus)
// return
// }
//}
} else {
log.Printf("Container %s was created under environment %s", cont.ID, environment)
}
return nil
return
}
func stopWorker(containername string) error {
@@ -169,7 +194,7 @@ func initializeImages(dockercli *dockerclient.Client) {
// Initial loop etc
func main() {
zombiecheck()
go zombiecheck()
log.Println("Setting up execution environment")
//FIXME
@@ -236,7 +261,7 @@ func main() {
log.Printf("Failed making request: %s", err)
zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout {
zombiecheck()
go zombiecheck()
zombiecounter = 0
}
time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -257,7 +282,7 @@ func main() {
log.Printf("Failed reading body: %s", err)
zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout {
zombiecheck()
go zombiecheck()
zombiecounter = 0
}
time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -271,7 +296,7 @@ func main() {
sleepTime = 10
zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout {
zombiecheck()
go zombiecheck()
zombiecounter = 0
}
time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -286,7 +311,7 @@ func main() {
if len(executionRequests.Data) == 0 {
zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout {
zombiecheck()
go zombiecheck()
zombiecounter = 0
}
time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -322,34 +347,9 @@ func main() {
env = append(env, fmt.Sprintf("DOCKER_API_VERSION=%s", dockerApiVersion))
}
err = deployWorker(dockercli, workerImage, containerName, env)
if err != nil {
stats, err := dockercli.ContainerInspect(context.Background(), containerName)
if err != nil {
log.Printf("Failed checking worker %s", execution.ExecutionId)
continue
}
go deployWorker(dockercli, workerImage, containerName, env)
containerStatus := stats.ContainerJSONBase.State.Status
if containerStatus != "running" {
log.Printf("Status of %s is %s. Should be running. Will reset", containerName, containerStatus)
err = stopWorker(containerName)
if err != nil {
log.Printf("Failed stopping worker %s", execution.ExecutionId)
continue
}
err = deployWorker(dockercli, workerImage, containerName, env)
if err != nil {
log.Printf("Failed executing worker %s in state %s", execution.ExecutionId, containerStatus)
}
} else {
// Should basically never hit here rofl
log.Printf("ERROR: I HAVE NO IDEA WHAT WENT WRONG. CHECK %s", containerName)
}
}
log.Printf("%s is deployed and to being removed from queue.", execution.ExecutionId)
log.Printf("%s is deployed and to be removed from queue.", execution.ExecutionId)
zombiecounter += 1
toBeRemoved.Data = append(toBeRemoved.Data, execution)
}
@@ -427,34 +427,54 @@ func zombiecheck() error {
All: true,
})
containerNames := map[string]string{}
stopContainers := []string{}
removeContainers := []string{}
for _, container := range containers {
// Skip random containers. Only handle things related to Shuffle.
if !strings.Contains(container.Image, baseimagename) {
shuffleFound := false
for _, item := range container.Labels {
if item == "shuffle" {
shuffleFound = true
break
}
}
// Check image name
if !shuffleFound {
continue
}
}
for _, name := range container.Names {
// FIXME - add name_version_uid_uid regex check as well
if !strings.HasPrefix(name, "/worker") {
if strings.HasPrefix(name, "/shuffle") {
continue
}
if container.State != "running" {
removeContainers = append(removeContainers, container.ID)
containerNames[container.ID] = name
}
// stopcontainer & removecontainer
currenttime := time.Now().Unix()
//log.Printf("Time: %d - %d", currenttime-container.Created, int64(workerTimeout))
if container.State == "running" && currenttime-container.Created > int64(workerTimeout) {
stopContainers = append(stopContainers, container.ID)
containerNames[container.ID] = name
}
}
}
// FIXME - add killing of apps with same execution ID too
for _, containername := range stopContainers {
if err := dockercli.ContainerStop(ctx, containername, nil); err != nil {
log.Printf("Unable to stop container: %s", err)
} else {
log.Printf("Stopped container %s", containername)
}
log.Printf("Stopping and removing container %s", containerNames[containername])
go dockercli.ContainerStop(ctx, containername, nil)
removeContainers = append(removeContainers, containername)
}
removeOptions := types.ContainerRemoveOptions{
@@ -463,11 +483,7 @@ func zombiecheck() error {
}
for _, containername := range removeContainers {
if err := dockercli.ContainerRemove(ctx, containername, removeOptions); err != nil {
log.Printf("Unable to remove container: %s", err)
} else {
log.Printf("Removed container %s", containername)
}
go dockercli.ContainerRemove(ctx, containername, removeOptions)
}
return nil
functions/onprem/worker/worker.bin
BIN
View File
Binary file not shown.