marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

BUG: Fixed resource overload when getting workflows

Browse Source
This commit is contained in:
frikky
2020-12-29 19:34:51 +01:00
parent 5beeadc1b1
commit 45665921a8
6 changed files with 125 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/go-app/walkoff.go
+1 -1
View File
@@ -507,7 +507,7 @@ func setWorkflowQueue(ctx context.Context, executionRequests ExecutionRequestWra
} }
func getWorkflowQueue(ctx context.Context, id string) (ExecutionRequestWrapper, error) { func getWorkflowQueue(ctx context.Context, id string) (ExecutionRequestWrapper, error) {
key := datastore.NameKey("workflowqueue", id, nil) key := datastore.NameKey("workflowqueue", id, nil).Limit(50)
workflows := ExecutionRequestWrapper{} workflows := ExecutionRequestWrapper{}
if err := dbclient.Get(ctx, key, &workflows); err != nil { if err := dbclient.Get(ctx, key, &workflows); err != nil {
return ExecutionRequestWrapper{}, err return ExecutionRequestWrapper{}, err
backend/tests/execute.sh
+26 -2
View File
@@ -1,5 +1,29 @@
#!/bin/sh #!/bin/sh
curl -XPOST http://localhost:5001/api/v1/workflows/1d9d8ce2-566e-4c3f-8a37-5d6c7d2000b5/execute -d '{"execution_argument":""}' -H "Authorization: Bearer 144308d0-6aab-4d4f-8bb2-75189281ee26" #curl -XPOST http://localhost:5001/api/v1/workflows/1d9d8ce2-566e-4c3f-8a37-5d6c7d2000b5/execute -d '{"execution_argument":""}' -H "Authorization: Bearer 144308d0-6aab-4d4f-8bb2-75189281ee26"
curl -XPOST http://localhost:5001/api/v1/workflows/1d9d8ce2-566e-4c3f-8a37-5d6c7d2000b5/execute -d '{"execution_argument":""}' -H "Authorization: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjYwZjQwNjBlNThkNzVmZDNmNzBiZWZmODhjNzk0YTc3NTMyN2FhMzEiLCJ0eXAiOiJKV1QifQ.eyJhdWQiOiJodHRwczovL3NodWZmbGVyLmlvL2FwaS92MS93b3JrZmxvd3MvMWQ5ZDhjZTItNTY2ZS00YzNmLThhMzctNWQ2YzdkMjAwMGI1L2V4ZWN1dGUiLCJhenAiOiIxMDMwNzY3ODIwNjE0MjQ2MTg0MjIiLCJlbWFpbCI6InNjaGVkdWxlckBzaHVmZmxlLTI0MTUxNy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJleHAiOjE1NjU1Mjc1NTEsImlhdCI6MTU2NTUyMzk1MSwiaXNzIjoiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tIiwic3ViIjoiMTAzMDc2NzgyMDYxNDI0NjE4NDIyIn0.r0EDq9fjhf_5CPTiltyfk_L3uYJp577Uy0yYPcCAl2nv50_z_oUtbWGBpQLL8gcj-NGd3g4E52Qur8k6hCMIQweLS6WAb1279vGffEoCNDfkWb3Oy-yJGP1kzwLvqFJqnHLkSWYXNWvSyWnEimW8Rryx_m1BXS5wcA8l4NIr83kS7fPZrTwjnwFSeGSThwk91DVARzapQb8r0GEgOUyHZ1aBXnV98mikzSUt-5xFKe9eMdD22YJAj0Ru-DxAxs5nOqghX4PMRysWjshjOMrlR1piPWxqAmewp8YKZDCQ5gXskpeAFBDoULT971Wsx_NCohnJsFqx1JfPS9ZYMTW2oQ" curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
docker-compose.yml
+3 -3
View File
@@ -1,8 +1,8 @@
version: '3' version: '3'
services: services:
frontend: frontend:
#build: ./frontend build: ./frontend
image: ghcr.io/frikky/shuffle-frontend:0.8.43 image: ghcr.io/frikky/shuffle-frontend:0.8.45
container_name: shuffle-frontend container_name: shuffle-frontend
hostname: shuffle-frontend hostname: shuffle-frontend
ports: ports:
@@ -17,7 +17,7 @@ services:
- backend - backend
backend: backend:
#build: ./backend #build: ./backend
image: ghcr.io/frikky/shuffle-backend:0.8.44 image: ghcr.io/frikky/shuffle-backend:0.8.45
container_name: shuffle-backend container_name: shuffle-backend
hostname: ${BACKEND_HOSTNAME} hostname: ${BACKEND_HOSTNAME}
# Here for debugging: # Here for debugging:
frontend/src/views/AngularWorkflow.jsx
+5 -15
View File
@@ -3681,7 +3681,7 @@ const AngularWorkflow = (props) => {
<div style={{display: "flex", minHeight: 40, marginBottom: 30}}> <div style={{display: "flex", minHeight: 40, marginBottom: 30}}>
<div style={{flex: 1}}> <div style={{flex: 1}}>
<h3 style={{marginBottom: 5}}>{selectedAction.app_name}</h3> <h3 style={{marginBottom: 5}}>{selectedAction.app_name}</h3>
<div style={{display: "flex", width: "100%",}}> <div style={{display: "flex",}}>
<IconButton style={{marginTop: "auto", marginBottom: "auto", height: 30, paddingLeft: 0, paddingRight: 0}} onClick={() => { <IconButton style={{marginTop: "auto", marginBottom: "auto", height: 30, paddingLeft: 0, paddingRight: 0}} onClick={() => {
console.log("FIND EXAMPLE RESULTS FOR ", selectedAction) console.log("FIND EXAMPLE RESULTS FOR ", selectedAction)
if (workflowExecutions.length > 0) { if (workflowExecutions.length > 0) {
@@ -3707,7 +3707,7 @@ const AngularWorkflow = (props) => {
<ArrowLeftIcon style={{color: "white"}}/> <ArrowLeftIcon style={{color: "white"}}/>
</Tooltip> </Tooltip>
</IconButton> </IconButton>
<span style={{width: "100%"}}> <span style={{}}>
<Typography style={{marginTop: 5, marginLeft: 10,}}><a href="https://shuffler.io/docs/workflows#nodes" target="_blank" style={{textDecoration: "none", color: "#f85a3e"}}>What are actions?</a></Typography> <Typography style={{marginTop: 5, marginLeft: 10,}}><a href="https://shuffler.io/docs/workflows#nodes" target="_blank" style={{textDecoration: "none", color: "#f85a3e"}}>What are actions?</a></Typography>
{selectedAction.errors !== null && selectedAction.errors.length > 0 ? {selectedAction.errors !== null && selectedAction.errors.length > 0 ?
<div> <div>
@@ -6294,17 +6294,6 @@ const AngularWorkflow = (props) => {
style={{pointerEvents: "none"}} style={{pointerEvents: "none"}}
hideBackdrop={true} hideBackdrop={true}
open={codeModalOpen} open={codeModalOpen}
onClose={() => {
//setCodeModalOpen(false)
console.log("CLOSE?")
}}
BackdropProps={{
invisible: true,
style: {
backgroundColor: "transparent",
pointerEvents: "none",
}
}}
PaperProps={{ PaperProps={{
style: { style: {
pointerEvents: "auto", pointerEvents: "auto",
@@ -6313,7 +6302,8 @@ const AngularWorkflow = (props) => {
minWidth: 750, minWidth: 750,
padding: 30, padding: 30,
maxHeight: 700, maxHeight: 700,
overflow: "auto", overflowY: "auto",
overflowX: "hidden",
//boxShadow: "none", //boxShadow: "none",
}, },
}} }}
@@ -6828,7 +6818,7 @@ const AngularWorkflow = (props) => {
id="copy_element_shuffle" id="copy_element_shuffle"
value={to_be_copied} value={to_be_copied}
disabled={true} disabled={true}
style={{height: 0, width: 0, margin: 0, padding: 0,}} style={{height: 0, width: 0, margin: 0, padding: 0, zIndex: -10, position: "fixed",}}
/> />
</div> </div>
: :
functions/onprem/orborus/build.sh
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=shuffle-orborus NAME=shuffle-orborus
VERSION=0.8.32 VERSION=0.8.40
echo "Running docker build with $NAME:$VERSION" echo "Running docker build with $NAME:$VERSION"
#docker rmi frikky/shuffle:$NAME --force #docker rmi frikky/shuffle:$NAME --force
functions/onprem/orborus/orborus.go
+89 -10
View File
@@ -29,9 +29,11 @@ import (
// Starts jobs in bulk, so this could be increased // Starts jobs in bulk, so this could be increased
var sleepTime = 3 var sleepTime = 3
var maxConcurrency = 50
// Timeout if something rashes // Timeout if something rashes
var workerTimeoutEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_TIMEOUT") var workerTimeoutEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_TIMEOUT")
var concurrencyEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY")
var appSdkVersion = os.Getenv("SHUFFLE_APP_SDK_VERSION") var appSdkVersion = os.Getenv("SHUFFLE_APP_SDK_VERSION")
var workerVersion = os.Getenv("SHUFFLE_WORKER_VERSION") var workerVersion = os.Getenv("SHUFFLE_WORKER_VERSION")
@@ -147,7 +149,7 @@ func deployWorker(image string, identifier string, env []string) {
Env: env, Env: env,
} }
log.Printf("[INFO] Identifier: %s", identifier) //log.Printf("[INFO] Identifier: %s", identifier)
cont, err := dockercli.ContainerCreate( cont, err := dockercli.ContainerCreate(
context.Background(), context.Background(),
config, config,
@@ -317,7 +319,18 @@ func main() {
log.Printf("[INFO] Cleanup process running every %d seconds", workerTimeout) log.Printf("[INFO] Cleanup process running every %d seconds", workerTimeout)
} }
go zombiecheck(workerTimeout) if concurrencyEnv != "" {
//var concurrencyEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY")
tmpInt, err := strconv.Atoi(concurrencyEnv)
if err == nil {
maxConcurrency = tmpInt
} else {
log.Printf("[WARNING] Env SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY must be a number, not %s. Defaulted to %d", workerTimeoutEnv, maxConcurrency)
}
}
ctx := context.Background()
go zombiecheck(ctx, workerTimeout)
log.Printf("[INFO] Running towards %s with Org %s", baseUrl, orgId) log.Printf("[INFO] Running towards %s with Org %s", baseUrl, orgId)
httpProxy := os.Getenv("HTTP_PROXY") httpProxy := os.Getenv("HTTP_PROXY")
@@ -383,12 +396,13 @@ func main() {
for { for {
//log.Printf("Prerequest") //log.Printf("Prerequest")
newresp, err := client.Do(req) newresp, err := client.Do(req)
executionCount := getRunningWorkers(ctx, workerTimeout)
//log.Printf("Postrequest") //log.Printf("Postrequest")
if err != nil { if err != nil {
log.Printf("[WARNING] Failed making request: %s", err) log.Printf("[WARNING] Failed making request: %s", err)
zombiecounter += 1 zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout { if zombiecounter*sleepTime > workerTimeout {
go zombiecheck(workerTimeout) go zombiecheck(ctx, workerTimeout)
zombiecounter = 0 zombiecounter = 0
} }
time.Sleep(time.Duration(sleepTime) * time.Second) time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -409,7 +423,7 @@ func main() {
log.Printf("[ERROR] Failed reading body: %s", err) log.Printf("[ERROR] Failed reading body: %s", err)
zombiecounter += 1 zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout { if zombiecounter*sleepTime > workerTimeout {
go zombiecheck(workerTimeout) go zombiecheck(ctx, workerTimeout)
zombiecounter = 0 zombiecounter = 0
} }
time.Sleep(time.Duration(sleepTime) * time.Second) time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -423,7 +437,7 @@ func main() {
sleepTime = 10 sleepTime = 10
zombiecounter += 1 zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout { if zombiecounter*sleepTime > workerTimeout {
go zombiecheck(workerTimeout) go zombiecheck(ctx, workerTimeout)
zombiecounter = 0 zombiecounter = 0
} }
time.Sleep(time.Duration(sleepTime) * time.Second) time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -438,13 +452,31 @@ func main() {
if len(executionRequests.Data) == 0 { if len(executionRequests.Data) == 0 {
zombiecounter += 1 zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout { if zombiecounter*sleepTime > workerTimeout {
go zombiecheck(workerTimeout) go zombiecheck(ctx, workerTimeout)
zombiecounter = 0 zombiecounter = 0
} }
time.Sleep(time.Duration(sleepTime) * time.Second) time.Sleep(time.Duration(sleepTime) * time.Second)
continue continue
} }
// Anything below here verifies concurrency virification
if executionCount >= maxConcurrency {
if zombiecounter*sleepTime > workerTimeout {
go zombiecheck(ctx, workerTimeout)
zombiecounter = 0
}
time.Sleep(time.Duration(sleepTime) * time.Second)
continue
}
log.Printf("Got %d new requests. Executing: %d. Max: %d", len(executionRequests.Data), executionCount, maxConcurrency)
allowed := maxConcurrency - executionCount
if len(executionRequests.Data) > allowed {
log.Printf("[WARNING] Throttle - Cutting down requests from %d to %d", len(executionRequests.Data), allowed)
executionRequests.Data = executionRequests.Data[0:allowed]
}
// New, abortable version. Should check executionid and remove everything else // New, abortable version. Should check executionid and remove everything else
var toBeRemoved ExecutionRequestWrapper var toBeRemoved ExecutionRequestWrapper
for _, execution := range executionRequests.Data { for _, execution := range executionRequests.Data {
@@ -481,7 +513,7 @@ func main() {
go deployWorker(workerImage, containerName, env) go deployWorker(workerImage, containerName, env)
log.Printf("[INFO] %s is deployed and to be removed from queue.", execution.ExecutionId) log.Printf("[INFO] %s was deployed and to be removed from queue.", execution.ExecutionId)
zombiecounter += 1 zombiecounter += 1
toBeRemoved.Data = append(toBeRemoved.Data, execution) toBeRemoved.Data = append(toBeRemoved.Data, execution)
} }
@@ -543,12 +575,59 @@ func main() {
} }
} }
// Is this ok to do with Docker? idk :)
func getRunningWorkers(ctx context.Context, workerTimeout int) int {
containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
All: true,
})
if err != nil {
log.Printf("Error getting containers: %s", err)
return 0
}
currenttime := time.Now().Unix()
counter := 0
for _, container := range containers {
// Skip random containers. Only handle things related to Shuffle.
if !strings.Contains(container.Image, baseimagename) {
shuffleFound := false
for _, item := range container.Labels {
if item == "shuffle" {
shuffleFound = true
break
}
}
// Check image name
if !shuffleFound {
continue
}
//} else {
// log.Printf("NAME: %s", container.Image)
}
for _, name := range container.Names {
// FIXME - add name_version_uid_uid regex check as well
if !strings.HasPrefix(name, "/worker") {
continue
}
//log.Printf("Time: %d - %d", currenttime-container.Created, int64(workerTimeout))
if container.State == "running" && currenttime-container.Created < int64(workerTimeout) {
counter += 1
break
}
}
}
return counter
}
// FIXME - add this to remove exited workers // FIXME - add this to remove exited workers
// Should it check what happened to the execution? idk // Should it check what happened to the execution? idk
func zombiecheck(workerTimeout int) error { func zombiecheck(ctx context.Context, workerTimeout int) error {
log.Println("[INFO] Looking for old containers") log.Println("[INFO] Looking for old containers")
ctx := context.Background()
containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{ containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
All: true, All: true,
}) })