marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

BUG: Fixed resource overload when getting workflows

Browse Source
This commit is contained in:
frikky
2020-12-29 19:34:51 +01:00
parent 5beeadc1b1
commit 45665921a8
6 changed files with 125 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/go-app/walkoff.go
+1 -1
View File
@@ -507,7 +507,7 @@ func setWorkflowQueue(ctx context.Context, executionRequests ExecutionRequestWra
}
func getWorkflowQueue(ctx context.Context, id string) (ExecutionRequestWrapper, error) {
key := datastore.NameKey("workflowqueue", id, nil)
key := datastore.NameKey("workflowqueue", id, nil).Limit(50)
workflows := ExecutionRequestWrapper{}
if err := dbclient.Get(ctx, key, &workflows); err != nil {
return ExecutionRequestWrapper{}, err
backend/tests/execute.sh
+26 -2
View File
@@ -1,5 +1,29 @@
#!/bin/sh
curl -XPOST http://localhost:5001/api/v1/workflows/1d9d8ce2-566e-4c3f-8a37-5d6c7d2000b5/execute -d '{"execution_argument":""}' -H "Authorization: Bearer 144308d0-6aab-4d4f-8bb2-75189281ee26"
#curl -XPOST http://localhost:5001/api/v1/workflows/1d9d8ce2-566e-4c3f-8a37-5d6c7d2000b5/execute -d '{"execution_argument":""}' -H "Authorization: Bearer 144308d0-6aab-4d4f-8bb2-75189281ee26"
curl -XPOST http://localhost:5001/api/v1/workflows/1d9d8ce2-566e-4c3f-8a37-5d6c7d2000b5/execute -d '{"execution_argument":""}' -H "Authorization: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjYwZjQwNjBlNThkNzVmZDNmNzBiZWZmODhjNzk0YTc3NTMyN2FhMzEiLCJ0eXAiOiJKV1QifQ.eyJhdWQiOiJodHRwczovL3NodWZmbGVyLmlvL2FwaS92MS93b3JrZmxvd3MvMWQ5ZDhjZTItNTY2ZS00YzNmLThhMzctNWQ2YzdkMjAwMGI1L2V4ZWN1dGUiLCJhenAiOiIxMDMwNzY3ODIwNjE0MjQ2MTg0MjIiLCJlbWFpbCI6InNjaGVkdWxlckBzaHVmZmxlLTI0MTUxNy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJleHAiOjE1NjU1Mjc1NTEsImlhdCI6MTU2NTUyMzk1MSwiaXNzIjoiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tIiwic3ViIjoiMTAzMDc2NzgyMDYxNDI0NjE4NDIyIn0.r0EDq9fjhf_5CPTiltyfk_L3uYJp577Uy0yYPcCAl2nv50_z_oUtbWGBpQLL8gcj-NGd3g4E52Qur8k6hCMIQweLS6WAb1279vGffEoCNDfkWb3Oy-yJGP1kzwLvqFJqnHLkSWYXNWvSyWnEimW8Rryx_m1BXS5wcA8l4NIr83kS7fPZrTwjnwFSeGSThwk91DVARzapQb8r0GEgOUyHZ1aBXnV98mikzSUt-5xFKe9eMdD22YJAj0Ru-DxAxs5nOqghX4PMRysWjshjOMrlR1piPWxqAmewp8YKZDCQ5gXskpeAFBDoULT971Wsx_NCohnJsFqx1JfPS9ZYMTW2oQ"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
docker-compose.yml
+3 -3
View File
@@ -1,8 +1,8 @@
version: '3'
services:
frontend:
#build: ./frontend
image: ghcr.io/frikky/shuffle-frontend:0.8.43
build: ./frontend
image: ghcr.io/frikky/shuffle-frontend:0.8.45
container_name: shuffle-frontend
hostname: shuffle-frontend
ports:
@@ -17,7 +17,7 @@ services:
- backend
backend:
#build: ./backend
image: ghcr.io/frikky/shuffle-backend:0.8.44
image: ghcr.io/frikky/shuffle-backend:0.8.45
container_name: shuffle-backend
hostname: ${BACKEND_HOSTNAME}
# Here for debugging:
frontend/src/views/AngularWorkflow.jsx
+5 -15
View File
@@ -3681,7 +3681,7 @@ const AngularWorkflow = (props) => {
<div style={{display: "flex", minHeight: 40, marginBottom: 30}}>
<div style={{flex: 1}}>
<h3 style={{marginBottom: 5}}>{selectedAction.app_name}</h3>
<div style={{display: "flex", width: "100%",}}>
<div style={{display: "flex",}}>
<IconButton style={{marginTop: "auto", marginBottom: "auto", height: 30, paddingLeft: 0, paddingRight: 0}} onClick={() => {
console.log("FIND EXAMPLE RESULTS FOR ", selectedAction)
if (workflowExecutions.length > 0) {
@@ -3707,7 +3707,7 @@ const AngularWorkflow = (props) => {
<ArrowLeftIcon style={{color: "white"}}/>
</Tooltip>
</IconButton>
<span style={{width: "100%"}}>
<span style={{}}>
<Typography style={{marginTop: 5, marginLeft: 10,}}><a href="https://shuffler.io/docs/workflows#nodes" target="_blank" style={{textDecoration: "none", color: "#f85a3e"}}>What are actions?</a></Typography>
{selectedAction.errors !== null && selectedAction.errors.length > 0 ?
<div>
@@ -6294,17 +6294,6 @@ const AngularWorkflow = (props) => {
style={{pointerEvents: "none"}}
hideBackdrop={true}
open={codeModalOpen}
onClose={() => {
//setCodeModalOpen(false)
console.log("CLOSE?")
}}
BackdropProps={{
invisible: true,
style: {
backgroundColor: "transparent",
pointerEvents: "none",
}
}}
PaperProps={{
style: {
pointerEvents: "auto",
@@ -6313,7 +6302,8 @@ const AngularWorkflow = (props) => {
minWidth: 750,
padding: 30,
maxHeight: 700,
overflow: "auto",
overflowY: "auto",
overflowX: "hidden",
//boxShadow: "none",
},
}}
@@ -6828,7 +6818,7 @@ const AngularWorkflow = (props) => {
id="copy_element_shuffle"
value={to_be_copied}
disabled={true}
style={{height: 0, width: 0, margin: 0, padding: 0,}}
style={{height: 0, width: 0, margin: 0, padding: 0, zIndex: -10, position: "fixed",}}
/>
</div>
:
functions/onprem/orborus/build.sh
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=shuffle-orborus
VERSION=0.8.32
VERSION=0.8.40
echo "Running docker build with $NAME:$VERSION"
#docker rmi frikky/shuffle:$NAME --force
functions/onprem/orborus/orborus.go
+89 -10
View File
@@ -29,9 +29,11 @@ import (
// Starts jobs in bulk, so this could be increased
var sleepTime = 3
var maxConcurrency = 50
// Timeout if something rashes
var workerTimeoutEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_TIMEOUT")
var concurrencyEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY")
var appSdkVersion = os.Getenv("SHUFFLE_APP_SDK_VERSION")
var workerVersion = os.Getenv("SHUFFLE_WORKER_VERSION")
@@ -147,7 +149,7 @@ func deployWorker(image string, identifier string, env []string) {
Env: env,
}
log.Printf("[INFO] Identifier: %s", identifier)
//log.Printf("[INFO] Identifier: %s", identifier)
cont, err := dockercli.ContainerCreate(
context.Background(),
config,
@@ -317,7 +319,18 @@ func main() {
log.Printf("[INFO] Cleanup process running every %d seconds", workerTimeout)
}
go zombiecheck(workerTimeout)
if concurrencyEnv != "" {
//var concurrencyEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY")
tmpInt, err := strconv.Atoi(concurrencyEnv)
if err == nil {
maxConcurrency = tmpInt
} else {
log.Printf("[WARNING] Env SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY must be a number, not %s. Defaulted to %d", workerTimeoutEnv, maxConcurrency)
}
}
ctx := context.Background()
go zombiecheck(ctx, workerTimeout)
log.Printf("[INFO] Running towards %s with Org %s", baseUrl, orgId)
httpProxy := os.Getenv("HTTP_PROXY")
@@ -383,12 +396,13 @@ func main() {
for {
//log.Printf("Prerequest")
newresp, err := client.Do(req)
executionCount := getRunningWorkers(ctx, workerTimeout)
//log.Printf("Postrequest")
if err != nil {
log.Printf("[WARNING] Failed making request: %s", err)
zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout {
go zombiecheck(workerTimeout)
go zombiecheck(ctx, workerTimeout)
zombiecounter = 0
}
time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -409,7 +423,7 @@ func main() {
log.Printf("[ERROR] Failed reading body: %s", err)
zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout {
go zombiecheck(workerTimeout)
go zombiecheck(ctx, workerTimeout)
zombiecounter = 0
}
time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -423,7 +437,7 @@ func main() {
sleepTime = 10
zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout {
go zombiecheck(workerTimeout)
go zombiecheck(ctx, workerTimeout)
zombiecounter = 0
}
time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -438,13 +452,31 @@ func main() {
if len(executionRequests.Data) == 0 {
zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout {
go zombiecheck(workerTimeout)
go zombiecheck(ctx, workerTimeout)
zombiecounter = 0
}
time.Sleep(time.Duration(sleepTime) * time.Second)
continue
}
// Anything below here verifies concurrency virification
if executionCount >= maxConcurrency {
if zombiecounter*sleepTime > workerTimeout {
go zombiecheck(ctx, workerTimeout)
zombiecounter = 0
}
time.Sleep(time.Duration(sleepTime) * time.Second)
continue
}
log.Printf("Got %d new requests. Executing: %d. Max: %d", len(executionRequests.Data), executionCount, maxConcurrency)
allowed := maxConcurrency - executionCount
if len(executionRequests.Data) > allowed {
log.Printf("[WARNING] Throttle - Cutting down requests from %d to %d", len(executionRequests.Data), allowed)
executionRequests.Data = executionRequests.Data[0:allowed]
}
// New, abortable version. Should check executionid and remove everything else
var toBeRemoved ExecutionRequestWrapper
for _, execution := range executionRequests.Data {
@@ -481,7 +513,7 @@ func main() {
go deployWorker(workerImage, containerName, env)
log.Printf("[INFO] %s is deployed and to be removed from queue.", execution.ExecutionId)
log.Printf("[INFO] %s was deployed and to be removed from queue.", execution.ExecutionId)
zombiecounter += 1
toBeRemoved.Data = append(toBeRemoved.Data, execution)
}
@@ -543,12 +575,59 @@ func main() {
}
}
// Is this ok to do with Docker? idk :)
func getRunningWorkers(ctx context.Context, workerTimeout int) int {
containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
All: true,
})
if err != nil {
log.Printf("Error getting containers: %s", err)
return 0
}
currenttime := time.Now().Unix()
counter := 0
for _, container := range containers {
// Skip random containers. Only handle things related to Shuffle.
if !strings.Contains(container.Image, baseimagename) {
shuffleFound := false
for _, item := range container.Labels {
if item == "shuffle" {
shuffleFound = true
break
}
}
// Check image name
if !shuffleFound {
continue
}
//} else {
// log.Printf("NAME: %s", container.Image)
}
for _, name := range container.Names {
// FIXME - add name_version_uid_uid regex check as well
if !strings.HasPrefix(name, "/worker") {
continue
}
//log.Printf("Time: %d - %d", currenttime-container.Created, int64(workerTimeout))
if container.State == "running" && currenttime-container.Created < int64(workerTimeout) {
counter += 1
break
}
}
}
return counter
}
// FIXME - add this to remove exited workers
// Should it check what happened to the execution? idk
func zombiecheck(workerTimeout int) error {
func zombiecheck(ctx context.Context, workerTimeout int) error {
log.Println("[INFO] Looking for old containers")
ctx := context.Background()
containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
All: true,
})