marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize: Made Workflow workers self-destruct at times

Browse Source
This commit is contained in:
frikky
2021-01-14 17:01:11 +01:00
parent 34097f971e
commit 3b6501c56c
12 changed files with 162 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+3
View File
@@ -40,3 +40,6 @@ SHUFFLE_PASS_WORKER_PROXY=TRUE
SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io
SHUFFLE_BASE_IMAGE_NAME=frikky
SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.3"
# Used for auto-cleanup of containers. REALLY important at scale.
SHUFFLE_CONTAINER_AUTO_CLEANUP=false
backend/app_sdk/app_base.py
+1 -1
View File
@@ -21,7 +21,7 @@ class AppBase:
# apikey is for the user / org
# authorization is for the specific workflow
self.url = os.getenv("CALLBACK_URL", "https://shuffler.io")
self.base_url = os.getenv("BASE_URL", "https://shuffler.io")
self.base_url = os.getenv("BASE_URL", "")
self.action = os.getenv("ACTION", "")
self.authorization = os.getenv("AUTHORIZATION", "")
self.current_execution_id = os.getenv("EXECUTIONID", "")
backend/app_sdk/build.sh
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/bash
NAME=shuffle-app_sdk
VERSION=0.8.52
VERSION=0.8.53
docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
backend/go-app/codegen.go
+5 -6
View File
@@ -397,12 +397,11 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet
verifyAddin,
)
/*
if strings.Contains(functionname, "search") {
log.Println(data)
log.Printf("Queries: %s", queryString)
}
*/
//log.Printf("FUNCTION: %s", data)
//if strings.Contains(functionname, "search") {
// log.Println(data)
// log.Printf("Queries: %s", queryString)
//}
//log.Printf(data)
return functionname, data
backend/go-app/walkoff.go
+9 -6
View File
@@ -917,12 +917,13 @@ func validateNewWorkerExecution(body []byte) error {
return errors.New("Bad authorization when validating execution")
}
// used to validate if it's actually the right marshal
if len(baseExecution.Workflow.Actions) != len(execution.Workflow.Actions) {
return errors.New(fmt.Sprintf("Bad length of actions: %d", len(execution.Workflow.Actions)))
return errors.New(fmt.Sprintf("Bad length of actions (probably normal app): %d", len(execution.Workflow.Actions)))
}
if len(baseExecution.Workflow.Triggers) != len(execution.Workflow.Triggers) {
return errors.New(fmt.Sprintf("Bad length of trigger: %d", len(execution.Workflow.Triggers)))
return errors.New(fmt.Sprintf("Bad length of trigger: %d (probably normal app)", len(execution.Workflow.Triggers)))
}
// FIXME: Add extra here
@@ -933,8 +934,11 @@ func validateNewWorkerExecution(body []byte) error {
//log.Printf("\n\nSHOULD SET BACKEND DATA FOR EXEC \n\n")
err = setWorkflowExecution(ctx, execution, true)
if err != nil {
log.Printf("Successfully set the execution to wait.")
if err == nil {
log.Printf("[INFO] Set workflowexecution based on new worker (>0.8.53) for execution %s", baseExecution.ExecutionId)
//log.Printf("[INFO] Successfully set the execution to wait.")
} else {
log.Printf("[WARNING] Failed to set the execution to wait.")
}
return nil
@@ -957,12 +961,11 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
//log.Printf("Actionresult unmarshal: %s", string(body))
err = validateNewWorkerExecution(body)
if err == nil {
log.Printf("[INFO] Set workflowexecution based on new worker")
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Success"}`)))
return
} else {
log.Printf("[WARNING] Failed to handle new execution variant: %s", err)
//log.Printf("[WARNING] Failed to handle new execution variant: %s", err)
}
var actionResult ActionResult
docker-compose.yml
+4 -3
View File
@@ -1,8 +1,8 @@
version: '3'
services:
frontend:
#build: ./frontend
image: ghcr.io/frikky/shuffle-frontend:0.8.51
build: ./frontend
image: ghcr.io/frikky/shuffle-frontend:0.8.52
container_name: shuffle-frontend
hostname: shuffle-frontend
ports:
@@ -17,7 +17,7 @@ services:
- backend
backend:
#build: ./backend
image: ghcr.io/frikky/shuffle-backend:0.8.51
image: ghcr.io/frikky/shuffle-backend:0.8.52
container_name: shuffle-backend
hostname: ${BACKEND_HOSTNAME}
# Here for debugging:
@@ -66,6 +66,7 @@ services:
- SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME}
- SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY}
- SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX}
- CLEANUP=${SHUFFLE_CONTAINER_AUTO_CLEANUP}
restart: unless-stopped
database:
#build: ./backend/database
frontend/src/views/AngularWorkflow.jsx
+56 -37
View File
@@ -4276,11 +4276,18 @@ const AngularWorkflow = (props) => {
color="primary"
defaultValue={data.value}
placeholder={placeholder}
helperText={data.value !== undefined && data.value !== null && data.value.includes(".#") ?
<span style={{color:"white", marginBottom: 5, marginleft: 5,}}>
Use "Shuffle Tools" app with "Filter List" action to handle loops
</span>
: null
}
onClick={() => {
console.log("CHANGE FIELD")
}}
onBlur={(e) => {
changeActionVariable(data.action_field, e.target.value)
setUpdate(Math.random())
}}
/>
@@ -4473,8 +4480,10 @@ const AngularWorkflow = (props) => {
}}
>
<FormControl>
<DialogTitle><div style={{color:"white"}}>Condition</div></DialogTitle>
<DialogContent style={{display: "flex"}}>
<DialogTitle><span style={{color:"white"}}>Condition</span>
</DialogTitle>
<DialogContent style={{}}>
<div style={{display: "flex"}}>
<Tooltip color="primary" title={conditionValue.configuration ? "Negated" : "Default"} placement="top">
<Button color="primary" variant={conditionValue.configuration ? "contained" : "outlined"} style={{margin: "auto", height: 50, marginBottom: "auto", marginTop: "auto", marginRight: 5}} onClick={(e) => {
conditionValue.configuration = !conditionValue.configuration
@@ -4556,38 +4565,43 @@ const AngularWorkflow = (props) => {
<div style={{flex: "2"}}>
<AppConditionHandler tmpdata={destinationValue} setData={setDestinationValue} type={"destination"} />
</div>
</div>
<span style={{marginTop: 15}}>
Conditions can't be used for loops [ .# ]. <a target="_blank" href="https://shuffler.io/docs/conditions" style={{textDecoration: "none", color: "#f85a3e"}}>Learn more</a>
</span>
</DialogContent>
<DialogActions>
<Button
style={{borderRadius: "0px"}}
onClick={() => {
setSelectedEdge({})
<Button
style={{borderRadius: "0px"}}
onClick={() => {
setSelectedEdge({})
var data = {
condition: conditionValue,
source: sourceValue,
destination: destinationValue,
}
setConditionsModalOpen(false)
if (selectedEdge.conditions === undefined) {
selectedEdge.conditions = [data]
} else {
const curedgeindex = selectedEdge.conditions.findIndex(data => data.source.id === sourceValue.id)
if (curedgeindex < 0) {
selectedEdge.conditions.push(data)
} else {
selectedEdge.conditions[curedgeindex] = data
var data = {
condition: conditionValue,
source: sourceValue,
destination: destinationValue,
}
setConditionsModalOpen(false)
if (selectedEdge.conditions === undefined) {
selectedEdge.conditions = [data]
} else {
const curedgeindex = selectedEdge.conditions.findIndex(data => data.source.id === sourceValue.id)
if (curedgeindex < 0) {
selectedEdge.conditions.push(data)
} else {
selectedEdge.conditions[curedgeindex] = data
}
}
}
setSelectedEdge(selectedEdge)
workflow.branches[selectedEdgeIndex] = selectedEdge
setWorkflow(workflow)
}} color="primary">
Submit
</Button>
</DialogActions>
setSelectedEdge(selectedEdge)
workflow.branches[selectedEdgeIndex] = selectedEdge
setWorkflow(workflow)
}} color="primary">
Submit
</Button>
</DialogActions>
</FormControl>
</Dialog>
@@ -6422,15 +6436,20 @@ const AngularWorkflow = (props) => {
:
<div style={{padding: 25, maxWidth: 365, overflowX: "hidden",}}>
<Breadcrumbs aria-label="breadcrumb" separator="›" style={{color: "white", fontSize: 16}}>
<h2 style={{color: "rgba(255,255,255,0.5)", cursor: "pointer"}} onClick={() => {
setExecutionRunning(false)
stop()
getWorkflowExecution(props.match.params.key)
setExecutionModalView(0)
<span style={{color: "rgba(255,255,255,0.5)", display: "flex"}} onClick={() => {
setExecutionRunning(false)
stop()
getWorkflowExecution(props.match.params.key)
setExecutionModalView(0)
}}>
<ArrowBackIcon style={{marginRight: 7, }} />
See other Executions
</h2>
<IconButton style={{paddingLeft: 0, marginTop: "auto", marginBottom: "auto", }} onClick={() => {}}>
<ArrowBackIcon style={{color: "rgba(255,255,255,0.5)",}} />
</IconButton>
<h2 style={{color: "rgba(255,255,255,0.5)", cursor: "pointer"}} onClick={() => {
}}>
See other Executions
</h2>
</span>
</Breadcrumbs>
<Divider style={{backgroundColor: "white", marginTop: 10, marginBottom: 10,}}/>
<h2>Executing Workflow</h2>
frontend/src/views/Apps.jsx
+1 -1
View File
@@ -341,7 +341,7 @@ const Apps = (props) => {
}
var description = data.description
const maxDescLen = 60
const maxDescLen = 58
if (description.length > maxDescLen) {
description = data.description.slice(0, maxDescLen)+"..."
}
functions/onprem/orborus/build.sh
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=shuffle-orborus
VERSION=0.8.52
VERSION=0.8.53
echo "Running docker build with $NAME:$VERSION"
#docker rmi frikky/shuffle:$NAME --force
functions/onprem/orborus/orborus.go
+10 -2
View File
@@ -49,6 +49,8 @@ var baseUrl = os.Getenv("BASE_URL")
var environment = os.Getenv("ENVIRONMENT_NAME")
var dockerApiVersion = os.Getenv("DOCKER_API_VERSION")
var runningMode = strings.ToLower(os.Getenv("RUNNING_MODE"))
var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP"))
var workerIds = []string{}
type ExecutionRequestWrapper struct {
Data []ExecutionRequest `json:"data"`
@@ -144,6 +146,10 @@ func deployWorker(image string, identifier string, env []string) {
//log.Printf("[INFO] Empty self container id, continue without NetworkMode")
}
if cleanupEnv == "true" {
hostConfig.AutoRemove = true
}
config := &container.Config{
Image: image,
Env: env,
@@ -212,6 +218,7 @@ func deployWorker(image string, identifier string, env []string) {
//}
} else {
log.Printf("[INFO] Container %s was created under environment %s", cont.ID, environment)
//workerIds = append(workerIds, cont.ID)
}
return
@@ -248,7 +255,7 @@ func initializeImages() {
log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion)
}
if workerVersion == "" {
workerVersion = "0.8.52"
workerVersion = "0.8.53"
log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion)
}
@@ -499,6 +506,7 @@ func main() {
fmt.Sprintf("EXECUTIONID=%s", execution.ExecutionId),
fmt.Sprintf("ENVIRONMENT_NAME=%s", environment),
fmt.Sprintf("BASE_URL=%s", baseUrl),
fmt.Sprintf("CLEANUP=%s", cleanupEnv),
}
if strings.ToLower(os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) != "false" {
@@ -512,7 +520,7 @@ func main() {
go deployWorker(workerImage, containerName, env)
log.Printf("[INFO] %s was deployed and to be removed from queue.", execution.ExecutionId)
log.Printf("[INFO] ExecutionID %s was deployed and to be removed from queue.", execution.ExecutionId)
zombiecounter += 1
toBeRemoved.Data = append(toBeRemoved.Data, execution)
}
functions/onprem/worker/build.sh
+2 -2
View File
@@ -1,5 +1,5 @@
NAME=shuffle-worker
VERSION=0.8.52
VERSION=0.8.53
echo "Running docker build with $NAME:$VERSION"
#CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin .
@@ -10,5 +10,5 @@ docker build . -t frikky/shuffle:$NAME -t frikky/shuffle:$NAME_$VERSION -t docke
#docker push frikky/shuffle:$NAME_$VERSION
#docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION
#docker tag frikky/shuffle:0.8.51 ghcr.io/frikky/shuffle-worker:0.8.5
#docker push ghcr.io/frikky/$NAME:$VERSION
docker tag frikky/shuffle:$NAME ghcr.io/frikky/shuffle-worker:0.8.52
docker push ghcr.io/frikky/$NAME:$VERSION
functions/onprem/worker/worker.go
+69 -98
View File
@@ -28,6 +28,7 @@ import (
var environment = os.Getenv("ENVIRONMENT_NAME")
var baseUrl = os.Getenv("BASE_URL")
var appCallbackUrl = os.Getenv("BASE_URL")
var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP"))
var baseimagename = "frikky/shuffle"
var registryName = "registry.hub.docker.com"
var fallbackName = "shuffle-orborus"
@@ -43,6 +44,7 @@ var children map[string][]string
var visited []string
var executed []string
var nextActions []string
var containerIds []string
var extra int
var startAction string
@@ -68,9 +70,9 @@ func getThisContainerId() string {
func init() {
containerId = getThisContainerId()
if len(containerId) == 0 {
log.Printf("[ERROR] No container ID found. Not running containerized?")
log.Printf("[WARNING] No container ID found. Not running containerized? This should only show during testing")
} else {
log.Printf("[INFO] Found container ID: %s", containerId)
log.Printf("[INFO] Found container ID for this worker: %s", containerId)
}
}
@@ -770,35 +772,28 @@ type AppExecutionExample struct {
// removes every container except itself (worker)
func shutdown(executionId, workflowId string) {
dockercli, err := dockerclient.NewEnvClient()
if err != nil {
log.Printf("[ERROR] Unable to create docker client: %s", err)
os.Exit(3)
}
log.Printf("[INFO] Shutdown started")
containerOptions := types.ContainerListOptions{
All: true,
}
// Might not be necessary because of cleanupEnv hostconfig autoremoval
if cleanupEnv == "true" && len(containerIds) > 0 {
ctx := context.Background()
dockercli, err := dockerclient.NewEnvClient()
if err == nil {
log.Printf("[INFO] Cleaning up %d containers", len(containerIds))
removeOptions := types.ContainerRemoveOptions{
RemoveVolumes: true,
Force: true,
}
containers, err := dockercli.ContainerList(context.Background(), containerOptions)
if err != nil {
panic(err)
}
_ = containers
for _, container := range containers {
for _, name := range container.Names {
if strings.Contains(name, executionId) {
// FIXME - reinstate - not here for debugging
//err = removeContainer(container.ID)
//if err != nil {
// log.Printf("Failed removing %s before shutdown.", name)
//}
break
for _, containername := range containerIds {
log.Printf("[INFO] Stopping and removing container %s", containername)
dockercli.ContainerStop(ctx, containername, nil)
dockercli.ContainerRemove(ctx, containername, removeOptions)
//removeContainers = append(removeContainers, containername)
}
}
} else {
log.Printf("[INFO] NOT cleaning up containers. IDS: %d, CLEANUP env: %s", len(containerIds), cleanupEnv)
}
fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowId, executionId)
@@ -844,7 +839,9 @@ func shutdown(executionId, workflowId string) {
log.Printf("[INFO] Failed abort request: %s", err)
}
log.Printf("[INFO] Finished shutdown.")
log.Printf("[INFO] Finished shutdown (after 15 seconds).")
// Allows everything to finish in subprocesses
time.Sleep(time.Duration(15) * time.Second)
os.Exit(3)
}
@@ -865,6 +862,10 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
log.Printf("[WARNING] Empty self container id, continue without NetworkMode")
}
if cleanupEnv == "true" {
hostConfig.AutoRemove = true
}
config := &container.Config{
Image: image,
Env: env,
@@ -892,6 +893,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
}
log.Printf("[INFO] Container %s is created for %s", cont.ID, identifier)
containerIds = append(containerIds, cont.ID)
return nil
}
@@ -1537,71 +1539,16 @@ func handleExecutionResult(workflowExecution WorkflowExecution) {
// FIXME - new request here
// FIXME - clean up stopped (remove) containers with this execution id
dockercli, err := dockerclient.NewEnvClient()
if err != nil {
log.Printf("Unable to create docker client: %s", err)
shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID)
}
if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extra {
shutdownCheck := true
ctx := context.Background()
for _, result := range workflowExecution.Results {
if result.Status == "EXECUTING" {
// Cleaning up executing stuff
shutdownCheck = false
// Check status
containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
All: true,
})
if err != nil {
log.Printf("Failed listing containers: %s", err)
continue
}
stopContainers := []string{}
removeContainers := []string{}
for _, container := range containers {
for _, name := range container.Names {
if !strings.Contains(name, result.Action.ID) {
continue
}
if container.State != "running" {
removeContainers = append(removeContainers, container.ID)
stopContainers = append(stopContainers, container.ID)
}
}
}
// FIXME - add killing of apps with same execution ID too
// FIXME - stahp
//for _, containername := range stopContainers {
// if err := dockercli.ContainerStop(ctx, containername, nil); err != nil {
// log.Printf("Unable to stop container: %s", err)
// } else {
// log.Printf("Stopped container %s", containername)
// }
//}
removeOptions := types.ContainerRemoveOptions{
RemoveVolumes: true,
Force: true,
}
_ = removeOptions
// FIXME - this
//for _, containername := range removeContainers {
// if err := dockercli.ContainerRemove(ctx, containername, removeOptions); err != nil {
// log.Printf("Unable to remove container: %s", err)
// } else {
// log.Printf("Removed container %s", containername)
// }
//}
// USED TO BE CONTAINER REMOVAL
// FIXME - send POST request to kill the container
log.Printf("Should remove (POST request) stopped containers")
//log.Printf("Should remove (POST request) stopped containers")
//ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result)
}
}
@@ -2511,7 +2458,7 @@ func validateFinished(workflowExecution WorkflowExecution) {
if err != nil {
log.Printf("[ERROR] Failed reading body: %s", err)
} else {
log.Printf("NEWRESP: %s", string(body))
log.Printf("[INFO] NEWRESP (from backend): %s", string(body))
}
}
}
@@ -2583,7 +2530,7 @@ func setWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecuti
}
// GetLocalIP returns the non loopback local IP of the host
func GetLocalIP() string {
func getLocalIP() string {
addrs, err := net.InterfaceAddrs()
if err != nil {
return ""
@@ -2599,22 +2546,46 @@ func GetLocalIP() string {
return ""
}
func webserverSetup() {
hostname := GetLocalIP()
func getAvailablePort() (net.Listener, error) {
listener, err := net.Listen("tcp", ":0")
if err != nil {
log.Printf("[WARNING] Failed to assign port by default. Defaulting to 5001")
//return ":5001"
return nil, err
}
log.Printf("\nStarting webserver on port 5001 with hostname: %s\n", hostname)
log.Printf("OLD HOSTNAME: %s", appCallbackUrl)
appCallbackUrl = fmt.Sprintf("http://%s:5001", hostname)
log.Printf("NEW HOSTNAME: %s", appCallbackUrl)
return listener, nil
//return fmt.Sprintf(":%d", port)
}
func runWebserver() {
func webserverSetup(workflowExecution WorkflowExecution) net.Listener {
hostname := getLocalIP()
// FIXME: This MAY not work because of speed between first
// container being launched and port being assigned to webserver
listener, err := getAvailablePort()
if err != nil {
log.Printf("Failed to created listener: %s", err)
shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID)
}
port := listener.Addr().(*net.TCPAddr).Port
log.Printf("\n\nStarting webserver on port %d with hostname: %s\n\n", port, hostname)
log.Printf("OLD HOSTNAME: %s", appCallbackUrl)
appCallbackUrl = fmt.Sprintf("http://%s:%d", hostname, port)
log.Printf("NEW HOSTNAME: %s", appCallbackUrl)
return listener
}
func runWebserver(listener net.Listener) {
r := mux.NewRouter()
r.HandleFunc("/api/v1/streams", handleWorkflowQueue).Methods("POST")
r.HandleFunc("/api/v1/streams/results", handleGetStreamResults).Methods("POST", "OPTIONS")
http.Handle("/", r)
log.Fatal(http.ListenAndServe(":5001", nil))
//log.Fatal(http.ListenAndServe(port, nil))
log.Fatal(http.Serve(listener, nil))
}
// Initial loop etc
@@ -2735,8 +2706,8 @@ func main() {
}
log.Printf("Environments: %s. 1 = webserver, 0 or >1 = default", environments)
if len(environments) == 1 {
webserverSetup()
if len(environments) == 1 { //&& len(workflowExecution.Actions)+len(workflowExecution.Triggers) > 1 {
listener := webserverSetup(workflowExecution)
err := executionInit(workflowExecution)
if err != nil {
log.Printf("[INFO] Workflow setup failed: %s", workflowExecution.ExecutionId, err)
@@ -2748,7 +2719,7 @@ func main() {
handleExecutionResult(workflowExecution)
}()
runWebserver()
runWebserver(listener)
//log.Printf("Before wait")
//wg := sync.WaitGroup{}
//wg.Add(1)