Added tracker of selected executions

backend/app_sdk/app_base.py

@@ -601,7 +601,7 @@ class AppBase:
 
 
         self.full_execution = fullexecution
-        self.logger.info("AFTER FULLEXEC stream result")
+        self.logger.info("AFTER FULLEXEC stream result (init)")
 
         # Gets the value at the parenthesis level you want
         def parse_nested_param(string, level):

backend/go-app/codegen.go

@@ -412,8 +412,8 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet
 		fileBalance,
 	)
 
-	//log.Printf("FUNCTION: %s", data)
+	if strings.Contains(functionname, "get_list_rulesssss") {
-	if strings.Contains(functionname, "filescan") {
+		//log.Printf("FUNCTION: %s", data)
 		log.Println(data)
 		log.Printf("Queries: %s", queryString)
 	}

backend/go-app/docker.go

@@ -190,7 +190,7 @@ func fixTags(tags []string) []string {
 */
 
 // Custom Docker image builder wrapper in memory
-func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string) error {
+func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string, downloadIfFail bool) error {
 	ctx := context.Background()
 	client, err := client.NewEnvClient()
 	if err != nil {

backend/go-app/walkoff.go

@@ -909,8 +909,6 @@ func validateNewWorkerExecution(body []byte) error {
 		return err
 	}
 
-	//log.Printf("LEN: %s", string(body))
-	//log.Printf("LEN: %d", len(string(body)))
 	baseExecution, err := getWorkflowExecution(ctx, execution.ExecutionId)
 	if err != nil {
 		log.Printf("[ERROR] Failed getting execution (workflowqueue) %s: %s", execution.ExecutionId, err)
@@ -6421,7 +6419,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
 	if len(extra) == 0 {
 		log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst))
 		for _, item := range buildLaterFirst {
-			err = buildImageMemory(fs, item.Tags, item.Extra)
+			err = buildImageMemory(fs, item.Tags, item.Extra, true)
 			if err != nil {
 				log.Printf("Failed image build memory: %s", err)
 			} else {
@@ -6435,7 +6433,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
 
 		log.Printf("[INFO] Starting build of %d skipped docker images", len(buildLaterList))
 		for _, item := range buildLaterList {
-			err = buildImageMemory(fs, item.Tags, item.Extra)
+			err = buildImageMemory(fs, item.Tags, item.Extra, true)
 			if err != nil {
 				log.Printf("[INFO] Failed image build memory: %s", err)
 			} else {
@@ -6647,15 +6645,19 @@ func getAllSchedules(ctx context.Context, orgId string) ([]ScheduleOld, error) {
 	return schedules, nil
 }
 
+//FIXME: Add cursor
 func getAllWorkflowApps(ctx context.Context) ([]WorkflowApp, error) {
 	var allworkflowapps []WorkflowApp
-	q := datastore.NewQuery("workflowapp").Order("-edited").Limit(50)
+	q := datastore.NewQuery("workflowapp").Order("-edited").Limit(40)
+	//Activated     bool   `json:"activated" yaml:"activated" required:false datastore:"activated"`
 	//Activated     bool   `json:"activated" yaml:"activated" required:false datastore:"activated"`
 
 	_, err := dbclient.GetAll(ctx, q, &allworkflowapps)
 	if err != nil {
 		if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") {
-			q := datastore.NewQuery("workflowapp").Limit(40).Order("-edited")
+			//datastore.NewQuery("workflowapp").Limit(30).Order("-edited")
+			q = datastore.NewQuery("workflowapp").Order("-edited").Limit(27)
+			//q := q.Limit(25)
 			_, err := dbclient.GetAll(ctx, q, &allworkflowapps)
 			if err != nil {
 				return []WorkflowApp{}, err

docker-compose.yml

@@ -16,7 +16,7 @@ services:
     depends_on:
       - backend
   backend:
-    #build: ./backend
+    build: ./backend
     image: ghcr.io/frikky/shuffle-backend:0.8.54
     container_name: shuffle-backend
     hostname: ${BACKEND_HOSTNAME}

frontend/src/views/AngularWorkflow.jsx

@@ -153,6 +153,7 @@ const AngularWorkflow = (props) => {
 	const [requiresAuthentication, setRequiresAuthentication] = React.useState(false)
 	const [rightSideBarOpen, setRightSideBarOpen] = React.useState(false)
 	const [showSkippedActions, setShowSkippedActions] = React.useState(false)
+	const [lastExecution, setLastExecution] = React.useState("")
 
 	const [selectedResult, setSelectedResult] = React.useState({})
 	const [codeModalOpen, setCodeModalOpen] = React.useState(false);
@@ -6380,13 +6381,14 @@ const AngularWorkflow = (props) => {
 				</Breadcrumbs>	
 				<Button 
 					style={{borderRadius: "0px"}}
+					variant="outlined"
 					onClick={() => {
 						getWorkflowExecution(props.match.params.key)
 					}} color="primary">
 					<CachedIcon style={{marginRight: 10}}/>
 					Refresh	executions
 				</Button>
-				<Divider style={{backgroundColor: "white", marginTop: 10, marginBottom: 10,}}/>
+				<Divider style={{backgroundColor: "rgba(255,255,255,0.5)", marginTop: 10, marginBottom: 10,}}/>
 				{workflowExecutions.length > 0 ? 
 					<div>
 						{workflowExecutions.map((data, index) => {
@@ -6405,6 +6407,7 @@ const AngularWorkflow = (props) => {
 							}
 
 							return (
+								<Tooltip title={data.result} placement="left-start">
 								<Paper elevation={5} key={data.execution_id} square style={executionPaperStyle} onMouseOver={() => {}} onMouseOut={() => {}} onClick={() => {
 
 									if (data.result === undefined || data.result === null || data.result.length === 0) {
@@ -6420,7 +6423,7 @@ const AngularWorkflow = (props) => {
 									setExecutionData(data)
 								}}>
 									<div style={{display: "flex", flex: 1}}>
-										<div style={{marginLeft: 5, width: 2, backgroundColor: statusColor, marginRight: 5}} />
+										<div style={{marginLeft: 0, width: lastExecution === data.execution_id ? 4 : 2, backgroundColor: statusColor, marginRight: 5}} />
 										<div style={{height: "100%", width: 40, borderColor: "white", marginRight: 15}}>
 											{getExecutionSourceImage(data)}
 										</div>
@@ -6436,9 +6439,14 @@ const AngularWorkflow = (props) => {
 											: null}
 									</div>
 									<Tooltip title={"Inspect execution"} placement="top">
-										<KeyboardArrowRightIcon style={{marginTop: "auto", marginBottom: "auto"}}/>
+										{lastExecution === data.execution_id ?
+											<KeyboardArrowRightIcon style={{color: "#f85a3e", marginTop: "auto", marginBottom: "auto"}}/>
+											:
+											<KeyboardArrowRightIcon style={{marginTop: "auto", marginBottom: "auto"}}/>
+										}
 									</Tooltip>
 								</Paper>
+								</Tooltip>
 							)
 							return 
 						})}
@@ -6457,6 +6465,7 @@ const AngularWorkflow = (props) => {
 							stop()
 							getWorkflowExecution(props.match.params.key)
 							setExecutionModalView(0)
+							setLastExecution(executionData.execution_id)
 					}}>
 						<IconButton style={{paddingLeft: 0, marginTop: "auto", marginBottom: "auto", }} onClick={() => {}}>
 							<ArrowBackIcon style={{color: "rgba(255,255,255,0.5)",}} />
@@ -6617,6 +6626,21 @@ const AngularWorkflow = (props) => {
 		validate.result = JSON.parse(validate.result)
 	}
 
+	//if (codeModalOpen && selectedResult.result.includes("file_id")) {
+	//	console.log("SHOW RESULT WITH FILES: ", selectedResult.result)
+	//	//const regex = "\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b"
+	//	//const regex = /^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}/i
+	//	const regex = /^[A-F\d]{8}-[A-F\d]{4}-4[A-F\d]{3}-[89AB][A-F\d]{3}-[A-F\d]{12}$/i
+	//	//const found = selectedResult.result.match(regex)
+	//	const found = "hello how are you cf80fa70-65cf-4963-b474-b459a6dead81 what".match(regex)
+	//	const regex = /\${(\w{8}-\w{4}-\w{3}-\w{3}-\w)}/g
+	//	const found = placeholder.match(regex)
+
+	//	console.log("FOUND: ", found)
+
+	//	//cf80fa70-65cf-4963-b474-b459a6dead81
+	//}
+
 	const codePopoutModal = !codeModalOpen ? null : 
 		<Draggable
 			onDrag={(e) => {

frontend/src/views/AppCreator.jsx

@@ -479,11 +479,12 @@ const AppCreator = (props) => {
 		for (let [path, pathvalue] of Object.entries(data.paths)) {
 			for (let [method, methodvalue] of Object.entries(pathvalue)) {
 				if (methodvalue === null) {
-					alert.info("Skipped method "+method)
+					alert.info("Skipped method (null)"+method)
 					continue
 				}
 
 				if (!allowedfunctions.includes(method.toUpperCase())) {
+					alert.info("Skipped method (not allowed) "+method)
 					continue
 				}
 
@@ -556,7 +557,7 @@ const AppCreator = (props) => {
 				}
 
 				// HAHAHA wtf is this.
-				if (methodvalue.responses !== undefined) {
+				if (methodvalue.responses !== undefined && methodvalue.responses !== null) {
 					if (methodvalue.responses.default !== undefined) {
 						if (methodvalue.responses.default.content !== undefined) {
 							if (methodvalue.responses.default.content["text/plain"] !== undefined) {
@@ -1818,7 +1819,7 @@ const AppCreator = (props) => {
 						addPathQuery()
 					}}>New query</Button> 				
 					{currentActionMethod === "POST" ?
-						<Button color="primary" variant={fileUploadEnabled ? "contained" : "outlined"} style={{marginLeft: 10, marginTop: "5px", marginBottom: "10px", borderRadius: "0px"}} onClick={() => {
+						<Button disabled color="primary" variant={fileUploadEnabled ? "contained" : "outlined"} style={{marginLeft: 10, marginTop: "5px", marginBottom: "10px", borderRadius: "0px"}} onClick={() => {
 							setFileUploadEnabled(!fileUploadEnabled)
 							if (fileUploadEnabled && currentAction["file_field"].length > 0) {
 								setActionField("file_field", "")

functions/onprem/orborus/build.sh

@@ -1,5 +1,5 @@
 NAME=shuffle-orborus
-VERSION=0.8.53
+VERSION=0.8.54
 
 echo "Running docker build with $NAME:$VERSION"
 #docker rmi frikky/shuffle:$NAME --force

functions/onprem/orborus/orborus.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
+	//"github.com/docker/docker/api/types/filters"
 	dockerclient "github.com/docker/docker/client"
 	"github.com/satori/go.uuid"
 	//network "github.com/docker/docker/api/types/network"
@@ -311,15 +312,12 @@ func getStats() {
 		return
 	}
 
-	fmt.Printf("[INFO] memory total: %d bytes\n", memory.Total)
-	fmt.Printf("[INFO] memory used: %d bytes\n", memory.Used)
-
 	before, err := cpu.Get()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%s\n", err)
 		return
 	}
-	time.Sleep(time.Duration(500) * time.Millisecond)
+	time.Sleep(time.Duration(250) * time.Millisecond)
 	after, err := cpu.Get()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%s\n", err)
@@ -327,6 +325,8 @@ func getStats() {
 	}
 	total := float64(after.Total - before.Total)
 
+	fmt.Printf("[INFO] memory total: %d bytes\n", memory.Total)
+	fmt.Printf("[INFO] memory used: %d bytes\n", memory.Used)
 	fmt.Printf("[INFO] cpu used  : %f%%\n", float64(after.User-before.User)/total*100)
 	fmt.Printf("[INFO] cpu system: %f%%\n", float64(after.System-before.System)/total*100)
 	fmt.Printf("[INFO] cpu idle  : %f%%\n", float64(after.Idle-before.Idle)/total*100)
@@ -408,7 +408,7 @@ func main() {
 		},
 	}
 
-	getStats()
+	//getStats()
 
 	if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" {
 		client = &http.Client{}
@@ -440,6 +440,7 @@ func main() {
 	hasStarted := false
 	for {
 		//log.Printf("Prerequest")
+		//go getStats()
 		newresp, err := client.Do(req)
 		executionCount := getRunningWorkers(ctx, workerTimeout)
 		//log.Printf("Postrequest")
@@ -626,10 +627,13 @@ func getRunningWorkers(ctx context.Context, workerTimeout int) int {
 	containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
 		All: true,
 	})
+	//Filters: filters.Args{
+	//	map[string][]string{"ancestor": {"<imagename>:<version>"}},
+	//},
 
 	if err != nil {
-		log.Printf("Error getting containers: %s", err)
+		log.Printf("[ERROR] Error getting containers: %s", err)
-		return 0
+		return maxConcurrency
 	}
 
 	currenttime := time.Now().Unix()

functions/onprem/worker/worker.go

@@ -852,6 +852,10 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
 			Type:   "json-file",
 			Config: map[string]string{},
 		},
+		Resources: container.Resources{
+			CPUShares: 256,
+			CPUPeriod: 10000,
+		},
 	}
 
 	// form container id and use it as network source if it's not empty