marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added tracker of selected executions

Browse Source
This commit is contained in:
frikky
2021-01-20 16:03:01 +01:00
parent 792a0b86ca
commit 42bfa92c81
10 changed files with 60 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app_sdk/app_base.py
+1 -1
View File
@@ -601,7 +601,7 @@ class AppBase:
self.full_execution = fullexecution
self.logger.info("AFTER FULLEXEC stream result")
self.logger.info("AFTER FULLEXEC stream result (init)")
# Gets the value at the parenthesis level you want
def parse_nested_param(string, level):
backend/go-app/codegen.go
+2 -2
View File
@@ -412,8 +412,8 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet
fileBalance,
)
//log.Printf("FUNCTION: %s", data)
if strings.Contains(functionname, "filescan") {
if strings.Contains(functionname, "get_list_rulesssss") {
//log.Printf("FUNCTION: %s", data)
log.Println(data)
log.Printf("Queries: %s", queryString)
}
backend/go-app/docker.go
+1 -1
View File
@@ -190,7 +190,7 @@ func fixTags(tags []string) []string {
*/
// Custom Docker image builder wrapper in memory
func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string) error {
func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string, downloadIfFail bool) error {
ctx := context.Background()
client, err := client.NewEnvClient()
if err != nil {
backend/go-app/walkoff.go
+8 -6
View File
@@ -909,8 +909,6 @@ func validateNewWorkerExecution(body []byte) error {
return err
}
//log.Printf("LEN: %s", string(body))
//log.Printf("LEN: %d", len(string(body)))
baseExecution, err := getWorkflowExecution(ctx, execution.ExecutionId)
if err != nil {
log.Printf("[ERROR] Failed getting execution (workflowqueue) %s: %s", execution.ExecutionId, err)
@@ -6421,7 +6419,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
if len(extra) == 0 {
log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst))
for _, item := range buildLaterFirst {
err = buildImageMemory(fs, item.Tags, item.Extra)
err = buildImageMemory(fs, item.Tags, item.Extra, true)
if err != nil {
log.Printf("Failed image build memory: %s", err)
} else {
@@ -6435,7 +6433,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
log.Printf("[INFO] Starting build of %d skipped docker images", len(buildLaterList))
for _, item := range buildLaterList {
err = buildImageMemory(fs, item.Tags, item.Extra)
err = buildImageMemory(fs, item.Tags, item.Extra, true)
if err != nil {
log.Printf("[INFO] Failed image build memory: %s", err)
} else {
@@ -6647,15 +6645,19 @@ func getAllSchedules(ctx context.Context, orgId string) ([]ScheduleOld, error) {
return schedules, nil
}
//FIXME: Add cursor
func getAllWorkflowApps(ctx context.Context) ([]WorkflowApp, error) {
var allworkflowapps []WorkflowApp
q := datastore.NewQuery("workflowapp").Order("-edited").Limit(50)
q := datastore.NewQuery("workflowapp").Order("-edited").Limit(40)
//Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"`
//Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"`
_, err := dbclient.GetAll(ctx, q, &allworkflowapps)
if err != nil {
if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") {
q := datastore.NewQuery("workflowapp").Limit(40).Order("-edited")
//datastore.NewQuery("workflowapp").Limit(30).Order("-edited")
q = datastore.NewQuery("workflowapp").Order("-edited").Limit(27)
//q := q.Limit(25)
_, err := dbclient.GetAll(ctx, q, &allworkflowapps)
if err != nil {
return []WorkflowApp{}, err
docker-compose.yml
+1 -1
View File
@@ -16,7 +16,7 @@ services:
depends_on:
- backend
backend:
#build: ./backend
build: ./backend
image: ghcr.io/frikky/shuffle-backend:0.8.54
container_name: shuffle-backend
hostname: ${BACKEND_HOSTNAME}
frontend/src/views/AngularWorkflow.jsx
+27 -3
View File
@@ -153,6 +153,7 @@ const AngularWorkflow = (props) => {
const [requiresAuthentication, setRequiresAuthentication] = React.useState(false)
const [rightSideBarOpen, setRightSideBarOpen] = React.useState(false)
const [showSkippedActions, setShowSkippedActions] = React.useState(false)
const [lastExecution, setLastExecution] = React.useState("")
const [selectedResult, setSelectedResult] = React.useState({})
const [codeModalOpen, setCodeModalOpen] = React.useState(false);
@@ -6380,13 +6381,14 @@ const AngularWorkflow = (props) => {
</Breadcrumbs>
<Button
style={{borderRadius: "0px"}}
variant="outlined"
onClick={() => {
getWorkflowExecution(props.match.params.key)
}} color="primary">
<CachedIcon style={{marginRight: 10}}/>
Refresh executions
</Button>
<Divider style={{backgroundColor: "white", marginTop: 10, marginBottom: 10,}}/>
<Divider style={{backgroundColor: "rgba(255,255,255,0.5)", marginTop: 10, marginBottom: 10,}}/>
{workflowExecutions.length > 0 ?
<div>
{workflowExecutions.map((data, index) => {
@@ -6405,6 +6407,7 @@ const AngularWorkflow = (props) => {
}
return (
<Tooltip title={data.result} placement="left-start">
<Paper elevation={5} key={data.execution_id} square style={executionPaperStyle} onMouseOver={() => {}} onMouseOut={() => {}} onClick={() => {
if (data.result === undefined || data.result === null || data.result.length === 0) {
@@ -6420,7 +6423,7 @@ const AngularWorkflow = (props) => {
setExecutionData(data)
}}>
<div style={{display: "flex", flex: 1}}>
<div style={{marginLeft: 5, width: 2, backgroundColor: statusColor, marginRight: 5}} />
<div style={{marginLeft: 0, width: lastExecution === data.execution_id ? 4 : 2, backgroundColor: statusColor, marginRight: 5}} />
<div style={{height: "100%", width: 40, borderColor: "white", marginRight: 15}}>
{getExecutionSourceImage(data)}
</div>
@@ -6436,9 +6439,14 @@ const AngularWorkflow = (props) => {
: null}
</div>
<Tooltip title={"Inspect execution"} placement="top">
<KeyboardArrowRightIcon style={{marginTop: "auto", marginBottom: "auto"}}/>
{lastExecution === data.execution_id ?
<KeyboardArrowRightIcon style={{color: "#f85a3e", marginTop: "auto", marginBottom: "auto"}}/>
:
<KeyboardArrowRightIcon style={{marginTop: "auto", marginBottom: "auto"}}/>
}
</Tooltip>
</Paper>
</Tooltip>
)
return
})}
@@ -6457,6 +6465,7 @@ const AngularWorkflow = (props) => {
stop()
getWorkflowExecution(props.match.params.key)
setExecutionModalView(0)
setLastExecution(executionData.execution_id)
}}>
<IconButton style={{paddingLeft: 0, marginTop: "auto", marginBottom: "auto", }} onClick={() => {}}>
<ArrowBackIcon style={{color: "rgba(255,255,255,0.5)",}} />
@@ -6617,6 +6626,21 @@ const AngularWorkflow = (props) => {
validate.result = JSON.parse(validate.result)
}
//if (codeModalOpen && selectedResult.result.includes("file_id")) {
// console.log("SHOW RESULT WITH FILES: ", selectedResult.result)
// //const regex = "\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b"
// //const regex = /^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}/i
// const regex = /^[A-F\d]{8}-[A-F\d]{4}-4[A-F\d]{3}-[89AB][A-F\d]{3}-[A-F\d]{12}$/i
// //const found = selectedResult.result.match(regex)
// const found = "hello how are you cf80fa70-65cf-4963-b474-b459a6dead81 what".match(regex)
// const regex = /\${(\w{8}-\w{4}-\w{3}-\w{3}-\w)}/g
// const found = placeholder.match(regex)
// console.log("FOUND: ", found)
// //cf80fa70-65cf-4963-b474-b459a6dead81
//}
const codePopoutModal = !codeModalOpen ? null :
<Draggable
onDrag={(e) => {
frontend/src/views/AppCreator.jsx
+4 -3
View File
@@ -479,11 +479,12 @@ const AppCreator = (props) => {
for (let [path, pathvalue] of Object.entries(data.paths)) {
for (let [method, methodvalue] of Object.entries(pathvalue)) {
if (methodvalue === null) {
alert.info("Skipped method "+method)
alert.info("Skipped method (null)"+method)
continue
}
if (!allowedfunctions.includes(method.toUpperCase())) {
alert.info("Skipped method (not allowed) "+method)
continue
}
@@ -556,7 +557,7 @@ const AppCreator = (props) => {
}
// HAHAHA wtf is this.
if (methodvalue.responses !== undefined) {
if (methodvalue.responses !== undefined && methodvalue.responses !== null) {
if (methodvalue.responses.default !== undefined) {
if (methodvalue.responses.default.content !== undefined) {
if (methodvalue.responses.default.content["text/plain"] !== undefined) {
@@ -1818,7 +1819,7 @@ const AppCreator = (props) => {
addPathQuery()
}}>New query</Button>
{currentActionMethod === "POST" ?
<Button color="primary" variant={fileUploadEnabled ? "contained" : "outlined"} style={{marginLeft: 10, marginTop: "5px", marginBottom: "10px", borderRadius: "0px"}} onClick={() => {
<Button disabled color="primary" variant={fileUploadEnabled ? "contained" : "outlined"} style={{marginLeft: 10, marginTop: "5px", marginBottom: "10px", borderRadius: "0px"}} onClick={() => {
setFileUploadEnabled(!fileUploadEnabled)
if (fileUploadEnabled && currentAction["file_field"].length > 0) {
setActionField("file_field", "")
functions/onprem/orborus/build.sh
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=shuffle-orborus
VERSION=0.8.53
VERSION=0.8.54
echo "Running docker build with $NAME:$VERSION"
#docker rmi frikky/shuffle:$NAME --force
functions/onprem/orborus/orborus.go
+11 -7
View File
@@ -21,6 +21,7 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
//"github.com/docker/docker/api/types/filters"
dockerclient "github.com/docker/docker/client"
"github.com/satori/go.uuid"
//network "github.com/docker/docker/api/types/network"
@@ -311,15 +312,12 @@ func getStats() {
return
}
fmt.Printf("[INFO] memory total: %d bytes\n", memory.Total)
fmt.Printf("[INFO] memory used: %d bytes\n", memory.Used)
before, err := cpu.Get()
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
return
}
time.Sleep(time.Duration(500) * time.Millisecond)
time.Sleep(time.Duration(250) * time.Millisecond)
after, err := cpu.Get()
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
@@ -327,6 +325,8 @@ func getStats() {
}
total := float64(after.Total - before.Total)
fmt.Printf("[INFO] memory total: %d bytes\n", memory.Total)
fmt.Printf("[INFO] memory used: %d bytes\n", memory.Used)
fmt.Printf("[INFO] cpu used : %f%%\n", float64(after.User-before.User)/total*100)
fmt.Printf("[INFO] cpu system: %f%%\n", float64(after.System-before.System)/total*100)
fmt.Printf("[INFO] cpu idle : %f%%\n", float64(after.Idle-before.Idle)/total*100)
@@ -408,7 +408,7 @@ func main() {
},
}
getStats()
//getStats()
if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" {
client = &http.Client{}
@@ -440,6 +440,7 @@ func main() {
hasStarted := false
for {
//log.Printf("Prerequest")
//go getStats()
newresp, err := client.Do(req)
executionCount := getRunningWorkers(ctx, workerTimeout)
//log.Printf("Postrequest")
@@ -626,10 +627,13 @@ func getRunningWorkers(ctx context.Context, workerTimeout int) int {
containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
All: true,
})
//Filters: filters.Args{
// map[string][]string{"ancestor": {"<imagename>:<version>"}},
//},
if err != nil {
log.Printf("Error getting containers: %s", err)
return 0
log.Printf("[ERROR] Error getting containers: %s", err)
return maxConcurrency
}
currenttime := time.Now().Unix()
functions/onprem/worker/worker.go
+4
View File
@@ -852,6 +852,10 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
Type: "json-file",
Config: map[string]string{},
},
Resources: container.Resources{
CPUShares: 256,
CPUPeriod: 10000,
},
}
// form container id and use it as network source if it's not empty