diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index a4e68653..8e77b6b0 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -73,13 +73,17 @@ class AppBase: self.logger.info("[DEBUG] Already JSON-like. Returning from magic") return input_data + if len(input_data) < 3: + self.logger.info("[DEBUG] Too short input data") + return input_data + # Don't touch large data. if len(input_data) > 100000: self.logger.info("[DEBUG] Value too large. Returning from magic") return input_data if not "\n" in input_data and not "," in input_data: - self.logger.info("[DEBUG] No data to autoparse") + self.logger.info("[DEBUG] No data to autoparse - requires newline or comma") return input_data new_input = input_data diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index ab1c9f38..7aa31f4b 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -3,7 +3,7 @@ ### DEFAULT NAME=shuffle-app_sdk -VERSION=0.9.40 +VERSION=0.9.44 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 0099d571..30d5e46e 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module main go 1.15 -//replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi //replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 07f56321..880f09c2 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -2846,82 +2846,6 @@ func getOpenapi(resp http.ResponseWriter, request *http.Request) { resp.Write(data) } -func echoOpenapiData(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - // Just here to verify that the user is logged in - user, err := shuffle.HandleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in validate swagger: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed authentication"}`)) - return - } - - if user.Role == "org-reader" { - log.Printf("[WARNING] Org-reader doesn't have access to echo OpenAPI data: %s (%s)", user.Username, user.Id) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Read only user"}`)) - return - } - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Bodyreader err: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`)) - return - } - - newbody := string(body) - newbody = strings.TrimSpace(newbody) - if strings.HasPrefix(newbody, "\"") { - newbody = newbody[1:len(newbody)] - } - - if strings.HasSuffix(newbody, "\"") { - newbody = newbody[0 : len(newbody)-1] - } - - req, err := http.NewRequest("GET", newbody, nil) - if err != nil { - log.Printf("[ERROR] Requestbuilder err: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed building request"}`)) - return - } - - httpClient := &http.Client{} - newresp, err := httpClient.Do(req) - if err != nil { - log.Printf("[ERROR] Grabbing error: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed making remote request to get the data"}`))) - return - } - defer newresp.Body.Close() - - urlbody, err := ioutil.ReadAll(newresp.Body) - if err != nil { - log.Printf("[ERROR] URLbody error: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't get data from selected uri"}`))) - return - } - - if newresp.StatusCode >= 400 { - resp.WriteHeader(201) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, urlbody))) - return - } - - resp.WriteHeader(200) - resp.Write(urlbody) -} - func handleSwaggerValidation(body []byte) (shuffle.ParsedOpenApi, error) { type versionCheck struct { Swagger string `datastore:"swagger" json:"swagger" yaml:"swagger"` @@ -5888,7 +5812,7 @@ func initHandlers() { // OpenAPI configuration r.HandleFunc("/api/v1/verify_swagger", verifySwagger).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/verify_openapi", verifySwagger).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/get_openapi_uri", echoOpenapiData).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/get_openapi_uri", shuffle.EchoOpenapiData).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/validate_openapi", shuffle.ValidateSwagger).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/get_openapi/{key}", getOpenapi).Methods("GET", "OPTIONS") diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index e31797d4..e40f91ad 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -2059,7 +2059,7 @@ const AngularWorkflow = (props) => { const parentNode = cy.getElementById(data.attachedTo); if (parentNode !== null && parentNode !== undefined) { var newNodeData = JSON.parse(JSON.stringify(parentNode.data())); - newNodeData.id = uuidv4.v4(); + newNodeData.id = uuidv4(); if (newNodeData.position !== undefined) { newNodeData.position = { x: newNodeData.position.x + 100, diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 69910640..1fbf5e93 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.9.42 +VERSION=0.9.44 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index bf8db0de..162794b4 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.9.43 +VERSION=0.9.44 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index a2965adf..a7253db8 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -85,7 +85,6 @@ var autoDeploy = map[string]string{ "testing:1.0.0": "frikky/shuffle:testing_1.0.0", } -//if !shuffle.ArrayContains(executedIds, //fmt.Sprintf("%s_%s", workflowExecution.ExecutionId, action.ID) // New Worker mappings @@ -855,6 +854,20 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { continue } + newExecId := fmt.Sprintf("%s_%s", workflowExecution.ExecutionId, nextAction) + if !shuffle.ArrayContains(executedIds, newExecId) { + executedIds = append(executedIds, newExecId) + toRemove = append(toRemove, index) + } else { + log.Printf("\n\n[DEBUG] %s is already executed. Continuing.", newExecId) + continue + } + + // max 1000 :o + if len(executedIds) >= 1000 { + executedIds = executedIds[900:999] + } + if action.AppName == "Shuffle Tools" && (action.Name == "skip_me" || action.Name == "router" || action.Name == "route") { err := runSkipAction(topClient, action, workflowExecution.Workflow.ID, workflowExecution.ExecutionId, workflowExecution.Authorization, "SKIPPED") if err != nil { @@ -2888,7 +2901,8 @@ func main() { } */ - _, err := shuffle.RunInit(datastore.Client{}, storage.Client{}, "", "", true, "") + // Elasticsearch necessary to ensure we'ren ot running with Datastore configurations for minimal/maximal data sizes + _, err := shuffle.RunInit(datastore.Client{}, storage.Client{}, "", "", true, "elasticsearch") if err != nil { log.Printf("[ERROR] Failed to run worker init: %s", err) } else {