diff --git a/.github/workflows/dockerbuild-nightly.yaml b/.github/workflows/dockerbuild-nightly.yaml new file mode 100644 index 00000000..cb7907c6 --- /dev/null +++ b/.github/workflows/dockerbuild-nightly.yaml @@ -0,0 +1,82 @@ +name: nightly-dockerbuild + +on: + workflow_dispatch: + push: + branches: + - nightly + paths: + - "**" + - "!.github/**" + - "!**.md" + - "!docker-compose.yml" +jobs: + main: + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + include: + - app: frontend + path: frontend + version: nightly + experimental: true + - app: backend + path: backend + version: nightly + experimental: true + - app: orborus + path: functions/onprem/orborus + version: nightly + experimental: true + - app: worker + path: functions/onprem/worker + version: nightly + experimental: true + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: "amd64,arm64,arm" + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to Ghcr + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Ghcr Build and push + id: docker_build + uses: docker/build-push-action@v4 + env: + BUILDX_NO_DEFAULT_LOAD: true + with: + logout: false + context: ${{ matrix.path }}/ + file: ${{ matrix.path }}/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + tags: | + ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ matrix.version }} + ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ matrix.version }} + frikky/shuffle-${{ matrix.app }}:${{ matrix.version }} + frikky/shuffle:${{ matrix.app }} + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/helm-release.yml b/.github/workflows/helm-release.yml index e365080e..4c3c6b6f 100644 --- a/.github/workflows/helm-release.yml +++ b/.github/workflows/helm-release.yml @@ -33,16 +33,21 @@ jobs: sudo apt-get install helm -y --no-install-recommends - name: Set versions - id: set_versions run: | if [[ ${{ github.event_name }} == 'release' ]]; then - CHART_VERSION="${{ github.event.release.tag_name }}" - APP_VERSION="${{ github.event.release.tag_name }}" + TAG_NAME="${{ github.event.release.tag_name }}" + + # Remove the v prefix + VERSION=${TAG_NAME#v} + + APP_VERSION="${VERSION}" + CHART_VERSION="${VERSION}" else - CHART_VERSION="0.0.0-nightly-untagged-latest" APP_VERSION="nightly" + CHART_VERSION="0.0.0-nightly-untagged-latest" fi + echo "APP_VERSION set to ${APP_VERSION}" echo "CHART_VERSION set to ${CHART_VERSION}. Validating..." # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string @@ -55,8 +60,8 @@ jobs: exit 1; fi - echo "CHART_VERSION=${CHART_VERSION}" >> $GITHUB_OUTPUT - echo "APP_VERSION=${APP_VERSION}" >> $GITHUB_OUTPUT + echo "CHART_VERSION=${CHART_VERSION}" >> "$GITHUB_ENV" + echo "APP_VERSION=${APP_VERSION}" >> "$GITHUB_ENV" - name: Update helm dependencies run: helm dependency update ./functions/kubernetes/charts/shuffle @@ -68,4 +73,4 @@ jobs: run: helm registry login ghcr.io --username ${{ github.actor }} --password ${{ secrets.GITHUB_TOKEN }} - name: Push helm chart - run: helm push ./functions/kubernetes/charts/shuffle-*.tgz oci://ghcr.io/shuffle/shuffle/charts + run: helm push ./functions/kubernetes/charts/shuffle-*.tgz oci://ghcr.io/shuffle/charts diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 11bbe14d..9f0c72e2 100755 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -604,11 +604,27 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { // return //} - body, err := ioutil.ReadAll(request.Body) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`)) - return + var err error + body := []byte{} + //log.Printf("IMAGE REQUEST BODY: %#v", request.Body) + if request.Body == nil || request.Body == http.NoBody { + // Check for the image query, otherwise we skip everything + imageQuery := request.URL.Query().Get("image") + if len(imageQuery) == 0 { + resp.WriteHeader(400) + resp.Write([]byte(`{"success": false, "reason": "No image query found"}`)) + return + } + + body = []byte(fmt.Sprintf(`{"name": "%s"}`, imageQuery)) + + } else { + body, err = ioutil.ReadAll(request.Body) + if err != nil { + resp.WriteHeader(400) + resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`)) + return + } } // This has to be done in a weird way because Datastore doesn't @@ -630,28 +646,48 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { return } + img := image.Summary{} + img2 := image.Summary{} + tagFound := "" + tagFound2 := "" + + // Old way of doing it + //alternativeNameSplit := strings.Split(version.Name, "/") + //alternativeName := version.Name + //if len(alternativeNameSplit) == 3 { + // alternativeName = strings.Join(alternativeNameSplit[1:3], "/") + //} + + appname, baseAppname, appnameSplit2, err := shuffle.GetAppNameSplit(version) + if err != nil { + log.Printf("[ERROR] Failed getting appname split: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't get the right docker image name"}`))) + return + } + + if len(version.Name) == 0 { + log.Printf("[ERROR] No image name provided for download: %s", version.Name) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "No image name"}`))) + return + + } + + log.Printf("[INFO] Trying to download image: '%s'. Appname: '%s'. BaseAppname: '%s', Split2: %s", version.Name, appname, baseAppname, appnameSplit2) + + alternativeName := appname ctx := context.Background() images, err := dockercli.ImageList(ctx, image.ListOptions{ All: true, }) - img := image.Summary{} - tagFound := "" - - img2 := image.Summary{} - tagFound2 := "" - - alternativeNameSplit := strings.Split(version.Name, "/") - alternativeName := version.Name - if len(alternativeNameSplit) == 3 { - alternativeName = strings.Join(alternativeNameSplit[1:3], "/") - } - - log.Printf("[INFO] Trying to download image: %s. Alt: %s", version.Name, alternativeName) - for _, image := range images { for _, tag := range image.RepoTags { - //log.Printf("[DEBUG] Tag: %s", tag) + if strings.Contains(tag, "") { + continue + } + if strings.ToLower(tag) == strings.ToLower(version.Name) { img = image tagFound = tag diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index a8e0b107..3db9ac92 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -20,7 +20,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.18 + github.com/shuffle/shuffle-shared v0.8.35 golang.org/x/crypto v0.36.0 google.golang.org/api v0.176.1 google.golang.org/grpc v1.68.1 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index d4b660fc..5d37f78b 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -333,8 +333,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.18 h1:7f7cV+P2pr/g44i+AI8P0UheEe8oG8O2V8loBfe9YSw= -github.com/shuffle/shuffle-shared v0.8.18/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.35 h1:3awc0TrsLLZiQeWD2XGIkTnFbczAG0cMfy1+cB/P7zg= +github.com/shuffle/shuffle-shared v0.8.35/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 6df286b9..746904aa 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3543,7 +3543,12 @@ func handleCloudJob(job shuffle.CloudSyncJob) error { return err } - redirectDomain := "localhost:5001" + backendPort := os.Getenv("BACKEND_PORT") + if backendPort == "" { + backendPort = "5001" + } + + redirectDomain := fmt.Sprintf("localhost:%s", backendPort) redirectUrl := fmt.Sprintf("http://%s/api/v1/triggers/outlook/register", redirectDomain) outlookClient, _, err := shuffle.GetOutlookClient(ctx, "", hook.OauthToken, redirectUrl) if err != nil { @@ -4197,20 +4202,26 @@ func runInitEs(ctx context.Context) { cleanupJob := func() func() { return func() { - log.Printf("[INFO] Running schedule for cleaning up or re-running unfinished workflows in %d environments.", len(environments)) + //log.Printf("[INFO] Running schedule for cleaning up or re-running unfinished workflows in %d environments.", len(environments)) + + backendPort := os.Getenv("BACKEND_PORT") + if backendPort == "" { + backendPort = "5001" + } for _, environment := range environments { // Allowed without PROXY management as it's localhost // client := shuffle.GetExternalClient(syncUrl) httpClient := &http.Client{} - url := fmt.Sprintf("http://localhost:5001/api/v1/environments/%s/stop", environment) + url := fmt.Sprintf("http://localhost:%s/api/v1/environments/%s/stop", backendPort, environment) req, err := http.NewRequest( "GET", url, nil, ) + // FIXME: This will stop working of the user rotates their key lol req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, parsedApikey)) if err != nil { log.Printf("[ERROR] Failed CREATING environment request for %s: %s", environment, err) @@ -4231,7 +4242,7 @@ func runInitEs(ctx context.Context) { } log.Printf("[DEBUG] Successfully ran workflow cleanup request for %s. Body: %s", environment, string(respBody)) - url = fmt.Sprintf("http://localhost:5001/api/v1/environments/%s/rerun", environment) + url = fmt.Sprintf("http://localhost:%s/api/v1/environments/%s/rerun", backendPort, environment) req, err = http.NewRequest( "GET", url, @@ -4256,7 +4267,8 @@ func runInitEs(ctx context.Context) { log.Printf("[ERROR] Failed setting respbody %s", err) continue } - log.Printf("[DEBUG] Successfully ran workflow RERUN request for %s. Body: %s", environment, string(respBody)) + + //log.Printf("[DEBUG] Ran workflow RERUN request for %s with the response. Body: %s", environment, string(respBody)) } } } @@ -5373,6 +5385,7 @@ func main() { if innerPort == "" { log.Printf("[DEBUG] Running on %s:5001", hostname) log.Fatal(http.ListenAndServe(":5001", nil)) + os.Setenv("BACKEND_PORT", "5001") } else { log.Printf("[DEBUG] Running on %s:%s", hostname, innerPort) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", innerPort), nil)) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index d2316b5d..35db256f 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -22,9 +22,6 @@ import ( dockerclient "github.com/docker/docker/client" "github.com/docker/docker/api/types/image" - - //gyaml "github.com/ghodss/yaml" - "github.com/h2non/filetype" uuid "github.com/satori/go.uuid" @@ -36,14 +33,6 @@ import ( "github.com/go-git/go-git/v5/storage/memory" "github.com/go-git/go-git/v5/plumbing" http2 "github.com/go-git/go-git/v5/plumbing/transport/http" - //http2 "gopkg.in/src-d/go-git.v5/plumbing/transport/http" - //http2 "github.com/go-git/go-git/plumbing/transport/http" - - //"github.com/gorilla/websocket" - //"google.golang.org/appengine" - //"google.golang.org/appengine/memcache" - //"cloud.google.com/go/firestore" - // "google.golang.org/api/option" gyaml "github.com/ghodss/yaml" ) @@ -208,7 +197,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque } // remove items from DB - parsedId := fmt.Sprintf("workflowqueue-%s", id) + parsedId := strings.ReplaceAll(fmt.Sprintf("workflowqueue-%s", id), " ", "-") ids := []string{} for _, execution := range removeExecutionRequests.Data { ids = append(ids, execution.ExecutionId) @@ -985,7 +974,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { if err != nil { log.Printf("[ERROR] Failed to list child workflows: %s", err) } else { - log.Printf("\n\n[DEBUG] Found %d child workflows for workflow %s\n\n", len(childWorkflows), workflow.ID) + //log.Printf("\n\n[DEBUG] Found %d child workflows for workflow %s\n\n", len(childWorkflows), workflow.ID) // Find cookies and append them to request.Header to replicate current request as closely as possible for _, childWorkflow := range childWorkflows { @@ -1141,507 +1130,6 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request return shuffle.WorkflowExecution{}, "Failed unmarshal during execution", err } - /* - makeNew := true - start, startok := request.URL.Query()["start"] - if request.Method == "POST" { - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("[ERROR] Failed request POST read: %s", err) - return shuffle.WorkflowExecution{}, "Failed getting body", err - } - - // This one doesn't really matter. - log.Printf("[INFO] Running POST execution with body of length %d for workflow %s", len(string(body)), workflowExecution.Workflow.ID) - - if len(body) >= 4 { - if body[0] == 34 && body[len(body)-1] == 34 { - body = body[1 : len(body)-1] - } - if body[0] == 34 && body[len(body)-1] == 34 { - body = body[1 : len(body)-1] - } - } - - sourceAuth, sourceAuthOk := request.URL.Query()["source_auth"] - if sourceAuthOk { - //log.Printf("\n\n\nSETTING SOURCE WORKFLOW AUTH TO %s!!!\n\n\n", sourceAuth[0]) - workflowExecution.ExecutionSourceAuth = sourceAuth[0] - } else { - //log.Printf("Did NOT get source workflow") - } - - sourceNode, sourceNodeOk := request.URL.Query()["source_node"] - if sourceNodeOk { - //log.Printf("\n\n\nSETTING SOURCE WORKFLOW NODE TO %s!!!\n\n\n", sourceNode[0]) - workflowExecution.ExecutionSourceNode = sourceNode[0] - } else { - //log.Printf("Did NOT get source workflow") - } - - //workflowExecution.ExecutionSource = "default" - sourceWorkflow, sourceWorkflowOk := request.URL.Query()["source_workflow"] - if sourceWorkflowOk { - //log.Printf("Got source workflow %s", sourceWorkflow) - workflowExecution.ExecutionSource = sourceWorkflow[0] - } else { - //log.Printf("Did NOT get source workflow") - } - - sourceExecution, sourceExecutionOk := request.URL.Query()["source_execution"] - if sourceExecutionOk { - //log.Printf("[INFO] Got source execution%s", sourceExecution) - workflowExecution.ExecutionParent = sourceExecution[0] - } else { - //log.Printf("Did NOT get source execution") - } - - if len(string(body)) < 50 { - //log.Println(body) - // String in string - //log.Println(body) - - //if string(body)[0] == "\"" && string(body)[string(body) - log.Printf("[DEBUG] Body: %s", string(body)) - } - - var execution shuffle.ExecutionRequest - err = json.Unmarshal(body, &execution) - if err != nil { - log.Printf("[WARNING] Failed execution POST unmarshalling for execution %s - continuing anyway: %s", execution.ExecutionId, err) - //return shuffle.WorkflowExecution{}, "", err - } - - if execution.Start == "" && len(body) > 0 { - execution.ExecutionArgument = string(body) - } - - // FIXME - this should have "execution_argument" from executeWorkflow frontend - //log.Printf("EXEC: %#v", execution) - if len(execution.ExecutionArgument) > 0 { - workflowExecution.ExecutionArgument = execution.ExecutionArgument - } - - if len(execution.ExecutionSource) > 0 { - workflowExecution.ExecutionSource = execution.ExecutionSource - } - - //log.Printf("Execution data: %#v", execution) - if len(execution.Start) == 36 && len(workflow.Actions) > 0 { - log.Printf("[INFO] Should start execution on node %s", execution.Start) - workflowExecution.Start = execution.Start - - found := false - for _, action := range workflow.Actions { - if action.ID == execution.Start { - found = true - break - } - } - - if !found { - log.Printf("[ERROR] Action %s was NOT found! Exiting execution.", execution.Start) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", workflow.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", workflow.Start)) - } - } else if len(execution.Start) > 0 { - //log.Printf("[INFO] !") - //log.Printf("[ERROR] START ACTION %s IS WRONG ID LENGTH %d!", execution.Start, len(execution.Start)) - //return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", execution.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", execution.Start)) - } - - if len(execution.ExecutionId) == 36 { - workflowExecution.ExecutionId = execution.ExecutionId - } else { - sessionToken := uuid.NewV4() - workflowExecution.ExecutionId = sessionToken.String() - } - } else { - // Check for parameters of start and ExecutionId - // This is mostly used for user input trigger - - answer, answerok := request.URL.Query()["answer"] - referenceId, referenceok := request.URL.Query()["reference_execution"] - if answerok && referenceok && len(answer) > 0 && len(referenceId) > 0 { - // If answer is false, reference execution with result - log.Printf("[INFO] Answer is OK AND reference is OK!") - if answer[0] == "false" { - log.Printf("Should update reference and return, no need for further execution!") - - // Get the reference execution - oldExecution, err := shuffle.GetWorkflowExecution(ctx, referenceId[0]) - if err != nil { - log.Printf("Failed getting execution (execution) %s: %s", referenceId[0], err) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err - } - - if oldExecution.Workflow.ID != id { - log.Println("Wrong workflowid!") - return shuffle.WorkflowExecution{}, fmt.Sprintf("Bad ID %s", referenceId), errors.New("Bad ID") - } - - newResults := []shuffle.ActionResult{} - //log.Printf("%#v", oldExecution.Results) - for _, result := range oldExecution.Results { - log.Printf("%s - %s", result.Action.ID, start[0]) - if result.Action.ID == start[0] { - note, noteok := request.URL.Query()["note"] - if noteok && len(note) > 0 { - result.Result = fmt.Sprintf("User note: %s", note[0]) - } else { - result.Result = fmt.Sprintf("User clicked %s", answer[0]) - } - - // Stopping the whole thing - result.CompletedAt = int64(time.Now().Unix()) - result.Status = "ABORTED" - oldExecution.Status = result.Status - oldExecution.Result = result.Result - oldExecution.LastNode = result.Action.ID - } - - newResults = append(newResults, result) - } - - oldExecution.Results = newResults - err = shuffle.SetWorkflowExecution(ctx, *oldExecution, true) - if err != nil { - log.Printf("Error saving workflow execution actionresult setting: %s", err) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed setting workflowexecution actionresult in execution: %s", err), err - } - - return shuffle.WorkflowExecution{}, "", nil - } - } - - if referenceok { - log.Printf("Handling an old execution continuation!") - // Will use the old name, but still continue with NEW ID - oldExecution, err := shuffle.GetWorkflowExecution(ctx, referenceId[0]) - if err != nil { - log.Printf("Failed getting execution (execution) %s: %s", referenceId[0], err) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err - } - - workflowExecution = *oldExecution - } - - if len(workflowExecution.ExecutionId) == 0 { - sessionToken := uuid.NewV4() - workflowExecution.ExecutionId = sessionToken.String() - } else { - log.Printf("Using the same executionId as before: %s", workflowExecution.ExecutionId) - makeNew = false - } - - // Don't override workflow defaults - } - - if startok { - //log.Printf("\n\n[INFO] Setting start to %s based on query!\n\n", start[0]) - //workflowExecution.Workflow.Start = start[0] - workflowExecution.Start = start[0] - } - - // FIXME - regex uuid, and check if already exists? - if len(workflowExecution.ExecutionId) != 36 { - log.Printf("Invalid uuid: %s", workflowExecution.ExecutionId) - return shuffle.WorkflowExecution{}, "Invalid uuid", err - } - - // FIXME - find owner of workflow - // FIXME - get the actual workflow itself and build the request - // MAYBE: Don't send the workflow within the pubsub, as this requires more data to be sent - // Check if a worker already exists for company, else run one with: - // locations, project IDs and subscription names - - // When app is executed: - // Should update with status execution (somewhere), which will trigger the next node - // IF action.type == internal, we need the internal watcher to be running and executing - // This essentially means the WORKER has to be the responsible party for new actions in the INTERNAL landscape - // Results are ALWAYS posted back to cloud@execution_id? - if makeNew { - workflowExecution.Type = "workflow" - //workflowExecution.Stream = "tmp" - //workflowExecution.WorkflowQueue = "tmp" - //workflowExecution.SubscriptionNameNodestream = "testcompany-nodestream" - //workflowExecution.Locations = []string{"europe-west2"} - workflowExecution.ProjectId = gceProject - workflowExecution.WorkflowId = workflow.ID - workflowExecution.StartedAt = int64(time.Now().Unix()) - workflowExecution.CompletedAt = 0 - workflowExecution.Authorization = uuid.NewV4().String() - - // Status for the entire workflow. - workflowExecution.Status = "EXECUTING" - } - - if len(workflowExecution.ExecutionSource) == 0 { - log.Printf("[INFO] No execution source (trigger) specified. Setting to default") - workflowExecution.ExecutionSource = "default" - } else { - log.Printf("[INFO] Execution source is %s for execution ID %s in workflow %s", workflowExecution.ExecutionSource, workflowExecution.ExecutionId, workflowExecution.Workflow.ID) - } - - workflowExecution.ExecutionVariables = workflow.ExecutionVariables - if len(workflowExecution.Start) == 0 && len(workflowExecution.Workflow.Start) > 0 { - workflowExecution.Start = workflowExecution.Workflow.Start - } - - startnodeFound := false - newStartnode := "" - for _, item := range workflowExecution.Workflow.Actions { - if item.ID == workflowExecution.Start { - startnodeFound = true - } - - if item.IsStartNode { - newStartnode = item.ID - } - } - - if !startnodeFound { - log.Printf("[INFO] Couldn't find startnode %s. Remapping to %#v", workflowExecution.Start, newStartnode) - - if len(newStartnode) > 0 { - workflowExecution.Start = newStartnode - } else { - return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode couldn't be found"), errors.New("Startnode isn't defined in this workflow..") - } - } - - childNodes := shuffle.FindChildNodes(workflowExecution.Workflow, workflowExecution.Start, []string{}, []string{}) - - startFound := false - newActions := []shuffle.Action{} - defaultResults := []shuffle.ActionResult{} - - for _, action := range workflowExecution.Workflow.Actions { - //action.LargeImage = "" - if action.ID == workflowExecution.Start { - startFound = true - } - //log.Println(action.Environment) - - if action.Environment == "" { - return shuffle.WorkflowExecution{}, fmt.Sprintf("Environment is not defined for %s", action.Name), errors.New("Environment not defined!") - } - - action.LargeImage = "" - if len(action.Label) == 0 { - action.Label = action.ID - } - //log.Printf("LABEL: %s", action.Label) - newActions = append(newActions, action) - - // If the node is NOT found, it's supposed to be set to SKIPPED, - // as it's not a childnode of the startnode - // This is a configuration item for the workflow itself. - if len(workflowExecution.Results) > 0 { - defaultResults = []shuffle.ActionResult{} - for _, result := range workflowExecution.Results { - if result.Status == "WAITING" { - result.Status = "FINISHED" - result.Result = "Continuing" - } - - defaultResults = append(defaultResults, result) - } - } else if len(workflowExecution.Results) == 0 && !workflowExecution.Workflow.Configuration.StartFromTop { - found := false - for _, nodeId := range childNodes { - if nodeId == action.ID { - //log.Printf("Found %s", action.ID) - found = true - } - } - - if !found { - if action.ID == workflowExecution.Start { - continue - } - - //log.Printf("[WARNING] Set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID) - curaction := shuffle.Action{ - AppName: action.AppName, - AppVersion: action.AppVersion, - Label: action.Label, - Name: action.Name, - ID: action.ID, - } - //action - //curaction.Parameters = [] - defaultResults = append(defaultResults, shuffle.ActionResult{ - Action: curaction, - ExecutionId: workflowExecution.ExecutionId, - Authorization: workflowExecution.Authorization, - Result: "Skipped because it's not under the startnode", - StartedAt: 0, - CompletedAt: 0, - Status: "SKIPPED", - }) - } - } - } - - removeTriggers := []string{} - for triggerIndex, trigger := range workflowExecution.Workflow.Triggers { - //log.Printf("[INFO] ID: %s vs %s", trigger.ID, workflowExecution.Start) - if trigger.ID == workflowExecution.Start { - if trigger.AppName == "User Input" { - startFound = true - break - } - } - - if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" { - found := false - for _, node := range childNodes { - if node == trigger.ID { - found = true - break - } - } - - if !found { - //log.Printf("SHOULD SET TRIGGER %s TO BE SKIPPED", trigger.ID) - - curaction := shuffle.Action{ - AppName: "shuffle-subflow", - AppVersion: trigger.AppVersion, - Label: trigger.Label, - Name: trigger.Name, - ID: trigger.ID, - } - - defaultResults = append(defaultResults, shuffle.ActionResult{ - Action: curaction, - ExecutionId: workflowExecution.ExecutionId, - Authorization: workflowExecution.Authorization, - Result: "Skipped because it's not under the startnode", - StartedAt: 0, - CompletedAt: 0, - Status: "SKIPPED", - }) - } else { - // Replaces trigger with the subflow - //if trigger.AppName == "Shuffle Workflow" { - // replaceActions := false - // workflowAction := "" - // for _, param := range trigger.Parameters { - // if param.Name == "argument" && !strings.Contains(param.Value, ".#") { - // replaceActions = true - // } - - // if param.Name == "startnode" { - // workflowAction = param.Value - // } - // } - - // if replaceActions { - // replacementNodes, newBranches, lastnode := shuffle.GetReplacementNodes(ctx, workflowExecution, trigger, trigger.Label) - // log.Printf("REPLACEMENTS: %d, %d", len(replacementNodes), len(newBranches)) - // if len(replacementNodes) > 0 { - // for _, action := range replacementNodes { - // found := false - - // for subActionIndex, subaction := range newActions { - // if subaction.ID == action.ID { - // found = true - // //newActions[subActionIndex].Name = action.Name - // newActions[subActionIndex].Label = action.Label - // break - // } - // } - - // if !found { - // action.SubAction = true - // newActions = append(newActions, action) - // } - - // // Check if it's already set to have a value - // for resultIndex, result := range defaultResults { - // if result.Action.ID == action.ID { - // defaultResults = append(defaultResults[:resultIndex], defaultResults[resultIndex+1:]...) - // break - // } - // } - // } - - // for _, branch := range newBranches { - // workflowExecution.Workflow.Branches = append(workflowExecution.Workflow.Branches, branch) - // } - - // // Append branches: - // // parent -> new inner node (FIRST one) - // for branchIndex, branch := range workflowExecution.Workflow.Branches { - // if branch.DestinationID == trigger.ID { - // log.Printf("REPLACE DESTINATION WITH %s!!", workflowAction) - // workflowExecution.Workflow.Branches[branchIndex].DestinationID = workflowAction - // } - - // if branch.SourceID == trigger.ID { - // log.Printf("REPLACE SOURCE WITH LASTNODE %s!!", lastnode) - // workflowExecution.Workflow.Branches[branchIndex].SourceID = lastnode - // } - // } - - // // Remove the trigger - // removeTriggers = append(removeTriggers, workflowExecution.Workflow.Triggers[triggerIndex].ID) - // } - - // log.Printf("NEW ACTION LENGTH %d, RESULT: %d, Triggers: %d, BRANCHES: %d", len(newActions), len(defaultResults), len(workflowExecution.Workflow.Triggers), len(workflowExecution.Workflow.Branches)) - // } - //} - _ = triggerIndex - } - } - } - - //newTriggers := []shuffle.Trigger{} - //for _, trigger := range workflowExecution.Workflow.Triggers { - // found := false - // for _, triggerId := range removeTriggers { - // if trigger.ID == triggerId { - // found = true - // break - // } - // } - - // if found { - // log.Printf("[WARNING] Removed trigger %s during execution", trigger.ID) - // continue - // } - - // newTriggers = append(newTriggers, trigger) - //} - //workflowExecution.Workflow.Triggers = newTriggers - - if !startFound { - if len(workflowExecution.Start) == 0 && len(workflowExecution.Workflow.Start) > 0 { - workflowExecution.Start = workflow.Start - } else if len(workflowExecution.Workflow.Actions) > 0 { - workflowExecution.Start = workflowExecution.Workflow.Actions[0].ID - } else { - log.Printf("[ERROR] Startnode %s doesn't exist!!", workflowExecution.Start) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow action %s doesn't exist in workflow", workflowExecution.Start), errors.New(fmt.Sprintf(`Workflow start node "%s" doesn't exist. Exiting!`, workflowExecution.Start)) - } - } - - //log.Printf("EXECUTION START: %s", workflowExecution.Start) - - // Verification for execution environments - workflowExecution.Results = defaultResults - workflowExecution.Workflow.Actions = newActions - onpremExecution := true - _ = onpremExecution - environments := []string{} - - if len(workflowExecution.ExecutionOrg) == 0 && len(workflow.ExecutingOrg.Id) > 0 { - workflowExecution.ExecutionOrg = workflow.ExecutingOrg.Id - } - */ - - //workflowExecution, execInfo, _, workflowExecErr := shuffle.PrepareWorkflowExecution(ctx, workflow, request, int64(maxExecutionDepth)) err = shuffle.SetWorkflowExecution(ctx, workflowExecution, true) if err != nil { log.Printf("[ERROR] Failed setting workflow execution during init (2): %s", err) @@ -1652,7 +1140,6 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request environments := execInfo.Environments var allEnvs []shuffle.Environment if len(workflowExecution.ExecutionOrg) > 0 { - //log.Printf("[INFO] Executing ORG: %s", workflowExecution.ExecutionOrg) allEnvironments, err := shuffle.GetEnvironments(ctx, workflowExecution.ExecutionOrg) if err != nil { @@ -3557,9 +3044,21 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { return } - time.Sleep(2 * time.Second) - log.Printf("[INFO] Starting validation of execution %s", workflowExecution.ExecutionId) + shouldRerun := false + rerun, rerunOk := query["rerun"] + if rerunOk && len(rerun) > 0 && rerun[0] == "true" { + shouldRerun = true + } + if shouldRerun { + log.Printf("[DEBUG] Returning single action execution ID for rerun: %s", workflowExecution.ExecutionId) + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization))) + return + } + + log.Printf("[INFO] Starting validation of execution %s", workflowExecution.ExecutionId) + time.Sleep(2 * time.Second) returnBody := shuffle.HandleRetValidation(ctx, workflowExecution, 1) returnBytes, err := json.Marshal(returnBody) if err != nil { diff --git a/frontend/src/components/AdminNavBar.jsx b/frontend/src/components/AdminNavBar.jsx index c6f70357..e0faec6e 100644 --- a/frontend/src/components/AdminNavBar.jsx +++ b/frontend/src/components/AdminNavBar.jsx @@ -33,6 +33,9 @@ const AdminNavBar = (props) => { const navigate = useNavigate(); + //const leadinfo = selectedOrganization.lead_info === undefined || selectedOrganization.lead_info === null || selectedOrganization.lead_info === "" ? "" : JSON.stringify(selectedOrganization.lead_info) + //const isPartner = leadinfo.includes("partner") + useEffect(() => { const queryParams = new URLSearchParams(location.search); const tabName = queryParams.get('tab'); diff --git a/frontend/src/components/Billing.jsx b/frontend/src/components/Billing.jsx index 44e36334..ef09b6f4 100644 --- a/frontend/src/components/Billing.jsx +++ b/frontend/src/components/Billing.jsx @@ -891,7 +891,7 @@ const Billing = memo((props) => { : null} - {showSupport ? + {/* {showSupport ? - : null} + : null} */} ) } @@ -2021,7 +2021,7 @@ const Billing = memo((props) => {
- {isCloud && + {/* {isCloud && selectedOrganization.subscriptions !== undefined && selectedOrganization.subscriptions !== null && selectedOrganization.subscriptions.length > 0 && @@ -2043,7 +2043,7 @@ const Billing = memo((props) => { /> ) }) - : null} + : null} */}
{isCloud && billingInfo.subscription !== undefined && billingInfo.subscription !== null ? isChildOrg ? null : diff --git a/frontend/src/components/CacheView.jsx b/frontend/src/components/CacheView.jsx index 34b4608f..f4b8b9cb 100644 --- a/frontend/src/components/CacheView.jsx +++ b/frontend/src/components/CacheView.jsx @@ -151,13 +151,16 @@ const CacheView = memo((props) => { if (fileCategories.length === 1 && fileCategories[0] === "default") { var newcategories = ["default"] for (var key in responseJson.keys) { - if (responseJson.keys[key].category !== undefined && responseJson.keys[key].category !== null && responseJson.keys[key].category !== "" && !fileCategories.includes(responseJson.keys[key].category)) { - newcategories.push(responseJson.keys[key].category); + var category = responseJson.keys[key].category + if (category !== undefined && category !== null && category !== ""){ + category = category.replaceAll(" ", "_") + + if (!newcategories.includes(category)) { + newcategories.push(category) + } } } - console.log("CATEGORIES: ", newcategories) - setFileCategories(newcategories) } } @@ -357,6 +360,7 @@ const CacheView = memo((props) => { { editCache ? "Edit Key" : "Add Key"}{selectedCategory === "" || selectedCategory === "default" ? "" : ` in category '${selectedCategory}'`} +
Key @@ -1041,13 +1045,13 @@ const CacheView = memo((props) => { { deleteCache(orgId, data.key); //deleteFile(orgId); diff --git a/frontend/src/components/EditWorkflow.jsx b/frontend/src/components/EditWorkflow.jsx index 5f571110..dcfa3057 100644 --- a/frontend/src/components/EditWorkflow.jsx +++ b/frontend/src/components/EditWorkflow.jsx @@ -545,6 +545,8 @@ const EditWorkflow = (props) => { }} > } label="Test" /> + } label="Staging" /> + } label="Pre-production" /> } label="Production" /> diff --git a/frontend/src/components/LeftSideBar.jsx b/frontend/src/components/LeftSideBar.jsx index baf60f06..92aeabe4 100644 --- a/frontend/src/components/LeftSideBar.jsx +++ b/frontend/src/components/LeftSideBar.jsx @@ -774,8 +774,8 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => { }, [window?.location?.pathname]); const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); - - + const showPartnerLogo = userdata?.org_status?.includes("integration_partner") && userdata?.active_org?.image !== undefined && userdata?.active_org?.image !== null && userdata?.active_org?.image.length > 0 + return (
{ } }} > - + Shuffle Logo @@ -1611,7 +1613,7 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => { }} > - {expandLeftNav && + {userdata?.licensed !== true && !userdata?.org_status?.includes("integration_partner") && expandLeftNav &&
- {subscription.active === true && !isScale && } + {subscription.active === true && !isScale && }
{top_text === "Base Cloud Access" && userdata.has_card_available === true && !isScale ? { color="primary" /> : null} - + {top_text} @@ -396,7 +453,7 @@ const LicencePopup = (props) => { }} /> : null} - {isCloud && highlight === true && top_text !== "Base Cloud Access" ? + {isCloud && highlight === true && top_text !== "Starter Plan" ? { {subscription.currency_text}{subscription.price} - / {subscription.interval} + {subscription.interval.length > 0 ? `/ ${subscription.interval}` : ""}
: null} @@ -517,23 +574,17 @@ const LicencePopup = (props) => { : null} - {subscription.name.includes("Scale") ? - "" - : - - userdata.has_card_available === true ? - "While you have a card attached to your account, Shuffle will no longer prevent workflows from running. Billing will occur at the start of each month." - : + { isCloud ? - userdata?.app_execution_limit && userdata?.app_execution_limit >= 300000 ? - "You have subscribed to the Enterprise plan, which includes " + (userdata?.app_execution_limit/1000) + "K app runs/month. You can increase the limit by upgrading current plan. Contact support@shuffler.io for more information." : - `You are not subscribed to any plan and are using the free plan with max 10,000 app runs per month. Upgrade to deactivate this limit.` + userdata?.app_execution_limit && userdata?.app_execution_limit !== 10000 ? + "You have already subscribed to the Scale plan, which includes " + (userdata?.app_execution_limit/1000) + "K app runs/month. You can increase the limit by upgrading current plan. Contact support@shuffler.io for more information." : + `You are using free Starter plan with max ${userdata?.app_execution_limit === 10000 ? "10,000" : "2,000"} runs per month. Upgrade to increase this limit.` : `You are not subscribed to any plan and are using the free, open source plan. This plan has no enforced limits, but scale issues may occur due to CPU congestion.` } - {isCloud && (userdata.has_card_available === true || selectedOrganization?.Billing?.Email?.length > 0 )? + {/* {isCloud && (userdata.has_card_available === true || selectedOrganization?.Billing?.Email?.length > 0 )?
Billing email: {BillingEmail}
- : null} + : null} */}
{isCloud ? ( ) : null} } - + color="primary">Recommended + + { + billingCycle === "annual" && + ( + + + 10% OFF + + + ) + } +
+
+ + {scaleValue > 300 ? "Enterprise Plan" : "Scale Plan"} + + + + + Monthly + + + Annual + + +
- {shuffleVariant === 1 ? "Scale" : "Enterprise"} - {shuffleVariant === 0 ? - "SaaS / Cloud - Per Month" - : - "Open Source + Scale License" - } + App Runs Units - { - - if (calculatedCores === "Get A Quote") { - console.log("Clicked on get a quote") - if (window.drift !== undefined) { - window.drift.api.startInteraction({ interactionId: 340785 }) - } - } - }}>{calculatedCost} - For {shuffleVariant === 1 ? `${selectedValue} CPU cores` : `${selectedValue}k App Runs`}: -
- - { - handleChange(event, newValue) +
+ + {scaleValue > 300 ? "Let's Talk" : `$${getPrice(32) * (scaleValue / 10)}`} + + 300 ? 1 : 0, }} - marks - value={selectedValue} - step={shuffleVariant === 0 ? 100 : 4} - min={shuffleVariant === 0 ? 100 : 8} - max={shuffleVariant === 0 ? 1000 : 32} - valueLabelDisplay="auto" - /> + > + {scaleValue > 300 ? `for ${scaleValue > 500 ? "500k+" : `${scaleValue}k`} App Runs` : `/month for ${scaleValue}k App Runs`} +
+ + { + if(value === 510){ + return "500k+" + } + return `${value}k` + }} + step={10} + min={10} + max={510} + marks + sx={{ + color: "#ff8544", + "& .MuiSlider-thumb": { + width: 15, + height: 15, + }, + "& .MuiSlider-valueLabel": { + backgroundColor: "rgba(33, 33, 33, 1)", + color: "rgba(241, 241, 241, 1)", + fontSize: 14, + borderRadius: "4px", + border: "1px solid rgba(73, 73, 73, 1)", + fontFamily: theme?.typography?.fontFamily, + }, + }} + /> +
{defaultTaskIcon} - Priority Support + Standard Email Support
@@ -1117,7 +1357,7 @@ const LicencePopup = (props) => {
{defaultTaskIcon} - Help with Workflow and App development + 30 Days workflow run history
@@ -1136,7 +1376,7 @@ const LicencePopup = (props) => { navigate("/pricing") } else { - window.open("https://shuffler.io/pricing?tab=onprem", "_blank") + window.open("https://shuffler.io/pricing?tab=Self-Hosted", "_blank") } }} color="primary" @@ -1149,6 +1389,10 @@ const LicencePopup = (props) => { style={{ borderRadius: 4, textTransform: "capitalize", color: "#1a1a1a", backgroundColor: "#ff8544", width: "100%", fontSize: 16}} onClick={() => { if (isCloud) { + if(scaleValue > 300){ + navigate("/contact?category=cloud_enterprise_plan") + return; + } ReactGA.event({ category: "header", action: "upgread_clicks_popup", @@ -1170,7 +1414,7 @@ const LicencePopup = (props) => { }} color="primary" > - Upgrade + {scaleValue > 300 ? "Let's Talk" : "Upgrade"}
diff --git a/frontend/src/components/Navbar.jsx b/frontend/src/components/Navbar.jsx index 21321b94..fa038a51 100644 --- a/frontend/src/components/Navbar.jsx +++ b/frontend/src/components/Navbar.jsx @@ -61,7 +61,7 @@ const menuData = { description: "Connect and run actions seamlessly between different platforms.", icon: "/images/logos/singul.svg", - path: "https://singul.io/", + path: "https://singul-docs.gitbook.io/singul/getting-started", gaData: { category: "navbar", action: "products_click", @@ -1247,7 +1247,7 @@ const Navbar = (props) => { letterSpacing: '0.5px', }} > - Coming Soon + Beta: Coming Soon )} @@ -1370,7 +1370,7 @@ const Navbar = (props) => { label: "go_to_pricing", }) } else { - window.open("https://shuffler.io/pricing", '_blank'); + window.open("https://shuffler.io/pricing?env=Self-Hosted", '_blank'); return; } }} diff --git a/frontend/src/components/OrgHeaderexpandedNew.jsx b/frontend/src/components/OrgHeaderexpandedNew.jsx index e5317da3..8d95b046 100644 --- a/frontend/src/components/OrgHeaderexpandedNew.jsx +++ b/frontend/src/components/OrgHeaderexpandedNew.jsx @@ -538,7 +538,7 @@ const OrgHeaderexpandedNew = (props) => { renderValue={(selected) => selected.join(', ')} MenuProps={MenuProps} > - {["contacted", "lead", "demo done", "pov", "customer", "open source", "student", "internal", "creator", "tech partner", "old customer", "old lead"].map((name) => ( + {["contacted", "lead", "demo done", "pov", "customer", "open source", "student", "internal", "creator", "tech partner", "integration partner", "distribution partner", "service partner", "old customer", "old lead"].map((name) => ( -1} /> diff --git a/frontend/src/components/OrganizationTab.jsx b/frontend/src/components/OrganizationTab.jsx index 1911bfad..56a1531d 100644 --- a/frontend/src/components/OrganizationTab.jsx +++ b/frontend/src/components/OrganizationTab.jsx @@ -120,7 +120,7 @@ const OrganizationTab = (props) => { isLoaded={isLoaded} /> ); - case 'branding(beta)': + case 'branding': return { return (
- {['Org Configuration', "sso", "Notifications", 'Billing & Stats', 'Branding (Beta)'].map((tabName, index) => ( + {['Org Configuration', "sso", "Notifications", 'Billing & Stats', 'Branding'].map((tabName, index) => ( { suborgWorkflows, originalWorkflow, + runFromHere, } = props; let navigate = useNavigate() @@ -233,11 +234,16 @@ const ParsedAction = (props) => { // auth, required, optional var changed = false if (selectedActionParameters === undefined || selectedActionParameters === null || selectedActionParameters.length === 0) { + console.log("Returning because no params") return } if (selectedApp !== undefined && selectedApp !== null && selectedApp.generated !== true) { - return + + if (isAgent || isIntegration) { + } else { + return + } } // Fixing required fields with a shitty structure :) @@ -346,13 +352,23 @@ const ParsedAction = (props) => { // 3. generated fields & all else - const newparams = auth + var newparams = auth .concat(bodyfield) .concat(required) .concat(generated_optional) .concat(special_optional) .concat(optional) + const dedupedParams = [] + for (var paramKey in newparams) { + const param = newparams[paramKey] + if (dedupedParams.find(item => item.name === param.name) === undefined) { + dedupedParams.push(param) + } + } + + newparams = dedupedParams + var newkeyorder = [] for (let paramkey in newparams) { //console.log("Param: ", newparams[paramkey]) @@ -666,11 +682,14 @@ const ParsedAction = (props) => { // Process workflowExecutions if (workflowExecutions.length > 0) { + var appended = false + var foundvalue = "" for (let execution of workflowExecutions) { const execArg = execution.execution_argument; if (execArg && execArg.length > 0) { const valid = validateJson(execArg); if (valid.valid) { + appended = true newActionList.push({ type: "Runtime Argument", name: "Runtime Argument", @@ -681,9 +700,23 @@ const ParsedAction = (props) => { }) break + } else { + foundvalue = execArg } } } + + if (!appended && foundvalue !== undefined && foundvalue !== "") { + newActionList.push({ + type: "Runtime Argument", + name: "Runtime Argument", + highlight: "exec", + autocomplete: "exec", + + value: foundvalue, + example: foundvalue, + }) + } } // Add default Runtime Argument if none were added @@ -772,6 +805,8 @@ const ParsedAction = (props) => { } labels.push(parentNode.label); + + var secondaryExample = "" let exampleData = parentNode.example ?? ""; if (parentNode?.app_name === "http") { exampleData = "" @@ -782,9 +817,22 @@ const ParsedAction = (props) => { const foundResult = exec.results?.find(result => result?.action?.id === parentNode?.id); if (foundResult) { const valid = validateJson(foundResult.result); - if (valid.valid && valid.result.success !== false) { - exampleData = valid.result - break + if (valid.valid) { + + // Check if array, and if first item is object + success + if (Array.isArray(valid.result) && valid.result.length > 0 && typeof valid.result[0] === "object") { + if (valid.result[0].success !== false) { + exampleData = valid.result[0] + break + } + } else { + if (valid.result.success !== false) { + exampleData = valid.result + break + } + } + } else { + secondaryExample = foundResult.result } } } @@ -818,6 +866,10 @@ const ParsedAction = (props) => { } } } + } + + if (exampleData === "" && secondaryExample !== "") { + exampleData = secondaryExample } if (parentNode.label === undefined) { @@ -1162,7 +1214,9 @@ const ParsedAction = (props) => { selectedAction.parameters[1].value = splitparsed[1] if (splitparsed.length > 2) { - toast.warn("Filter list only supports filtering at the first level. If you want multi-level filtering, please use the 'execute python' action with the 'filter a list' function in the code editor.") + toast.warn("Filter list only supports filtering on the first list. If you want multi-level filtering, please use the 'execute python' action with the 'filter a list' function in the code editor.", { + autoClose: 10000, + }) } else if (selectedAction.parameters[1].value.includes(".#")) { toast.warn("This filter may not work due to using .# indexing. Please use the 'execute python' action and try the 'filter a list' function in the code editor.") } @@ -1814,6 +1868,38 @@ const ParsedAction = (props) => { + + Rerun this action with results from previous executions. Built for testing individual actions in the middle of workflows. + + } + placement="top" + > + + + {(selectedAction?.generated === true && selectedAction?.app_version === "1.0.0") || (selectedAction?.app_name === "Shuffle Tools" && selectedAction?.app_version !== "1.2.0") ?
@@ -4737,6 +4823,7 @@ const ParsedAction = (props) => { showDropdownNumber === count && data.variant === "STATIC_VALUE" && jsonList.length > 0 ? ( + { color="secondary" style={{ height: 50, + textTransform: "none", }} onClick={() => { dismissNotification(data.id); }} > - Mark Read + Mark as Read ) : null} diff --git a/frontend/src/components/ShuffleCodeEditor1.jsx b/frontend/src/components/ShuffleCodeEditor1.jsx index 8f6d7970..dea674f5 100644 --- a/frontend/src/components/ShuffleCodeEditor1.jsx +++ b/frontend/src/components/ShuffleCodeEditor1.jsx @@ -46,6 +46,7 @@ import { DragIndicator as DragIndicatorIcon, RestartAlt as RestartAltIcon, ArrowForward as ArrowForwardIcon, + KeyboardReturn as KeyboardReturnIcon, } from '@mui/icons-material'; @@ -94,9 +95,12 @@ const pythonFilters = [ { "name": "Get full execution details", "value": `print(self.full_execution)`, "example": `` }, { "name": "Use files", "value": `# Create a sample file\nfiles = [{\n \"filename\": \"test.txt\",\n \"data\": \"Testdata\"\n}]\nret = self.set_files(files)\n\n# Get the content of the file from Shuffle storage\n# Originally a byte string in the \"data\" key\nfile_content = (self.get_file(ret[0])[\"data\"]).decode()\nprint(file_content)`, "example": `` }, - { "name": "Use datastore", "value": `key = \"testkey\"\nvalue = \"The value of the testkey\"\n\nself.set_cache(key, value)\n\n# Print the details of the key after it's been updated\n# To get the value, use self.get_cache(key)[\"value\"]\nprint(self.get_cache(key))`, "example": `` }, - { "name": "Run an App Action", "value": `response = shuffle.run_app(app_id="app", action="action_name", auth="authentication_id", params={})\nprint(response)`, "example": ``, "disabled": true, }, - { "name": "Run a Singul AI Action", "value": `response = singul.create_ticket(app="jira/iris/ticketingsystem", fields={"title": "Test ticket!"})\nprint(response)`, "example": ``, "disabled": true, }, + { "name": "Use datastore", "value": `key = \"testkey\"\nvalue = \"The value of the testkey\"\n\nself.set_key(key, value)\n\n# Print the details of the key after it's been updated\n# To get the value, use self.get_key(key)[\"value\"]\nprint(self.get_key(key))`, "example": `` }, + + { "name": "Run a Subflow", "value": `response = shuffle.run_workflow(workflow_id="", start_command="Runtime arg here!", wait=True)\nprint(response)`, "example": ``, "disabled": false, }, + { "name": "Run an App Action", "value": `response = shuffle.run_app(app_id="app", action="action_name", auth="authentication_id", params={})\nprint(response)`, "example": ``, "disabled": false, }, + { "name": "Run a Singul AI Action", "value": `response = singul.cases.create_ticket(app="jira", fields={"title": "Test ticket!"})\nprint(response)`, "example": ``, "disabled": false, }, + ] @@ -299,8 +303,19 @@ const CodeEditor = (props) => { setMainVariables(tmpVariables) } + const handleKeyDown = (event) => { + if ((event.metaKey || event.ctrlKey) && event.key === 'Enter') { + event.preventDefault() + const tryItButton = document.getElementById("try-it-button") + if (tryItButton !== undefined && tryItButton !== null) { + tryItButton.click() + } + } + } + // Remove the original useEffect for actionlist since we'll update on action/trigger changes useEffect(() => { + document.addEventListener("keydown", handleKeyDown) updateAvailableVariables(actionlist) }, []) @@ -887,6 +902,8 @@ const CodeEditor = (props) => { // Whelp this is inefficient af. Single loop pls // When the found array is empty. if (found !== null && found !== undefined) { + + //console.log("FOUND: ", found) try { for (var i = 0; i < found.length; i++) { try { @@ -921,28 +938,72 @@ const CodeEditor = (props) => { } } } + + // Find the location to ensure replacements happen correctly + var foundlocation = -1 + for (var j = 0; j < input.length; j++) { + const foundStringSize = fixedVariable.length + const foundslice = input.slice(j, j + foundStringSize) + //console.log("FOUNDSLICE: ", foundslice) + if (fixedVariable !== foundslice) { + continue + } + + // Check if it matches EXACTLY or not, as there may be more AFTER the found[i] + const nextchar = input.slice(j + foundStringSize, j + foundStringSize + 1) + if (nextchar === ".") { + continue + } + + foundlocation = j + break + } + // FIXME: There is something wrong here with: + // $variable.# + // vs + // $variable.#.subvalue + // if you put both of those lines in the same editor, then it will replace both (somehow). Make sure $variable.#.subvalue exists while testing. + console.log("FOUNDLOC: ", fixedVariable, foundlocation) for (var j = 0; j < actionlist.length; j++) { if (fixedVariable.slice(1,).toLowerCase() !== actionlist[j].autocomplete.toLowerCase()) { continue } + // Look for the location of found[i] in the input, as to make sure to skip parts of the input in the replace. Find ALL spots for it valuefound = true + var newvalue = "" try { - if (typeof actionlist[j].example === "object") { - input = input.replace(found[i], JSON.stringify(actionlist[j].example), -1); + if (typeof actionlist[j].example === "object") { + newvalue = JSON.stringify(actionlist[j].example) } else if (actionlist[j].example.trim().startsWith("{") || actionlist[j].example.trim().startsWith("[")) { - input = input.replace(found[i], JSON.stringify(actionlist[j].example), -1); + + newvalue = JSON.stringify(actionlist[j].example) } else { const newExample = fixStringInput(actionlist[j].example) - input = input.replace(found[i], newExample, -1) + + newvalue = newExample } } catch (e) { - input = input.replace(found[i], actionlist[j].example, -1) + newvalue = actionlist[j].example } + try { + console.log("REPLACE: ", foundlocation, fixedVariable, newvalue) + if (newvalue !== "") { + if (foundlocation === -1) { + input = input.replace(fixedVariable, newvalue, 1) + } else { + // Ensures we don't just randomly replace the first value we find + const replacedSlice = input.slice(foundlocation, input.length).replace(fixedVariable, newvalue, 1) + input = input.slice(0, foundlocation) + replacedSlice + } + } + } catch (e) { + console.log("Replace error: ", e) + } } if (!valuefound) { @@ -1721,13 +1782,13 @@ const CodeEditor = (props) => { color="secondary" style={{ textTransform: "none", - width: 120, + width: 145, }} onClick={(event) => { setAnchorEl3(event.currentTarget); }} > - Python Code + Python Examples diff --git a/frontend/src/components/TenantsTab.jsx b/frontend/src/components/TenantsTab.jsx index 907c982e..e5552e3e 100644 --- a/frontend/src/components/TenantsTab.jsx +++ b/frontend/src/components/TenantsTab.jsx @@ -30,6 +30,8 @@ import { Apps as AppsIcon, Business as BusinessIcon, Flag, + ArrowDropDown as ArrowDropDownIcon, + } from "@mui/icons-material"; import { toast } from 'react-toastify'; @@ -54,9 +56,12 @@ const TenantsTab = memo((props) => { const [cloudSyncModalOpen, setCloudSyncModalOpen] = React.useState(false); const [modalOpen, setModalOpen] = React.useState(false); const [parentOrg, setParentOrg] = React.useState(null); - const [parentOrgFlag, setParentOrgFlag] = React.useState(null); + const [parentOrgFlag, setParentOrgFlag] = React.useState("gb"); + const [parentOrgRegionName, setParentOrgRegionName] = React.useState("UK"); const [loadOrgs, setLoadOrgs] = React.useState(true); const [, forceUpdate] = React.useState(); + const [suborglistOpen, setSuborglistOpen] = React.useState(false); + const [allTenantsOpen, setAllTenantsOpen] = React.useState(false); const itemColor = "black"; useEffect(() => { @@ -82,9 +87,10 @@ const TenantsTab = memo((props) => { } } setParentOrgFlag(regionCode); + setParentOrgRegionName(regiontag); } } - }, [parentOrg]); + }, [parentOrg, parentOrgFlag]); var syncList = [ { @@ -164,7 +170,8 @@ const TenantsTab = memo((props) => { regionCode = "ca"; } } - setParentOrgFlag(regionCode); + setParentOrgFlag(regionCode); + setParentOrgRegionName(regiontag); } } }) @@ -479,8 +486,8 @@ const TenantsTab = memo((props) => { const createSubOrg = (currentOrgId, name) => { const data = { name: name, org_id: currentOrgId }; - console.log(data); const url = globalUrl + `/api/v1/orgs/${currentOrgId}/create_sub_org`; + setSuborglistOpen(true) fetch(url, { mode: "cors", @@ -788,7 +795,7 @@ const TenantsTab = memo((props) => { Create, manage and change to sub-organizations (tenants)! {" "} {isCloud ? "You can only make a sub organization if you are a customer of shuffle or running a POC of the platform. Please contact support@shuffler.io to try it out." - : ''} + : ''}  { src={`https://flagcdn.com/w20/${parentOrgFlag}.png`} style={{ width: "30px", height: "20px", marginRight: "5px" }} /> - +
} style={{ display: "table-cell", padding: 8, verticalAlign: "middle" }} @@ -1147,168 +1154,212 @@ const TenantsTab = memo((props) => { overflowX: "auto", paddingBottom: 0, }}> - - - - {isCloud && ( - - )} - - - - {subOrgs.map((data, index) => { - let regiontag = "UK"; - let regionCode = "gb"; + {!suborglistOpen ? + 0) { - const regionsplit = data.region_url.split("."); - if (regionsplit.length > 2 && !regionsplit[0].includes("shuffler")) { - const namesplit = regionsplit[0].split("/"); - regiontag = namesplit[namesplit.length - 1]; - if (regiontag === "california") { - regiontag = "US"; - regionCode = "us"; - } else if (regiontag === "frankfurt") { - regiontag = "EU-2"; - regionCode = "eu"; - } else if (regiontag === "ca") { - regiontag = "CA"; - regionCode = "ca"; + }} + > + setSuborglistOpen(true)} + > + Show Sub-Organizations + + } + style={{ + width: 100, + minWidth: 100, + maxWidth: 100, + paddingLeft: 20, + display: "table-cell", + padding: "0px 8px 8px 8px", + textAlign: "center", + borderBottom: "1px solid #494949", + verticalAlign: "middle", + }} + /> + + : + + + + + {isCloud && ( + + )} + + + + {subOrgs.map((data, index) => { + let regiontag = "UK"; + let regionCode = "gb"; + + if (data.region_url?.length > 0) { + const regionsplit = data.region_url.split("."); + if (regionsplit.length > 2 && !regionsplit[0].includes("shuffler")) { + const namesplit = regionsplit[0].split("/"); + regiontag = namesplit[namesplit.length - 1]; + if (regiontag === "california") { + regiontag = "US"; + regionCode = "us"; + } else if (regiontag === "frankfurt") { + regiontag = "EU-2"; + regionCode = "eu"; + } else if (regiontag === "ca") { + regiontag = "CA"; + regionCode = "ca"; + } } } - } - return ( - - } style={{ width: 100, - minWidth: 100, - maxWidth: 100, - display: "table-cell", - padding: "8px 8px 8px 20px", - textAlign: "center", }} /> - + return ( + + } style={{ width: 100, + minWidth: 100, + maxWidth: 100, + display: "table-cell", + padding: "8px 8px 8px 20px", + textAlign: "center", }} /> + - {isCloud && ( - - {regiontag} - -
- } - style={{ display: "table-cell", padding: 8, verticalAlign: "middle" }} - /> - )} - + {isCloud && ( + + {regiontag} + +
+ } + style={{ display: "table-cell", padding: 8, verticalAlign: "middle" }} + /> + )} + + + + + + } + style={{ display: "table-cell", verticalAlign: "middle" }} + /> + + )})} + + } - - - - } - style={{ display: "table-cell", verticalAlign: "middle" }} - /> - - )})}
@@ -1363,247 +1414,289 @@ const TenantsTab = memo((props) => { paddingBottom: 0, }} > - - - - {isCloud && ( - - )} - - - + {!allTenantsOpen ? + ( - - {Array(7) - .fill() - .map((_, colIndex) => ( - - - - ))} - - )) - ) : ( - userdata?.orgs?.length > 0 && - userdata.orgs.map((data, index) => { - let regiontag = "UK"; - let regionCode = "gb"; - - if (data.region_url?.length > 0) { - const regionsplit = data.region_url.split("."); - if (regionsplit.length > 2 && !regionsplit[0].includes("shuffler")) { - const namesplit = regionsplit[0].split("/"); - regiontag = namesplit[namesplit.length - 1]; - - if (regiontag === "california") { - regiontag = "US"; - regionCode = "us"; - } else if (regiontag === "frankfurt") { - regiontag = "EU-2"; - regionCode = "eu"; - } else if (regiontag === "ca") { - regiontag = "CA"; - regionCode = "ca"; + }} + > + setAllTenantsOpen(true)} + > + Show ALL your tenants + } - } - } + style={{ + width: 100, + minWidth: 100, + maxWidth: 100, + paddingLeft: 20, + display: "table-cell", + padding: "0px 8px 8px 8px", + textAlign: "center", + borderBottom: "1px solid #494949", + verticalAlign: "middle", + }} + /> + + : + + + + + {isCloud && ( + + )} + + + - return ( - - - } - style={{ - width: 100, - minWidth: 100, - maxWidth: 100, - display: "table-cell", - padding: "8px 8px 8px 20px", - textAlign: "center", - }} - /> - - {isCloud ? ( - - {regiontag} + {userdata?.orgs?.length <= 0 ? ( + [...Array(6)].map((_, rowIndex) => ( + + {Array(7) + .fill() + .map((_, colIndex) => ( + + + + ))} + + )) + ) : ( + userdata?.orgs?.length > 0 && + userdata.orgs.map((data, index) => { + let regiontag = "UK"; + let regionCode = "gb"; - - - } - style={{ - display: "table-cell", - padding: 8, - verticalAlign: "middle", - }} - > - ) : null} - - { - handleClickChangeOrg(data?.id); - }} - > - Change Active Org - - } - style={{ - display: "table-cell", - padding: 8, - verticalAlign: "middle", - }} - > - - ); - }) - )} + if (data.region_url?.length > 0) { + const regionsplit = data.region_url.split("."); + if (regionsplit.length > 2 && !regionsplit[0].includes("shuffler")) { + const namesplit = regionsplit[0].split("/"); + regiontag = namesplit[namesplit.length - 1]; + + if (regiontag === "california") { + regiontag = "US"; + regionCode = "us"; + } else if (regiontag === "frankfurt") { + regiontag = "EU-2"; + regionCode = "eu"; + } else if (regiontag === "ca") { + regiontag = "CA"; + regionCode = "ca"; + } + } + } + + return ( + + + } + style={{ + width: 100, + minWidth: 100, + maxWidth: 100, + display: "table-cell", + padding: "8px 8px 8px 20px", + textAlign: "center", + }} + /> + + {isCloud ? ( + + {regiontag} + + + + } + style={{ + display: "table-cell", + padding: 8, + verticalAlign: "middle", + }} + > + ) : null} + + { + handleClickChangeOrg(data?.id); + }} + > + Change Active Org + + } + style={{ + display: "table-cell", + padding: 8, + verticalAlign: "middle", + }} + > + + ); + }) + )} + } diff --git a/frontend/src/components/UserManagmentTab.jsx b/frontend/src/components/UserManagmentTab.jsx index 8aff24a5..bb6870ff 100644 --- a/frontend/src/components/UserManagmentTab.jsx +++ b/frontend/src/components/UserManagmentTab.jsx @@ -248,6 +248,14 @@ const UserManagmentTab = memo((props) => { return; } + if (event.target.value.includes("ALL")) { + toast.info("Adding to available all sub-organizations. This may take a minute.") + event.target.value = selectedOrganization.child_orgs.map((org) => org.id) + } else if (event.target.value.includes("None")) { + toast.info("Removing from all sub-organizations. This may take a minute") + event.target.value = [] + } + console.log("event: ", event.target.value); setMatchingOrganizations(event.target.value); // Workaround for empty orgs @@ -286,6 +294,14 @@ const UserManagmentTab = memo((props) => { }} MenuProps={MenuProps} > + + + + + + + + {selectedOrganization.child_orgs.map((org, index) => ( -1} /> diff --git a/frontend/src/components/WorkflowValidationTimeline.jsx b/frontend/src/components/WorkflowValidationTimeline.jsx index 78e6b766..2d724cfc 100644 --- a/frontend/src/components/WorkflowValidationTimeline.jsx +++ b/frontend/src/components/WorkflowValidationTimeline.jsx @@ -608,8 +608,6 @@ const WorkflowValidationTimeline = (props) => { const ballsize = 8 const topMargin = 20 - console.log("CHIP: ", index, chipColor, chipBackground) - const chipStyle = { height: 40, minWidth: 125, diff --git a/frontend/src/defaultCytoscapeStyle.jsx b/frontend/src/defaultCytoscapeStyle.jsx index d7a6b68e..8a2623f6 100644 --- a/frontend/src/defaultCytoscapeStyle.jsx +++ b/frontend/src/defaultCytoscapeStyle.jsx @@ -95,23 +95,56 @@ const data = [ { selector: `node[type="COMMENT"]`, css: { - label: function(element) { - return element.data("label") - }, + label: function (element) { + return element.data("label") + }, shape: "roundrectangle", color: "data(color)", width: "data(width)", height: "data(height)", - padding: "0px", + padding: "5px", margin: "0px", "background-color": "data(backgroundcolor)", "background-image": "data(backgroundimage)", "border-color": "#ffffff", - "text-margin-x": "0px", + "text-margin-x": "data(textMarginX)", + "text-margin-y": "data(textMarginY)", "z-index": 4999, "border-radius": "5px", "background-opacity": "0.5", - "text-wrap": "wrap", + "text-wrap": "wrap", + "text-max-width": "data(width)", + "text-halign": function(element) { + const align = element?.data("textHalign") + if (align === null || align === undefined || align === "") { + return "center" + } + + return align + }, + "text-valign": function(element) { + const align = element?.data("textValign") + if (align === null || align === undefined || align === "") { + return "center" + } + + return align + } + }, + }, + { + selector: `node[type="RESIZE-HANDLE"]`, + css: { + shape: "ellipse", + width: "8px", + height: "8px", + "border-width": 1, + "border-color": "white", + "z-index": 5002, + "overlay-opacity": 0, + "cursor": "nwse-resize", + "opacity": 0, + "pointer-events": "auto", }, }, { diff --git a/frontend/src/theme.jsx b/frontend/src/theme.jsx index 4897e2ca..1621dc42 100644 --- a/frontend/src/theme.jsx +++ b/frontend/src/theme.jsx @@ -27,7 +27,7 @@ const theme = createTheme(adaptV4Theme({ distributionColor: "#40E0D0", green: "#5cc879", - borderRadius: 10, + borderRadius: 8, defaultBorder: "1px solid rgba(255,255,255,0.3)", //jsonTheme: "brewer", @@ -112,39 +112,39 @@ const theme = createTheme(adaptV4Theme({ MuiCssBaseline: { styleOverrides: ` @font-face { - font-family: 'Roboto'; + font-family: 'Inter'; font-style: normal; font-display: swap; font-weight: 300; - src: local('Roboto Light'), local('Roboto-Light'); + src: local('Inter Light'), local('Inter-Light'); } @font-face { - font-family: 'Roboto'; + font-family: 'Inter'; font-style: normal; font-display: swap; font-weight: 400; - src: local('Roboto'), local('Roboto-Regular'); + src: local('Inter Regular'), local('Inter-Regular'); } @font-face { - font-family: 'Roboto'; + font-family: 'Inter'; font-style: normal; font-display: swap; font-weight: 500; - src: local('Roboto Medium'), local('Roboto-Medium'); + src: local('Inter Medium'), local('Inter-Medium'); } @font-face { - font-family: 'Roboto'; + font-family: 'Inter'; font-style: normal; font-display: swap; font-weight: 600; - src: local('Roboto SemiBold'), local('Roboto-SemiBold'); + src: local('Inter SemiBold'), local('Inter-SemiBold'); } @font-face { - font-family: 'Roboto'; + font-family: 'Inter'; font-style: normal; font-display: swap; font-weight: 700; - src: local('Roboto Bold'), local('Roboto-Bold'); + src: local('Inter Bold'), local('Inter-Bold'); } `, }, diff --git a/frontend/src/views/Admin2.jsx b/frontend/src/views/Admin2.jsx index 45d933b9..30a356e5 100644 --- a/frontend/src/views/Admin2.jsx +++ b/frontend/src/views/Admin2.jsx @@ -76,6 +76,18 @@ const Admin2 = (props) => { leads.push("tech partner"); } + if (responseJson.lead_info.integration_partner) { + leads.push("integration partner"); + } + + if (responseJson.lead_info.distribution_partner) { + leads.push("distribution partner"); + } + + if (responseJson.lead_info.service_partner) { + leads.push("service partner"); + } + if (responseJson.lead_info.creator) { leads.push("creator"); } diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 1f32c6ec..df92a1cd 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -475,6 +475,7 @@ const AngularWorkflow = (defaultprops) => { const [authGroups, setAuthGroups] = React.useState([]) const curpath = typeof window === "undefined" || window.location === undefined ? "" : window.location.pathname; + const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; // 0 = normal, 1 = just done, 2 = normal @@ -652,6 +653,31 @@ const AngularWorkflow = (defaultprops) => { ], "multiselect": true, }, + { + "name": "memory", + "value": "", + "required": true, + "description": "Whether to store the conversation in memory", + "options": [ + "Nothing", + "Shuffle Datastore", + ], + "multiselect": false, + "disabled": true, + }, + { + "name": "knowledge", + "value": "", + "required": true, + "description": "The knowledge we should inject into the context window", + "options": [ + "Nothing", + "Shuffle Files", + ], + "multiselect": false, + "disabled": true, + }, + ] }], large_image: theme.palette.singulBlackWhite, @@ -772,7 +798,7 @@ const AngularWorkflow = (defaultprops) => { const [loadedApps, setLoadedApps] = React.useState([]) - const loadAppConfig = (appId, select) => { + const loadAppConfig = (appId, select, skipAppLoad) => { if (appId === undefined || appId === null || appId.length === 0) { console.log("No appId to load") return @@ -868,7 +894,7 @@ const AngularWorkflow = (defaultprops) => { setSelectedApp(foundapp) } - if (apps === undefined || apps === null || apps.length === 0) { + if ((apps === undefined || apps === null || apps.length === 0) && skipAppLoad !== true) { console.log("No apps to update :(") getApps() return @@ -888,7 +914,7 @@ const AngularWorkflow = (defaultprops) => { break } } else { - console.log("Found app, but no actions: ", foundapp) + //console.log("Found app, but no actions: ", foundapp) } if (cy !== undefined && cy !== null) { @@ -1130,7 +1156,8 @@ const AngularWorkflow = (defaultprops) => { !el.data("isButton") && !el.data("isDescriptor") && !el.data("isSuggestion") && - el.data("type") !== "COMMENT") { + el.data("type") !== "COMMENT" && + el.data("type") !== "RESIZE-HANDLE") { return true } @@ -1769,7 +1796,6 @@ const AngularWorkflow = (defaultprops) => { const newkeys = sortByKey(responseJson.executions, "-started_at"); setWorkflowExecutions(newkeys); - const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; var tmpView = new URLSearchParams(cursearch).get("execution_id"); if (execution_id !== undefined && execution_id !== null && execution_id.length > 0 && (tmpView === undefined || tmpView === null || tmpView.length === 0)) { tmpView = execution_id; @@ -1825,7 +1851,6 @@ const AngularWorkflow = (defaultprops) => { } } } else { - const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; var tmpView = new URLSearchParams(cursearch).get("execution_id"); if (tmpView === undefined || tmpView === null || tmpView.length === 0) { const execution_id = tmpView; @@ -1868,7 +1893,6 @@ const AngularWorkflow = (defaultprops) => { //toast("Failed loading the workflow run") console.log("Status not 200 for stream results :O!"); - //const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; //const newitem = removeParam("execution_id", cursearch); //navigate(curpath + newitem) } @@ -2301,6 +2325,7 @@ const AngularWorkflow = (defaultprops) => { var newBranches = []; var newVBranches = []; var newComments = []; + var newResizes = []; for (let cyelementsKey in cyelements) { if (cyelements[cyelementsKey].data === undefined) { continue; @@ -2485,7 +2510,40 @@ const AngularWorkflow = (defaultprops) => { //console.log(curworkflowComment) newComments.push(curworkflowComment); - } else { + } else if (type === "RESIZE-HANDLE") { + if (useworkflow.resizes === undefined || useworkflow.resizes === null) { + useworkflow.resizes = []; + } + + var curworkflowResize = useworkflow.resizes.find( + (a) => a.id === cyelements[cyelementsKey].data()["id"] + ); + + if (curworkflowResize === undefined) { + curworkflowResize = cyelements[cyelementsKey].data(); + } + + // Ensure width and height are properly parsed + const parsedHeight = parseInt(curworkflowResize["height"]); + if (!isNaN(parsedHeight)) { + curworkflowResize.height = parsedHeight; + } else { + curworkflowResize.height = 150; // Default value if parsing fails + } + + const parsedWidth = parseInt(curworkflowResize["width"]); + if (!isNaN(parsedWidth)) { + curworkflowResize.width = parsedWidth; + } else { + curworkflowResize.width = 200; // Default value if parsing fails + } + + // Update position from Cytoscape + curworkflowResize.position = cyelements[cyelementsKey].position(); + + newResizes.push(curworkflowResize); + } + else { toast("No handler for type: " + type); } } @@ -2785,6 +2843,92 @@ const AngularWorkflow = (defaultprops) => { return true; }; + const runFromHere = (curAction) => { + if (curAction.app_id === undefined || curAction.app_id === null || curAction.app_id.length === 0) { + toast.error("No app id found for action. Please contact support@shuffler.io if this persists") + return + } + + if (workflow.id !== undefined && workflow.id !== null && workflow.id.length > 0) { + curAction.source_workflow = workflow.id + } + + // Based on the previous execution id + // Look for the "execution_id" parameter + const execFound = new URLSearchParams(cursearch).get("execution_id"); + if (execFound !== undefined && execFound !== null && execFound.length > 0) { + toast.info("Rerunning based on previously watched execution id") + curAction.source_execution = execFound + } else if (workflowExecutions !== undefined && workflowExecutions !== null && workflowExecutions.length > 0) { + curAction.source_execution = workflowExecutions[0].execution_id + } else { + toast.error("No previous execution found. Please run the workflow first.") + return + } + + setExecutionRunning(true) + setExecutionRequestStarted(true) + var headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + } + + if (workflow.org_id !== undefined && workflow.org_id !== null && workflow.org_id.length > 0) { + headers["Org-Id"] = workflow.org_id + } + + // Rerun makes it return execution_id + authorization + const appRunUrl = `${globalUrl}/api/v1/apps/${curAction.app_id}/run?rerun=true` + fetch(appRunUrl, { + method: 'POST', + headers: headers, + body: JSON.stringify(curAction), + credentials: "include", + }) + .then((response) => { + setExecutionRunning(false) + setExecutionRequestStarted(false) + + if (response.status !== 200) { + console.log("Status not 200 for stream results :O!") + } + + return response.json() + }) + .then((responseJson) => { + setExecutionRequestStarted(false) + + if (responseJson?.success === false) { + setExecutionRunning(false) + + if (responseJson?.reason !== undefined && responseJson?.reason !== null && responseJson?.reason.length > 0) { + toast.error(responseJson.reason) + } else { + toast.error("Failed to run the action. Please try again or contact support@shuffler.io") + } + + return + } else if (responseJson?.success === true && responseJson?.execution_id !== undefined && responseJson?.execution_id !== null && responseJson?.execution_id.length > 0) { + navigate(`?execution_id=${responseJson.execution_id}&node=${curAction.id}&rerun=true`) + setExecutionRequest({ + execution_id: responseJson.execution_id, + authorization: responseJson.authorization, + }) + + setExecutionData({}) + setExecutionModalOpen(true) + setExecutionModalView(1) + start() + } + }) + .catch((error) => { + toast.error("Failed to run the action. "+error.toString()) + + setExecutionRunning(false) + setExecutionRequestStarted(false) + }) + } + const executeWorkflow = (executionArgument, startNode, hasSaved, skip_popup) => { if (hasSaved === false) { @@ -3203,6 +3347,13 @@ const AngularWorkflow = (defaultprops) => { return } + for (var key in responseJson) { + const curapp = responseJson[key] + if (curapp?.actions === undefined || curapp?.actions === null || curapp?.actions?.length === 0 || curapp?.actions?.length === 1) { + loadAppConfig(curapp?.id, false, true) + } + } + // Find app with ID "794e51c3c1a8b24b89ccc573a3defc47" (gmail) to force-break it, // Find app with ID "3e2bdf9d5069fe3f4746c29d68785a6a" (shuffle tools) to force-break it, // as to ensure the autocorrect works. @@ -4196,14 +4347,19 @@ const AngularWorkflow = (defaultprops) => { // Check for execution_id in URL // don't redirect if it exists - const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; var execFound = new URLSearchParams(cursearch).get("execution_id"); var sessionToken = new URLSearchParams(cursearch).get("session_token"); if (execFound === null && sessionToken === null) { - toast.error(`You don't have access to this workflow or loading failed. Redirecting to workflows in a few seconds. If you recently deleted this workflow, speak with support@shuffler.io to recover it from a revision.`, { - autoClose: 10000, - }) + if (isCloud) { + toast.error(`You don't have access to this workflow or loading failed. Redirecting to workflows in a few seconds. If you recently deleted this workflow, speak with support@shuffler.io to recover it from a revision.`, { + autoClose: 10000, + }) + } else { + toast.error(`You don't have access to this workflow or loading failed. Redirecting to workflows in a few seconds. Contact support@shuffler.io if this is unexpected.`, { + autoClose: 10000, + }) + } setTimeout(() => { window.location.pathname = "/workflows"; @@ -4858,7 +5014,7 @@ const AngularWorkflow = (defaultprops) => { } } - if (nodedata.app_name === "Webhook" || nodedata.app_name === "Schedule" || nodedata.app_name === "Gmail" || nodedata.app_name === "Office365") { + if (nodedata.app_name === "Webhook" || nodedata.app_name === "Schedule") { if (!found) { //console.log("Find amount of executions for the specific nodetype: ", nodedata.app_name, "Executions: ", workflowExecutions) // Find how many executions it has @@ -4887,6 +5043,7 @@ const AngularWorkflow = (defaultprops) => { } } else { // Readding the icon after moving the node + /* if (!found) { const iconInfo = GetIconInfo(nodedata); const svg_pin = ``; @@ -4914,6 +5071,7 @@ const AngularWorkflow = (defaultprops) => { } else { //console.log("Node already exists - don't add descriptor node"); } + */ } } @@ -6552,6 +6710,21 @@ const AngularWorkflow = (defaultprops) => { } setSelectedComment(data); + } else if (data.type === "RESIZE-HANDLE") { + + const parentNode = cy.getElementById(data.attachedTo); + if (parentNode) { + console.log("Resizing parent node:", parentNode); + + // Get the parent node's data + const parentData = parentNode.data(); + if (parentData?.type === "COMMENT") { + + // Set the parent node's data as the selected comment + setSelectedComment(parentData); + } + } + return; // Exit after handling the resize handle } else { toast("Can't handle node type " + data.type); return; @@ -7516,25 +7689,9 @@ const AngularWorkflow = (defaultprops) => { } break; case 86: - if (event.ctrlKey) { - //console.log("CTRL+V") - // The below parts are handled in the function handlePaste() - /* - const clipboard = navigator.clipboard - if (clipboard === undefined || window === undefined || window === null) { - toast("Can only use cliboard over HTTPS (port 3443)") - return - } - - console.log("CLIPBOARD: ", window.clipboardData) - const pastedData = window.clipboardData.getData('Text'); - console.log("PASTED: ", pastedData) - - - //var tmpAuth = JSON.parse(JSON.stringify(appAuthentication)) - var jsonvalid = true - var parsedjson = [] - */ + console.log("CTRL+V? ctrl: ", event.ctrlKey) + if (event.ctrlKey) { + // Paste is handled in the handlePaste() function. } break; case 88: @@ -7558,30 +7715,25 @@ const AngularWorkflow = (defaultprops) => { }; const handlePaste = (event) => { - if ( - event.path !== undefined && - event.path !== null && - event.path.length > 0 - ) { + if (event.path !== undefined && event.path !== null && event.path.length > 0) { if (event.path[0].localName !== "body") { - return; + console.log("Skipping paste because body is not targeted") + return; } } - if ( - event.target !== undefined && - event.target !== null - ) { - if (event.target.localName !== "body") { - return; - } + if (event.target !== undefined && event.target !== null) { + // If it's an input area, skip paste + if (event.target.localName === "input" || event.target.localName === "textarea") { + console.log("Skipping paste because body is not targeted (1). Target: ", event?.target?.localName) + return; + } } - - event.preventDefault(); - const clipboard = (event.originalEvent || event).clipboardData.getData( - "text/plain" - ); + // Does this stop things? + //event.preventDefault() + const clipboard = (event.originalEvent || event).clipboardData.getData("text/plain") + console.log("CLIPBOARD TO PASTE: ", clipboard) try { const allnodes = cy.nodes().jsons() @@ -9590,8 +9742,6 @@ const AngularWorkflow = (defaultprops) => { setLeftSideBarOpenByClick(false) localStorage.setItem("expandLeftNav", false) - const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; - // FIXME: Don't check specific one here const tmpExec = new URLSearchParams(cursearch).get("execution_highlight"); if ( @@ -9677,7 +9827,8 @@ const AngularWorkflow = (defaultprops) => { !el.data("isButton") && !el.data("isDescriptor") && !el.data("isSuggestion") && - el.data("type") !== "COMMENT") { + el.data("type") !== "COMMENT" && + el.data("type") !== "RESIZE-HANDLE") { return true } @@ -9759,6 +9910,27 @@ const AngularWorkflow = (defaultprops) => { cy.on("mouseover", "node", (e) => onNodeHover(e)); cy.on("mouseout", "node", (e) => onNodeHoverOut(e)); + cy.on("mouseover", "node[type='RESIZE-HANDLE']", (e) => { + const nodeId = e.target.id(); + + // Check the node ID to determine the cursor style based on position + if (nodeId.includes("bottom-right")) { + cy.container().style.cursor = "nwse-resize"; // Bottom-right resize cursor + } else if (nodeId.includes("top-right")) { + cy.container().style.cursor = "nesw-resize"; // Top-right resize cursor + } else if (nodeId.includes("top-left")) { + cy.container().style.cursor = "nwse-resize"; // Top-left resize cursor + } else if (nodeId.includes("bottom-left")) { + cy.container().style.cursor = "nesw-resize"; // Bottom-left resize cursor + } else { + cy.container().style.cursor = "default"; // Default cursor for other cases + } + }); + + cy.on("mouseout", "node[type='RESIZE-HANDLE']", (e) => { + cy.container().style.cursor = ""; + }); + // Handles dragging cy.on("drag", "node", (e) => onNodeDrag(e, selectedAction)); cy.on("free", "node", (e) => onNodeDragStop(e, selectedAction)); @@ -11433,7 +11605,7 @@ const AngularWorkflow = (defaultprops) => { const positionInfo = document.activeElement.getBoundingClientRect() const outerlistitemStyle = { - width: "100%", + width: "90%", overflowX: "hidden", overflowY: "hidden", borderBottom: "1px solid rgba(255,255,255,0.4)", @@ -13580,14 +13752,18 @@ const AngularWorkflow = (defaultprops) => { setSourceValue({ ...sourceValue, value: value - }); + }) + + setUpdate(Math.random()) } else if (fieldType === "destination") { setDestinationValue({ ...destinationValue, value: value - }); - } - }; + }) + + setUpdate(Math.random()) + } + } const conditionsModal = ( @@ -14763,6 +14939,7 @@ const AngularWorkflow = (defaultprops) => { ] const handleSubflowParamChange = (triggerId, triggerField, newData) => { + var updateFail = "" if (workflow !== undefined && workflow !== null) { // Find the trigger with matching id @@ -14778,9 +14955,20 @@ const AngularWorkflow = (defaultprops) => { setWorkflow(workflow); setSelectedTriggerValue(newData) setLastSaved(false); - } - } - } + setUpdate(Math.random()) + } else { + updateFail = "Parameter is undefined or null" + } + } else { + updateFail = "Trigger is undefined or null" + } + } else { + updateFail = "Workflow is undefined or null" + } + + if (updateFail !== "") { + toast.error(updateFail + " - Failed to update subflow parameter value. Please try again.") + } } const SubflowSidebar = Object.getOwnPropertyNames(selectedTrigger).length === 0 || workflow.triggers[selectedTriggerIndex] === undefined || selectedTrigger.trigger_type !== "SUBFLOW" ? null : @@ -14970,28 +15158,6 @@ const AngularWorkflow = (defaultprops) => { Select a workflow to run - - {workflow.triggers[selectedTriggerIndex].parameters[0].value - .length === 0 ? null : workflow.triggers[selectedTriggerIndex] - .parameters[0].value === props.match.params.key ? - null - : ( -
- - - -
- )} {workflows === undefined || @@ -15103,13 +15269,36 @@ const AngularWorkflow = (defaultprops) => { }} renderInput={(params) => { return ( - - ); +
+ + {workflow.triggers[selectedTriggerIndex].parameters[0].value + .length === 0 ? null : workflow.triggers[selectedTriggerIndex] + .parameters[0].value === props.match.params.key ? + null + : ( +
+ + + +
+ )} +
+ ) }} /> )} @@ -15737,6 +15926,79 @@ const AngularWorkflow = (defaultprops) => { setSelectedComment(selectedComment); }} /> +
+
+
Justify
+ +
+
+
Align
+ +
+
Height
@@ -15841,7 +16103,6 @@ const AngularWorkflow = (defaultprops) => { defaultValue={selectedComment["backgroundimage"]} onChange={(event) => { selectedComment.backgroundimage = event.target.value; - console.log("Comment: ", selectedComment) setSelectedComment(selectedComment); }} /> @@ -17329,12 +17590,32 @@ const AngularWorkflow = (defaultprops) => { }} renderInput={(params) => { return ( - +
+ + + {subworkflow === null || subworkflow === undefined || subworkflow?.id === undefined || subworkflow?.id === null || subworkflow?.id.length === 0 ? null : + + + + + + } +
); }} /> @@ -17406,7 +17687,6 @@ const AngularWorkflow = (defaultprops) => { workflow?.triggers[selectedTriggerIndex].parameters[2] && workflow?.triggers[selectedTriggerIndex].parameters[2].value && ( - workflow?.triggers[selectedTriggerIndex].parameters[2].value.includes("email") || workflow?.triggers[selectedTriggerIndex].parameters[2].value.includes("sms") ) ? ( { if (!workflow.public && executionModalOpen) { setExecutionRunning(false); stop() - const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; const newitem = removeParam("execution_id", cursearch); navigate(curpath + newitem) setExecutionModalView(0); @@ -19223,7 +19502,6 @@ const AngularWorkflow = (defaultprops) => { if (!workflow.public && executionModalOpen) { setExecutionRunning(false); stop() - const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; const newitem = removeParam("execution_id", cursearch); navigate(curpath + newitem) setExecutionModalView(0); @@ -19738,12 +20016,115 @@ const AngularWorkflow = (defaultprops) => { ); }; + const setupResizeHandlers = (cy, nodeId) => { + let height; + let width; + let resizeTimeout; + + cy.on("drag", ".resize-handle", (event) => { + if (resizeTimeout) return; + + resizeTimeout = setTimeout(() => { + resizeTimeout = null; + const handle = event.target; + const parent = cy.$(`#${nodeId}`); // Fetch the main node directly + + // Check if the parent node exists + if (!parent || parent.empty()) { + console.warn(`Parent node (${nodeId}) not found.`); + return; + } + + if (!handle?.position()) { + console.warn(`Handle position is undefined for ${handle.id()}`); + return; + } + + const handlePos = handle.position(); + const parentPos = parent.position(); + + if (!parentPos) { + console.warn(`Parent position is undefined for ${nodeId}`); + return; + } + + // Calculate new width & height based on handle movement + const newWidth = Math.abs(handlePos.x - parentPos.x) * 2; + const newHeight = Math.abs(handlePos.y - parentPos.y) * 2; + + // Apply min/max constraints + const constrainedWidth = Math.max(100, Math.min(newWidth, 500)); + const constrainedHeight = Math.max(50, Math.min(newHeight, 300)); + + // Update node size + parent.style({ + width: constrainedWidth, + height: constrainedHeight, + }); + + // Store dimensions for state update on drag end + height = Math.floor(constrainedHeight); + width = Math.floor(constrainedWidth); + + // Update handle positions + cy.$(".resize-handle").forEach((corner) => { + if (!corner?.id() || !corner.position()) return; + + const { x, y } = parent.position(); + const offsetX = corner.id().includes("left") ? -constrainedWidth / 2 : constrainedWidth / 2; + const offsetY = corner.id().includes("top") ? -constrainedHeight / 2 : constrainedHeight / 2; + + corner.position({ x: x + offsetX, y: y + offsetY }); + }); + }, 16); // Throttle to ~60 FPS + }); + + // Update state when resizing ends + cy.on("free", ".resize-handle", (event) => { + const data = event.target.data(); + const parentNode = cy.getElementById(data.attachedTo); + + if (parentNode) { + + parentNode.data({ + ...parentNode.data(), + width: Math.floor(width), + height: Math.floor(height), + }); + setSelectedComment((prev) => ({ + ...prev, + width: Math.floor(width), + height: Math.floor(height), + })); + } + }); + + }; + + + const setupNodeDragHandler = (cy, nodeId) => { + cy.on("drag", `#${nodeId}`, (event) => { + const node = event.target; + const { x, y } = node.position(); + const width = parseFloat(node.style("width")); + const height = parseFloat(node.style("height")); + + // Move resize handles with the node + cy.$(".resize-handle").forEach((corner) => { + const offsetX = corner.id().includes("left") ? -width / 2 : width / 2; + const offsetY = corner.id().includes("top") ? -height / 2 : height / 2; + + corner.position({ x: x + offsetX, y: y + offsetY }); + }); + }); + }; + const addCommentNode = () => { const newId = uuidv4(); - const position = { - x: 300, - y: 300, - }; + const position = { x: 300, y: 300 }; + const width = 250; + const height = 150; + const handleOffset = 10; // Move handles outside the node cy.add({ group: "nodes", @@ -19753,20 +20134,51 @@ const AngularWorkflow = (defaultprops) => { type: "COMMENT", is_valid: true, decorator: true, - width: 250, - height: 150, - position: position, + width, + height, + position, backgroundcolor: "#1f2023", color: "#ffffff", + textHalign: "center", + textValign: "center", + textMarginX: "0px", + textMarginY: "0px", }, - position: position, + position, }); + + // Define corner positions relative to the main node + const corners = [ + { id: `${newId}-top-left`, dx: -width / 2 - handleOffset, dy: -height / 2 - handleOffset }, + { id: `${newId}-top-right`, dx: width / 2 + handleOffset, dy: -height / 2 - handleOffset }, + { id: `${newId}-bottom-left`, dx: -width / 2 - handleOffset, dy: height / 2 + handleOffset }, + { id: `${newId}-bottom-right`, dx: width / 2 + handleOffset, dy: height / 2 + handleOffset }, + ]; + + // Add resize handles **without** the parent property + corners.forEach((corner) => { + cy.add({ + group: "nodes", + data: { + id: corner.id, + type: "RESIZE-HANDLE", + is_valid: true, + attachedTo: newId, + decorator: true, + }, // No parent to avoid edges + position: { x: position.x + corner.dx, y: position.y + corner.dy }, + classes: "resize-handle", + }); + }); + + setupResizeHandlers(cy, newId); + setupNodeDragHandler(cy, newId); }; const RightSideBar = (props) => { var defaultReturn = null - if (Object.getOwnPropertyNames(selectedComment).length > 0) { + if (Object.getOwnPropertyNames(selectedComment).length > 0 && selectedComment.hasOwnProperty('id')) { defaultReturn = } else if (Object.getOwnPropertyNames(selectedTrigger).length > 0) { if (selectedTrigger.trigger_type === undefined) { @@ -20288,7 +20700,7 @@ const AngularWorkflow = (defaultprops) => { > @@ -20794,6 +21206,7 @@ const AngularWorkflow = (defaultprops) => { const envStatus = !(executionData.workflow !== undefined && executionData.workflow !== null && executionData.workflow.actions !== undefined && executionData.workflow.actions !== null && executionData.workflow.actions.length > 0) ? "loading" : "success" var executionDelay = -75 + const executionModal = ( { onClose={() => { setExecutionModalOpen(false) - const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; - const newitem = removeParam("execution_id", cursearch); - navigate(curpath + newitem) + //const newitem = removeParam("execution_id", cursearch); + //navigate(curpath + newitem) }} style={{ resize: "both", @@ -20876,20 +21288,18 @@ const AngularWorkflow = (defaultprops) => {
- - - + @@ -21142,7 +21552,7 @@ const AngularWorkflow = (defaultprops) => { onClick={(e) => { e.preventDefault() e.stopPropagation() - window.open(`/admin?admin_tab=notifications&workflow=${data.workflow.id}&execution_id=${data.execution_id}`, "_blank") + window.open(`/admin?org_id=${workflow.org_id}&admin_tab=notifications&workflow=${data.workflow.id}&execution_id=${data.execution_id}`, "_blank") }} /> @@ -21229,20 +21639,17 @@ const AngularWorkflow = (defaultprops) => { > -

{ - const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; const newitem = removeParam("execution_id", cursearch); navigate(curpath + newitem) setExecutionRunning(false); stop() }} > - See more runs + Back to all runs

- {

Details

+ Rerun workflow. Uses same startnode as the original. Runs from scratch. + + } + placement="left" style={{ zIndex: 50000 }} > @@ -21558,7 +21969,7 @@ const AngularWorkflow = (defaultprops) => {
: null} - {userdata.support === true && executionData.workflow !== undefined && executionData.workflow !== null && executionData.status !== "EXECUTING" ? + {userdata.support === true && executionData.workflow !== undefined && executionData.workflow !== null && executionData.status !== "EXECUTING" && executionData.status !== "ABORTED" ?
{ } /> ) : null} +
{executionData.status !== undefined && @@ -21634,6 +22046,7 @@ const AngularWorkflow = (defaultprops) => { ) : null}
+ { executionData.results === undefined || executionData.results === null || @@ -21654,6 +22067,14 @@ const AngularWorkflow = (defaultprops) => { return null; } + const showRerun = new URLSearchParams(cursearch).get("rerun") + if (showRerun === "true") { + const showNode = new URLSearchParams(cursearch).get("node") + if (data.action.id !== showNode) { + return null + } + } + // FIXME: The latter replace doens't really work if ' is used in a string var showResult = data.result.trim(); const validate = validateJson(showResult); @@ -21733,10 +22154,10 @@ const AngularWorkflow = (defaultprops) => { ); } - if (data.action.app_name === "User Input") { + if (data?.action?.app_name === "User Input" || data?.action?.name === "run_userinput") { actionimg = ( {"Shuffle { } } - const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; const chosenNodeId = new URLSearchParams(cursearch).get("node"); const highlightNode = chosenNodeId !== null && chosenNodeId !== undefined && chosenNodeId !== "" && chosenNodeId === data.action.id var relevant_errors = [] @@ -21941,7 +22361,7 @@ const AngularWorkflow = (defaultprops) => { color="primary" title={ - Expand result window. Errors: {relevant_errors.length} + Expand debug window. Errors: {relevant_errors.length} } placement="top" @@ -22580,7 +23000,7 @@ const AngularWorkflow = (defaultprops) => { {curapp === null ? null : ( {selectedResult.action.app_name} { suborgWorkflows={suborgWorkflows} originalWorkflow={originalWorkflow} + runFromHere={runFromHere} />
@@ -23344,11 +23765,8 @@ const AngularWorkflow = (defaultprops) => { // Automatically mapping fields that already exist (predefined). // Warning if fields are NOT filled for (let paramkey in selectedApp.authentication.parameters) { - if ( - authenticationOption.fields[ - selectedApp.authentication.parameters[paramkey].name - ].length === 0 - ) { + if (authenticationOption.fields[selectedApp.authentication.parameters[paramkey].name].length === 0) { + if ( selectedApp.authentication.parameters[paramkey].value !== undefined && selectedApp.authentication.parameters[paramkey].value !== null && @@ -23397,8 +23815,30 @@ const AngularWorkflow = (defaultprops) => { var newAuthOption = JSON.parse(JSON.stringify(authenticationOption)); var newFields = []; + + var warningsent = false for (let authkey in newAuthOption.fields) { - const value = newAuthOption.fields[authkey]; + var value = newAuthOption.fields[authkey]; + + if (value?.toLowerCase().includes("secret. replace")) { + value = "" + + if (authkey === "url") { + // Use default value of the url + const urlparam = selectedApp.authentication.parameters.find((data) => data.name === "url") + if (urlparam !== undefined && urlparam !== null) { + if (urlparam.example !== undefined && urlparam.example !== null && urlparam.example.length > 0) { + value = urlparam.example + } + } + } else { + if (!warningsent) { + warningsent = true + toast("Warning: As you didn't fill in all fields, be aware that the authentication may fail.") + } + } + } + newFields.push({ "key": authkey, "value": value, @@ -23551,18 +23991,18 @@ const AngularWorkflow = (defaultprops) => { }} fullWidth type={ - data.example !== undefined && data.example.includes("***") + data.example !== undefined && data.example.includes("**") ? "password" : "text" } color="primary" defaultValue={ - data.value !== undefined && data.value !== null && !data.value.includes("Secret. Replace") ? data.value : "" + data.value !== undefined && data.value !== null && !data.value.includes("Secret. Replace") ? data.value : + data?.example !== undefined && data?.example !== null && data?.example !== "" && !data?.example.includes("*") ? data.example : "" } placeholder={data.example} onChange={(event) => { - authenticationOption.fields[data.name] = - event.target.value; + authenticationOption.fields[data.name] = event.target.value; }} id={`${data.name}_auth`} /> @@ -24731,6 +25171,8 @@ const AngularWorkflow = (defaultprops) => { } const handleActionParamChange = (actionId, fieldName, newData) => { + var updateFail = "" + if (workflow !== undefined) { // Find the action with matching id const actionIndex = workflow?.actions.findIndex(action => action.id === actionId); @@ -24745,9 +25187,20 @@ const AngularWorkflow = (defaultprops) => { // Update workflow state to trigger re-render setWorkflow({...workflow}); setLastSaved(false); - } - } - } + setUpdate(Math.random()) + } else { + updateFail = "Parameter is undefined or null" + } + } else { + updateFail = "Trigger is undefined or null" + } + } else { + updateFail = "Workflow is undefined or null" + } + + if (updateFail !== "") { + toast.error(updateFail + " - Failed to update subflow parameter value. Please try again.") + } } /* var foundusecase = {} diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 50106831..ae234e9e 100755 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -943,8 +943,8 @@ const AppCreator = (defaultprops) => { if (newaction.url !== undefined && newaction.url !== null && newaction.url.includes("_shuffle_replace_")) { //const regex = /_shuffle_replace_\d/i; const regex = /_shuffle_replace_\d+/i - - newaction.url = newaction.url.replaceAll(new RegExp(regex, 'g'), "") + const newurl = newaction.url.replaceAll(new RegExp(regex, 'g'), "") + newaction.url = newurl } // Finding category @@ -956,7 +956,6 @@ const AppCreator = (defaultprops) => { if (pathsplit[splitkey].includes("_shuffle_replace_")) { //const regex = /_shuffle_replace_\d/i; const regex = /_shuffle_replace_\d+/i - //console.log("NEW: ", pathsplit[splitkey] = pathsplit[splitkey].replaceAll(new RegExp(regex, 'g'), "") } @@ -2037,7 +2036,7 @@ const AppCreator = (defaultprops) => { for (let actionkey in actions) { var item = JSON.parse(JSON.stringify(actions[actionkey])) if (item.errors.length > 0) { - toast("Saving with error in action " + item.name); + //toast("Saving with error in action " + item.name); } if (item.name === undefined && item.description !== undefined) { @@ -3858,7 +3857,7 @@ const AppCreator = (defaultprops) => { if (currentAction.url === "" && actions !== undefined && actions !== null && actions.length > 0) { for (var i = 0; i < actions.length; i++) { if (actions[i].name.toLowerCase() === e.target.value.toLowerCase()) { - toast("Action with name " + e.target.value + " already exists. If you keep this, it will be overwritten.") + //toast("Action with name " + e.target.value + " already exists. If you keep this, it will be overwritten.") break } } diff --git a/frontend/src/views/AppExplorer.jsx b/frontend/src/views/AppExplorer.jsx index d37709ee..b19e5cdd 100644 --- a/frontend/src/views/AppExplorer.jsx +++ b/frontend/src/views/AppExplorer.jsx @@ -1088,14 +1088,13 @@ const AppExplorer = (props) => { setNewWorkflowTags(newWorkflowTags); } - // This is annoying (: - var securitySchemes = data.components.securityDefinitions; + var securitySchemes = data?.components?.securityDefinitions; if (securitySchemes === undefined) { - securitySchemes = data.securitySchemes; + securitySchemes = data?.securitySchemes; } if (securitySchemes === undefined) { - securitySchemes = data.components.securitySchemes; + securitySchemes = data?.components?.securitySchemes; } const allowedfunctions = [ @@ -1681,7 +1680,7 @@ const AppExplorer = (props) => { setExecutionResult({ valid: false, result: - "Couldn't finish execution (2). Please fill all the required fields, and validate the execution.", + "Couldn't finish execution OR no result was returned (2). Please fill all the required fields, and validate the execution.", }); } diff --git a/frontend/src/views/Docs.jsx b/frontend/src/views/Docs.jsx index 8b56b2ce..3e2cfe1b 100755 --- a/frontend/src/views/Docs.jsx +++ b/frontend/src/views/Docs.jsx @@ -343,10 +343,15 @@ const Docs = (defaultprops) => { return } + console.log("PROPKEY: ", propkey) if (location.pathname.includes("/docs/")) { if (propkey === "cookie_policy" || propkey === "compliance" || propkey === "privacy_policy" || propkey === "terms_of_service") { navigate(`/legal/${propkey}`) } + + if (propkey === "app_creation") { + navigate('/docs/apps#app-creation-introduction') + } } }, [location]); @@ -932,7 +937,7 @@ const Docs = (defaultprops) => { } if (responseJson.reason !== undefined && responseJson.reason !== null && responseJson.reason.includes("404: Not Found") && !isArticlePage) { - navigate("/docs") + //navigate("/docs") return } @@ -1109,6 +1114,7 @@ const Docs = (defaultprops) => { marginTop: 25, } + const showPartnerLogo = userdata?.org_status?.includes("integration_partner") && userdata?.active_org?.image !== undefined && userdata?.active_org?.image !== null && userdata?.active_org?.image.length > 0 const mainpageInfo =
{ Documentation -
- /> - link="https://discord.gg/B2CBzUm" /> -
+ {showPartnerLogo === true ? null : +
+ /> + link="https://discord.gg/B2CBzUm" /> +
+ }
Tutorial diff --git a/frontend/src/views/LoginPage.jsx b/frontend/src/views/LoginPage.jsx index 9bcc6966..6291e6a3 100755 --- a/frontend/src/views/LoginPage.jsx +++ b/frontend/src/views/LoginPage.jsx @@ -323,9 +323,17 @@ const LoginPage = props => { if (document !== undefined) { if (register) { - document.title = "Login to Shuffle SaaS" + if (isCloud) { + document.title = "Login to Shuffle SaaS" + } else { + document.title = "Login to Shuffle" + } } else { - document.title = "Register to Shuffle SaaS" + if (isCloud) { + document.title = "Register to Shuffle SaaS" + } else { + document.title = "Register to Shuffle" + } } } @@ -375,7 +383,7 @@ const LoginPage = props => { padding: "40px", flex: 1, maxWidth: isMobile ? "100%" : 410, - minWidth: 410, + minWidth: isCloud ? 410 : 475, background: "#212121", borderRadius: "12px", display: "flex", diff --git a/frontend/src/views/RunWorkflow.jsx b/frontend/src/views/RunWorkflow.jsx index 919d3551..e7c617b8 100644 --- a/frontend/src/views/RunWorkflow.jsx +++ b/frontend/src/views/RunWorkflow.jsx @@ -19,6 +19,7 @@ import RecentWorkflow from "../components/RecentWorkflow.jsx"; import { Tooltip, + Fade, Select, IconButton, CircularProgress, @@ -33,6 +34,7 @@ import { DialogTitle, DialogContent, MenuItem, + Autocomplete, } from '@mui/material'; import { @@ -44,6 +46,7 @@ import { LockOpen as LockOpenIcon, OpenInNew as OpenInNewIcon, Edit as EditIcon, + Polyline as PolylineIcon, } from '@mui/icons-material'; const hrefStyle = { @@ -74,6 +77,7 @@ const RunWorkflow = (defaultprops) => { const [sharingOpen, setSharingOpen] = React.useState(false) const [realtimeMarkdown, setRealtimeMarkdown] = React.useState("") const [forms, setForms] = React.useState([]) + const [workflows, setWorkflows] = React.useState([]) const [boxWidth, setBoxWidth] = React.useState(500) const [inputQuestions, setInputQuestions] = React.useState([]) @@ -180,6 +184,37 @@ const RunWorkflow = (defaultprops) => { return true } + const getWorkflows = () => { + const url = `${globalUrl}/api/v1/workflows` + fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for org forms"); + } + + return response.json() + }) + .then((responseJson) => { + if (responseJson.success === false) { + toast.error("Failed saving workflow. Please try again.") + } else { + if (responseJson?.length > 0) { + setWorkflows(responseJson) + } + } + }) + .catch((error) => { + //toast.error("Load form error: " + error) + }) + } + const loadForms = (orgId) => { const url = `${globalUrl}/api/v1/orgs/${orgId}/forms` fetch(url, { @@ -462,6 +497,8 @@ const RunWorkflow = (defaultprops) => { } else { console.log("Started execution") + start() + setExecutionRunning(true); if (answer !== undefined && answer !== null) { console.log("Skipping start") } else { @@ -875,6 +912,7 @@ const RunWorkflow = (defaultprops) => { // Just use this one? var url = execution_id !== undefined && authorization !== undefined ? `${globalUrl}/api/v1/orgs/${orgId}?reference_execution=${execution_id}&authorization=${authorization}` : `${globalUrl}/api/v1/orgs/${orgId}`; + getWorkflows() loadForms(orgId) fetch(url, { @@ -1055,7 +1093,7 @@ const RunWorkflow = (defaultprops) => { if (parsedresult.click_info !== undefined && parsedresult.click_info !== null) { if (parsedresult.click_info.user !== undefined && parsedresult.click_info.user !== null && parsedresult.click_info.user.length > 0) { - setMessage("Already answered by " + parsedresult.click_info.user) + setMessage("Answered by " + parsedresult.click_info.user) } } else { @@ -1149,10 +1187,100 @@ const RunWorkflow = (defaultprops) => { No Forms Found - Every Workflow is a form, and can be accessed by going to /forms/{`{workflow_id}`}. You can control the form by editing the workflow details in the "Forms" section. - + ALL Workflows are forms, and can be accessed by going to /forms/{`{workflow_id}`}. You can control the form by editing the workflow details in the "Forms" section. +
} + + {workflows === undefined || workflows === null || workflows.length === 0 ? null : + option.id === value.id} + getOptionLabel={(option) => { + if ( + option === undefined || + option === null || + option.name === undefined || + option.name === null + ) { + return "No Workflow Selected"; + } + + const newname = ( + option.name.charAt(0).toUpperCase() + option.name.substring(1) + ).replaceAll("_", " "); + return newname; + }} + options={workflows} + fullWidth + style={{ + backgroundColor: theme.palette.inputColor, + borderRadius: theme.palette?.borderRadius, + marginTop: 75, + }} + renderOption={(props, data, state) => { + if (data.id === workflow.id) { + data = workflow; + } + + //key={index} + return ( + + {data.image !== undefined && data.image !== null && data.image.length > 0 ? + {data.name} + : null} + + Choose Subflow '{data.name}' + + + }> + { + window.location.href = `/forms/${data.id}` + }} + value={data} + > + + {data.name} + + + ) + }} + renderInput={(params) => { + return ( +
+ +
+ ) + }} + /> + }
) } @@ -1325,7 +1453,8 @@ const RunWorkflow = (defaultprops) => { })} : - answer !== undefined && answer !== null ? null : + (answer !== undefined && answer !== null) || message !== "" ? null : + Runtime Argument
@@ -1334,7 +1463,13 @@ const RunWorkflow = (defaultprops) => { style={{backgroundColor: theme.palette.inputColor, marginTop: 5, }} multiLine maxRows={2} + type="text" + autoComplete="off" InputProps={{ + autocomplete: "off", + form: { + autocomplete: "off", + }, style:{ height: "50px", color: "white", @@ -1375,9 +1510,11 @@ const RunWorkflow = (defaultprops) => { {message}. You may close this window. : - - {disabledButtons ? "Answered. You may close this window." : ""} - + + + {disabledButtons ? "Answered. You may close this window." : ""} + + } {disabledButtons ? null : @@ -1387,25 +1524,47 @@ const RunWorkflow = (defaultprops) => { }
- + onSubmit(null, execution_id, authorization, true) + }}> + Continue  or  - + onSubmit(null, execution_id, authorization, false) + }}> + Stop +
: @@ -1416,6 +1575,9 @@ const RunWorkflow = (defaultprops) => { color="primary" fullWidth disabled={!handleValidateForm(executionArgument) || executionLoading} + style={{ + textTransform: "none", + }} > {executionLoading ? @@ -1601,7 +1763,10 @@ const RunWorkflow = (defaultprops) => { disabled={workflow.id === undefined || workflow.id === null} variant={"outlined"} color={"secondary"} - style={{marginRight: 10, }} + style={{ + marginRight: 10, + textTransform: "none", + }} onClick={() => { window.open(`/workflows/${workflow.id}`, "_blank") }} @@ -1614,7 +1779,10 @@ const RunWorkflow = (defaultprops) => { disabled={workflow.id === undefined || workflow.id === null} variant={workflow.sharing === "form" ? "outlined" : "contained"} color={"secondary"} - style={{marginRight: 10, }} + style={{ + marginRight: 10, + textTransform: "none", + }} onClick={() => { setSharingOpen(true) }} @@ -1636,7 +1804,9 @@ const RunWorkflow = (defaultprops) => { disabled={workflow.id === undefined || workflow.id === null} variant={"contained"} color={"primary"} - style={{}} + style={{ + textTransform: "none", + }} onClick={() => { setEditWorkflowModalOpen(true) }} diff --git a/frontend/src/views/SettingsPage.jsx b/frontend/src/views/SettingsPage.jsx index a507857f..c19cd987 100755 --- a/frontend/src/views/SettingsPage.jsx +++ b/frontend/src/views/SettingsPage.jsx @@ -527,6 +527,8 @@ const Settings = (props) => { }; const generateApikey = () => { + toast.info("Generating new API key. This may take a bit."); + fetch(globalUrl + "/api/v1/generateapikey", { method: "GET", headers: { @@ -538,7 +540,7 @@ const Settings = (props) => { .then((response) => { if (response.status !== 200) { console.log("Status not 200 for WORKFLOW EXECUTION :O!"); - } + } return response.json(); }) @@ -836,7 +838,12 @@ const Settings = (props) => { variant="outlined" />
diff --git a/functions/extensions/k8s/shuffle/Chart.yaml b/functions/extensions/k8s/shuffle/Chart.yaml index bd42981f..eb53a0b5 100755 --- a/functions/extensions/k8s/shuffle/Chart.yaml +++ b/functions/extensions/k8s/shuffle/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.2.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.16.0" +appVersion: "2.0.0" diff --git a/functions/kubernetes/charts/shuffle/Chart.yaml b/functions/kubernetes/charts/shuffle/Chart.yaml index f11f1726..0a4e4fa3 100644 --- a/functions/kubernetes/charts/shuffle/Chart.yaml +++ b/functions/kubernetes/charts/shuffle/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: shuffle description: A Helm chart for deploying Shuffle on Kubernetes type: application -version: 0.0.0 -appVersion: 0.0.0 +version: 0.0.0 # Set during publishing in GitHub actions +appVersion: nightly # Overwritten during publishing in GitHub actions dependencies: - name: common version: ^2.23.0 diff --git a/functions/kubernetes/charts/shuffle/README.md b/functions/kubernetes/charts/shuffle/README.md index 495d2690..e2025f94 100644 --- a/functions/kubernetes/charts/shuffle/README.md +++ b/functions/kubernetes/charts/shuffle/README.md @@ -148,6 +148,7 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------- | | `backend.image.registry` | backend image registry | `ghcr.io` | | `backend.image.repository` | backend image repository | `shuffle/shuffle-backend` | +| `backend.image.tag` | backend image tag (immutable tags are recommended, defaults to appVersion) | `""` | | `backend.image.digest` | backend image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) | `""` | | `backend.image.pullPolicy` | backend image pull policy | `IfNotPresent` | | `backend.image.pullSecrets` | backend image pull secrets | `[]` | @@ -184,8 +185,8 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | `backend.podSecurityContext.fsGroup` | Set fsGroup in backend pods' Security Context | `1001` | | `backend.containerSecurityContext.enabled` | Enabled backend container' Security Context | `true` | | `backend.containerSecurityContext.seLinuxOptions` | Set SELinux options in backend container | `{}` | -| `backend.containerSecurityContext.runAsUser` | Set runAsUser in backend container' Security Context | `1000` | -| `backend.containerSecurityContext.runAsGroup` | Set runAsGroup in backend container' Security Context | `1000` | +| `backend.containerSecurityContext.runAsUser` | Set runAsUser in backend container' Security Context | `1001` | +| `backend.containerSecurityContext.runAsGroup` | Set runAsGroup in backend container' Security Context | `1001` | | `backend.containerSecurityContext.runAsNonRoot` | Set runAsNonRoot in backend container' Security Context | `true` | | `backend.containerSecurityContext.readOnlyRootFilesystem` | Set readOnlyRootFilesystem in backend container' Security Context | `true` | | `backend.containerSecurityContext.privileged` | Set privileged in backend container' Security Context | `false` | @@ -196,9 +197,7 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | `backend.args` | Override default backend container args (useful when using custom images) | `[]` | | `backend.automountServiceAccountToken` | Mount Service Account token in backend pods | `true` | | `backend.hostAliases` | backend pods host aliases | `[]` | -| `backend.daemonsetAnnotations` | Annotations for backend daemonset | `{}` | | `backend.deploymentAnnotations` | Annotations for backend deployment | `{}` | -| `backend.statefulsetAnnotations` | Annotations for backend statefulset | `{}` | | `backend.podLabels` | Extra labels for backend pods | `{}` | | `backend.podAnnotations` | Annotations for backend pods | `{}` | | `backend.podAffinityPreset` | Pod affinity preset. Ignored if `backend.affinity` is set. Allowed values: `soft` or `hard` | `""` | @@ -210,8 +209,6 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | `backend.nodeSelector` | Node labels for backend pods assignment | `{}` | | `backend.tolerations` | Tolerations for backend pods assignment | `[]` | | `backend.updateStrategy.type` | backend deployment strategy type | `RollingUpdate` | -| `backend.updateStrategy.type` | backend statefulset strategy type | `RollingUpdate` | -| `backend.podManagementPolicy` | Pod management policy for backend statefulset | `OrderedReady` | | `backend.priorityClassName` | backend pods' priorityClassName | `""` | | `backend.topologySpreadConstraints` | Topology Spread Constraints for backend pod assignment spread across your cluster among failure-domains | `[]` | | `backend.schedulerName` | Name of the k8s scheduler (other than default) for backend pods | `""` | @@ -265,6 +262,7 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------- | | `frontend.image.registry` | frontend image registry | `ghcr.io` | | `frontend.image.repository` | frontend image repository | `shuffle/shuffle-frontend` | +| `frontend.image.tag` | frontend image tag (immutable tags are recommended, defaults to appVersion) | `""` | | `frontend.image.digest` | frontend image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) | `""` | | `frontend.image.pullPolicy` | frontend image pull policy | `IfNotPresent` | | `frontend.image.pullSecrets` | frontend image pull secrets | `[]` | @@ -302,8 +300,8 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | `frontend.podSecurityContext.fsGroup` | Set fsGroup in frontend pods' Security Context | `1001` | | `frontend.containerSecurityContext.enabled` | Enabled frontend container' Security Context | `false` | | `frontend.containerSecurityContext.seLinuxOptions` | Set SELinux options in frontend container | `{}` | -| `frontend.containerSecurityContext.runAsUser` | Set runAsUser in frontend container' Security Context | `101` | -| `frontend.containerSecurityContext.runAsGroup` | Set runAsGroup in frontend container' Security Context | `101` | +| `frontend.containerSecurityContext.runAsUser` | Set runAsUser in frontend container' Security Context | `1001` | +| `frontend.containerSecurityContext.runAsGroup` | Set runAsGroup in frontend container' Security Context | `1001` | | `frontend.containerSecurityContext.runAsNonRoot` | Set runAsNonRoot in frontend container' Security Context | `true` | | `frontend.containerSecurityContext.readOnlyRootFilesystem` | Set readOnlyRootFilesystem in frontend container' Security Context | `true` | | `frontend.containerSecurityContext.privileged` | Set privileged in frontend container' Security Context | `false` | @@ -314,9 +312,7 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | `frontend.args` | Override default frontend container args (useful when using custom images) | `[]` | | `frontend.automountServiceAccountToken` | Mount Service Account token in frontend pods | `false` | | `frontend.hostAliases` | frontend pods host aliases | `[]` | -| `frontend.daemonsetAnnotations` | Annotations for frontend daemonset | `{}` | | `frontend.deploymentAnnotations` | Annotations for frontend deployment | `{}` | -| `frontend.statefulsetAnnotations` | Annotations for frontend statefulset | `{}` | | `frontend.podLabels` | Extra labels for frontend pods | `{}` | | `frontend.podAnnotations` | Annotations for frontend pods | `{}` | | `frontend.podAffinityPreset` | Pod affinity preset. Ignored if `frontend.affinity` is set. Allowed values: `soft` or `hard` | `""` | @@ -328,8 +324,6 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | `frontend.nodeSelector` | Node labels for frontend pods assignment | `{}` | | `frontend.tolerations` | Tolerations for frontend pods assignment | `[]` | | `frontend.updateStrategy.type` | frontend deployment strategy type | `RollingUpdate` | -| `frontend.updateStrategy.type` | frontend statefulset strategy type | `RollingUpdate` | -| `frontend.podManagementPolicy` | Pod management policy for frontend statefulset | `OrderedReady` | | `frontend.priorityClassName` | frontend pods' priorityClassName | `""` | | `frontend.topologySpreadConstraints` | Topology Spread Constraints for frontend pod assignment spread across your cluster among failure-domains | `[]` | | `frontend.schedulerName` | Name of the k8s scheduler (other than default) for frontend pods | `""` | @@ -373,6 +367,7 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | | `orborus.image.registry` | orborus image registry | `ghcr.io` | | `orborus.image.repository` | orborus image repository | `shuffle/shuffle-orborus` | +| `orborus.image.tag` | orborus image tag (immutable tags are recommended, defaults to appVersion) | `""` | | `orborus.image.digest` | orborus image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) | `""` | | `orborus.image.pullPolicy` | orborus image pull policy | `IfNotPresent` | | `orborus.image.pullSecrets` | orborus image pull secrets | `[]` | @@ -409,8 +404,8 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | `orborus.podSecurityContext.fsGroup` | Set fsGroup in orborus pods' Security Context | `1001` | | `orborus.containerSecurityContext.enabled` | Enabled orborus container' Security Context | `true` | | `orborus.containerSecurityContext.seLinuxOptions` | Set SELinux options in orborus container | `{}` | -| `orborus.containerSecurityContext.runAsUser` | Set runAsUser in orborus container' Security Context | `101` | -| `orborus.containerSecurityContext.runAsGroup` | Set runAsGroup in orborus container' Security Context | `101` | +| `orborus.containerSecurityContext.runAsUser` | Set runAsUser in orborus container' Security Context | `1001` | +| `orborus.containerSecurityContext.runAsGroup` | Set runAsGroup in orborus container' Security Context | `1001` | | `orborus.containerSecurityContext.runAsNonRoot` | Set runAsNonRoot in orborus container' Security Context | `true` | | `orborus.containerSecurityContext.readOnlyRootFilesystem` | Set readOnlyRootFilesystem in orborus container' Security Context | `true` | | `orborus.containerSecurityContext.privileged` | Set privileged in orborus container' Security Context | `false` | @@ -421,9 +416,7 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | `orborus.args` | Override default orborus container args (useful when using custom images) | `[]` | | `orborus.automountServiceAccountToken` | Mount Service Account token in orborus pods | `true` | | `orborus.hostAliases` | orborus pods host aliases | `[]` | -| `orborus.daemonsetAnnotations` | Annotations for orborus daemonset | `{}` | | `orborus.deploymentAnnotations` | Annotations for orborus deployment | `{}` | -| `orborus.statefulsetAnnotations` | Annotations for orborus statefulset | `{}` | | `orborus.podLabels` | Extra labels for orborus pods | `{}` | | `orborus.podAnnotations` | Annotations for orborus pods | `{}` | | `orborus.podAffinityPreset` | Pod affinity preset. Ignored if `orborus.affinity` is set. Allowed values: `soft` or `hard` | `""` | @@ -435,8 +428,6 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | `orborus.nodeSelector` | Node labels for orborus pods assignment | `{}` | | `orborus.tolerations` | Tolerations for orborus pods assignment | `[]` | | `orborus.updateStrategy.type` | orborus deployment strategy type | `RollingUpdate` | -| `orborus.updateStrategy.type` | orborus statefulset strategy type | `RollingUpdate` | -| `orborus.podManagementPolicy` | Pod management policy for orborus statefulset | `OrderedReady` | | `orborus.priorityClassName` | orborus pods' priorityClassName | `""` | | `orborus.topologySpreadConstraints` | Topology Spread Constraints for orborus pod assignment spread across your cluster among failure-domains | `[]` | | `orborus.schedulerName` | Name of the k8s scheduler (other than default) for orborus pods | `""` | @@ -481,6 +472,7 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | | `worker.image.registry` | worker image registry | `ghcr.io` | | `worker.image.repository` | worker image repository | `shuffle/shuffle-worker` | +| `worker.image.tag` | worker image tag (immutable tags are recommended, defaults to appVersion) | `""` | | `worker.image.digest` | worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) | `""` | | `worker.serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | | `worker.serviceAccount.name` | The name of the ServiceAccount to use. | `""` | @@ -582,6 +574,7 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | `volumePermissions.enabled` | Enable init container that changes the owner/group of the PV mount point to `runAsUser:fsGroup` | `false` | | `volumePermissions.image.registry` | OS Shell + Utility image registry | `docker.io` | | `volumePermissions.image.repository` | OS Shell + Utility image repository | `bitnami/os-shell` | +| `volumePermissions.image.tag` | OS Shell + Utility image tag (immutable tags are recommended) | `12-debian-12-r30` | | `volumePermissions.image.pullPolicy` | OS Shell + Utility image pull policy | `IfNotPresent` | | `volumePermissions.image.pullSecrets` | OS Shell + Utility image pull secrets | `[]` | | `volumePermissions.resourcesPreset` | Set init container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if volumePermissions.resources is set (volumePermissions.resources is recommended for production). | `nano` | @@ -605,3 +598,4 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" ### Other Parameters + diff --git a/functions/kubernetes/charts/shuffle/templates/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/_helpers.tpl index 6bd29238..dc93dfc2 100644 --- a/functions/kubernetes/charts/shuffle/templates/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/_helpers.tpl @@ -119,14 +119,14 @@ app.kubernetes.io/name: shuffle-app Return the proper image name (for the init container volume-permissions image) */}} {{- define "shuffle.volumePermissions.image" -}} -{{- include "common.images.image" ( dict "imageRoot" .Values.volumePermissions.image "global" .Values.global ) -}} +{{- include "common.images.image" ( dict "imageRoot" .Values.volumePermissions.image "global" .Values.global "chart" .Chart ) -}} {{- end -}} {{/* Return the proper Shuffle backend image name */}} {{- define "shuffle.backend.image" -}} -{{- include "common.images.image" ( dict "imageRoot" .Values.backend.image "global" .Values.global ) -}} +{{- include "common.images.image" ( dict "imageRoot" .Values.backend.image "global" .Values.global "chart" .Chart ) -}} {{- end -}} {{/* @@ -140,7 +140,7 @@ Return the proper Docker Image Registry Secret Names for the backend pod Return the proper Shuffle frontend image name */}} {{- define "shuffle.frontend.image" -}} -{{- include "common.images.image" ( dict "imageRoot" .Values.frontend.image "global" .Values.global ) -}} +{{- include "common.images.image" ( dict "imageRoot" .Values.frontend.image "global" .Values.global "chart" .Chart ) -}} {{- end -}} {{/* @@ -154,7 +154,7 @@ Return the proper Docker Image Registry Secret Names for the frontend pod Return the proper Shuffle orborus image name */}} {{- define "shuffle.orborus.image" -}} -{{- include "common.images.image" ( dict "imageRoot" .Values.orborus.image "global" .Values.global ) -}} +{{- include "common.images.image" ( dict "imageRoot" .Values.orborus.image "global" .Values.global "chart" .Chart ) -}} {{- end -}} {{/* @@ -168,7 +168,7 @@ Return the proper Docker Image Registry Secret Names for the orborus pod Return the proper Shuffle worker image name */}} {{- define "shuffle.worker.image" -}} -{{- include "common.images.image" ( dict "imageRoot" .Values.worker.image "global" .Values.global ) -}} +{{- include "common.images.image" ( dict "imageRoot" .Values.worker.image "global" .Values.global "chart" .Chart ) -}} {{- end -}} {{/* diff --git a/functions/kubernetes/charts/shuffle/values.schema.json b/functions/kubernetes/charts/shuffle/values.schema.json index 7a85b9fa..b785b76e 100644 --- a/functions/kubernetes/charts/shuffle/values.schema.json +++ b/functions/kubernetes/charts/shuffle/values.schema.json @@ -155,6 +155,11 @@ "description": "backend image repository", "default": "shuffle/shuffle-backend" }, + "tag": { + "type": "string", + "description": "backend image tag (immutable tags are recommended, defaults to appVersion)", + "default": "" + }, "digest": { "type": "string", "description": "backend image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended)", @@ -367,12 +372,12 @@ "runAsUser": { "type": "number", "description": "Set runAsUser in backend container' Security Context", - "default": 1000 + "default": 1001 }, "runAsGroup": { "type": "number", "description": "Set runAsGroup in backend container' Security Context", - "default": 1000 + "default": 1001 }, "runAsNonRoot": { "type": "boolean", @@ -444,21 +449,11 @@ "default": [], "items": {} }, - "daemonsetAnnotations": { - "type": "object", - "description": "Annotations for backend daemonset", - "default": {} - }, "deploymentAnnotations": { "type": "object", "description": "Annotations for backend deployment", "default": {} }, - "statefulsetAnnotations": { - "type": "object", - "description": "Annotations for backend statefulset", - "default": {} - }, "podLabels": { "type": "object", "description": "Extra labels for backend pods", @@ -521,16 +516,11 @@ "properties": { "type": { "type": "string", - "description": "backend statefulset strategy type", + "description": "backend deployment strategy type", "default": "RollingUpdate" } } }, - "podManagementPolicy": { - "type": "string", - "description": "Pod management policy for backend statefulset", - "default": "OrderedReady" - }, "priorityClassName": { "type": "string", "description": "backend pods' priorityClassName", @@ -839,6 +829,11 @@ "description": "frontend image repository", "default": "shuffle/shuffle-frontend" }, + "tag": { + "type": "string", + "description": "frontend image tag (immutable tags are recommended, defaults to appVersion)", + "default": "" + }, "digest": { "type": "string", "description": "frontend image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended)", @@ -1056,12 +1051,12 @@ "runAsUser": { "type": "number", "description": "Set runAsUser in frontend container' Security Context", - "default": 101 + "default": 1001 }, "runAsGroup": { "type": "number", "description": "Set runAsGroup in frontend container' Security Context", - "default": 101 + "default": 1001 }, "runAsNonRoot": { "type": "boolean", @@ -1133,21 +1128,11 @@ "default": [], "items": {} }, - "daemonsetAnnotations": { - "type": "object", - "description": "Annotations for frontend daemonset", - "default": {} - }, "deploymentAnnotations": { "type": "object", "description": "Annotations for frontend deployment", "default": {} }, - "statefulsetAnnotations": { - "type": "object", - "description": "Annotations for frontend statefulset", - "default": {} - }, "podLabels": { "type": "object", "description": "Extra labels for frontend pods", @@ -1210,16 +1195,11 @@ "properties": { "type": { "type": "string", - "description": "frontend statefulset strategy type", + "description": "frontend deployment strategy type", "default": "RollingUpdate" } } }, - "podManagementPolicy": { - "type": "string", - "description": "Pod management policy for frontend statefulset", - "default": "OrderedReady" - }, "priorityClassName": { "type": "string", "description": "frontend pods' priorityClassName", @@ -1463,6 +1443,11 @@ "description": "orborus image repository", "default": "shuffle/shuffle-orborus" }, + "tag": { + "type": "string", + "description": "orborus image tag (immutable tags are recommended, defaults to appVersion)", + "default": "" + }, "digest": { "type": "string", "description": "orborus image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended)", @@ -1675,12 +1660,12 @@ "runAsUser": { "type": "number", "description": "Set runAsUser in orborus container' Security Context", - "default": 101 + "default": 1001 }, "runAsGroup": { "type": "number", "description": "Set runAsGroup in orborus container' Security Context", - "default": 101 + "default": 1001 }, "runAsNonRoot": { "type": "boolean", @@ -1752,21 +1737,11 @@ "default": [], "items": {} }, - "daemonsetAnnotations": { - "type": "object", - "description": "Annotations for orborus daemonset", - "default": {} - }, "deploymentAnnotations": { "type": "object", "description": "Annotations for orborus deployment", "default": {} }, - "statefulsetAnnotations": { - "type": "object", - "description": "Annotations for orborus statefulset", - "default": {} - }, "podLabels": { "type": "object", "description": "Extra labels for orborus pods", @@ -1829,16 +1804,11 @@ "properties": { "type": { "type": "string", - "description": "orborus statefulset strategy type", + "description": "orborus deployment strategy type", "default": "RollingUpdate" } } }, - "podManagementPolicy": { - "type": "string", - "description": "Pod management policy for orborus statefulset", - "default": "OrderedReady" - }, "priorityClassName": { "type": "string", "description": "orborus pods' priorityClassName", @@ -2092,6 +2062,11 @@ "description": "worker image repository", "default": "shuffle/shuffle-worker" }, + "tag": { + "type": "string", + "description": "worker image tag (immutable tags are recommended, defaults to appVersion)", + "default": "" + }, "digest": { "type": "string", "description": "worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended)", @@ -2591,6 +2566,11 @@ "description": "OS Shell + Utility image repository", "default": "bitnami/os-shell" }, + "tag": { + "type": "string", + "description": "OS Shell + Utility image tag (immutable tags are recommended)", + "default": "12-debian-12-r30" + }, "pullPolicy": { "type": "string", "description": "OS Shell + Utility image pull policy", diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 35492f94..23c5f8ea 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -96,7 +96,7 @@ backend: ## backend image ## @param backend.image.registry backend image registry ## @param backend.image.repository backend image repository - ## @skip backend.image.tag backend image tag (immutable tags are recommended) + ## @param backend.image.tag backend image tag (immutable tags are recommended, defaults to appVersion) ## @param backend.image.digest backend image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) ## @param backend.image.pullPolicy backend image pull policy ## @param backend.image.pullSecrets backend image pull secrets @@ -104,7 +104,7 @@ backend: image: registry: ghcr.io repository: shuffle/shuffle-backend - tag: nightly + tag: "" digest: "" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' @@ -235,8 +235,8 @@ backend: containerSecurityContext: enabled: true seLinuxOptions: {} - runAsUser: 1000 - runAsGroup: 1000 + runAsUser: 1001 + runAsGroup: 1001 runAsNonRoot: true readOnlyRootFilesystem: true privileged: false @@ -260,18 +260,10 @@ backend: ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ ## hostAliases: [] - ## @param backend.daemonsetAnnotations Annotations for backend daemonset - ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ - ## - daemonsetAnnotations: {} ## @param backend.deploymentAnnotations Annotations for backend deployment ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ ## deploymentAnnotations: {} - ## @param backend.statefulsetAnnotations Annotations for backend statefulset - ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ - ## - statefulsetAnnotations: {} ## @param backend.podLabels Extra labels for backend pods ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ ## @@ -321,22 +313,11 @@ backend: ## ONLY FOR DEPLOYMENTS: ## @param backend.updateStrategy.type backend deployment strategy type ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy - ## ONLY FOR STATEFULSETS: - ## @param backend.updateStrategy.type backend statefulset strategy type - ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies ## updateStrategy: - ## ONLY FOR DEPLOYMENTS: ## Can be set to RollingUpdate or Recreate - ## ONLY FOR STATEFULSETS: - ## Can be set to RollingUpdate or OnDelete ## type: RollingUpdate - ## ONLY FOR STATEFULSETS: - ## @param backend.podManagementPolicy Pod management policy for backend statefulset - ## Ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies - ## - podManagementPolicy: OrderedReady ## @param backend.priorityClassName backend pods' priorityClassName ## priorityClassName: "" @@ -565,7 +546,7 @@ frontend: ## frontend image ## @param frontend.image.registry frontend image registry ## @param frontend.image.repository frontend image repository - ## @skip frontend.image.tag frontend image tag (immutable tags are recommended) + ## @param frontend.image.tag frontend image tag (immutable tags are recommended, defaults to appVersion) ## @param frontend.image.digest frontend image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) ## @param frontend.image.pullPolicy frontend image pull policy ## @param frontend.image.pullSecrets frontend image pull secrets @@ -573,7 +554,7 @@ frontend: image: registry: ghcr.io repository: shuffle/shuffle-frontend - tag: nightly + tag: "" digest: "" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' @@ -683,11 +664,12 @@ frontend: ## @param frontend.podSecurityContext.fsGroup Set fsGroup in frontend pods' Security Context ## podSecurityContext: - enabled: false + enabled: false # The default shuffle frontend image does not support running as non-root, because /etc/nginx/nginx.conf is written on startup fsGroupChangePolicy: Always sysctls: [] supplementalGroups: [] fsGroup: 1001 + ## Configure Container Security Context ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container ## @param frontend.containerSecurityContext.enabled Enabled frontend container' Security Context @@ -702,10 +684,10 @@ frontend: ## @param frontend.containerSecurityContext.seccompProfile.type Set seccomp profile in frontend container ## containerSecurityContext: - enabled: false + enabled: false # The default shuffle frontend image does not support running as non-root, because /etc/nginx/nginx.conf is written on startup seLinuxOptions: {} - runAsUser: 101 - runAsGroup: 101 + runAsUser: 1001 + runAsGroup: 1001 runAsNonRoot: true readOnlyRootFilesystem: true privileged: false @@ -728,18 +710,10 @@ frontend: ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ ## hostAliases: [] - ## @param frontend.daemonsetAnnotations Annotations for frontend daemonset - ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ - ## - daemonsetAnnotations: {} ## @param frontend.deploymentAnnotations Annotations for frontend deployment ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ ## deploymentAnnotations: {} - ## @param frontend.statefulsetAnnotations Annotations for frontend statefulset - ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ - ## - statefulsetAnnotations: {} ## @param frontend.podLabels Extra labels for frontend pods ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ ## @@ -786,25 +760,13 @@ frontend: ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ ## tolerations: [] - ## ONLY FOR DEPLOYMENTS: ## @param frontend.updateStrategy.type frontend deployment strategy type ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy - ## ONLY FOR STATEFULSETS: - ## @param frontend.updateStrategy.type frontend statefulset strategy type - ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies ## updateStrategy: - ## ONLY FOR DEPLOYMENTS: ## Can be set to RollingUpdate or Recreate - ## ONLY FOR STATEFULSETS: - ## Can be set to RollingUpdate or OnDelete ## type: RollingUpdate - ## ONLY FOR STATEFULSETS: - ## @param frontend.podManagementPolicy Pod management policy for frontend statefulset - ## Ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies - ## - podManagementPolicy: OrderedReady ## @param frontend.priorityClassName frontend pods' priorityClassName ## priorityClassName: "" @@ -970,7 +932,7 @@ orborus: ## orborus image ## @param orborus.image.registry orborus image registry ## @param orborus.image.repository orborus image repository - ## @skip orborus.image.tag orborus image tag (immutable tags are recommended) + ## @param orborus.image.tag orborus image tag (immutable tags are recommended, defaults to appVersion) ## @param orborus.image.digest orborus image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) ## @param orborus.image.pullPolicy orborus image pull policy ## @param orborus.image.pullSecrets orborus image pull secrets @@ -978,7 +940,7 @@ orborus: image: registry: ghcr.io repository: shuffle/shuffle-orborus - tag: nightly + tag: "" digest: "" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' @@ -1107,8 +1069,8 @@ orborus: containerSecurityContext: enabled: true seLinuxOptions: {} - runAsUser: 101 - runAsGroup: 101 + runAsUser: 1001 + runAsGroup: 1001 runAsNonRoot: true readOnlyRootFilesystem: true privileged: false @@ -1132,18 +1094,10 @@ orborus: ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ ## hostAliases: [] - ## @param orborus.daemonsetAnnotations Annotations for orborus daemonset - ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ - ## - daemonsetAnnotations: {} ## @param orborus.deploymentAnnotations Annotations for orborus deployment ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ ## deploymentAnnotations: {} - ## @param orborus.statefulsetAnnotations Annotations for orborus statefulset - ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ - ## - statefulsetAnnotations: {} ## @param orborus.podLabels Extra labels for orborus pods ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ ## @@ -1190,25 +1144,13 @@ orborus: ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ ## tolerations: [] - ## ONLY FOR DEPLOYMENTS: ## @param orborus.updateStrategy.type orborus deployment strategy type ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy - ## ONLY FOR STATEFULSETS: - ## @param orborus.updateStrategy.type orborus statefulset strategy type - ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies ## updateStrategy: - ## ONLY FOR DEPLOYMENTS: ## Can be set to RollingUpdate or Recreate - ## ONLY FOR STATEFULSETS: - ## Can be set to RollingUpdate or OnDelete ## type: RollingUpdate - ## ONLY FOR STATEFULSETS: - ## @param orborus.podManagementPolicy Pod management policy for orborus statefulset - ## Ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies - ## - podManagementPolicy: OrderedReady ## @param orborus.priorityClassName orborus pods' priorityClassName ## priorityClassName: "" @@ -1377,13 +1319,13 @@ worker: ## worker image ## @param worker.image.registry worker image registry ## @param worker.image.repository worker image repository - ## @skip worker.image.tag worker image tag (immutable tags are recommended) + ## @param worker.image.tag worker image tag (immutable tags are recommended, defaults to appVersion) ## @param worker.image.digest worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) ## image: registry: ghcr.io repository: shuffle/shuffle-worker - tag: nightly + tag: "" digest: "" ## ServiceAccount configuration @@ -1748,7 +1690,7 @@ volumePermissions: ## ref: https://hub.docker.com/r/bitnami/os-shell/tags/ ## @param volumePermissions.image.registry OS Shell + Utility image registry ## @param volumePermissions.image.repository OS Shell + Utility image repository - ## @skip volumePermissions.image.tag OS Shell + Utility image tag (immutable tags are recommended) + ## @param volumePermissions.image.tag OS Shell + Utility image tag (immutable tags are recommended) ## @param volumePermissions.image.pullPolicy OS Shell + Utility image pull policy ## @param volumePermissions.image.pullSecrets OS Shell + Utility image pull secrets ## diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 343efc26..cd796147 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v27.5.0+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.18 + github.com/shuffle/shuffle-shared v0.8.35 k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 ) diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 8a1a0443..cbbfad08 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -301,8 +301,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.18 h1:7f7cV+P2pr/g44i+AI8P0UheEe8oG8O2V8loBfe9YSw= -github.com/shuffle/shuffle-shared v0.8.18/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.35 h1:3awc0TrsLLZiQeWD2XGIkTnFbczAG0cMfy1+cB/P7zg= +github.com/shuffle/shuffle-shared v0.8.35/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 18f3b9aa..6e031d4a 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1,13 +1,8 @@ package main /* - Orborus exists to listen for new jobs which are deployed as workers. + Orborus exists to listen for new jobs from Shuffle. This is to run workflows, pipelines, and other tasks. */ - -// Potential issues: -// Default network could be same as on the host -// Ingress network may not exist (default) - import ( "archive/zip" "bytes" @@ -60,7 +55,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) -// Starts jobs in bulk, so this could be increased +// Starts jobs in bulk, so this could be increased or decreased based on who the user is var sleepTime = 2 // Making it work on low-end machines even during busy times :) @@ -75,9 +70,13 @@ var workerVersion = os.Getenv("SHUFFLE_WORKER_VERSION") var newWorkerImage = os.Getenv("SHUFFLE_WORKER_IMAGE") var dockerSwarmBridgeMTU = os.Getenv("SHUFFLE_SWARM_BRIDGE_DEFAULT_MTU") var dockerSwarmBridgeInterface = os.Getenv("SHUFFLE_SWARM_BRIDGE_DEFAULT_INTERFACE") +var maxCPUPercent = 90 + +// Kubernetes settings var isKubernetes = os.Getenv("IS_KUBERNETES") var kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") -var maxCPUPercent = 90 +var workerServiceAccountName = os.Getenv("SHUFFLE_WORKER_SERVICE_ACCOUNT_NAME") +var appServiceAccountName = os.Getenv("SHUFFLE_APP_SERVICE_ACCOUNT_NAME") // var baseimagename = "docker.pkg.github.com/shuffle/shuffle" // var baseimagename = "ghcr.io/frikky" @@ -217,8 +216,7 @@ func skipCheckInCleanup(name string) bool { func cleanupExistingNodes(ctx context.Context) error { if isKubernetes == "true" { - // of course, this doesn't clean up "nodes" but - // rather pods, services, roles etc. + // Cleanup all workers created by orborus and all apps created by workers. if kubernetesNamespace == "" { kubernetesNamespace = "default" @@ -230,62 +228,38 @@ func cleanupExistingNodes(ctx context.Context) error { return err } - // Delete all pods - pods, err := clientset.CoreV1().Pods(kubernetesNamespace).List(context.Background(), metav1.ListOptions{}) - if err != nil { - log.Printf("[ERROR] Failed listing pods: %s", err) - return err - } - - for _, pod := range pods.Items { - // check if pod.Name starts with: - // "backend-", "frontend-", "orborus-", "opensearch-" or "memcached-" - if skipCheckInCleanup(pod.Name) { - continue - } - - err := clientset.CoreV1().Pods(kubernetesNamespace).Delete(context.Background(), pod.Name, metav1.DeleteOptions{}) - if err != nil { - log.Printf("[ERROR] Failed deleting pod %s: %s", pod.Name, err) - } - } - // Delete all services - services, err := clientset.CoreV1().Services(kubernetesNamespace).List(context.Background(), metav1.ListOptions{}) + services, err := clientset.CoreV1().Services(kubernetesNamespace).List(context.Background(), metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name in (shuffle-worker, shuffle-app),app.kubernetes.io/managed-by in (shuffle-orborus, shuffle-worker)", + }) if err != nil { log.Printf("[ERROR] Failed listing services: %s", err) return err } for _, service := range services.Items { - if skipCheckInCleanup(service.Name) { - continue - } - err := clientset.CoreV1().Services(kubernetesNamespace).Delete(context.Background(), service.Name, metav1.DeleteOptions{}) if err != nil { log.Printf("[ERROR] Failed deleting service %s: %s", service.Name, err) } } - deployments, err := clientset.AppsV1().Deployments(kubernetesNamespace).List(context.Background(), metav1.ListOptions{}) + deployments, err := clientset.AppsV1().Deployments(kubernetesNamespace).List(context.Background(), metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name in (shuffle-worker, shuffle-app),app.kubernetes.io/managed-by in (shuffle-orborus, shuffle-worker)", + }) if err != nil { log.Printf("[ERROR] Failed listing deployments: %s", err) return err } for _, deployment := range deployments.Items { - if skipCheckInCleanup(deployment.Name) { - continue - } - err := clientset.AppsV1().Deployments(kubernetesNamespace).Delete(context.Background(), deployment.Name, metav1.DeleteOptions{}) if err != nil { log.Printf("[ERROR] Failed deleting deployment %s: %s", deployment.Name, err) } } - log.Printf("[INFO] Cleaned up all pods and services in namespace %s. Waiting 10 seconds for cleanup to reflect", kubernetesNamespace) + log.Printf("[INFO] Cleaned up all services and deployments in namespace %s. Waiting 10 seconds for cleanup to reflect", kubernetesNamespace) time.Sleep(10 * time.Second) @@ -457,19 +431,19 @@ func deployServiceWorkers(image string) { } /* - isMemcachedRunning, err := checkMemcached(ctx, dockercli) - if err != nil { - log.Printf("[ERROR] Failed checking memcached: %s", err) - } - if isMemcachedRunning == false { - log.Printf("[ERROR] Memcached is not running. Will try to deploy it.") - deployMemcached(dockercli) - } + isMemcachedRunning, err := checkMemcached(ctx, dockercli) + if err != nil { + log.Printf("[ERROR] Failed checking memcached: %s", err) + } + if isMemcachedRunning == false { + log.Printf("[ERROR] Memcached is not running. Will try to deploy it.") + deployMemcached(dockercli) + } - ip := "shuffle-cache" - if len(os.Getenv("SHUFFLE_MEMCACHED")) == 0 { - os.Setenv("SHUFFLE_MEMCACHED", fmt.Sprintf("%s:11211", ip)) - } + ip := "shuffle-cache" + if len(os.Getenv("SHUFFLE_MEMCACHED")) == 0 { + os.Setenv("SHUFFLE_MEMCACHED", fmt.Sprintf("%s:11211", ip)) + } */ defaultNetworkAttach := false @@ -725,7 +699,6 @@ func deployServiceWorkers(image string) { if err == nil { log.Printf("[DEBUG] Successfully deployed workers with %d replica(s) on %d node(s)", replicas, cnt) - //time.Sleep(time.Duration(10) * time.Second) //log.Printf("[DEBUG] Servicecreate request: %#v %#v", service, err) } else { if !strings.Contains(fmt.Sprintf("%s", err), "Already Exists") && !strings.Contains(fmt.Sprintf("%s", err), "is already in use by service") { @@ -769,10 +742,10 @@ func handleBackendImageDownload(ctx context.Context, images string) error { // Remove the image handled := []string{} - log.Printf("[DEBUG] Removing existing image (s): %s. Waiting 30 seconds before starting to ensure backend has the latest images built and ready to distribute.", images) - //time.Sleep(time.Duration(30) * time.Second) - + //log.Printf("[DEBUG] Removing existing image (s): %s", images) newImages := []string{} + + successful := []string{} for _, curimage := range strings.Split(images, ",") { curimage = strings.TrimSpace(curimage) if shuffle.ArrayContains(handled, curimage) { @@ -787,25 +760,36 @@ func handleBackendImageDownload(ctx context.Context, images string) error { newImages = append(newImages, curimage) // Force remove the current image to avoid cached layers - _, err := dockercli.ImageRemove(ctx, curimage, image.RemoveOptions{ - Force: true, - PruneChildren: true, - }) + if swarmConfig == "run" || swarmConfig == "swarm" { + _, err := dockercli.ImageRemove(ctx, curimage, image.RemoveOptions{ + Force: true, + PruneChildren: true, + }) - if err != nil { - log.Printf("[ERROR] Failed removing image for re-download: %s", err) + if err != nil { + log.Printf("[ERROR] Failed removing image for re-download: %s", err) + } else { + log.Printf("[DEBUG] Removed image: %s", curimage) + } } else { - log.Printf("[DEBUG] Removed image: %s", curimage) + //log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig) } - err = shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, curimage) + err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, curimage) if err != nil { - log.Printf("[ERROR] Failed downloading image: %s", err) + //log.Printf("[ERROR] Failed downloading image: %s", err) } else { - log.Printf("[DEBUG] Downloaded image: %s", curimage) + //log.Printf("[DEBUG] Downloaded image: %s", curimage) + successful = append(successful, curimage) } } + if len(successful) == 0 { + log.Printf("[ERROR] Failed downloading image copies: %s. This means the app may not have been updated.", strings.Join(handled, ", ")) + } else { + log.Printf("[DEBUG] Successfully downloaded image copies: %s", strings.Join(successful, ", ")) + } + if swarmConfig == "run" || swarmConfig == "swarm" { log.Printf("[DEBUG] Should update service with new image after updating(s): %s. \n\nBETA REPLACEMENT IMPLEMENTATION: Contact support@shuffler.io for support.", strings.Join(newImages, "\n")) @@ -853,7 +837,7 @@ func handleBackendImageDownload(ctx context.Context, images string) error { if !strings.Contains(fmt.Sprintf("%s", resp), "error") { break } else { - found = true + found = true log.Printf("[ERROR] Failed updating service %s with the new image %s: %s. Resp: %#v", service.Spec.Annotations.Name, image, err, resp) } } @@ -1010,6 +994,10 @@ func deployK8sWorker(image string, identifier string, env []string) error { env = append(env, fmt.Sprintf("SHUFFLE_USE_GHCR_OVERRIDE_FOR_AUTODEPLOY=%s", os.Getenv("SHUFFLE_USE_GHCR_OVERRIDE_FOR_AUTODEPLOY"))) } + if len(appServiceAccountName) > 0 { + env = append(env, fmt.Sprintf("SHUFFLE_APP_SERVICE_ACCOUNT_NAME=%s", appServiceAccountName)) + } + clientset, _, err := shuffle.GetKubernetesClient() if err != nil { log.Printf("[ERROR] Error getting kubernetes client:", err) @@ -1018,18 +1006,6 @@ func deployK8sWorker(image string, identifier string, env []string) error { //env = append(env, fmt.Sprintf("KUBERNETES_CONFIG=%s", config.String())) - // FIXME: When a service account is used, the account is also mounted in the pod - // The volume mount location is: - // /var/run/secrets/kubernetes.io/serviceaccount - - // Look for if there is a default service account in use - if len(os.Getenv("KUBERNETES_SERVICE_ACCOUNT")) > 0 { - log.Printf("[DEBUG] Using Kubernetes service account %s", os.Getenv("KUBERNETES_SERVICE_ACCOUNT")) - env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_ACCOUNT=%s", os.Getenv("KUBERNETES_SERVICE_ACCOUNT"))) - - // use k8s downward API to find it if we are in a pod - } - // Check if namespace exist as variable. If so, make it if len(os.Getenv("KUBERNETES_NAMESPACE")) > 0 && !namespacemade { kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") @@ -1088,10 +1064,21 @@ func deployK8sWorker(image string, identifier string, env []string) error { } } - containerLabels := map[string]string{ + labels := map[string]string{ + "app.kubernetes.io/name": "shuffle-worker", + "app.kubernetes.io/instance": identifier, + // "app.kubernetes.io/version": "", + "app.kubernetes.io/part-of": "shuffle", + "app.kubernetes.io/managed-by": "shuffle-orborus", + // Keep legacy labels for backward compatibility "container": "shuffle-worker", } + matchLabels := map[string]string{ + "app.kubernetes.io/name": "shuffle-worker", + "app.kubernetes.io/instance": identifier, + } + containerAttachment := corev1.Container{ Name: identifier, Image: kubernetesImage, @@ -1194,22 +1181,24 @@ func deployK8sWorker(image string, identifier string, env []string) error { deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: identifier, + Name: identifier, + Labels: labels, }, Spec: appsv1.DeploymentSpec{ Replicas: &replicaNumberInt32, Selector: &metav1.LabelSelector{ - MatchLabels: containerLabels, + MatchLabels: matchLabels, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: containerLabels, + Labels: labels, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ containerAttachment, }, - DNSPolicy: corev1.DNSClusterFirst, + DNSPolicy: corev1.DNSClusterFirst, + ServiceAccountName: workerServiceAccountName, }, }, }, @@ -1224,10 +1213,11 @@ func deployK8sWorker(image string, identifier string, env []string) error { // kubectl expose deployment shuffle-workers --type=NodePort --port=33333 --target-port=33333 service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: identifier, + Name: identifier, + Labels: labels, }, Spec: corev1.ServiceSpec{ - Selector: containerLabels, + Selector: matchLabels, Ports: []corev1.ServicePort{ { Protocol: "TCP", @@ -1250,7 +1240,6 @@ func deployK8sWorker(image string, identifier string, env []string) error { func deployWorker(image string, identifier string, env []string, executionRequest shuffle.ExecutionRequest) error { - if len(os.Getenv("REGISTRY_URL")) > 0 && os.Getenv("REGISTRY_URL") != "" { env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL"))) } @@ -1280,18 +1269,17 @@ func deployWorker(image string, identifier string, env []string, executionReques Resources: container.Resources{}, } - certPath := "/certs" // This is just to test the mounting locally so // I can control from what source I'm mounting // the certs to. Default behaviour is: // /certs:/certs. + certPath := "/certs" if os.Getenv("SHUFFLE_CERT_PATH") != "" { certPath = os.Getenv("SHUFFLE_CERT_PATH") } _, err := os.ReadDir(certPath) - if certPath != "" && err == nil { certVol := mount.Mount{ Type: mount.TypeBind, @@ -1310,7 +1298,6 @@ func deployWorker(image string, identifier string, env []string, executionReques } } - //var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG") parsedUuid := uuid.NewV4() @@ -1322,7 +1309,7 @@ func deployWorker(image string, identifier string, env []string, executionReques if isKubernetes != "true" { hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) - if strings.ToLower(cleanupEnv) != "false" { + if strings.ToLower(cleanupEnv) == "true" { hostConfig.AutoRemove = true } } @@ -1351,11 +1338,11 @@ func deployWorker(image string, identifier string, env []string, executionReques ) if err != nil { - log.Printf("[ERROR] Container create error(2): %s", err) + log.Printf("[ERROR][%s] Container create error(2): %s", executionRequest.ExecutionId, err) return err } } else { - log.Printf("[ERROR] Container create error: %s", err) + log.Printf("[ERROR][%s] Container create error: %s", executionRequest.ExecutionId, err) return err } } @@ -1380,47 +1367,47 @@ func deployWorker(image string, identifier string, env []string, executionReques identifier+"-2", ) if err != nil { - log.Printf("[ERROR] Failed to CREATE container (2): %s", err) + log.Printf("[ERROR][%s] Failed to CREATE container (2): %s", executionRequest.ExecutionId, err) } err = dockercli.ContainerStart(context.Background(), cont.ID, containerStartOptions) if err != nil { - log.Printf("[ERROR] Failed to start container (2): %s", err) + log.Printf("[ERROR][%s] Failed to start container (2): %s", executionRequest.ExecutionId, err) } } else { - log.Printf("[ERROR] Failed initial container start. Quitting as this is NOT a simple network issue. Err: %s", err) + log.Printf("[ERROR][%s] Failed initial container start. Quitting as this is NOT a simple network issue. Err: %s", executionRequest.ExecutionId, err) } if err != nil { - log.Printf("[ERROR] Failed to start worker container in environment '%s': %s", environment, err) + log.Printf("[ERROR][%s] Failed to start worker container in environment '%s': %s", executionRequest.ExecutionId, environment, err) return err } else { - log.Printf("[INFO][%s] Worker Container created (2). Environment %s: docker logs %s", executionRequest.ExecutionId, environment, cont.ID) + log.Printf("[INFO][%s] Worker Container created (2). Runtime Location '%s': docker logs -f %s", executionRequest.ExecutionId, environment, cont.ID) } stats, err := dockercli.ContainerInspect(ctx, cont.ID) if err != nil { - log.Printf("[WARNING] Failed checking worker '%s': %s", cont.ID, err) - return nil + log.Printf("[WARNING][%s] Failed checking worker '%s': %s", executionRequest.ExecutionId, cont.ID, err) + return nil } containerStatus := stats.ContainerJSONBase.State.Status if containerStatus != "running" { - log.Printf("[ERROR] Status of %s is %s. Should be running. Contact support@shuffler.io if this persists.", cont.ID, containerStatus) + log.Printf("[ERROR][%s] Status of %s is %s. Should be running. Contact support@shuffler.io if this persists.", executionRequest.ExecutionId, cont.ID, containerStatus) } - /* - err = stopWorker(containerName) - if err != nil { - log.Printf("Failed stopping worker %s", execution.ExecutionId) - return nil - } + /* + err = stopWorker(containerName) + if err != nil { + log.Printf("Failed stopping worker %s", execution.ExecutionId) + return nil + } - err = deployWorker(dockercli, workerImage, containerName, env) - if err != nil { - log.Printf("Failed executing worker %s in state %s", execution.ExecutionId, containerStatus) - return nil + err = deployWorker(dockercli, workerImage, containerName, env) + if err != nil { + log.Printf("Failed executing worker %s in state %s", execution.ExecutionId, containerStatus) + return nil + } } - } */ } else { log.Printf("[INFO][%s] New Worker created. Environment %s: docker logs %s", executionRequest.ExecutionId, environment, cont.ID) @@ -1468,31 +1455,43 @@ func initializeImages() { } if baseimageregistry == "" { + //baseimageregistry = "ghcr.io" // Github baseimageregistry = "docker.io" // Dockerhub - baseimageregistry = "ghcr.io" // Github - log.Printf("[DEBUG] Setting baseimageregistry to %#v", baseimageregistry) + + if len(os.Getenv("REGISTRY_URL")) > 0 { + baseimageregistry = os.Getenv("REGISTRY_URL") + } else { + os.Setenv("REGISTRY_URL", baseimageregistry) + } + + os.Setenv("SHUFFLE_BASE_IMAGE_REGISTRY", baseimageregistry) + + log.Printf("[WARNING] Setting baseimageregistry to %#v", baseimageregistry) } if baseimagename == "" { + // FIXME: This is probably the problem for image names tbh + //baseimagename = "shuffle" // Github (ghcr.io) baseimagename = "frikky/shuffle" // Dockerhub - baseimagename = "shuffle" // Github (ghcr.io) - log.Printf("[DEBUG] Setting baseimagename to %#v", baseimagename) + + os.Setenv("SHUFFLE_BASE_IMAGE_NAME", baseimagename) + log.Printf("[WARNING] Setting baseimagename to %#v", baseimagename) } log.Printf("[DEBUG] Setting swarm config to %#v. Default is empty.", swarmConfig) - newWorker := fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion) + // This is now always static + newWorker := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) if len(newWorkerImage) > 0 { newWorker = newWorkerImage } - // check whether they are the same first - - if os.Getenv("SHUFFLE_AUTO_IMAGE_DOWNLOAD") != "true" { + // Check whether they are the same first + if os.Getenv("SHUFFLE_AUTO_IMAGE_DOWNLOAD") == "false" { + log.Printf("[DEBUG] Skipping image download as SHUFFLE_AUTO_IMAGE_DOWNLOAD is set to false") + } else { images := []string{ fmt.Sprintf("frikky/shuffle:app_sdk"), - fmt.Sprintf("shuffle/shuffle:app_sdk"), - fmt.Sprintf("%s/%s/shuffle-app_sdk:%s", baseimageregistry, baseimagename, appSdkVersion), newWorker, } @@ -1513,8 +1512,6 @@ func initializeImages() { log.Printf("[DEBUG] Successfully downloaded and built %s", image) } } - } else { - log.Printf("[DEBUG] Skipping image download as SHUFFLE_AUTO_IMAGE_DOWNLOAD is set to true") } } @@ -1931,7 +1928,7 @@ func main() { } // Handle Cleanup - made it cleanup by default - if strings.ToLower(os.Getenv("SHUFFLE_CONTAINER_AUTO_CLEANUP")) != "false" { + if strings.ToLower(os.Getenv("SHUFFLE_CONTAINER_AUTO_CLEANUP")) != "false" && os.Getenv("CLEANUP") == "" { cleanupEnv = "true" } @@ -2008,7 +2005,7 @@ func main() { log.Printf("[INFO] Setting up Docker environment. Downloading worker and App SDK!") initializeImages() - workerImage := fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion) + workerImage := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) if len(newWorkerImage) > 0 { workerImage = newWorkerImage } @@ -2119,6 +2116,7 @@ func main() { // Marshal and set body orborusStats := getOrborusStats(ctx) + pipelinePayload, pipelineerr := sendPipelineHealthStatus() if pipelineerr != nil { @@ -2248,7 +2246,7 @@ func main() { if incRequest.Type == "PIPELINE_CREATE" || incRequest.Type == "PIPELINE_START" || incRequest.Type == "PIPELINE_STOP" || incRequest.Type == "PIPELINE_DELETE" { os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - tenzirDisabled = false + tenzirDisabled = false // Running NEW or editing pipelines err := handlePipeline(incRequest) @@ -2258,12 +2256,9 @@ func main() { toBeRemoved.Data = append(toBeRemoved.Data, incRequest) } else if incRequest.Type == "DOCKER_IMAGE_DOWNLOAD" { - log.Printf("[INFO] Re-downloading new image(s): %#v", incRequest.ExecutionArgument) + log.Printf("[INFO] Re-downloading new image(s) due to backend request: %#v", incRequest.ExecutionArgument) if len(incRequest.ExecutionArgument) > 0 { - // FIXME: Wait X seconds before running this as the image build may not be done yet. This is shitty, but may be ok to do in Orborus. Easy fix for the future: Just let it run through jobs 5-10 times before actually picking it up - - // Run after 25 seconds in the goroutine instead go handleBackendImageDownload(ctx, incRequest.ExecutionArgument) } else { log.Printf("[ERROR] No image name provided for download. Removing job from queue.") @@ -2273,7 +2268,7 @@ func main() { } else if incRequest.Type == "CATEGORY_UPDATE" { os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - tenzirDisabled = false + tenzirDisabled = false err = handleFileCategoryChange() if err != nil { @@ -2318,7 +2313,7 @@ func main() { // Manual command = overrides to allow starting of Tenzir from the frontend anyway. os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - tenzirDisabled = false + tenzirDisabled = false // Removed either way toBeRemoved.Data = append(toBeRemoved.Data, incRequest) @@ -2328,7 +2323,7 @@ func main() { if strings.Contains(fmt.Sprintf("%s", err), "node available") { // Disabling until UI is updated os.Setenv("SHUFFLE_SKIP_PIPELINES", "true") - tenzirDisabled = true + tenzirDisabled = true log.Printf("[ERROR] Failed to start tenzir, reason: %s", err) err = shuffle.CreateOrgNotification( @@ -2426,11 +2421,11 @@ func main() { } if execution.Status == "ABORT" || execution.Status == "FAILED" { - log.Printf("[INFO] Executionstatus issue: ", execution.Status) + log.Printf("[INFO][%s] Executionstatus issue: ", execution.ExecutionId, execution.Status) } if shuffle.ArrayContains(executionIds, execution.ExecutionId) { - log.Printf("[INFO] Execution already handled (rerun of old executions?): %s", execution.ExecutionId) + log.Printf("[INFO][%s] Execution already handled (rerunning old execution)", execution.ExecutionId) toBeRemoved.Data = append(toBeRemoved.Data, execution) // Should check when last this was ran, and if it's more than 10 minutes ago and it's not finished, we should run it again? @@ -2490,7 +2485,7 @@ func main() { // Look for volume binds if len(os.Getenv("SHUFFLE_VOLUME_BINDS")) > 0 { - log.Printf("[DEBUG] Added volume binds: %s", os.Getenv("SHUFFLE_VOLUME_BINDS")) + //log.Printf("[DEBUG] Added volume binds: %s", os.Getenv("SHUFFLE_VOLUME_BINDS")) env = append(env, fmt.Sprintf("SHUFFLE_VOLUME_BINDS=%s", os.Getenv("SHUFFLE_VOLUME_BINDS"))) } @@ -2520,10 +2515,24 @@ func main() { toBeRemoved.Data = append(toBeRemoved.Data, execution) executionIds = append(executionIds, execution.ExecutionId) } else { - log.Printf("[WARNING] Execution ID '%s' failed to deploy: %s", execution.ExecutionId, err) + log.Printf("[WARNING][%s] Failed to deploy: %s", execution.ExecutionId, err) + if strings.Contains(err.Error(), "already exists") { toBeRemoved.Data = append(toBeRemoved.Data, execution) executionIds = append(executionIds, execution.ExecutionId) + } else if strings.Contains(err.Error(), "No such image") { + // Download the image + + if isKubernetes == "true" { + log.Printf("[DEBUG] Skipping image pull of '%s' because Kubernetes does it in realtime instead", workerImage) + } else { + log.Printf("[DEBUG] Re-pulling image %s as it doesn't exist, and is necessary for worker to run (autofix)", workerImage) + pullOptions := image.PullOptions{} + _, err = dockercli.ImagePull(ctx, workerImage, pullOptions) + if err != nil { + log.Printf("[ERROR] Failed to pull image %s: %s", workerImage, err) + } + } } } } @@ -2647,7 +2656,7 @@ func deployTenzirNode() error { // return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") //log.Printf("[INFO] Pipelines are enabled by user") } else { - return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") + return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") } if isKubernetes == "true" { @@ -2847,12 +2856,12 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri }, } - // FIXME: Is this necessary? Seems to screw up networking: + // FIXME: Is this necessary? Seems to screw up networking: // conflicting options: hostname and the network mode /* - if isKubernetes != "true" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" { - hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) - } + if isKubernetes != "true" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" { + hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) + } */ resp, err := dockercli.ContainerCreate(ctx, config, hostConfig, networkingConfig, nil, containerName) @@ -3060,7 +3069,6 @@ func createPipeline(command, identifier string) (string, error) { return "", err } - body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Printf("[ERROR] Failed reading response body: %s", err) @@ -3078,8 +3086,8 @@ func createPipeline(command, identifier string) (string, error) { } type PipelineResponse struct { - ID string `json:"id"` - Message string `json:"message"` + ID string `json:"id"` + Message string `json:"message"` Severity string `json:"severity"` } @@ -3215,10 +3223,28 @@ func deletePipeline(pipelineId string) error { func listPipelines() ([]shuffle.PipelineInfo, error) { responseData := shuffle.PipelineInfoWrapper{} + if tenzirDisabled { + return responseData.Pipelines, errors.New("Tenzir is disabled") + } + var reqBody []byte url := fmt.Sprintf("%s/api/v0/pipeline/list", pipelineUrl) - resp, err := http.Post(url, "application/json", bytes.NewBuffer(reqBody)) + client := http.Client{ + Timeout: 2 * time.Second, + } + req, err := http.NewRequest( + "POST", + url, + bytes.NewBuffer(reqBody), + ) + + if err != nil { + return responseData.Pipelines, err + } + + req.Header.Set("Content-Type", "application/json") + resp, err := client.Do(req) if err != nil { return responseData.Pipelines, err } @@ -3302,7 +3328,7 @@ func handleFileCategoryChange() error { tenzirStorageFolder = "/tmp/" } - tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/") + tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/") sigmaPath := fmt.Sprintf("%s/sigma_rules", tenzirStorageFolder) err = extractZIP("files.zip", sigmaPath) if err != nil { @@ -3310,7 +3336,6 @@ func handleFileCategoryChange() error { return err } - log.Printf("[DEBUG] Detection files copied to '%s' successfully.", sigmaPath) return nil @@ -3399,7 +3424,7 @@ func removeFileCategory() error { tenzirStorageFolder = "/tmp/" } - tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/") + tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/") //sigmaPath := "/var/lib/tenzir/sigma_rules/*" sigmaPath := fmt.Sprintf("%s/sigma_rules", tenzirStorageFolder) @@ -3446,6 +3471,10 @@ func sendPipelineHealthStatus() (shuffle.LakeConfig, error) { Pipelines: []shuffle.PipelineInfoMini{}, } + if tenzirDisabled { + return pipelinePayload, nil + } + // To not spam down the list API too much randint := rand.Intn(5) if len(pipelines) == 0 || randint == 0 { @@ -3468,10 +3497,6 @@ func sendPipelineHealthStatus() (shuffle.LakeConfig, error) { pipelinePayload.Pipelines = pipelines } - if tenzirDisabled { - return pipelinePayload, nil - } - err := deployTenzirNode() if err != nil { if (!strings.Contains(err.Error(), "SHUFFLE_SKIP_PIPELINES") && !strings.Contains(err.Error(), "Kubernetes not implemented for Tenzir node")) && !strings.Contains(err.Error(), "Tenzir Node is already running") && !strings.Contains(err.Error(), "docker daemon") { @@ -3564,9 +3589,8 @@ func getRunningWorkers(ctx context.Context, workerTimeout int) int { return 0 } - labelSelector := "app=shuffle-worker" pods, podErr := clientset.CoreV1().Pods(kubernetesNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: labelSelector, + LabelSelector: "app.kubernetes.io/name=shuffle-worker", }) if podErr != nil { log.Printf("[ERROR] Failed getting running workers: %s", podErr) @@ -3655,15 +3679,12 @@ func zombiecheck(ctx context.Context, workerTimeout int) error { All: true, }) - //log.Printf("Len: %d", len(containers)) - if err != nil { log.Printf("[ERROR] Failed creating Containerlist: %s", err) return err } containerNames := map[string]string{} - stopContainers := []string{} removeContainers := []string{} log.Printf("[INFO] Baseimage: %s, Workertimeout: %d", baseimagename, int64(workerTimeout)) @@ -3812,8 +3833,7 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, if err != nil { log.Printf("[ERROR] Failed creating worker request: %s", err) if strings.Contains(fmt.Sprintf("%s", err), "connection refused") || strings.Contains(fmt.Sprintf("%s", err), "EOF") { - workerImage := fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion) - + workerImage := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) if len(newWorkerImage) > 0 { workerImage = newWorkerImage } @@ -3837,8 +3857,7 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, log.Printf("[ERROR] Error running worker request to %s (1): %s", streamUrl, err) if strings.Contains(fmt.Sprintf("%s", err), "connection refused") || strings.Contains(fmt.Sprintf("%s", err), "EOF") { - workerImage := fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion) - + workerImage := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) if len(newWorkerImage) > 0 { workerImage = newWorkerImage } diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 797a3de9..83de43e0 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -8,7 +8,7 @@ require ( github.com/docker/docker v27.5.0+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.18 + github.com/shuffle/shuffle-shared v0.8.35 k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 k8s.io/client-go v0.30.2 diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index 9abeb169..49b14df1 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -294,8 +294,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.18 h1:7f7cV+P2pr/g44i+AI8P0UheEe8oG8O2V8loBfe9YSw= -github.com/shuffle/shuffle-shared v0.8.18/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.35 h1:3awc0TrsLLZiQeWD2XGIkTnFbczAG0cMfy1+cB/P7zg= +github.com/shuffle/shuffle-shared v0.8.35/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 12ff86c6..803aaac2 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -23,8 +23,8 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" - "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/image" + "github.com/docker/docker/api/types/mount" dockerclient "github.com/docker/docker/client" // This is for automatic removal of certain code :) @@ -56,6 +56,7 @@ var logsDisabled = os.Getenv("SHUFFLE_LOGS_DISABLED") var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP")) var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME") var dockerApiVersion = strings.ToLower(os.Getenv("DOCKER_API_VERSION")) +var appServiceAccountName = os.Getenv("SHUFFLE_APP_SERVICE_ACCOUNT_NAME") var kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") var executionCount int64 @@ -104,9 +105,9 @@ var window = shuffle.NewTimeWindow(10 * time.Second) // Images to be autodeployed in the latest version of Shuffle. var autoDeploy = map[string]string{ - "http:1.4.0": "frikky/shuffle:http_1.4.0", - "shuffle-tools:1.2.0": "frikky/shuffle:shuffle-tools_1.2.0", - "shuffle-subflow:1.1.0": "frikky/shuffle:shuffle-subflow_1.1.0", + "http:1.4.0": "frikky/shuffle:http_1.4.0", + "shuffle-tools:1.2.0": "frikky/shuffle:shuffle-tools_1.2.0", + "shuffle-subflow:1.1.0": "frikky/shuffle:shuffle-subflow_1.1.0", // "shuffle-tools-fork:1.0.0": "frikky/shuffle:shuffle-tools-fork_1.0.0", } @@ -298,7 +299,7 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason } // Might not be necessary because of cleanupEnv hostconfig autoremoval - if cleanupEnv == "true" && (os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm") { + if strings.ToLower(cleanupEnv) == "true" && (os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm") { /* ctx := context.Background() dockercli, err := dockerclient.NewEnvClient() @@ -482,9 +483,24 @@ func deployk8sApp(image string, identifier string, env []string) error { //fix naming convention // podUuid := uuid.NewV4().String() - // podName := fmt.Sprintf("%s-%s", value, podUuid) + // name := fmt.Sprintf("%s-%s", value, podUuid) // replace identifier "_" with "-" - podName := strings.ReplaceAll(identifier, "_", "-") + name := strings.ReplaceAll(identifier, "_", "-") + + labels := map[string]string{ + "app.kubernetes.io/name": "shuffle-app", + "app.kubernetes.io/instance": name, + // "app.kubernetes.io/version": "", + "app.kubernetes.io/part-of": "shuffle", + "app.kubernetes.io/managed-by": "shuffle-worker", + // Keep legacy labels for backward compatibility + "app": name, + } + + matchLabels := map[string]string{ + "app.kubernetes.io/name": "shuffle-app", + "app.kubernetes.io/instance": name, + } // pod := &corev1.Pod{ // ObjectMeta: metav1.ObjectMeta{ @@ -564,20 +580,17 @@ func deployk8sApp(image string, identifier string, env []string) error { deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: podName, + Name: name, + Labels: labels, }, Spec: appsv1.DeploymentSpec{ Replicas: &replicaNumberInt32, Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": podName, - }, + MatchLabels: matchLabels, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": podName, - }, + Labels: labels, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -587,6 +600,8 @@ func deployk8sApp(image string, identifier string, env []string) error { Env: buildEnvVars(envMap), }, }, + DNSPolicy: corev1.DNSClusterFirst, + ServiceAccountName: appServiceAccountName, }, }, }, @@ -601,12 +616,11 @@ func deployk8sApp(image string, identifier string, env []string) error { // kubectl expose deployment {podName} --type=NodePort --port=80 --target-port=80 service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: podName, + Name: name, + Labels: labels, }, Spec: corev1.ServiceSpec{ - Selector: map[string]string{ - "app": podName, - }, + Selector: matchLabels, Ports: []corev1.ServicePort{ { Protocol: "TCP", @@ -870,7 +884,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] } // Removing because log extraction should happen first - if cleanupEnv == "true" { + if strings.ToLower(cleanupEnv) == "true" { hostConfig.AutoRemove = true } @@ -880,19 +894,30 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] if len(volumeBindString) > 0 { volumeBindSplit := strings.Split(volumeBindString, ",") for _, volumeBind := range volumeBindSplit { - if strings.Contains(volumeBind, ":") { - volumeBinds = append(volumeBinds, volumeBind) - } else { - log.Printf("[ERROR] Volume bind '%s' is invalid.", volumeBind) + if volumeBind == "srcfolder=dstfolder" || volumeBind == "srcfolder:dstfolder" || volumeBind == "/srcfolder:/dstfolder" { + log.Printf("[DEBUG] Volume bind '%s' is invalid.", volumeBind) + continue } + + if !strings.HasPrefix(volumeBind, "/") { + log.Printf("[ERROR] Volume bind '%s' is invalid. Use absolute paths.", volumeBind) + continue + } + + if !strings.Contains(volumeBind, ":") { + log.Printf("[ERROR] Volume bind '%s' is invalid. Use absolute paths with colon inbetween them (/srcpath:dstpath/", volumeBind) + continue + } + + volumeBinds = append(volumeBinds, volumeBind) } } // Add more volume binds if possible if len(volumeBinds) > 0 { - log.Printf("[DEBUG] Setting up binds for container. Got %d volume binds.", len(volumeBinds)) - hostConfig.Binds = volumeBinds + // Only use mounts, not direct binds + hostConfig.Binds = []string{} hostConfig.Mounts = []mount.Mount{} for _, bind := range volumeBinds { if !strings.Contains(bind, ":") || strings.Contains(bind, "..") || strings.HasPrefix(bind, "~") { @@ -900,15 +925,27 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] continue } - log.Printf("[DEBUG] Appending bind %s to app container", bind) + log.Printf("[DEBUG] Appending bind %s to App container", bind) bindSplit := strings.Split(bind, ":") sourceFolder := bindSplit[0] destinationFolder := bindSplit[1] - hostConfig.Mounts = append(hostConfig.Mounts, mount.Mount{ + + readOnly := false + if len(bindSplit) > 2 { + mode := bindSplit[2] + if mode == "ro" { + readOnly = true + } + } + + builtMount := mount.Mount{ Type: mount.TypeBind, Source: sourceFolder, Target: destinationFolder, - }) + ReadOnly: readOnly, + } + + hostConfig.Mounts = append(hostConfig.Mounts, builtMount) } } @@ -954,7 +991,8 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] func cleanupKubernetesExecution(clientset *kubernetes.Clientset, workflowExecution shuffle.WorkflowExecution, namespace string) error { // workerName := fmt.Sprintf("worker-%s", workflowExecution.ExecutionId) - labelSelector := fmt.Sprintf("app=shuffle-app,executionId=%s", workflowExecution.ExecutionId) + // FIXME: The executionId label is currently not set + labelSelector := fmt.Sprintf("app.kubernetes.io/name=shuffle-app,executionId=%s", workflowExecution.ExecutionId) podList, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{ LabelSelector: labelSelector, @@ -1035,17 +1073,24 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont err = cli.ContainerStart(ctx, cont.ID, container.StartOptions{}) if err != nil { if strings.Contains(fmt.Sprintf("%s", err), "cannot join network") || strings.Contains(fmt.Sprintf("%s", err), "No such container") { + // Remove the "CREATED" one from the previous: + removeErr := cli.ContainerRemove(ctx, cont.ID, container.RemoveOptions{}) + if removeErr != nil { + log.Printf("[ERROR] Failed to remove container %s: %s", cont.ID, removeErr) + } + + log.Printf("[WARNING] Failed deploying App on first attempt: %s. Removing some HostConfig configs.", err) parsedUuid := uuid.NewV4() identifier = fmt.Sprintf("%s-%s-nonetwork", identifier, parsedUuid) - hostConfig = &container.HostConfig{ - LogConfig: container.LogConfig{ - Type: "json-file", - Config: map[string]string{ - "max-size": "10m", - }, + + hostConfig.NetworkMode = container.NetworkMode("") + hostConfig.LogConfig = container.LogConfig{ + Type: "json-file", + Config: map[string]string{ + "max-size": "10m", }, - Resources: container.Resources{}, } + hostConfig.Resources = container.Resources{} cont, err = cli.ContainerCreate( context.Background(), @@ -1067,12 +1112,12 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont return err } - log.Printf("[DEBUG] Running secondary check without network with worker") + //log.Printf("[DEBUG] Running secondary check without network with worker") err = cli.ContainerStart(ctx, cont.ID, container.StartOptions{}) } if err != nil { - log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err) + log.Printf("[ERROR] Failed to start container (2) in runtime location %s: %s", environment, err) cacheErr := shuffle.DeleteCache(ctx, actionExecId) if cacheErr != nil { @@ -1232,6 +1277,7 @@ func getWorkerURLs() ([]string, error) { } func askOtherWorkersToDownloadImage(image string) { + // Why wouldn't it happen on swarm? Hmm if os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm" { return } @@ -1474,10 +1520,6 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { env = append(env, fmt.Sprintf("SHUFFLE_APP_SDK_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_SDK_TIMEOUT"))) } - // Fixes issue: - // standard_go init_linux.go:185: exec user process caused "argument list too long" - // https://devblogs.microsoft.com/oldnewthing/20100203-00/?p=15083 - // FIXME: Ensure to NEVER do this anymore // This potentially breaks too much stuff. Better to have the app poll the data. _ = executionData @@ -1501,9 +1543,11 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { fmt.Sprintf("%s:%s_%s", baseimagename, parsedAppname, action.AppVersion), } - // If cleanup is set, it should run for efficiency + // This is the weirdest shit ever looking back at + // Needs optimization lol + pullOptions := image.PullOptions{} - if cleanupEnv == "true" { + if strings.ToLower(cleanupEnv) == "true" { err = deployApp(dockercli, images[0], identifier, env, workflowExecution, action) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") {