diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index d946307b..60e483cd 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -4,8 +4,8 @@ go 1.24.0 toolchain go1.24.3 +replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared //replace github.com/frikky/schemaless => ../../../schemaless -//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 51a1a856..a84462b4 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -63,6 +63,7 @@ var registryName = "registry.hub.docker.com" var runningEnvironment = "onprem" var syncUrl = "https://shuffler.io" +var debug = false //var syncUrl = "http://localhost:5002" type retStruct struct { @@ -5488,6 +5489,10 @@ func initHandlers() { // Had to move away from mux, which means Method is fucked up right now. func main() { + if os.Getenv("DEBUG") == "true" { + debug = true + } + initHandlers() hostname, err := os.Hostname() if err != nil { diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index c7d3658d..38e767da 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -257,7 +257,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { return } - // This is really the environment's name - NOT org-id + // This is really the environment's name - NOT OrgId environment := request.Header.Get("Org-Id") if len(environment) == 0 { log.Printf("[AUDIT] No org-id header set") @@ -266,6 +266,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { return } + // Org => Org ID here orgId := request.Header.Get("Org") if len(orgId) == 0 { //log.Printf("[AUDIT] No 'org' header set (get workflow queue). ") @@ -275,9 +276,6 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { return */ } - - orborusLabel := request.Header.Get("x-orborus-label") - _ = orborusLabel // This section is cloud custom for now auth := request.Header.Get("Authorization") @@ -290,12 +288,10 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { */ } - //log.Printf("[AUDIT] Get workflow queue for org %s, env %s, orborus label %s", orgId, environment, orborusLabel) - ctx := shuffle.GetContext(request) envs, err := shuffle.GetEnvironments(ctx, orgId) if err != nil || len(envs) == 0 { - log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", environment, err) + //log.Printf("[WARNING] No env found for orgId %s during queue loading", orgId) } var env *shuffle.Environment @@ -308,6 +304,8 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } } + // Only works onprem - shared queues across tenants + // without tenancy if !found { env, err = shuffle.GetEnvironment(ctx, environment, "") if err != nil { @@ -315,22 +313,21 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } } - timeNow := time.Now().Unix() + // Handles failover control between Orborus' + // Further tracks checkin time to ensure this works properly + // across instances err = shuffle.HandleOrborusFailover(ctx, request, resp, env) if err != nil { log.Printf("[WARNING] Failed handling Orborus failover: %s", err) } - //log.Printf("Found env: %#v", env) if len(env.OrgId) > 0 { orgId = env.OrgId } executionRequests, err := shuffle.GetWorkflowQueue(ctx, environment, 100) if err != nil { - // Skipping as this comes up over and over - //log.Printf("(2) Failed reading body for workflowqueue: %s", err) - resp.WriteHeader(401) + resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } @@ -339,10 +336,11 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { if len(executionRequests.Data) == 0 { executionRequests.Data = []shuffle.ExecutionRequest{} } else { - //log.Printf("In workflowqueue with %d", len(executionRequests.Data)) - // Try again :) + // Try again? I don't think this is necessary, and shouldn't really ever occur. + /* if len(env.Id) == 0 && len(env.Name) == 0 { + timeNow := int64(time.Now().Unix()) foundId := "" for _, requestData := range executionRequests.Data { execution, err := shuffle.GetWorkflowExecution(ctx, requestData.ExecutionId) @@ -373,6 +371,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } } } + */ if len(executionRequests.Data) > 50 { executionRequests.Data = executionRequests.Data[0:49] diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index c8cac862..c12e3ae0 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -18661,9 +18661,10 @@ const AngularWorkflow = (defaultprops) => { - const defaultEnvironment = environments.find( + + const defaultEnvironment = environments?.find( (env) => env.default && env.Name.toLowerCase() !== "cloud" - ); + ) if (selectedTrigger.trigger_type === "PIPELINE" && selectedTrigger.environment === "onprem" && defaultEnvironment !== undefined) { selectedTrigger.environment = defaultEnvironment.Name @@ -19269,15 +19270,15 @@ const AngularWorkflow = (defaultprops) => { const TopCytoscapeBar = (props) => { const [hovered, setHovered] = useState(false) - if (workflow.public === true) { + if (workflow?.public === true) { return null } - if (userdata.active_org === undefined || userdata.active_org === null) { + if (userdata?.active_org === undefined || userdata?.active_org === null) { return null } - const isCorrectOrg = workflow.public === true || userdata.active_org.id === undefined || userdata.active_org.id === null || workflow.org_id === null || workflow.org_id === undefined || workflow.org_id.length === 0 || userdata.active_org.id === workflow.org_id + const isCorrectOrg = workflow?.public === true || userdata?.active_org.id === undefined || userdata?.active_org.id === null || workflow?.org_id === null || workflow?.org_id === undefined || workflow?.org_id?.length === 0 || userdata?.active_org?.id === workflow?.org_id return (
@@ -19305,7 +19306,7 @@ const AngularWorkflow = (defaultprops) => { setLastSaved(false) }} > - {workflow.name !== undefined && workflow.name !== null && workflow.name.length > 0 ? + {workflow?.name !== undefined && workflow?.name !== null && workflow?.name?.length > 0 ? : null @@ -19361,7 +19362,7 @@ const AngularWorkflow = (defaultprops) => { {originalWorkflow?.suborg_distribution === undefined || originalWorkflow?.suborg_distribution === null || originalWorkflow?.suborg_distribution?.length === 0 || originalWorkflow?.suborg_distribution.includes("none") ?
- {originalWorkflow?.parentorg_workflow !== undefined && originalWorkflow?.parentorg_workflow !== null && originalWorkflow?.parentorg_workflow.length > 0 || workflow?.parentorg_workflow !== undefined && workflow?.parentorg_workflow !== null && workflow?.parentorg_workflow.length > 0 ? + {originalWorkflow?.parentorg_workflow !== undefined && originalWorkflow?.parentorg_workflow !== null && originalWorkflow?.parentorg_workflow?.length > 0 || workflow?.parentorg_workflow !== undefined && workflow?.parentorg_workflow !== null && workflow?.parentorg_workflow?.length > 0 ? null @@ -19391,7 +19392,7 @@ const AngularWorkflow = (defaultprops) => { : null} - {userdata !== undefined && userdata !== null && userdata.orgs !== undefined && userdata.orgs !== null && userdata.orgs.length > 1 && workflow?.id !== undefined && workflow?.id && workflow?.id?.length > 0 && userdata?.active_org?.creator_org?.length === 0 && userdata?.active_org?.id == workflow?.org_id ? + {userdata !== undefined && userdata !== null && userdata?.orgs !== undefined && userdata?.orgs !== null && userdata?.orgs?.length > 1 && workflow?.id !== undefined && workflow?.id && workflow?.id?.length > 0 && userdata?.active_org?.creator_org?.length === 0 && userdata?.active_org?.id == workflow?.org_id ?
- {showEnvironment === true && environments.length > 0 && selectedActionEnvironment !== undefined && selectedActionEnvironment !== null && selectedActionEnvironment.Name !== undefined && selectedActionEnvironment.Name !== null ? + {showEnvironment === true && environments?.length > 0 && selectedActionEnvironment !== undefined && selectedActionEnvironment !== null && selectedActionEnvironment.Name !== undefined && selectedActionEnvironment.Name !== null ? { const [isLoadingWorkflow, setIsLoadingWorkflow] = useState(false); const [isLoadingPublicWorkflow, setIsLoadingPublicWorkflow] = useState(false); const [view, setView] = useState(localStorage?.getItem("workflowView") || "grid"); - const [showExecutionStats, setShowExecutionStats] = React.useState(localStorage?.getItem("showExecutionStats") === "true" || false); + + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + const [showExecutionStats, setShowExecutionStats] = React.useState(localStorage?.getItem("showExecutionStats") === "true" || isCloud) const imgSize = 60; @@ -906,7 +908,7 @@ const Workflows2 = (props) => { if (!alreadyDismissed && !aiAnnouncementModalOpen) { // Show the banner as the user hasn't seen it yet - setAiAnnouncementModalOpen(true); + //setAiAnnouncementModalOpen(true); } } }, [isLoggedIn, userdata]); @@ -944,10 +946,6 @@ const Workflows2 = (props) => { }); }; - //const isCloud = - // window.location.host === "localhost:3002" || - // window.location.host === "shuffler.io"; - const isCloud = false const findWorkflow = (filters) => { console.log("Using filters: ", filters) @@ -3186,7 +3184,7 @@ const Workflows2 = (props) => { {showExecutionStats === true && foundTimeline !== undefined && foundTimeline?.timeline?.length > 0 && -
+
{ */} - {/* {modalView} */} {deleteModal} {exportVerifyModal} {publishModal} diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index cb02b199..eacbbba3 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -496,7 +496,8 @@ func deployServiceWorkers(image string) { //} } - replicas := uint64(1) + // Running 2 by default instead of 1. Higher scale mechanisms - es + replicas := uint64(2) scaleReplicas := os.Getenv("SHUFFLE_SCALE_REPLICAS") if len(scaleReplicas) > 0 { tmpInt, err := strconv.Atoi(scaleReplicas) @@ -521,7 +522,7 @@ func deployServiceWorkers(image string) { } appReplicas := os.Getenv("SHUFFLE_APP_REPLICAS") - appReplicaCnt := 1 + appReplicaCnt := 2 if len(appReplicas) > 0 { newCnt, err := strconv.Atoi(appReplicas) if err != nil { diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 35314834..b6693d08 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -612,8 +612,8 @@ func deployk8sApp(image string, identifier string, env []string) error { // use deployment instead of pod // then expose a service similarly. // number of replicas can be set to os.Getenv("SHUFFLE_SCALE_REPLICAS") + replicaNumber := 2 replicaNumberStr := os.Getenv("SHUFFLE_SCALE_REPLICAS") - replicaNumber := 1 if len(replicaNumberStr) > 0 { tmpInt, err := strconv.Atoi(replicaNumberStr) if err != nil {