marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed environment management and minor bugs onprem

Browse Source
This commit is contained in:
Frikky
2025-08-26 19:32:30 +02:00
parent dedd00e521
commit d2fcc2c3eb
7 changed files with 46 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/go-app/go.mod
+1 -1
View File
@@ -4,8 +4,8 @@ go 1.24.0
toolchain go1.24.3 toolchain go1.24.3
replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared
//replace github.com/frikky/schemaless => ../../../schemaless //replace github.com/frikky/schemaless => ../../../schemaless
//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared
//replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi
backend/go-app/main.go
+5
View File
@@ -63,6 +63,7 @@ var registryName = "registry.hub.docker.com"
var runningEnvironment = "onprem" var runningEnvironment = "onprem"
var syncUrl = "https://shuffler.io" var syncUrl = "https://shuffler.io"
var debug = false
//var syncUrl = "http://localhost:5002" //var syncUrl = "http://localhost:5002"
type retStruct struct { type retStruct struct {
@@ -5488,6 +5489,10 @@ func initHandlers() {
// Had to move away from mux, which means Method is fucked up right now. // Had to move away from mux, which means Method is fucked up right now.
func main() { func main() {
if os.Getenv("DEBUG") == "true" {
debug = true
}
initHandlers() initHandlers()
hostname, err := os.Hostname() hostname, err := os.Hostname()
if err != nil { if err != nil {
backend/go-app/walkoff.go
+13 -14
View File
@@ -257,7 +257,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
return return
} }
// This is really the environment's name - NOT org-id // This is really the environment's name - NOT OrgId
environment := request.Header.Get("Org-Id") environment := request.Header.Get("Org-Id")
if len(environment) == 0 { if len(environment) == 0 {
log.Printf("[AUDIT] No org-id header set") log.Printf("[AUDIT] No org-id header set")
@@ -266,6 +266,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
return return
} }
// Org => Org ID here
orgId := request.Header.Get("Org") orgId := request.Header.Get("Org")
if len(orgId) == 0 { if len(orgId) == 0 {
//log.Printf("[AUDIT] No 'org' header set (get workflow queue). ") //log.Printf("[AUDIT] No 'org' header set (get workflow queue). ")
@@ -275,9 +276,6 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
return return
*/ */
} }
orborusLabel := request.Header.Get("x-orborus-label")
_ = orborusLabel
// This section is cloud custom for now // This section is cloud custom for now
auth := request.Header.Get("Authorization") auth := request.Header.Get("Authorization")
@@ -290,12 +288,10 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
*/ */
} }
//log.Printf("[AUDIT] Get workflow queue for org %s, env %s, orborus label %s", orgId, environment, orborusLabel)
ctx := shuffle.GetContext(request) ctx := shuffle.GetContext(request)
envs, err := shuffle.GetEnvironments(ctx, orgId) envs, err := shuffle.GetEnvironments(ctx, orgId)
if err != nil || len(envs) == 0 { if err != nil || len(envs) == 0 {
log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", environment, err) //log.Printf("[WARNING] No env found for orgId %s during queue loading", orgId)
} }
var env *shuffle.Environment var env *shuffle.Environment
@@ -308,6 +304,8 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
} }
} }
// Only works onprem - shared queues across tenants
// without tenancy
if !found { if !found {
env, err = shuffle.GetEnvironment(ctx, environment, "") env, err = shuffle.GetEnvironment(ctx, environment, "")
if err != nil { if err != nil {
@@ -315,22 +313,21 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
} }
} }
timeNow := time.Now().Unix() // Handles failover control between Orborus'
// Further tracks checkin time to ensure this works properly
// across instances
err = shuffle.HandleOrborusFailover(ctx, request, resp, env) err = shuffle.HandleOrborusFailover(ctx, request, resp, env)
if err != nil { if err != nil {
log.Printf("[WARNING] Failed handling Orborus failover: %s", err) log.Printf("[WARNING] Failed handling Orborus failover: %s", err)
} }
//log.Printf("Found env: %#v", env)
if len(env.OrgId) > 0 { if len(env.OrgId) > 0 {
orgId = env.OrgId orgId = env.OrgId
} }
executionRequests, err := shuffle.GetWorkflowQueue(ctx, environment, 100) executionRequests, err := shuffle.GetWorkflowQueue(ctx, environment, 100)
if err != nil { if err != nil {
// Skipping as this comes up over and over resp.WriteHeader(500)
//log.Printf("(2) Failed reading body for workflowqueue: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return return
} }
@@ -339,10 +336,11 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
if len(executionRequests.Data) == 0 { if len(executionRequests.Data) == 0 {
executionRequests.Data = []shuffle.ExecutionRequest{} executionRequests.Data = []shuffle.ExecutionRequest{}
} else { } else {
//log.Printf("In workflowqueue with %d", len(executionRequests.Data))
// Try again :) // Try again? I don't think this is necessary, and shouldn't really ever occur.
/*
if len(env.Id) == 0 && len(env.Name) == 0 { if len(env.Id) == 0 && len(env.Name) == 0 {
timeNow := int64(time.Now().Unix())
foundId := "" foundId := ""
for _, requestData := range executionRequests.Data { for _, requestData := range executionRequests.Data {
execution, err := shuffle.GetWorkflowExecution(ctx, requestData.ExecutionId) execution, err := shuffle.GetWorkflowExecution(ctx, requestData.ExecutionId)
@@ -373,6 +371,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
} }
} }
} }
*/
if len(executionRequests.Data) > 50 { if len(executionRequests.Data) > 50 {
executionRequests.Data = executionRequests.Data[0:49] executionRequests.Data = executionRequests.Data[0:49]
frontend/src/views/AngularWorkflow.jsx
+18 -17
View File
@@ -18661,9 +18661,10 @@ const AngularWorkflow = (defaultprops) => {
</div> </div>
</div> </div>
const defaultEnvironment = environments.find(
const defaultEnvironment = environments?.find(
(env) => env.default && env.Name.toLowerCase() !== "cloud" (env) => env.default && env.Name.toLowerCase() !== "cloud"
); )
if (selectedTrigger.trigger_type === "PIPELINE" && selectedTrigger.environment === "onprem" && defaultEnvironment !== undefined) { if (selectedTrigger.trigger_type === "PIPELINE" && selectedTrigger.environment === "onprem" && defaultEnvironment !== undefined) {
selectedTrigger.environment = defaultEnvironment.Name selectedTrigger.environment = defaultEnvironment.Name
@@ -19269,15 +19270,15 @@ const AngularWorkflow = (defaultprops) => {
const TopCytoscapeBar = (props) => { const TopCytoscapeBar = (props) => {
const [hovered, setHovered] = useState(false) const [hovered, setHovered] = useState(false)
if (workflow.public === true) { if (workflow?.public === true) {
return null return null
} }
if (userdata.active_org === undefined || userdata.active_org === null) { if (userdata?.active_org === undefined || userdata?.active_org === null) {
return null return null
} }
const isCorrectOrg = workflow.public === true || userdata.active_org.id === undefined || userdata.active_org.id === null || workflow.org_id === null || workflow.org_id === undefined || workflow.org_id.length === 0 || userdata.active_org.id === workflow.org_id const isCorrectOrg = workflow?.public === true || userdata?.active_org.id === undefined || userdata?.active_org.id === null || workflow?.org_id === null || workflow?.org_id === undefined || workflow?.org_id?.length === 0 || userdata?.active_org?.id === workflow?.org_id
return ( return (
<div style={topBarStyle}> <div style={topBarStyle}>
@@ -19305,7 +19306,7 @@ const AngularWorkflow = (defaultprops) => {
setLastSaved(false) setLastSaved(false)
}} }}
> >
{workflow.name !== undefined && workflow.name !== null && workflow.name.length > 0 ? {workflow?.name !== undefined && workflow?.name !== null && workflow?.name?.length > 0 ?
<EditIcon style={{ position: "absolute", top: 7, height: 20, width: 20, }} /> <EditIcon style={{ position: "absolute", top: 7, height: 20, width: 20, }} />
: :
null null
@@ -19361,7 +19362,7 @@ const AngularWorkflow = (defaultprops) => {
{originalWorkflow?.suborg_distribution === undefined || originalWorkflow?.suborg_distribution === null || originalWorkflow?.suborg_distribution?.length === 0 || originalWorkflow?.suborg_distribution.includes("none") ? {originalWorkflow?.suborg_distribution === undefined || originalWorkflow?.suborg_distribution === null || originalWorkflow?.suborg_distribution?.length === 0 || originalWorkflow?.suborg_distribution.includes("none") ?
<div> <div>
{originalWorkflow?.parentorg_workflow !== undefined && originalWorkflow?.parentorg_workflow !== null && originalWorkflow?.parentorg_workflow.length > 0 || workflow?.parentorg_workflow !== undefined && workflow?.parentorg_workflow !== null && workflow?.parentorg_workflow.length > 0 ? {originalWorkflow?.parentorg_workflow !== undefined && originalWorkflow?.parentorg_workflow !== null && originalWorkflow?.parentorg_workflow?.length > 0 || workflow?.parentorg_workflow !== undefined && workflow?.parentorg_workflow !== null && workflow?.parentorg_workflow?.length > 0 ?
null null
@@ -19391,7 +19392,7 @@ const AngularWorkflow = (defaultprops) => {
: null} : null}
{userdata !== undefined && userdata !== null && userdata.orgs !== undefined && userdata.orgs !== null && userdata.orgs.length > 1 && workflow?.id !== undefined && workflow?.id && workflow?.id?.length > 0 && userdata?.active_org?.creator_org?.length === 0 && userdata?.active_org?.id == workflow?.org_id ? {userdata !== undefined && userdata !== null && userdata?.orgs !== undefined && userdata?.orgs !== null && userdata?.orgs?.length > 1 && workflow?.id !== undefined && workflow?.id && workflow?.id?.length > 0 && userdata?.active_org?.creator_org?.length === 0 && userdata?.active_org?.id == workflow?.org_id ?
<Button <Button
color="secondary" color="secondary"
variant="outlined" variant="outlined"
@@ -19518,10 +19519,10 @@ const AngularWorkflow = (defaultprops) => {
//toast("Loading correct info for suborg") //toast("Loading correct info for suborg")
} }
if (originalWorkflow.childorg_workflow_ids === undefined || originalWorkflow.childorg_workflow_ids === null || originalWorkflow.childorg_workflow_ids.length === 0) { if (originalWorkflow?.childorg_workflow_ids === undefined || originalWorkflow?.childorg_workflow_ids === null || originalWorkflow?.childorg_workflow_ids?.length === 0) {
//console.log("Childorg doesn't exist (?). Suborgworkflows: ", suborgWorkflows) //console.log("Childorg doesn't exist (?). Suborgworkflows: ", suborgWorkflows)
if (suborgWorkflows !== undefined && suborgWorkflows !== null && suborgWorkflows.length > 0) { if (suborgWorkflows !== undefined && suborgWorkflows !== null && suborgWorkflows?.length > 0) {
var found = false var found = false
for (var suborgkey in suborgWorkflows) { for (var suborgkey in suborgWorkflows) {
const suborgWorkflow = suborgWorkflows[suborgkey] const suborgWorkflow = suborgWorkflows[suborgkey]
@@ -19623,8 +19624,8 @@ const AngularWorkflow = (defaultprops) => {
marginLeft: marginLeft:
data.creator_org !== undefined && data.creator_org !== undefined &&
data.creator_org !== null && data.creator_org !== null &&
data.creator_org.length > 0 data.creator_org?.length > 0
? data.id === userdata.active_org.id ? data?.id === userdata?.active_org?.id
? 0 ? 0
: 0 : 0
: 0, : 0,
@@ -19670,7 +19671,7 @@ const AngularWorkflow = (defaultprops) => {
title={ title={
<div style={{ overflow: "auto", }}> <div style={{ overflow: "auto", }}>
<Typography variant="body1" style={{margin: 16, }}> <Typography variant="body1" style={{margin: 16, }}>
<b>{foundMatchingWorkflow.errors.length} Workflow Issue{foundMatchingWorkflow.errors.length > 1 ? "s" : ""}</b> <b>{foundMatchingWorkflow?.errors?.length} Workflow Issue{foundMatchingWorkflow?.errors?.length > 1 ? "s" : ""}</b>
</Typography> </Typography>
</div> </div>
} }
@@ -19697,7 +19698,7 @@ const AngularWorkflow = (defaultprops) => {
} }
{orgDiff?.actions?.length > 0 && {orgDiff?.actions?.length > 0 &&
<span>- Actions ({orgDiff.actions.length}): <br/> <span>- Actions ({orgDiff.actions?.length}): <br/>
{orgDiff.actions.map((orgDiffAction, index) => { {orgDiff.actions.map((orgDiffAction, index) => {
//console.log("DIFF2: ", orgDiffAction) //console.log("DIFF2: ", orgDiffAction)
var formattedError = "" var formattedError = ""
@@ -19705,7 +19706,7 @@ const AngularWorkflow = (defaultprops) => {
var paramchanges = "" var paramchanges = ""
for (var diffActionKey in orgDiffAction) { for (var diffActionKey in orgDiffAction) {
if (diffActionKey !== "id" && diffActionKey !== "label" && diffActionKey !== "parameters" && diffActionKey !== "params" && diffActionKey !== "large_image") { if (diffActionKey !== "id" && diffActionKey !== "label" && diffActionKey !== "parameters" && diffActionKey !== "params" && diffActionKey !== "large_image") {
if (formattedError.length > 0) { if (formattedError?.length > 0) {
formattedError += ", " formattedError += ", "
} }
formattedError += diffActionKey formattedError += diffActionKey
@@ -19716,7 +19717,7 @@ const AngularWorkflow = (defaultprops) => {
} }
} }
if (paramchanges.length > 0) { if (paramchanges?.length > 0) {
formattedError += paramchanges formattedError += paramchanges
} }
@@ -19747,7 +19748,7 @@ const AngularWorkflow = (defaultprops) => {
} }
</div> </div>
{showEnvironment === true && environments.length > 0 && selectedActionEnvironment !== undefined && selectedActionEnvironment !== null && selectedActionEnvironment.Name !== undefined && selectedActionEnvironment.Name !== null ? {showEnvironment === true && environments?.length > 0 && selectedActionEnvironment !== undefined && selectedActionEnvironment !== null && selectedActionEnvironment.Name !== undefined && selectedActionEnvironment.Name !== null ?
<Tooltip <Tooltip
arrow arrow
title={ title={
frontend/src/views/Workflows2.jsx
+5 -8
View File
@@ -687,7 +687,9 @@ const Workflows2 = (props) => {
const [isLoadingWorkflow, setIsLoadingWorkflow] = useState(false); const [isLoadingWorkflow, setIsLoadingWorkflow] = useState(false);
const [isLoadingPublicWorkflow, setIsLoadingPublicWorkflow] = useState(false); const [isLoadingPublicWorkflow, setIsLoadingPublicWorkflow] = useState(false);
const [view, setView] = useState(localStorage?.getItem("workflowView") || "grid"); const [view, setView] = useState(localStorage?.getItem("workflowView") || "grid");
const [showExecutionStats, setShowExecutionStats] = React.useState(localStorage?.getItem("showExecutionStats") === "true" || false);
const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io";
const [showExecutionStats, setShowExecutionStats] = React.useState(localStorage?.getItem("showExecutionStats") === "true" || isCloud)
const imgSize = 60; const imgSize = 60;
@@ -906,7 +908,7 @@ const Workflows2 = (props) => {
if (!alreadyDismissed && !aiAnnouncementModalOpen) { if (!alreadyDismissed && !aiAnnouncementModalOpen) {
// Show the banner as the user hasn't seen it yet // Show the banner as the user hasn't seen it yet
setAiAnnouncementModalOpen(true); //setAiAnnouncementModalOpen(true);
} }
} }
}, [isLoggedIn, userdata]); }, [isLoggedIn, userdata]);
@@ -944,10 +946,6 @@ const Workflows2 = (props) => {
}); });
}; };
//const isCloud =
// window.location.host === "localhost:3002" ||
// window.location.host === "shuffler.io";
const isCloud = false
const findWorkflow = (filters) => { const findWorkflow = (filters) => {
console.log("Using filters: ", filters) console.log("Using filters: ", filters)
@@ -3186,7 +3184,7 @@ const Workflows2 = (props) => {
</Grid> </Grid>
{showExecutionStats === true && foundTimeline !== undefined && foundTimeline?.timeline?.length > 0 && {showExecutionStats === true && foundTimeline !== undefined && foundTimeline?.timeline?.length > 0 &&
<div style={{ margin: "40px 15px 0px 15px", paddingTop: 0, borderTop: "1px solid rgba(255,255,255,0.3)", }}> <div style={{ margin: "40px 10px 0px 10px", paddingTop: 0, borderTop: "1px solid rgba(255,255,255,0.3)", }}>
<LineChartWrapper <LineChartWrapper
inputname={""} inputname={""}
keys={foundTimeline?.timeline} keys={foundTimeline?.timeline}
@@ -5534,7 +5532,6 @@ const Workflows2 = (props) => {
</ShepherdTour> </ShepherdTour>
*/} */}
<DropzoneWrapper onDrop={uploadFile} WorkflowView={WorkflowView} /> <DropzoneWrapper onDrop={uploadFile} WorkflowView={WorkflowView} />
{/* {modalView} */}
{deleteModal} {deleteModal}
{exportVerifyModal} {exportVerifyModal}
{publishModal} {publishModal}
functions/onprem/orborus/orborus.go
+3 -2
View File
@@ -496,7 +496,8 @@ func deployServiceWorkers(image string) {
//} //}
} }
replicas := uint64(1) // Running 2 by default instead of 1. Higher scale mechanisms - es
replicas := uint64(2)
scaleReplicas := os.Getenv("SHUFFLE_SCALE_REPLICAS") scaleReplicas := os.Getenv("SHUFFLE_SCALE_REPLICAS")
if len(scaleReplicas) > 0 { if len(scaleReplicas) > 0 {
tmpInt, err := strconv.Atoi(scaleReplicas) tmpInt, err := strconv.Atoi(scaleReplicas)
@@ -521,7 +522,7 @@ func deployServiceWorkers(image string) {
} }
appReplicas := os.Getenv("SHUFFLE_APP_REPLICAS") appReplicas := os.Getenv("SHUFFLE_APP_REPLICAS")
appReplicaCnt := 1 appReplicaCnt := 2
if len(appReplicas) > 0 { if len(appReplicas) > 0 {
newCnt, err := strconv.Atoi(appReplicas) newCnt, err := strconv.Atoi(appReplicas)
if err != nil { if err != nil {
functions/onprem/worker/worker.go
+1 -1
View File
@@ -612,8 +612,8 @@ func deployk8sApp(image string, identifier string, env []string) error {
// use deployment instead of pod // use deployment instead of pod
// then expose a service similarly. // then expose a service similarly.
// number of replicas can be set to os.Getenv("SHUFFLE_SCALE_REPLICAS") // number of replicas can be set to os.Getenv("SHUFFLE_SCALE_REPLICAS")
replicaNumber := 2
replicaNumberStr := os.Getenv("SHUFFLE_SCALE_REPLICAS") replicaNumberStr := os.Getenv("SHUFFLE_SCALE_REPLICAS")
replicaNumber := 1
if len(replicaNumberStr) > 0 { if len(replicaNumberStr) > 0 {
tmpInt, err := strconv.Atoi(replicaNumberStr) tmpInt, err := strconv.Atoi(replicaNumberStr)
if err != nil { if err != nil {