Merge branch '2.0.0' of https://github.com/Shuffle/Shuffle into mergingIssue
This commit is contained in:
monilprajapati
@@ -714,7 +714,10 @@ const Billing = (props) => {
|
||||
userdata.has_card_available === true ?
|
||||
"While you have a card attached to your account, Shuffle will no longer prevent workflows from running. Billing will occur at the start of each month."
|
||||
:
|
||||
`You are not subscribed to any plan and are using the free plan with max 10,000 app runs per month. Upgrade to deactivate this limit.`
|
||||
isCloud ?
|
||||
`You are not subscribed to any plan and are using the free plan with max 10,000 app runs per month. Upgrade to deactivate this limit.`
|
||||
:
|
||||
`You are not subscribed to any plan and are using the free, open source plan. This plan has no enforced limits, but scale issues may occur due to CPU congestion.`
|
||||
}
|
||||
</Typography>
|
||||
<div style={{ display: 'flex', flexDirection: 'row' }}>
|
||||
|
||||
@@ -152,6 +152,7 @@ const CreatorGrid = props => {
|
||||
flexWrap: "wrap",
|
||||
alignContent: "space-between",
|
||||
marginTop: 5,
|
||||
padding:"0px 180px",
|
||||
}
|
||||
|
||||
const Hits = ({ hits }) => {
|
||||
|
||||
@@ -80,7 +80,7 @@ const DiscordChat = props => {
|
||||
event.preventDefault();
|
||||
}
|
||||
}}
|
||||
placeholder="Search Discord Chats"
|
||||
placeholder="Search Discord Chats..."
|
||||
style={{ backgroundColor: theme.palette.inputColor, borderRadius: borderRadius, margin: 10, width: "100%", }}
|
||||
InputProps={{
|
||||
style: {
|
||||
@@ -148,7 +148,7 @@ const DiscordChat = props => {
|
||||
const CustomHits = connectHits(Hits);
|
||||
|
||||
return (
|
||||
<div style={{ width: "100%", textAlign:"center", position: "relative", height: "100%", }}>
|
||||
<div style={{textAlign:"center", position: "relative", height: "100%", padding:"0px 240px" }}>
|
||||
<InstantSearch searchClient={searchClient} indexName="discord_chat">
|
||||
<div style={{ maxWidth: 450, margin: "auto", marginTop: 15, marginBottom: 5, }}>
|
||||
<CustomSearchBox />
|
||||
|
||||
@@ -279,7 +279,7 @@ const DocsGrid = props => {
|
||||
</Button>
|
||||
</div>
|
||||
*/}
|
||||
<div style={{width: "100%", position: "relative", height: "100%",}}>
|
||||
<div style={{width: "100%", position: "relative", height: "100%", padding: "0px 180px"}}>
|
||||
<InstantSearch searchClient={searchClient} indexName="documentation">
|
||||
<div style={{maxWidth: 450, margin: "auto", marginTop: 15, marginBottom: 15, }}>
|
||||
<CustomSearchBox />
|
||||
|
||||
@@ -563,7 +563,10 @@ const ParsedAction = (props) => {
|
||||
|
||||
useEffect(() => {
|
||||
selectedNameChange(appActionName)
|
||||
actionDelayChange(delay)
|
||||
|
||||
if (actionDelayChange !== undefined) {
|
||||
actionDelayChange(delay)
|
||||
}
|
||||
},[appActionName,delay])
|
||||
|
||||
const handleParamChange = (event, count,data) => {
|
||||
|
||||
@@ -204,6 +204,9 @@ const AppGrid = props => {
|
||||
style={{backgroundColor: theme.palette.inputColor, borderRadius: borderRadius, margin: 10, width: "100%",}}
|
||||
InputProps={{
|
||||
style:{
|
||||
color: "white",
|
||||
fontSize: "1em",
|
||||
height: 50,
|
||||
},
|
||||
startAdornment: (
|
||||
<InputAdornment position="start">
|
||||
@@ -238,6 +241,8 @@ const AppGrid = props => {
|
||||
flexWrap: "wrap",
|
||||
alignContent: "space-between",
|
||||
marginTop: 5,
|
||||
padding: "0px 180px",
|
||||
width:"auto"
|
||||
}
|
||||
|
||||
var workflowDelay = -50
|
||||
|
||||
@@ -76,6 +76,7 @@ import {
|
||||
Help as HelpIcon,
|
||||
Flag as FlagIcon,
|
||||
FmdGood as FmdGoodIcon,
|
||||
Warning as WarningIcon,
|
||||
} from "@mui/icons-material";
|
||||
|
||||
//import { useAlert
|
||||
@@ -5369,10 +5370,37 @@ If you're interested, please let me know a time that works for you, or set up a
|
||||
}
|
||||
}
|
||||
|
||||
console.log("AUTH: ", data)
|
||||
var validIcon = <CheckCircleIcon style={{ color: "green" }} />
|
||||
if (data.validation !== null && data.validation !== undefined && data.validation.valid === false) {
|
||||
|
||||
if (data.validation.changed_at === 0) {
|
||||
// Warning
|
||||
validIcon = "" // <WarningIcon style={{ color: "" }} />
|
||||
} else {
|
||||
validIcon = <CancelIcon style={{ color: "red" }} />
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<ListItem key={index} style={{ backgroundColor: bgColor }}>
|
||||
<ListItemText
|
||||
primary=
|
||||
<Tooltip title={data.validation.valid === true ? "Valid. Click to explore." : "Configuration failed. Click to learn why"} placement="top">
|
||||
<IconButton>
|
||||
{validIcon}
|
||||
</IconButton>
|
||||
</Tooltip>
|
||||
style={{ minWidth: 65, maxWidth: 65, }}
|
||||
onClick={() => {
|
||||
if (data.validation.workflow_id === undefined || data.validation.workflow_id === null || data.validation.workflow_id.length === 0) {
|
||||
toast.warn("No workflow runs found for this auth yet. Check back later.")
|
||||
return
|
||||
}
|
||||
|
||||
const url = `/workflows/${data.validation.workflow_id}?execution_id=${data.validation.execution_id}&node=${data.validation.node_id}`
|
||||
window.open(url, "_blank")
|
||||
}}
|
||||
/>
|
||||
<ListItemText
|
||||
primary=<img
|
||||
alt=""
|
||||
@@ -5497,17 +5525,17 @@ If you're interested, please let me know a time that works for you, or set up a
|
||||
.join(", ")
|
||||
}
|
||||
style={{
|
||||
minWidth: 125,
|
||||
maxWidth: 125,
|
||||
minWidth: 140,
|
||||
maxWidth: 140,
|
||||
overflow: "auto",
|
||||
marginRight: 10,
|
||||
}}
|
||||
/>
|
||||
<ListItemText
|
||||
style={{
|
||||
maxWidth: 230,
|
||||
minWidth: 230,
|
||||
overflow: "hidden",
|
||||
maxWidth: 150,
|
||||
minWidth: 150,
|
||||
overflow: "auto",
|
||||
}}
|
||||
primary={new Date(data.edited * 1000).toISOString()}
|
||||
/>
|
||||
@@ -5803,6 +5831,10 @@ If you're interested, please let me know a time that works for you, or set up a
|
||||
/>
|
||||
<List>
|
||||
<ListItem>
|
||||
<ListItemText
|
||||
primary="Valid"
|
||||
style={{ minWidth: 65, maxWidth: 65 }}
|
||||
/>
|
||||
<ListItemText
|
||||
primary="Icon"
|
||||
style={{ minWidth: 75, maxWidth: 75 }}
|
||||
@@ -5831,11 +5863,11 @@ If you're interested, please let me know a time that works for you, or set up a
|
||||
*/}
|
||||
<ListItemText
|
||||
primary="Fields"
|
||||
style={{ minWidth: 135, maxWidth: 135, overflow: "hidden" }}
|
||||
style={{ minWidth: 140, maxWidth: 140, overflow: "hidden" }}
|
||||
/>
|
||||
<ListItemText
|
||||
primary="Edited"
|
||||
style={{ minWidth: 230, maxWidth: 230, overflow: "hidden" }}
|
||||
style={{ minWidth: 150, maxWidth: 150, overflow: "hidden" }}
|
||||
/>
|
||||
<ListItemText
|
||||
primary="Actions"
|
||||
|
||||
@@ -432,7 +432,7 @@ const AngularWorkflow = (defaultprops) => {
|
||||
const [history, setHistory] = React.useState([]);
|
||||
const [historyIndex, setHistoryIndex] = React.useState(history.length);
|
||||
const [variableInfo, setVariableInfo] = React.useState({})
|
||||
|
||||
const [selectedVersion, setSelectedVersion] = React.useState(null)
|
||||
const [appAuthentication, setAppAuthentication] = React.useState(undefined);
|
||||
const [variablesModalOpen, setVariablesModalOpen] = React.useState(false);
|
||||
const [aiQueryModalOpen, setAiQueryModalOpen] = React.useState(false)
|
||||
@@ -472,7 +472,6 @@ const AngularWorkflow = (defaultprops) => {
|
||||
const [conditionValue, setConditionValue] = React.useState({});
|
||||
const [dragging, setDragging] = React.useState(false);
|
||||
const [showWorkflowRevisions, setShowWorkflowRevisions] = React.useState(false);
|
||||
const [selectedRevision, setSelectedRevision] = useState({})
|
||||
const [dragPosition, setDragPosition] = React.useState({
|
||||
x: 0,
|
||||
y: 0,
|
||||
@@ -2098,6 +2097,7 @@ const releaseToConnectLabel = "Release to Connect"
|
||||
setTimeout(() => {
|
||||
setSavingState(0);
|
||||
}, 1500);
|
||||
getRevisionHistory(useworkflow.id)
|
||||
}
|
||||
})
|
||||
.catch((error) => {
|
||||
@@ -7967,6 +7967,7 @@ const releaseToConnectLabel = "Release to Connect"
|
||||
}
|
||||
|
||||
setAllRevisions(responseJson)
|
||||
setSelectedVersion(responseJson[0])
|
||||
})
|
||||
.catch((error) => {
|
||||
console.log("Error getting workflow revisions: ", error)
|
||||
@@ -16571,7 +16572,6 @@ const releaseToConnectLabel = "Release to Connect"
|
||||
console.log("Show workflow revisions key pressed")
|
||||
if (!workflow.public) {
|
||||
setShowWorkflowRevisions(true)
|
||||
setSelectedRevision(workflow)
|
||||
//setOriginalWorkflow(workflow)
|
||||
}
|
||||
}
|
||||
@@ -16624,7 +16624,6 @@ const releaseToConnectLabel = "Release to Connect"
|
||||
console.log("Show workflow revisions key pressed")
|
||||
if (!workflow.public) {
|
||||
setShowWorkflowRevisions(true)
|
||||
setSelectedRevision(workflow)
|
||||
//setOriginalWorkflow(workflow)
|
||||
}
|
||||
}
|
||||
@@ -17099,7 +17098,6 @@ const releaseToConnectLabel = "Release to Connect"
|
||||
variant={"outlined"}
|
||||
onClick={() => {
|
||||
setShowWorkflowRevisions(true)
|
||||
setSelectedRevision(workflow)
|
||||
//setOriginalWorkflow(workflow)
|
||||
}}
|
||||
>
|
||||
@@ -21408,14 +21406,12 @@ const releaseToConnectLabel = "Release to Connect"
|
||||
}*/
|
||||
|
||||
const RevisionBox = (props) => {
|
||||
const { revision, } = props
|
||||
|
||||
if (revision === undefined || revision === null || revision === {}) {
|
||||
const { revision, } = props
|
||||
if (revision === undefined || revision === null) {
|
||||
return null
|
||||
}
|
||||
|
||||
var newrevision = JSON.parse(JSON.stringify(revision))
|
||||
|
||||
// Make unix timestamp into ISO timestamp in the format July 27th, 3:05 AM
|
||||
// Format: July 27th, 3:05 AM
|
||||
//console.log("Edited time: ", revision.edited)
|
||||
@@ -21440,18 +21436,17 @@ const releaseToConnectLabel = "Release to Connect"
|
||||
|
||||
return (
|
||||
<Paper
|
||||
style={{padding: "15px 15px 15px 25px", minHeight: 105, maxHeight: 105, cursor: "pointer", backgroundColor: newrevision.edited === selectedRevision.edited ? "rgba(255,255,255,0.3)" : theme.palette.surfaceColor, border: "1px solid rgba(255,255,255,0.3)", marginBottom: 10,
|
||||
style={{padding: "15px 15px 15px 25px", minHeight: 105, maxHeight: 105, cursor: "pointer", backgroundColor: newrevision.edited === selectedVersion.edited ? "rgba(255,255,255,0.3)" : theme.palette.surfaceColor, border: "1px solid rgba(255,255,255,0.3)", marginBottom: 10,
|
||||
}} onClick={(e) => {
|
||||
if (newrevision.edited === selectedRevision.edited) {
|
||||
if (newrevision.edited === selectedVersion.edited) {
|
||||
console.log("Same revision! No setting.")
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
// Should render if it's not the same as workflow.edited
|
||||
console.log("Clicked revision: ", newrevision)
|
||||
setLastSaved(false)
|
||||
setSelectedRevision(newrevision)
|
||||
setLastSaved(false)
|
||||
setSelectedVersion(newrevision);
|
||||
setWorkflow(newrevision)
|
||||
setSelectedAction({});
|
||||
setSelectedApp({})
|
||||
@@ -21474,7 +21469,11 @@ const releaseToConnectLabel = "Release to Connect"
|
||||
//cy.remove('*')
|
||||
setElements([])
|
||||
}
|
||||
|
||||
|
||||
// Remove all edges
|
||||
cy.edges().remove()
|
||||
cy.nodes().remove()
|
||||
|
||||
// Remove all cy nodes
|
||||
setTimeout(() => {
|
||||
setupGraph(newrevision)
|
||||
@@ -21525,6 +21524,7 @@ const releaseToConnectLabel = "Release to Connect"
|
||||
<span style={{flex: 5, }}>
|
||||
<Typography variant="body1">
|
||||
{translatedDate}
|
||||
{/* {newrevision.edited.toString().slice(6,10)} | {newrevision.revision_id.slice(0,5)} */}
|
||||
</Typography>
|
||||
</span>
|
||||
<span style={{flex: 2, }}>
|
||||
@@ -21607,39 +21607,64 @@ const releaseToConnectLabel = "Release to Connect"
|
||||
</Paper>
|
||||
)
|
||||
}
|
||||
|
||||
const drawerData = originalWorkflow !== undefined && originalWorkflow !== null && originalWorkflow !== {} ?
|
||||
<div style={{padding: "0px 25px 100px 25px", }}>
|
||||
<Typography variant="h6" style={{marginTop: 10, marginBottom: 15, }}>
|
||||
//! Logs
|
||||
console.log("Workflow state", workflow)
|
||||
console.log("All revision", allRevisions)
|
||||
const drawerData = originalWorkflow !== undefined && originalWorkflow !== null ?
|
||||
<div style={{ height: "100%"}}>
|
||||
<Typography variant="h5" style={{ paddingLeft: 25, paddingTop:25, backgroundColor: theme.palette.surfaceColor, height: "8%" }}>
|
||||
Version History (Beta)
|
||||
</Typography>
|
||||
<div style={{height: "92%" }}>
|
||||
<div style={{paddingLeft: "25px", paddingRight: "25px", paddingTop: "10px"}}>
|
||||
<div style={{marginBottom: "20px", }}>
|
||||
<Typography variant="h6" style={{marginTop: 10, marginBottom: 5, }}>
|
||||
Current Version
|
||||
</Typography>
|
||||
<RevisionBox
|
||||
revision={selectedVersion}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<RevisionBox
|
||||
revision={originalWorkflow}
|
||||
/>
|
||||
<Divider
|
||||
style={{
|
||||
marginBottom: 15,
|
||||
height: 1,
|
||||
width: "100%",
|
||||
backgroundColor: "rgb(91, 96, 100)",
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{allRevisions.length > 0 ?
|
||||
allRevisions.map((revision, index) => {
|
||||
if (revision.edited === originalWorkflow.edited) {
|
||||
return null
|
||||
}
|
||||
|
||||
return (
|
||||
<RevisionBox
|
||||
revision={revision}
|
||||
key={index}
|
||||
index={index}
|
||||
/>
|
||||
)
|
||||
})
|
||||
:
|
||||
<div style={{padding: 5, }}>
|
||||
<Typography variant="body2">
|
||||
No other revisions found. Save your workflow with changes to create a revision.
|
||||
</Typography>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
{allRevisions.length > 0 ?
|
||||
<div style={{overflow: "auto", width: "100%" , height: "75%", paddingLeft: "25px", paddingRight: "20px", paddingTop: "10px", paddingBottom: "10px"}}>
|
||||
{
|
||||
allRevisions.map((revision, index) => {
|
||||
if(revision.edited === selectedVersion.edited){
|
||||
return null
|
||||
}
|
||||
|
||||
return (
|
||||
<RevisionBox
|
||||
revision={revision}
|
||||
key={index}
|
||||
index={index}
|
||||
/>
|
||||
)
|
||||
})
|
||||
}
|
||||
</div>
|
||||
|
||||
:
|
||||
<div style={{padding: 5, }}>
|
||||
<Typography variant="body2">
|
||||
No other revisions found. Save your workflow with changes to create a revision.
|
||||
</Typography>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
: null
|
||||
|
||||
const workflowRevisions = !showWorkflowRevisions ? null :
|
||||
@@ -21650,7 +21675,7 @@ const releaseToConnectLabel = "Release to Connect"
|
||||
onClose={() => {
|
||||
//setShowWorkflowRevisions(false)
|
||||
}}
|
||||
style={{ resize: "both", overflow: "auto", zIndex: 10005 }}
|
||||
style={{ resize: "both", overflow: "hidden", zIndex: 10005 }}
|
||||
hideBackdrop={true}
|
||||
variant="persistent"
|
||||
BackdropProps={{
|
||||
@@ -21661,7 +21686,7 @@ const releaseToConnectLabel = "Release to Connect"
|
||||
PaperProps={{
|
||||
style: {
|
||||
resize: "both",
|
||||
overflow: "auto",
|
||||
overflow: "hidden",
|
||||
minWidth: isMobile ? "100%" : 360,
|
||||
maxWidth: isMobile ? "100%" : 360,
|
||||
backgroundColor: theme.palette.platformColor,
|
||||
@@ -21669,7 +21694,6 @@ const releaseToConnectLabel = "Release to Connect"
|
||||
fontSize: 18,
|
||||
zIndex: 15001,
|
||||
borderRight: theme.palette.defaultBorder,
|
||||
paddingTop: 15,
|
||||
},
|
||||
}}
|
||||
>
|
||||
@@ -21692,7 +21716,7 @@ const releaseToConnectLabel = "Release to Connect"
|
||||
</div>
|
||||
<div style={{textAlign: "center", color: "white", flex: 1, paddingTop: 20, }}>
|
||||
<Typography variant="h6">
|
||||
{selectedRevision.name}
|
||||
{selectedVersion.name}
|
||||
</Typography>
|
||||
</div>
|
||||
{/* Cross icon to close it */}
|
||||
|
||||
@@ -32,8 +32,6 @@ import {
|
||||
FileCopy as FileCopyIcon
|
||||
} from "@mui/icons-material";
|
||||
import { fontGrid } from "@mui/material/styles/cssUtils.js";
|
||||
import { active } from "d3";
|
||||
import style from "./../index.css";
|
||||
|
||||
const Body = {
|
||||
//maxWidth: 1000,
|
||||
|
||||
@@ -43,11 +43,10 @@ const Search = (props) => {
|
||||
const params = Object.fromEntries(urlSearchParams.entries());
|
||||
const foundTab = params["tab"];
|
||||
if (foundTab !== null && foundTab !== undefined) {
|
||||
for (var key in Object.keys(views)) {
|
||||
for (let key in views) {
|
||||
const value = views[key];
|
||||
console.log(key, value);
|
||||
if (value === foundTab) {
|
||||
setConfig("", key);
|
||||
setConfig(null, key);
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -130,7 +129,7 @@ const Search = (props) => {
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
overflowX: "hidden",
|
||||
minHeight: 400,
|
||||
minHeight: 400
|
||||
};
|
||||
|
||||
const views = {
|
||||
@@ -138,7 +137,9 @@ const Search = (props) => {
|
||||
1: "workflows",
|
||||
2: "docs",
|
||||
3: "creators",
|
||||
4: "discord"
|
||||
};
|
||||
|
||||
const setConfig = (event, inputValue) => {
|
||||
const newValue = parseInt(inputValue);
|
||||
|
||||
@@ -216,7 +217,7 @@ const Search = (props) => {
|
||||
<div style={boxStyle}>
|
||||
<Tabs
|
||||
style={{
|
||||
width: 741,
|
||||
width: 757,
|
||||
margin: isHeader ? null : "auto",
|
||||
marginTop: hidemargins === true ? 0 : isHeader ? null : 25,
|
||||
backgroundColor: "rgba(33, 33, 33, 1)",
|
||||
@@ -227,8 +228,8 @@ const Search = (props) => {
|
||||
textColor="secondary"
|
||||
onChange={setConfig}
|
||||
aria-label="disabled tabs example"
|
||||
variant="scrollable"
|
||||
scrollButtons="auto"
|
||||
// variant="scrollable"
|
||||
// scrollButtons="auto"
|
||||
classes={{ indicator: classes.hideIndicator, root: classes.customTab }}
|
||||
>
|
||||
<StyledTab
|
||||
|
||||
@@ -4,16 +4,16 @@ go 1.22.0
|
||||
|
||||
toolchain go1.22.2
|
||||
|
||||
replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared
|
||||
//replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared
|
||||
|
||||
require (
|
||||
github.com/docker/docker v26.1.0+incompatible
|
||||
github.com/docker/go-connections v0.5.0
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
github.com/shuffle/shuffle-shared v0.6.37
|
||||
k8s.io/api v0.30.0
|
||||
k8s.io/apimachinery v0.30.0
|
||||
k8s.io/client-go v0.30.0
|
||||
github.com/shuffle/shuffle-shared v0.6.50
|
||||
k8s.io/api v0.30.2
|
||||
k8s.io/apimachinery v0.30.2
|
||||
k8s.io/client-go v0.30.2
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -31,7 +31,7 @@ require (
|
||||
github.com/algolia/algoliasearch-client-go/v3 v3.18.1 // indirect
|
||||
github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 // indirect
|
||||
github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect
|
||||
github.com/cloudflare/circl v1.3.3 // indirect
|
||||
github.com/cloudflare/circl v1.3.7 // indirect
|
||||
github.com/containerd/log v0.1.0 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
@@ -41,7 +41,7 @@ require (
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/frikky/kin-openapi v0.41.0 // indirect
|
||||
github.com/frikky/schemaless v0.0.11 // indirect
|
||||
github.com/frikky/schemaless v0.0.13 // indirect
|
||||
github.com/ghodss/yaml v1.0.0 // indirect
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.5.0 // indirect
|
||||
@@ -82,6 +82,8 @@ require (
|
||||
github.com/pjbgf/sha1cd v0.3.0 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/sashabaranov/go-openai v1.19.2 // indirect
|
||||
github.com/sendgrid/rest v2.6.9+incompatible // indirect
|
||||
github.com/sendgrid/sendgrid-go v3.14.0+incompatible // indirect
|
||||
github.com/sergi/go-diff v1.1.0 // indirect
|
||||
github.com/skeema/knownhosts v1.2.1 // indirect
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
|
||||
|
||||
@@ -97,6 +97,8 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=
|
||||
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
|
||||
github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
|
||||
github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ=
|
||||
@@ -139,6 +141,8 @@ github.com/frikky/schemaless v0.0.9 h1:RzNLPkJq5c4nlm5iLiTndFcbeQxdMGJIj266wSGt2
|
||||
github.com/frikky/schemaless v0.0.9/go.mod h1:mooDxY+D6weHjhKvjy3+IE9S7P4g4cpNnidkdRv/cHQ=
|
||||
github.com/frikky/schemaless v0.0.11 h1:c4r6CJX30XI+SoJdT9RlUd9qYSQlx6hvwGRtsypu+uM=
|
||||
github.com/frikky/schemaless v0.0.11/go.mod h1:mooDxY+D6weHjhKvjy3+IE9S7P4g4cpNnidkdRv/cHQ=
|
||||
github.com/frikky/schemaless v0.0.13 h1:ARiN9V7wr2VZXAr9JK5wvTbyPgpGrgeiL1VhR5MlgaQ=
|
||||
github.com/frikky/schemaless v0.0.13/go.mod h1:mooDxY+D6weHjhKvjy3+IE9S7P4g4cpNnidkdRv/cHQ=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
|
||||
@@ -385,6 +389,10 @@ github.com/sashabaranov/go-openai v1.19.2 h1:+dkuCADSnwXV02YVJkdphY8XD9AyHLUWwk6
|
||||
github.com/sashabaranov/go-openai v1.19.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
|
||||
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0=
|
||||
github.com/sendgrid/rest v2.6.9+incompatible/go.mod h1:kXX7q3jZtJXK5c5qK83bSGMdV6tsOE70KbHoqJls4lE=
|
||||
github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fcif2i7P7wzISv1sU6DUA=
|
||||
github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8=
|
||||
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
|
||||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/shuffle/shuffle-shared v0.6.16 h1:dQBDRmb2Wgl3pEuewqjDvN6v6nUKr+1EvGSEja9zG6s=
|
||||
@@ -395,6 +403,8 @@ github.com/shuffle/shuffle-shared v0.6.27 h1:q4qZD6bGZFIvZ5Y10unGr3N3rZ7OryWyvva
|
||||
github.com/shuffle/shuffle-shared v0.6.27/go.mod h1:rWkh1eWdIx7OqQzJ1+JzF3Hck1X/Ty1WkUtjLrp+CU4=
|
||||
github.com/shuffle/shuffle-shared v0.6.37 h1:IB8tJqubJmJwwpLYbXNMVWck5jgVO9SKcKo/BBu+wTc=
|
||||
github.com/shuffle/shuffle-shared v0.6.37/go.mod h1:rWkh1eWdIx7OqQzJ1+JzF3Hck1X/Ty1WkUtjLrp+CU4=
|
||||
github.com/shuffle/shuffle-shared v0.6.50 h1:MBeGAiBNkw9Eg+3YTJIlBOuskWntGvT0uefFUYOBhbY=
|
||||
github.com/shuffle/shuffle-shared v0.6.50/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
||||
@@ -908,10 +918,16 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
|
||||
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
k8s.io/api v0.30.0 h1:siWhRq7cNjy2iHssOB9SCGNCl2spiF1dO3dABqZ8niA=
|
||||
k8s.io/api v0.30.0/go.mod h1:OPlaYhoHs8EQ1ql0R/TsUgaRPhpKNxIMrKQfWUp8QSE=
|
||||
k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI=
|
||||
k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI=
|
||||
k8s.io/apimachinery v0.30.0 h1:qxVPsyDM5XS96NIh9Oj6LavoVFYff/Pon9cZeDIkHHA=
|
||||
k8s.io/apimachinery v0.30.0/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
|
||||
k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg=
|
||||
k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
|
||||
k8s.io/client-go v0.30.0 h1:sB1AGGlhY/o7KCyCEQ0bPWzYDL0pwOZO4vAtTSh/gJQ=
|
||||
k8s.io/client-go v0.30.0/go.mod h1:g7li5O5256qe6TYdAMyX/otJqMhIiGgTapdLchhmOaY=
|
||||
k8s.io/client-go v0.30.2 h1:sBIVJdojUNPDU/jObC+18tXWcTJVcwyqS9diGdWHk50=
|
||||
k8s.io/client-go v0.30.2/go.mod h1:JglKSWULm9xlJLx4KCkfLLQ7XwtlbflV6uFFSHTMgVs=
|
||||
k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw=
|
||||
k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
|
||||
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag=
|
||||
|
||||
+367
-173
@@ -50,6 +50,9 @@ import (
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
rbacv1 "k8s.io/api/rbac/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
|
||||
)
|
||||
|
||||
// Starts jobs in bulk, so this could be increased
|
||||
@@ -659,166 +662,209 @@ func handleBackendImageDownload(ctx context.Context, images string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func deployK8sWorker(image string, identifier string, env []string) error {
|
||||
env = append(env, fmt.Sprintf("IS_KUBERNETES=true"))
|
||||
env = append(env, fmt.Sprintf("KUBERNETES_NAMESPACE=%s", os.Getenv("KUBERNETES_NAMESPACE")))
|
||||
|
||||
if len(os.Getenv("KUBERNETES_SERVICE_HOST")) > 0 {
|
||||
env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_HOST=%s", os.Getenv("KUBERNETES_SERVICE_HOST")))
|
||||
}
|
||||
|
||||
if len(os.Getenv("SHUFFLE_MEMCACHED")) > 0 {
|
||||
env = append(env, fmt.Sprintf("SHUFFLE_MEMCACHED=%s", os.Getenv("SHUFFLE_MEMCACHED")))
|
||||
}
|
||||
|
||||
if len(os.Getenv("KUBERNETES_SERVICE_PORT")) > 0 {
|
||||
env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_PORT=%s", os.Getenv("KUBERNETES_SERVICE_PORT")))
|
||||
}
|
||||
|
||||
clientset, _, err := shuffle.GetKubernetesClient()
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Error getting kubernetes client:", err)
|
||||
return err
|
||||
}
|
||||
|
||||
//env = append(env, fmt.Sprintf("KUBERNETES_CONFIG=%s", config.String()))
|
||||
|
||||
// FIXME: When a service account is used, the account is also mounted in the pod
|
||||
// The volume mount location is:
|
||||
// /var/run/secrets/kubernetes.io/serviceaccount
|
||||
|
||||
// Look for if there is a default service account in use
|
||||
if len(os.Getenv("KUBERNETES_SERVICE_ACCOUNT")) > 0 {
|
||||
log.Printf("[DEBUG] Using Kubernetes service account %s", os.Getenv("KUBERNETES_SERVICE_ACCOUNT"))
|
||||
env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_ACCOUNT=%s", os.Getenv("KUBERNETES_SERVICE_ACCOUNT")))
|
||||
|
||||
// use k8s downward API to find it if we are in a pod
|
||||
}
|
||||
|
||||
|
||||
// Check if namespace exist as variable. If so, make it
|
||||
if len(os.Getenv("KUBERNETES_NAMESPACE")) > 0 && !namespacemade {
|
||||
kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE")
|
||||
|
||||
// Make the namespace
|
||||
namespace := &corev1.Namespace{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: os.Getenv("KUBERNETES_NAMESPACE"),
|
||||
},
|
||||
}
|
||||
|
||||
_, err := clientset.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
if !strings.Contains(strings.ToLower(fmt.Sprintf("%s", err)), "already exists") {
|
||||
log.Printf("[ERROR] Failed creating Kubernetes namespace: %s", err)
|
||||
} else {
|
||||
namespacemade = true
|
||||
}
|
||||
} else {
|
||||
namespacemade = true
|
||||
}
|
||||
}
|
||||
|
||||
env = append(env, fmt.Sprintf("BASE_URL=%s", baseUrl))
|
||||
env = append(env, fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", swarmConfig))
|
||||
|
||||
if len(kubernetesNamespace) == 0 {
|
||||
foundNamespace, err := shuffle.GetKubernetesNamespace()
|
||||
if err != nil {
|
||||
//log.Printf("[ERROR] Failed getting Kubernetes namespace: %s", err)
|
||||
}
|
||||
|
||||
if len(foundNamespace) > 0 {
|
||||
kubernetesNamespace = foundNamespace
|
||||
os.Setenv("KUBERNETES_NAMESPACE", kubernetesNamespace)
|
||||
}
|
||||
}
|
||||
|
||||
if len(kubernetesNamespace) == 0 {
|
||||
kubernetesNamespace = "default"
|
||||
}
|
||||
|
||||
kubernetesImage := os.Getenv("SHUFFLE_KUBERNETES_WORKER")
|
||||
if len(kubernetesImage) == 0 {
|
||||
kubernetesImage = image
|
||||
}
|
||||
log.Printf("[DEBUG] Using Kubernetes worker image '%s'", kubernetesImage)
|
||||
// image = "shuffle-worker:v1" //hard coded image name to test locally
|
||||
|
||||
envMap := make(map[string]string)
|
||||
for _, envStr := range env {
|
||||
parts := strings.SplitN(envStr, "=", 2)
|
||||
if len(parts) == 2 {
|
||||
envMap[parts[0]] = parts[1]
|
||||
}
|
||||
}
|
||||
|
||||
containerLabels := map[string]string{
|
||||
"container": "shuffle-worker",
|
||||
}
|
||||
|
||||
containerAttachment := corev1.Container{
|
||||
Name: identifier,
|
||||
Image: kubernetesImage,
|
||||
Env: buildEnvVars(envMap),
|
||||
|
||||
//ImagePullPolicy: "Never",
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
}
|
||||
|
||||
podname := shuffle.GetPodName()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
if len(podname) > 0 {
|
||||
_, err := shuffle.GetCurrentPodNetworkConfig(ctx, clientset, kubernetesNamespace, podname)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed getting current pod network: %s", err)
|
||||
} else {
|
||||
log.Printf("[DEBUG] Current pod found!")
|
||||
// currentPodStatus = k8s.io/api/core/v1.PodStatus
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// While testing:
|
||||
// kubectl delete pods --all --all-namespaces; kubectl delete services --all --all-namespaces
|
||||
pod := &corev1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: identifier,
|
||||
Labels: containerLabels,
|
||||
},
|
||||
Spec: corev1.PodSpec{
|
||||
RestartPolicy: "Never",
|
||||
// DNSPolicy: "Default",
|
||||
DNSPolicy: corev1.DNSClusterFirst,
|
||||
// NodeSelector: map[string]string{
|
||||
// "node": "master",
|
||||
// },
|
||||
Containers: []corev1.Container{
|
||||
containerAttachment,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Check if running on ARM or x86 to download the correct image
|
||||
|
||||
// Get current pod's network so we can make the pod in it
|
||||
|
||||
_, err = clientset.CoreV1().Pods(kubernetesNamespace).List(context.Background(), metav1.ListOptions{})
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed listing pods: %s", err)
|
||||
}
|
||||
|
||||
|
||||
createdPod, err := clientset.CoreV1().Pods(kubernetesNamespace).Create(context.Background(), pod, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
//log.Printf("[ERROR] Failed creating pod: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("[INFO] Created pod %q in namespace %q\n", createdPod.Name, createdPod.Namespace)
|
||||
|
||||
// kubectl expose pod shuffle-workers --type=LoadBalancer --port=33333
|
||||
service := &corev1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: identifier,
|
||||
},
|
||||
Spec: corev1.ServiceSpec{
|
||||
Selector: map[string]string{
|
||||
"container": "shuffle-worker",
|
||||
},
|
||||
Ports: []corev1.ServicePort{
|
||||
{
|
||||
Protocol: "TCP",
|
||||
Port: 33333,
|
||||
TargetPort: intstr.FromInt(33333),
|
||||
},
|
||||
},
|
||||
Type: corev1.ServiceTypeLoadBalancer,
|
||||
},
|
||||
}
|
||||
|
||||
_, err = clientset.CoreV1().Services(kubernetesNamespace).Create(context.TODO(), service, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed creating service: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
func deployWorker(image string, identifier string, env []string, executionRequest shuffle.ExecutionRequest) error {
|
||||
if len(os.Getenv("REGISTRY_URL")) > 0 && os.Getenv("REGISTRY_URL") != "" {
|
||||
env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL")))
|
||||
}
|
||||
|
||||
if isKubernetes == "true" {
|
||||
env = append(env, fmt.Sprintf("IS_KUBERNETES=true"))
|
||||
env = append(env, fmt.Sprintf("KUBERNETES_NAMESPACE=%s", os.Getenv("KUBERNETES_NAMESPACE")))
|
||||
// if isKubernetes == "true" {
|
||||
// err := deployK8sWorker(image, identifier, env, executionRequest)
|
||||
// if err != nil {
|
||||
// log.Printf("[ERROR] Failed deploying Kubernetes worker: %s", err)
|
||||
// }
|
||||
|
||||
if len(os.Getenv("KUBERNETES_SERVICE_HOST")) > 0 {
|
||||
env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_HOST=%s", os.Getenv("KUBERNETES_SERVICE_HOST")))
|
||||
}
|
||||
|
||||
if len(os.Getenv("KUBERNETES_SERVICE_PORT")) > 0 {
|
||||
env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_PORT=%s", os.Getenv("KUBERNETES_SERVICE_PORT")))
|
||||
}
|
||||
|
||||
clientset, _, err := shuffle.GetKubernetesClient()
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Error getting kubernetes client:", err)
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
//env = append(env, fmt.Sprintf("KUBERNETES_CONFIG=%s", config.String()))
|
||||
|
||||
// FIXME: When a service account is used, the account is also mounted in the pod
|
||||
// The volume mount location is:
|
||||
// /var/run/secrets/kubernetes.io/serviceaccount
|
||||
|
||||
// Look for if there is a default service account in use
|
||||
if len(os.Getenv("KUBERNETES_SERVICE_ACCOUNT")) > 0 {
|
||||
log.Printf("[DEBUG] Using Kubernetes service account %s", os.Getenv("KUBERNETES_SERVICE_ACCOUNT"))
|
||||
env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_ACCOUNT=%s", os.Getenv("KUBERNETES_SERVICE_ACCOUNT")))
|
||||
|
||||
// use k8s downward API to find it if we are in a pod
|
||||
}
|
||||
|
||||
|
||||
// Check if namespace exist as variable. If so, make it
|
||||
if len(os.Getenv("KUBERNETES_NAMESPACE")) > 0 && !namespacemade {
|
||||
kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE")
|
||||
|
||||
// Make the namespace
|
||||
namespace := &corev1.Namespace{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: os.Getenv("KUBERNETES_NAMESPACE"),
|
||||
},
|
||||
}
|
||||
|
||||
_, err := clientset.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
if !strings.Contains(strings.ToLower(fmt.Sprintf("%s", err)), "already exists") {
|
||||
log.Printf("[ERROR] Failed creating Kubernetes namespace: %s", err)
|
||||
} else {
|
||||
namespacemade = true
|
||||
}
|
||||
} else {
|
||||
namespacemade = true
|
||||
}
|
||||
}
|
||||
|
||||
if len(kubernetesNamespace) == 0 {
|
||||
foundNamespace, err := shuffle.GetKubernetesNamespace()
|
||||
if err != nil {
|
||||
//log.Printf("[ERROR] Failed getting Kubernetes namespace: %s", err)
|
||||
}
|
||||
|
||||
if len(foundNamespace) > 0 {
|
||||
kubernetesNamespace = foundNamespace
|
||||
os.Setenv("KUBERNETES_NAMESPACE", kubernetesNamespace)
|
||||
}
|
||||
}
|
||||
|
||||
if len(kubernetesNamespace) == 0 {
|
||||
kubernetesNamespace = "default"
|
||||
}
|
||||
|
||||
kubernetesImage := os.Getenv("SHUFFLE_KUBERNETES_WORKER")
|
||||
if len(kubernetesImage) == 0 {
|
||||
kubernetesImage = image
|
||||
}
|
||||
log.Printf("[DEBUG] Using Kubernetes worker image '%s'", kubernetesImage)
|
||||
// image = "shuffle-worker:v1" //hard coded image name to test locally
|
||||
|
||||
envMap := make(map[string]string)
|
||||
for _, envStr := range env {
|
||||
parts := strings.SplitN(envStr, "=", 2)
|
||||
if len(parts) == 2 {
|
||||
envMap[parts[0]] = parts[1]
|
||||
}
|
||||
}
|
||||
|
||||
containerLabels := map[string]string{
|
||||
"container": "shuffle-worker",
|
||||
}
|
||||
|
||||
containerAttachment := corev1.Container{
|
||||
Name: identifier,
|
||||
Image: kubernetesImage,
|
||||
Env: buildEnvVars(envMap),
|
||||
|
||||
//ImagePullPolicy: "Never",
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
}
|
||||
|
||||
podname := shuffle.GetPodName()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
if len(podname) > 0 {
|
||||
currentPodStatus, err := shuffle.GetCurrentPodNetworkConfig(ctx, clientset, kubernetesNamespace, podname)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed getting current pod network: %s", err)
|
||||
} else {
|
||||
log.Printf("[DEBUG] Current pod found!")
|
||||
// currentPodStatus = k8s.io/api/core/v1.PodStatus
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// While testing:
|
||||
// kubectl delete pods --all --all-namespaces; kubectl delete services --all --all-namespaces
|
||||
pod := &corev1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: identifier,
|
||||
Labels: containerLabels,
|
||||
},
|
||||
Spec: corev1.PodSpec{
|
||||
RestartPolicy: "Never",
|
||||
DNSPolicy: "Default",
|
||||
// NodeSelector: map[string]string{
|
||||
// "node": "master",
|
||||
// },
|
||||
Containers: []corev1.Container{
|
||||
containerAttachment,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Check if running on ARM or x86 to download the correct image
|
||||
|
||||
// Get current pod's network so we can make the pod in it
|
||||
|
||||
networks, err := clientset.CoreV1().Pods(kubernetesNamespace).List(context.Background(), metav1.ListOptions{})
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed listing pods: %s", err)
|
||||
}
|
||||
|
||||
|
||||
createdPod, err := clientset.CoreV1().Pods(kubernetesNamespace).Create(context.Background(), pod, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
//log.Printf("[ERROR] Failed creating pod: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("[INFO] Created pod %q in namespace %q\n", createdPod.Name, createdPod.Namespace)
|
||||
return nil
|
||||
}
|
||||
// return err
|
||||
// }
|
||||
|
||||
// Binds is the actual "-v" volume.
|
||||
// Max 20% CPU every second
|
||||
@@ -844,23 +890,25 @@ func deployWorker(image string, identifier string, env []string, executionReques
|
||||
}
|
||||
}
|
||||
|
||||
hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId))
|
||||
if strings.ToLower(cleanupEnv) != "false" {
|
||||
hostConfig.AutoRemove = true
|
||||
}
|
||||
|
||||
config := &container.Config{
|
||||
Image: image,
|
||||
Env: env,
|
||||
}
|
||||
|
||||
if isKubernetes != "true" {
|
||||
hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId))
|
||||
if strings.ToLower(cleanupEnv) != "false" {
|
||||
hostConfig.AutoRemove = true
|
||||
}
|
||||
}
|
||||
|
||||
//var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG")
|
||||
parsedUuid := uuid.NewV4()
|
||||
if swarmConfig == "run" || swarmConfig == "swarm" {
|
||||
if swarmConfig == "run" || swarmConfig == "swarm" || isKubernetes == "true" {
|
||||
// FIXME: Should we handle replies properly?
|
||||
// In certain cases, a workflow may e.g. be aborted already. If it's aborted, that returns
|
||||
// a 401 from the worker, which returns an error here
|
||||
go sendWorkerRequest(executionRequest)
|
||||
go sendWorkerRequest(executionRequest, image, env)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1401,6 +1449,115 @@ func main() {
|
||||
log.Printf("[INFO] Running inside k8s cluster")
|
||||
}
|
||||
|
||||
if isKubernetes == "true" {
|
||||
clientset, _, err := shuffle.GetKubernetesClient()
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Error getting kubernetes client: %s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
kubernetesNamespace := "default"
|
||||
|
||||
// Check if namespace exist as variable. If so, make it
|
||||
if len(os.Getenv("KUBERNETES_NAMESPACE")) > 0 && !namespacemade {
|
||||
kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE")
|
||||
}
|
||||
|
||||
// fix roles
|
||||
// check if "service-creator" role is assigned to the service account "default"
|
||||
roleBindingName := "service-creator-binding"
|
||||
serviceAccountName := "default"
|
||||
// Check if the RoleBinding exists
|
||||
roleBinding, err := clientset.RbacV1().RoleBindings(kubernetesNamespace).Get(context.TODO(), roleBindingName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
log.Printf("[WARNING] Failed to get RoleBinding %s: %s", roleBindingName, err)
|
||||
// create role and rolebinding
|
||||
role := &rbacv1.Role{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: roleBindingName,
|
||||
},
|
||||
Rules: []rbacv1.PolicyRule{
|
||||
{
|
||||
APIGroups: []string{""},
|
||||
Resources: []string{"services"},
|
||||
Verbs: []string{"create"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.TODO()
|
||||
|
||||
_, err := clientset.RbacV1().Roles(kubernetesNamespace).Create(ctx, role, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed to create Role %s: %s", roleBindingName, err)
|
||||
if !strings.Contains(fmt.Sprintf("%s", err), "already exists") {
|
||||
log.Printf("[INFO] role %s already exists", roleBindingName)
|
||||
}
|
||||
}
|
||||
|
||||
roleBinding := &rbacv1.RoleBinding{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: roleBindingName,
|
||||
},
|
||||
Subjects: []rbacv1.Subject{
|
||||
{
|
||||
Kind: "ServiceAccount",
|
||||
Name: serviceAccountName,
|
||||
Namespace: kubernetesNamespace,
|
||||
},
|
||||
},
|
||||
RoleRef: rbacv1.RoleRef{
|
||||
Kind: "Role",
|
||||
Name: roleBindingName,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
||||
_, err = clientset.RbacV1().RoleBindings(kubernetesNamespace).Create(ctx, roleBinding, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed to create RoleBinding %s: %s", roleBindingName, err)
|
||||
if !strings.Contains(fmt.Sprintf("%s", err), "already exists") {
|
||||
log.Printf("[INFO] rolebinding %s already exists", roleBindingName)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
log.Printf("[INFO] Created Role %s and RoleBinding %s", roleBindingName, roleBindingName)
|
||||
} else {
|
||||
log.Printf("[INFO] RoleBinding %s exists", roleBindingName)
|
||||
}
|
||||
|
||||
// Check if the RoleBinding is assigned to the service account
|
||||
var found bool
|
||||
for _, subject := range roleBinding.Subjects {
|
||||
if subject.Kind == "ServiceAccount" && subject.Name == serviceAccountName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
log.Printf("[WARNING] Service account %s is not assigned to RoleBinding %s\n", serviceAccountName, roleBindingName)
|
||||
// assign the service account to the rolebinding
|
||||
roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{
|
||||
Kind: "ServiceAccount",
|
||||
Name: serviceAccountName,
|
||||
Namespace: kubernetesNamespace,
|
||||
})
|
||||
|
||||
ctx := context.TODO()
|
||||
|
||||
_, err := clientset.RbacV1().RoleBindings(kubernetesNamespace).Update(ctx, roleBinding, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed to update RoleBinding %s: %s", roleBindingName, err)
|
||||
if !strings.Contains(fmt.Sprintf("%s", err), "already exists") {
|
||||
log.Printf("[INFO] rolebinding %s already exists", roleBindingName)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
startupDelay := os.Getenv("SHUFFLE_ORBORUS_STARTUP_DELAY")
|
||||
if len(startupDelay) > 0 {
|
||||
log.Printf("[DEBUG] Setting startup delay to %#v", startupDelay)
|
||||
@@ -1501,16 +1658,25 @@ func main() {
|
||||
workerImage = newWorkerImage
|
||||
}
|
||||
|
||||
if swarmConfig == "run" || swarmConfig == "swarm" {
|
||||
checkSwarmService(ctx)
|
||||
|
||||
log.Printf("[DEBUG] Cleaning up containers from previous run")
|
||||
cleanupExistingNodes(ctx)
|
||||
time.Sleep(time.Duration(5) * time.Second)
|
||||
if swarmConfig == "run" || swarmConfig == "swarm" || isKubernetes == "true" {
|
||||
if isKubernetes != "true" {
|
||||
checkSwarmService(ctx)
|
||||
log.Printf("[DEBUG] Cleaning up containers from previous run")
|
||||
cleanupExistingNodes(ctx)
|
||||
time.Sleep(time.Duration(5) * time.Second)
|
||||
}
|
||||
|
||||
log.Printf("[DEBUG] Deploying worker image %s to swarm", workerImage)
|
||||
deployServiceWorkers(workerImage)
|
||||
log.Printf("[DEBUG] Waiting 45 seconds to ensure workers are deployed. Run: \"docker service ls\" for more info")
|
||||
|
||||
runString := "Run: \"docker service ls\" for more info"
|
||||
|
||||
if isKubernetes != "true" {
|
||||
deployServiceWorkers(workerImage)
|
||||
} else {
|
||||
deployK8sWorker(workerImage, "shuffle-workers", []string{})
|
||||
runString = "Run: \"kubectl get pods\" for more info"
|
||||
}
|
||||
log.Printf("[DEBUG] Waiting 45 seconds to ensure workers are deployed. %s", runString)
|
||||
time.Sleep(time.Duration(45) * time.Second)
|
||||
|
||||
//deployServiceWorkers(workerImage)
|
||||
@@ -1520,7 +1686,12 @@ func main() {
|
||||
|
||||
client := shuffle.GetExternalClient(baseUrl)
|
||||
fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl)
|
||||
log.Printf("[INFO] Finished configuring docker environment. Connecting to %s", fullUrl)
|
||||
|
||||
if isKubernetes == "true" {
|
||||
log.Printf("[INFO] Finished configuring kubernetes environment. Connecting to %s", fullUrl)
|
||||
} else {
|
||||
log.Printf("[INFO] Finished configuring docker environment. Connecting to %s", fullUrl)
|
||||
}
|
||||
|
||||
forwardData := bytes.NewBuffer([]byte{})
|
||||
forwardMethod := "POST"
|
||||
@@ -2785,7 +2956,7 @@ func zombiecheck(ctx context.Context, workerTimeout int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest) error {
|
||||
func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, env []string) error {
|
||||
parsedRequest := shuffle.OrborusExecutionRequest{
|
||||
ExecutionId: workflowExecution.ExecutionId,
|
||||
Authorization: workflowExecution.Authorization,
|
||||
@@ -2818,6 +2989,15 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest) error {
|
||||
streamUrl = fmt.Sprintf("%s:33333/api/v1/execute", parsedBaseurl)
|
||||
}
|
||||
|
||||
identifier := "shuffle-workers"
|
||||
|
||||
if isKubernetes == "true" {
|
||||
if shuffle.IsRunningInCluster() {
|
||||
log.Printf("[INFO] Running in Kubernetes cluster")
|
||||
// try getting the k8s worker server url
|
||||
}
|
||||
}
|
||||
|
||||
if len(workerServerUrl) > 0 {
|
||||
streamUrl = fmt.Sprintf("%s:33333/api/v1/execute", workerServerUrl)
|
||||
}
|
||||
@@ -2842,7 +3022,12 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest) error {
|
||||
if len(newWorkerImage) > 0 {
|
||||
workerImage = newWorkerImage
|
||||
}
|
||||
deployServiceWorkers(workerImage)
|
||||
|
||||
if isKubernetes == "true" {
|
||||
deployK8sWorker(workerImage, identifier, env)
|
||||
} else {
|
||||
deployServiceWorkers(workerImage)
|
||||
}
|
||||
|
||||
time.Sleep(time.Duration(10) * time.Second)
|
||||
//err = sendWorkerRequest(executionRequest)
|
||||
@@ -2861,7 +3046,11 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest) error {
|
||||
workerImage = newWorkerImage
|
||||
}
|
||||
|
||||
deployServiceWorkers(workerImage)
|
||||
if isKubernetes == "true" {
|
||||
deployK8sWorker(workerImage, identifier, env)
|
||||
} else {
|
||||
deployServiceWorkers(workerImage)
|
||||
}
|
||||
|
||||
time.Sleep(time.Duration(10) * time.Second)
|
||||
//err = sendWorkerRequest(executionRequest)
|
||||
@@ -2890,6 +3079,11 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest) error {
|
||||
|
||||
_ = body
|
||||
|
||||
log.Printf("[DEBUG] Ran worker from request with execution ID: %s. Worker URL: %s. DEBUGGING:\ndocker service logs shuffle-workers 2>&1 -f | grep %s", workflowExecution.ExecutionId, streamUrl, workflowExecution.ExecutionId)
|
||||
debugCommand := fmt.Sprintf("docker service logs shuffle-workers 2>&1 -f | grep %s", workflowExecution.ExecutionId)
|
||||
if isKubernetes == "true" {
|
||||
debugCommand = fmt.Sprintf("kubectl logs -n %s %s | grep %s", kubernetesNamespace, identifier, workflowExecution.ExecutionId)
|
||||
}
|
||||
|
||||
log.Printf("[DEBUG] Ran worker from request with execution ID: %s. Worker URL: %s. DEBUGGING:\n%s", workflowExecution.ExecutionId, streamUrl, debugCommand)
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user