diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 664123ab..02239000 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -5,6 +5,7 @@ go 1.24.0 toolchain go1.24.3 //replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared + //replace github.com/frikky/schemaless => ../../../schemaless //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi @@ -25,8 +26,8 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.40 - github.com/shuffle/singul v0.0.19 + github.com/shuffle/shuffle-shared v0.9.51 + github.com/shuffle/singul v0.0.20 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 google.golang.org/grpc v1.72.2 @@ -74,7 +75,7 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frikky/schemaless v0.0.23 // indirect + github.com/frikky/schemaless v0.0.24 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect diff --git a/frontend/src/components/CacheView.jsx b/frontend/src/components/CacheView.jsx index 3c19da36..564af782 100644 --- a/frontend/src/components/CacheView.jsx +++ b/frontend/src/components/CacheView.jsx @@ -38,8 +38,8 @@ import { Avatar, } from "@mui/material"; -import { - DataGrid, +import { + DataGrid, GridColDef, } from '@mui/x-data-grid'; @@ -70,15 +70,15 @@ import { VisibilityOff as VisibilityOffIcon, Clear as ClearIcon, Add as AddIcon, - Rocket as RocketIcon, - Webhook as WebhookIcon, - Air as AirIcon, - RocketLaunch as RocketLaunchIcon, + Rocket as RocketIcon, + Webhook as WebhookIcon, + Air as AirIcon, + RocketLaunch as RocketLaunchIcon, Send as SendIcon, SmartToy as SmartToyIcon, Settings as SettingsIcon, FilterAlt as FilterAltIcon, - CompareArrows as CompareArrowsIcon, + CompareArrows as CompareArrowsIcon, } from "@mui/icons-material"; import { Context } from "../context/ContextApi.jsx"; @@ -104,7 +104,7 @@ const useStyles = makeStyles({ }, }); -// +// //const CacheView = (props) => { const CacheView = memo((props) => { @@ -130,7 +130,7 @@ const CacheView = memo((props) => { const [cursors, setCursors] = useState({ 0: "", }) - const [_, setUpdate] = useState(Math.random()) + const [_, setUpdate] = useState(Math.random()) const [selectedRows, setSelectedRows] = useState([]); // Direct category migration from ../components/Files.jsx const [selectAllChecked, setSelectAllChecked] = React.useState(false) @@ -170,7 +170,7 @@ const CacheView = memo((props) => { "key": "workflow_id", "value": "", }], - "icon": , + "icon": , "enabled": false, }, { @@ -203,7 +203,7 @@ const CacheView = memo((props) => { "key": "webhook_url", "value": "", }], - "icon": , + "icon": , "enabled": false, }, @@ -237,7 +237,7 @@ const CacheView = memo((props) => { const [categoryAutomations, setCategoryAutomations] = useState(defaultAutomation) const [categoryConfig, setCategoryConfig] = useState(undefined) - + const { themeMode, brandColor } = useContext(Context); const theme = getTheme(themeMode, brandColor); const classes = useStyles(); @@ -312,9 +312,9 @@ const CacheView = memo((props) => { useEffect(() => { getWorkflows() - getApps() + getApps() - var chosenCategory = selectedCategory + var chosenCategory = selectedCategory const urlParams = new URLSearchParams(window.location.search) const categoryParam = urlParams.get("category") if (categoryParam && categoryParam !== undefined && categoryParam !== "default" && categoryParam !== "") { @@ -335,7 +335,7 @@ const CacheView = memo((props) => { if (event.key === 'Escape'){ // not working for some reasons console.log('escape pressed') - setRenderTextBox(false); + setRenderTextBox(false); } } @@ -359,7 +359,7 @@ const CacheView = memo((props) => { if (page !== undefined && page !== null && page >= 0) { if (cursors[page-1] !== undefined && cursors[page-1] !== null && cursors[page-1] !== "") { - url += "&cursor=" + cursors[page-1] + url += "&cursor=" + cursors[page-1] } } @@ -405,8 +405,8 @@ const CacheView = memo((props) => { } // Especially important during first load - if (index < 2 && (category === "default" || category === "" || category === undefined)) { - // If it exists and isn't blank/default, load it + if (index < 2 && (category === "default" || category === "" || category === undefined)) { + // If it exists and isn't blank/default, load it const urlParams = new URLSearchParams(window.location.search); const categoryParam = urlParams.get("category"); if (categoryParam && categoryParam !== undefined && categoryParam !== "default" && categoryParam !== "") { @@ -414,7 +414,7 @@ const CacheView = memo((props) => { if (index === undefined || index === null) { index = 0 } - + listOrgCache(orgId, categoryParam, index+1, amount, page) } else { setSelectedCategory("default"); @@ -478,10 +478,10 @@ const CacheView = memo((props) => { } else { setCategoryAutomations(defaultAutomation) } - } + } } else { //toast.warn("Failed to load keys. Please try again or contact support@shuffler if this persists.") - + if (category !== undefined && category !== null && category !== "" && category !== "default") { toast.info(`No keys to load in category ${category}`) setSelectedCategory(category) @@ -495,6 +495,9 @@ const CacheView = memo((props) => { const deleteEntry = (orgId, key, itemCategory, refreshList) => { + + console.log(`Deleting entry for orgId: ${orgId}, key: ${key}, category: ${itemCategory}`); + const method = "POST" const url = `${globalUrl}/api/v1/orgs/${orgId}/delete_cache` var parsed = { @@ -534,8 +537,8 @@ const CacheView = memo((props) => { }; const editOrgCache = (orgId) => { - var entry = { - key: dataValue.key, + var entry = { + key: dataValue.key, value: value, category: selectedCategory, } @@ -587,10 +590,10 @@ const CacheView = memo((props) => { }; const addOrgCache = (orgId) => { - const cache = { - key: key, + const cache = { + key: key, value: value, - category: selectedCategory, + category: selectedCategory, } setCacheInput([cache]); @@ -709,7 +712,7 @@ const CacheView = memo((props) => { fullWidth={true} autoComplete="Key" placeholder="abc" - id="keyfield" + id="keyfield" margin="normal" variant="outlined" value={editCache ? dataValue.key : key} @@ -728,7 +731,7 @@ const CacheView = memo((props) => { autoFixJson(value) }} > - + @@ -758,7 +761,7 @@ const CacheView = memo((props) => { /> - {editCache ? + {editCache ?
Created: {timestamp(dataValue?.created)} @@ -841,7 +844,7 @@ const CacheView = memo((props) => { } }; - const changeDistribution = (id, selectedSubOrg) => { + const changeDistribution = (id, selectedSubOrg) => { editFileConfig(id, [...new Set(selectedSubOrg)], selectedCategory) } @@ -855,8 +858,8 @@ const CacheView = memo((props) => { category: category === undefined || category === "" || category === "default" ? "" : category, } - console.log("data: ", data); - + console.log("data: ", data); + const url = `${globalUrl}/api/v1/orgs/${orgId}/cache/config`; fetch(url, { @@ -989,7 +992,7 @@ const CacheView = memo((props) => { const originalIcon = automation.icon if (typeof automation.icon !== "string") { automation.icon = ""; - } + } }) const url = `${globalUrl}/api/v2/datastore/automate` @@ -999,7 +1002,7 @@ const CacheView = memo((props) => { } if (settings !== undefined && settings !== null && Object.keys(settings).length > 0) { - data["settings"] = settings + data["settings"] = settings } fetch(url, { @@ -1049,7 +1052,7 @@ const CacheView = memo((props) => { } } - const runSave = () => { + const runSave = () => { setUpdated(false) const newAutomations = [...categoryAutomations] @@ -1062,11 +1065,11 @@ const CacheView = memo((props) => { } return ( -
setHovered(true)} onMouseLeave={() => setHovered(false)} > @@ -1085,15 +1088,15 @@ const CacheView = memo((props) => { setUpdatedAutomation(updatedAutomation) setUpdated(true) - setUpdate(Math.random()) + setUpdate(Math.random()) }} /> -
{ @@ -1109,30 +1112,30 @@ const CacheView = memo((props) => { } }} > - {typeof automation?.icon === "string" && automation?.icon?.length > 0 ? + {typeof automation?.icon === "string" && automation?.icon?.length > 0 ? {automation.name} : automation.icon } - {automation.name}
- {automation?.disabled !== true ? + {automation?.disabled !== true ? - + : null}
@@ -1957,14 +1960,14 @@ const CacheView = memo((props) => { style={{textTransform: isSelectedDataStore ? 'capitalize':null, width : isSelectedDataStore ? 120 : null, height:isSelectedDataStore?40:null}} variant="contained" color="primary" - onClick={() =>{ + onClick={() =>{ setEditCache(false) setModalOpen(true) setValue("") }} > - Add Key + Add Key - + } {renderTextBox && { const foundValue = event.target.value.trim(); if(event.key === 'Enter' && foundValue?.length > 0){ setUpdateToThisCategory(event.target.value) - + listOrgCache(orgId, event.target.value, 0, pageSize, 0) setPage(0) } - + }} style={{ height: 35, @@ -2151,10 +2154,10 @@ const CacheView = memo((props) => { - + - + - {showAutomationMenu || showSettingsMenu ? + {showAutomationMenu || showSettingsMenu ? { @@ -2227,7 +2230,7 @@ const CacheView = memo((props) => { - {showSettingsMenu === true ? + {showSettingsMenu === true ?
@@ -2237,7 +2240,7 @@ const CacheView = memo((props) => {
- Timeout + Timeout You can set a timeout for the category. This will delete all keys in this category after the specified time. Timeout is in seconds and based on last EDITED time. @@ -2266,7 +2269,7 @@ const CacheView = memo((props) => { toast.info("Timeout must be a number. Setting to 0.") } else { timeoutValue = parseInt(e.target.value, 10) - if (timeoutValue < 60) { + if (timeoutValue < 60) { toast.info("Timeout must be between 60 seconds or more. Setting to 0.") } @@ -2283,7 +2286,7 @@ const CacheView = memo((props) => {
- {categoryConfig?.settings?.public === true ? "" : "NOT"} Public + {categoryConfig?.settings?.public === true ? "" : "NOT"} Public This will make the url for this category public. Metadata will be cleared, except for timestamps. Types: keys,ndjson,csv,values,json,meta @@ -2330,7 +2333,7 @@ const CacheView = memo((props) => { When - A key is edited + A key is edited @@ -2339,16 +2342,16 @@ const CacheView = memo((props) => { {categoryAutomations.map((automation, index) => { return ( - + automation={automation} + index={index} + /> ) })}
} - +
: null} @@ -2374,17 +2377,17 @@ const CacheView = memo((props) => { setSelectedRows(newSelection); }} keepNonExistentRowsSelected={false} - getRowId={(row) => `${row?.key}_${row?.category}`} + getRowId={(row) => row?.category ? `${row.key}_${row.category}` : row.key} autoHeight={true} sx={{ - marginTop: 1, + marginTop: 1, height: listCache.length*52+500, width: "100%", '.MuiTablePagination-selectLabel, .MuiTablePagination-select, .MuiTablePagination-selectIcon': { display: 'none', }, - marginBottom: 20, + marginBottom: 20, }} loading={cachedLoaded === false} @@ -2404,7 +2407,7 @@ const CacheView = memo((props) => { setCursors({ 0: "", - }) + }) }} @@ -2441,20 +2444,20 @@ const CacheView = memo((props) => { }} >
{page * pageSize + 1} - {Math.min((page + 1) * pageSize, totalAmount)} of {totalAmount} - + { var disabled = false if (item?.type === "page") { if (cursors[item.page-1] === undefined) { - disabled = true + disabled = true } } if (item?.type === "previous") { @@ -2470,7 +2473,7 @@ const CacheView = memo((props) => { } if (cachedLoaded === false) { - disabled = true + disabled = true } return ( @@ -2530,10 +2533,10 @@ const CacheView = memo((props) => { variant={"outlined"} color="secondary" > - Delete {selectedRows.length} Key{ selectedRows.length > 1 ? "s" : "" } diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index aebad855..36ff423a 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.30 + github.com/shuffle/shuffle-shared v0.9.51 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 ) @@ -58,7 +58,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/kin-openapi v0.42.0 // indirect - github.com/frikky/schemaless v0.0.20 // indirect + github.com/frikky/schemaless v0.0.24 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect @@ -93,8 +93,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/opensearch-project/opensearch-go v1.1.0 // indirect - github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect github.com/osteele/liquid v1.7.0 // indirect github.com/osteele/tuesday v1.0.3 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect @@ -105,6 +103,7 @@ require ( github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/sendgrid/sendgrid-go v3.16.1+incompatible // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/shuffle/opensearch-go/v4 v4.0.0 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 29bf0bc2..8f77fe56 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -70,20 +70,6 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/aws/aws-sdk-go v1.42.27/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= -github.com/aws/aws-sdk-go v1.44.263/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4= -github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf h1:TqhNAT4zKbTdLa62d2HDBFdvgSbIGB3eJE8HqhgiL9I= github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf/go.mod h1:r5xuitiExdLAJ09PR7vBVENGvp4ZuTBeWTGtxuX3K+c= github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 h1:/P9/RL0xgWE+ehnCUUN5h3RpG3dmoMCOONO1CCvq23Y= @@ -142,8 +128,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.20 h1:S/A2pQcRN9qa2RnufvxwCeM06trjG0JLTF3urt1tFQI= -github.com/frikky/schemaless v0.0.20/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= +github.com/frikky/schemaless v0.0.24 h1:1bNg96gm82yLbRlDS0haQH/96X3GlCAEuTCSasXIJZE= +github.com/frikky/schemaless v0.0.24/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -209,7 +195,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -246,8 +231,6 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -295,10 +278,6 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/opensearch-project/opensearch-go v1.1.0 h1:eG5sh3843bbU1itPRjA9QXbxcg8LaZ+DjEzQH9aLN3M= -github.com/opensearch-project/opensearch-go v1.1.0/go.mod h1:+6/XHCuTH+fwsMJikZEWsucZ4eZMma3zNSeLrTtVGbo= -github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ= -github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8= github.com/osteele/liquid v1.7.0 h1:VsbPSchE5D5S5scylAIvERET4dnCxsO6IDri2oSJ5Dk= github.com/osteele/liquid v1.7.0/go.mod h1:xU0Z2dn2hOQIEFEWNmeltOmCtfhtoW/2fCyiNQeNG+U= github.com/osteele/tuesday v1.0.3 h1:SrCmo6sWwSgnvs1bivmXLvD7Ko9+aJvvkmDjB5G4FTU= @@ -328,8 +307,10 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.9.30 h1:3CYvNyD7sTxdxoZjTVrtaDqFvSWQRKAFGaga6rPGf8A= -github.com/shuffle/shuffle-shared v0.9.30/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw= +github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU= +github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0= +github.com/shuffle/shuffle-shared v0.9.51 h1:jOlFr0cdhPkJxRxlFHddBSIucm62T8e2DVM2PGDkahY= +github.com/shuffle/shuffle-shared v0.9.51/go.mod h1:g03P/ZM8pUwBbdxW0l9A1siMMuemEQshgDpO9hF23eE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -354,10 +335,19 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/wI2L/jsondiff v0.7.0 h1:1lH1G37GhBPqCfp/lrs91rf/2j3DktX6qYAKZkLuCQQ= +github.com/wI2L/jsondiff v0.7.0/go.mod h1:KAEIojdQq66oJiHhDyQez2x+sRit0vIzC9KeK0yizxM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= @@ -452,9 +442,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= @@ -497,13 +485,11 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= @@ -515,7 +501,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index e2dfac44..1d9d8fed 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -115,8 +115,8 @@ var memcached = os.Getenv("SHUFFLE_MEMCACHED") var queuePerMinute = os.Getenv("SHUFFLE_EXECUTION_PER_MINIUTE") var queuePerMinuteInt int -// For it to download from Sigma? -var pipelineApikey = os.Getenv("SHUFFLE_PIPELINE_AUTH") +// Used to download file categories. Not required since 2.1.1 +var pipelineApikey = "" var pipelineUrl = os.Getenv("SHUFFLE_PIPELINE_URL") var executionIds = []string{} @@ -2182,6 +2182,18 @@ func main() { return } + if os.Getenv("SHUFFLE_PIPELINE_STANDALONE") == "true" { + log.Printf("[INFO] Allowing use of standalone pipeline (tenzir). URL: %s", pipelineUrl) + + //if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "false" { + // os.Setenv("SHUFFLE_SKIP_PIPELINES", "true") + //} + + //if os.Getenv("SHUFFLE_PIPELINE_ENABLED") == "true" { + // os.Setenv("SHUFFLE_PIPELINE_ENABLED", "false") + //} + } + // Block until a signal is received if shuffle.IsRunningInCluster() { log.Printf("[INFO] Running inside k8s cluster") @@ -3003,6 +3015,7 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { } func deployTenzirNode() error { + // Disabled all pipeline features if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "false" || os.Getenv("SHUFFLE_PIPELINE_ENABLED") == "true" { // return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") //log.Printf("[INFO] Pipelines are enabled by user") @@ -3010,8 +3023,13 @@ func deployTenzirNode() error { return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") } + // Specifically for standalone tenzir + if os.Getenv("SHUFFLE_PIPELINE_STANDALONE") == "true" { + return nil + } + if isKubernetes == "true" { - return errors.New("Kubernetes not implemented for Tenzir node") + return errors.New("Tenzir not implemented for k8s") } err := checkTenzirNode() @@ -3217,12 +3235,21 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri hostConfig.Mounts = []mount.Mount{} } + //networkingConfig := &network.NetworkingConfig{ + // EndpointsConfig: map[string]*network.EndpointSettings{ + // "tenzir-network": { + // IPAMConfig: &network.EndpointIPAMConfig{ + // IPv4Address: "192.168.102.100", + // }, + // }, + // }, + //} + networkingConfig := &network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ "tenzir-network": { - IPAMConfig: &network.EndpointIPAMConfig{ - IPv4Address: "192.168.102.100", - }, + IPAMConfig: nil, + Aliases: []string{"tenzir-node"}, }, }, } @@ -3277,7 +3304,7 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri log.Printf("[INFO] Successfully deployed Tenzir Node! Setting up default syslog listener on TCP/1514 AND UDP/1514") - command := `from "tcp://0.0.0.0:1514" { read_syslog } | import` + command := `load_tcp "0.0.0.0:1514" { read_syslog } | import` _, err = createPipeline(command, "default-syslog-tcp-514") if err != nil { log.Printf("[ERROR] Failed to create tcp syslog pipeline: %s", err)