From 7bbfda9e4fd99d1e667cb69b6aafe318f315f71a Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 27 Apr 2022 20:48:41 +0200 Subject: [PATCH 001/142] Minor changes for launch --- backend/app_sdk/app_base.py | 4 ++-- backend/app_sdk/build.sh | 2 +- backend/go-app/go.mod | 4 ++-- docker-compose.yml | 4 ++-- frontend/package.json | 3 ++- frontend/src/App.jsx | 2 +- functions/onprem/orborus/build.sh | 2 +- 7 files changed, 11 insertions(+), 10 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 3c42c42e..50c17656 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -2527,7 +2527,7 @@ class AppBase: check, sourcevalue, is_loop = parse_params(action, fullexecution, condition["source"], self) if check: continue - return False, {"success": False, "reason": "Failed condition (1): %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)} + return False, {"success": False, "reason": "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)} #sourcevalue = sourcevalue.encode("utf-8") sourcevalue = parse_wrapper_start(sourcevalue, self) @@ -2536,7 +2536,7 @@ class AppBase: check, destinationvalue, is_loop = parse_params(action, fullexecution, condition["destination"], self) if check: continue - return False, {"success": False, "reason": "Failed condition (2): %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)} + return False, {"success": False, "reason": "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)} #destinationvalue = destinationvalue.encode("utf-8") destinationvalue = parse_wrapper_start(destinationvalue, self) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 22416f8b..c0c768ec 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -2,7 +2,7 @@ ### DEFAULT NAME=shuffle-app_sdk -VERSION=0.9.70 +VERSION=1.0.0 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 25119c40..a2dfd189 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module main go 1.16 -replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi //replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch @@ -24,7 +24,7 @@ require ( github.com/h2non/filetype v1.1.3 github.com/nirasan/go-oauth-pkce-code-verifier v0.0.0-20170819232839-0fbfe93532da // indirect github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.2.29 + github.com/shuffle/shuffle-shared v0.2.38 go4.org v0.0.0-20201209231011-d4a079459e60 // indirect golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce google.golang.org/api v0.65.0 diff --git a/docker-compose.yml b/docker-compose.yml index 7353a12c..d177ea2e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - #build: ./frontend + build: ./frontend image: ghcr.io/frikky/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - build: ./backend + #build: ./backend image: ghcr.io/frikky/shuffle-backend:nightly container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} diff --git a/frontend/package.json b/frontend/package.json index e7ea8871..55e61c59 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "shuffler", "homepage": "https://shuffler.io", - "version": "0.9.61", + "version": "1.0.0", "private": true, "dependencies": { "@babel/core": "^7.15.8", @@ -54,6 +54,7 @@ "react-device-detect": "^1.9.10", "react-dom": "^16.14.0", "react-draggable": "^3.3.2", + "react-driftjs": "^1.2.2", "react-dropzone": "^10.1.10", "react-ga": "^2.7.0", "react-iframe": "^1.8.0", diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index e1b800a4..a4b5e132 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -310,7 +310,7 @@ const App = (message, props) => { Date: Mon, 2 May 2022 19:03:25 +0530 Subject: [PATCH 002/142] Update install-guide.md --- .github/install-guide.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/install-guide.md b/.github/install-guide.md index df1e5a87..45ddde3a 100644 --- a/.github/install-guide.md +++ b/.github/install-guide.md @@ -87,9 +87,13 @@ http://localhost:5001 - REST API - requires [>=go1.13](https://golang.org/dl/) ```bash export SHUFFLE_OPENSEARCH_URL="http://localhost:9200" export SHUFFLE_ELASTIC=true +export SHUFFLE_OPENSEARCH_USERNAME=admin +export SHUFFLE_OPENSEARCH_PASSWORD=admin +export SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY=true cd backend/go-app -go run *.go +go run main.go walkoff.go docker.go ``` +**WINDOWS USERS:** Follow [this guide](https://www.wikihow.com/Create-an-Environment-Variable-in-Windows-10) to add environment variables in your machine. Large portions of the backend is written in another repository - [shuffle-shared](https://github.com/frikky/shuffle-shared). If you want to update any of this code and test in realtime, we recommend following these steps: 1. Clone shuffle-shared to a local repository @@ -101,8 +105,6 @@ Large portions of the backend is written in another repository - [shuffle-shared 4. Make the changes you want, then restart the backend server! 5. With your changes made, make a pull request :fire: -**WINDOWS USERS:** You'll have to to add the "export" part as an environment variable. - ## Database - Opensearch Make sure this is running through the docker-compose, and that the backend points to it with SHUFFLE_OPENSEARCH_URL defined @@ -122,6 +124,4 @@ export BASE_URL=http://YOUR-IP:5001 export DOCKER_API_VERSION=1.40 ``` -**WINDOWS USERS:** You'll have to to add the "export" part as an environment variable. - AND THAT's it - hopefully it worked. If it didn't please email [frikky@shuffler.io](mailto:frikky@shuffler.io) From c50cb2ce4d8c656a2662ffbd813fe82b57eb2c03 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 10 May 2022 21:02:44 +0200 Subject: [PATCH 003/142] Minor fixes for frontend --- backend/go-app/go.mod | 2 +- backend/go-app/walkoff.go | 12 +- docker-compose.yml | 2 +- frontend/src/App.jsx | 1 - frontend/src/components/ParsedAction.jsx | 4 +- frontend/src/components/ShuffleCodeEditor.jsx | 70 ++- frontend/src/views/Admin.jsx | 512 +++++++++++++++++- frontend/src/views/AngularWorkflow.jsx | 2 +- frontend/src/views/Apps.jsx | 2 +- frontend/src/views/Docs.jsx | 2 +- 10 files changed, 591 insertions(+), 18 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index a2dfd189..287967fb 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module main go 1.16 -//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi //replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index ffdf9b4f..c83fb664 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1048,6 +1048,10 @@ func cloudExecuteAction(execution shuffle.WorkflowExecution) error { return nil } +// 1. Check CORS +// 2. Check authentication +// 3. Check authorization +// 4. Run the actual function func executeWorkflow(resp http.ResponseWriter, request *http.Request) { cors := shuffle.HandleCors(resp, request) if cors { @@ -1104,8 +1108,8 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { executionAuthValid, newOrgId = shuffle.RunExecuteAccessValidation(request, workflow) if !executionAuthValid { - log.Printf("[INFO] Api authentication failed in execute workflow: %s", userErr) - resp.WriteHeader(401) + log.Printf("[INFO] Api authorization failed in execute workflow: %s", userErr) + resp.WriteHeader(403) resp.Write([]byte(`{"success": false}`)) return } else { @@ -1122,7 +1126,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { log.Printf("[AUDIT] Letting user %s execute %s because they're admin of the same org", user.Username, workflow.ID) } else { log.Printf("[AUDIT] Wrong user (%s) for workflow %s (execute)", user.Username, workflow.ID) - resp.WriteHeader(401) + resp.WriteHeader(403) resp.Write([]byte(`{"success": false}`)) return } @@ -2606,7 +2610,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("[INFO] Execution: %s should execute onprem with execution environment \"%s\". Workflow: %s", workflowExecution.ExecutionId, environment, workflowExecution.Workflow.ID) + log.Printf("[INFO] Execution (single action): %s should execute onprem with execution environment \"%s\". Workflow: %s", workflowExecution.ExecutionId, environment, workflowExecution.Workflow.ID) executionRequest := shuffle.ExecutionRequest{ ExecutionId: workflowExecution.ExecutionId, diff --git a/docker-compose.yml b/docker-compose.yml index fd2973f3..3f818755 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - build: ./frontend + #build: ./frontend image: ghcr.io/frikky/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index a4b5e132..e403f7c5 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -319,7 +319,6 @@ const App = (message, props) => { }, ]} /> - }
{ isCloud, lastSaved, setLastSaved, + //expansionModalOpen, + //setExpansionModalOpen, } = props; //const theme = useTheme(); @@ -1395,7 +1397,7 @@ const ParsedAction = (props) => { }} /> - + { diff --git a/frontend/src/components/ShuffleCodeEditor.jsx b/frontend/src/components/ShuffleCodeEditor.jsx index 5afb4bd3..41456960 100644 --- a/frontend/src/components/ShuffleCodeEditor.jsx +++ b/frontend/src/components/ShuffleCodeEditor.jsx @@ -7,7 +7,10 @@ import { DialogTitle, DialogContent, Typography, - Paper + Paper, + Menu, + MenuItem, + Button, } from '@material-ui/core'; import Checkbox from '@mui/material/Checkbox'; @@ -34,11 +37,16 @@ import 'codemirror/theme/gruvbox-dark.css'; import 'codemirror/theme/duotone-light.css'; import { padding, textAlign } from '@mui/system'; +const liquidFilters = [ + {"name": "Size", "value": "size", "example": ""}, + {"name": "Date", "value": "date", "example": `{{ "now" | date: "%s" }}`}, +] + const CodeEditor = (props) => { const { fieldCount, setFieldCount, actionlist, changeActionParameterCodeMirror, expansionModalOpen, setExpansionModalOpen, codedata, setcodedata } = props const [localcodedata, setlocalcodedata] = React.useState(codedata === undefined || codedata === null || codedata.length === 0 ? "" : codedata); // const {codelang, setcodelang} = props - const theme = useTheme(); + const theme = useTheme(); const [validation, setValidation] = React.useState(false); const [expOutput, setExpOutput] = React.useState(" "); const [linewrap, setlinewrap] = React.useState(true); @@ -51,6 +59,9 @@ const CodeEditor = (props) => { const [variableOccurences, setVariableOccurences] = React.useState([]); const [currentLocation, setCurrentLocation] = React.useState([]); const [currentVariable, setCurrentVariable] = React.useState(""); + const [anchorEl, setAnchorEl] = React.useState(null); + const liquidOpen = Boolean(anchorEl); + // useEffect(() => { // console.log(currentLocation) // }, [currentLocation]) @@ -340,11 +351,23 @@ const CodeEditor = (props) => { } } + + const handleClick = (item) => { + if (item === undefined || item.value === undefined) { + return + } + + setlocalcodedata(localcodedata+" "+item.value) + setAnchorEl(null) + } + return ( { console.log("In closer") @@ -352,7 +375,6 @@ const CodeEditor = (props) => { //setExpansionModalOpen(false) }} PaperComponent={PaperComponent} - aria-labelledby="draggable-dialog-title" PaperProps={{ style: { backgroundColor: theme.palette.surfaceColor, @@ -400,10 +422,46 @@ const CodeEditor = (props) => { - + + + { + setAnchorEl(null); + }} + MenuListProps={{ + 'aria-labelledby': 'basic-button', + }} + > + {liquidFilters.map((item, index) => { + return ( + { + handleClick(item) + }}>{item.name} + ) + })} + { const [loading, setLoading] = React.useState(false); const [selectedOrganization, setSelectedOrganization] = React.useState({}); + const [selectedDealModalOpen, setSelectedDealModalOpen] = React.useState(false); //console.log("Selected: ", selectedOrganization) const [organizationFeatures, setOrganizationFeatures] = React.useState({}); const [loginInfo, setLoginInfo] = React.useState(""); @@ -140,6 +147,17 @@ const Admin = (props) => { const [secret2FA, setSecret2FA] = React.useState(""); const [show2faSetup, setShow2faSetup] = useState(false); + const [dealName, setDealName] = React.useState(""); + const [dealAddress, setDealAddress] = React.useState(""); + const [dealType, setDealType] = React.useState("MSSP"); + const [dealCountry, setDealCountry] = React.useState("United States"); + const [dealCurrency, setDealCurrency] = React.useState("USD"); + const [dealStatus, setDealStatus] = React.useState("initiated"); + const [dealValue, setDealValue] = React.useState(""); + const [dealDiscount, setDealDiscount] = React.useState(""); + const [dealerror, setDealerror] = React.useState(""); + const [dealList, setDealList] = React.useState([]); + useEffect(() => { if (isDropzone) { //redirectOpenApi(); @@ -648,6 +666,45 @@ const Admin = (props) => { }); }; + const handleGetDeals = (orgId) => { + console.log("Get deals!") + + if (orgId.length === 0) { + alert.error( + "Organization ID not defined. Please contact us on https://shuffler.io if this persists logout." + ); + return; + } + + const url = `${globalUrl}/api/v1/orgs/${orgId}/deals` + fetch(url, { + method: "GET", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => { + if (response.status !== 200) { + console.log("Bad status code in get deals: ", response.status) + } + + return response.json(); + }) + .then((responseJson) => { + console.log("Got deals: ", responseJson) + if (responseJson.success === false) { + alert.error("Failed loading deals. Contact support if this persists") + } else { + setDealList(responseJson) + } + }) + .catch((error) => { + console.log("Error getting org deals: ", error); + alert.error("Failed getting deals for your org. Contact support if this persists."); + }); + } + const handleGetOrg = (orgId) => { if (orgId.length === 0) { alert.error( @@ -674,7 +731,7 @@ const Admin = (props) => { }) .then((responseJson) => { if (responseJson["success"] === false) { - alert.error("Failed getting org: ", responseJson.readon); + alert.error("Failed getting your org: ", responseJson.readon); } else { if ( responseJson.sync_features === undefined || @@ -682,6 +739,12 @@ const Admin = (props) => { ) { responseJson.sync_features = {}; } + + if (isCloud && responseJson.partner_info !== undefined && responseJson.partner_info.reseller === true) { + handleGetDeals(orgId) + } + + setSelectedOrganization(responseJson); var lists = { active: { @@ -1682,6 +1745,235 @@ const Admin = (props) => { : null + + const products = [ + { code: '', label: 'MSSP', phone: '' }, + { code: '', label: 'Enterprise', phone: '' }, + { code: '', label: 'Consultancy', phone: '' }, + { code: '', label: 'Support', phone: '' }, + ] + + const addDealModal = ( + { + setSelectedDealModalOpen(false); + + }} + PaperProps={{ + style: { + backgroundColor: theme.palette.surfaceColor, + color: "white", + minWidth: "800px", + minHeight: "320px", + }, + }} + > + + + Register new deal + + + +
+ { + setDealName(e.target.value) + }} + /> + { + setDealAddress(e.target.value) + }} + /> +
+
+ { + setDealValue(e.target.value) + }} + /> + option.label} + onChange={(event, newValue) => { + setDealCountry(newValue.label) + }} + renderOption={(props, option) => ( + img': { mr: 2, flexShrink: 0 } }} {...props}> + + {option.label} ({option.code}) +{option.phone} + + )} + renderInput={(params) => ( + + )} + /> + { + setDealType(newValue) + }} + getOptionLabel={(option) => option.label} + renderOption={(props, option) => ( + img': { mr: 2, flexShrink: 0 } }} {...props}> + {option.label} + + )} + renderInput={(params) => ( + + )} + /> +
+ {dealerror.length > 0 ? + + error registering: {dealerror} + + : null} +
+ + +
+ + + ); + const editUserModal = ( { ); + const submitDeal = (dealName, dealAddress, dealCountry, dealValue) => { + if (dealerror.length > 0) { + setDealerror("") + } + + const orgId = selectedOrganization.id; + const data = { + reseller_org: orgId, + name: dealName, + address: dealAddress, + country: dealCountry, + value: dealValue, + }; + + const url = `${globalUrl}/api/v1/orgs/${orgId}/deals` + fetch(url, { + mode: "cors", + method: "POST", + body: JSON.stringify(data), + credentials: "include", + crossDomain: true, + withCredentials: true, + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + }) + .then(function (response) { + if (response.status !== 200) { + console.log("Error in response"); + } + + return response.json(); + }) + .then(function (responseJson) { + if (responseJson.success === true) { + setSelectedDealModalOpen(false); + alert.success("Added new deal! We will be in touch shortly with an update."); + + setDealName("") + setDealAddress("") + setDealValue("") + setDealCountry("United States") + setDealType("MSSP") + } else { + setDealerror(responseJson.reason) + } + }) + .catch(function (error) { + //console.log("Error: ", error); + setDealerror(error.toString()) + alert.error("Failed adding deal reg: ", error) + }); + } + const cancelSubscriptions = (subscription_id) => { console.log(selectedOrganization); const orgId = selectedOrganization.id; @@ -2450,6 +2796,169 @@ const Admin = (props) => { backgroundColor: theme.palette.inputColor, }} /> + + {isCloud && selectedOrganization.partner_info !== undefined && selectedOrganization.partner_info.reseller === true ? +
+ + Reseller dashboard + + + + + + + + + + + + + + + + + + {dealList.length === 0 ? + + No deals registered yet. Click "Add deal" to register one + + : + dealList.map((deal, index) => { + var bgColor = "#27292d"; + if (index % 2 === 0) { + bgColor = "#1f2023"; + } + + return ( + + + + + + + + + + + + + ) + }) + } + + + +
+ : null} + + {isCloud && selectedOrganization.subscriptions !== undefined && selectedOrganization.subscriptions !== null && @@ -4291,6 +4800,7 @@ const Admin = (props) => { {modalView} {cloudSyncModal} {editUserModal} + {addDealModal} {editAuthenticationModal} {data} { variant="temporary" BackdropProps={{ style: { - backgroundColor: "transparent", + //backgroundColor: "transparent", } }} PaperProps={{ diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 5ce2fcbc..912acaa5 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -1511,7 +1511,7 @@ const Apps = (props) => { fullWidth color="primary" id="app_search_field" - placeholder={"Search apps"} + placeholder={"Search your apps"} onChange={(event) => { handleSearchChange(event.target.value); setCursearch(event.target.value); diff --git a/frontend/src/views/Docs.jsx b/frontend/src/views/Docs.jsx index 8dba1aa2..a58da57c 100644 --- a/frontend/src/views/Docs.jsx +++ b/frontend/src/views/Docs.jsx @@ -598,7 +598,7 @@ const Docs = (defaultprops) => { Documentation
- link="https://support.shuffler.io" /> + link="mailto:support@shuffler.io" /> link="https://discord.gg/B2CBzUm" />
From cb3d2fbe0624f8b474996be41b2fe6307c11b916 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 10 May 2022 21:51:50 +0200 Subject: [PATCH 004/142] Added docker socket proxy fix for swarm config in orborus --- docker-compose.yml | 227 ++++++++++++++-------------- functions/onprem/orborus/build.sh | 2 +- functions/onprem/orborus/go.mod | 2 +- functions/onprem/orborus/go.sum | 2 + functions/onprem/orborus/orborus.go | 38 ++++- 5 files changed, 153 insertions(+), 118 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3f818755..973e295b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,120 +1,121 @@ version: '3' services: - frontend: - #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:nightly - container_name: shuffle-frontend - hostname: shuffle-frontend - ports: - - "${FRONTEND_PORT}:80" - - "${FRONTEND_PORT_HTTPS}:443" - networks: - - shuffle - environment: - - BACKEND_HOSTNAME=${BACKEND_HOSTNAME} - restart: unless-stopped - depends_on: - - backend - backend: - #build: ./backend - image: ghcr.io/frikky/shuffle-backend:nightly - container_name: shuffle-backend - hostname: ${BACKEND_HOSTNAME} - # Here for debugging: - ports: - - "${BACKEND_PORT}:5001" - networks: - - shuffle - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z - - ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z - #- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate - env_file: .env - environment: - - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - - SHUFFLE_FILE_LOCATION=/shuffle-files - restart: unless-stopped - #depends_on: - #- opensearch #Not necessary because dependancy is handled within the backend itself instead - #- database - orborus: - image: ghcr.io/frikky/shuffle-orborus:latest - container_name: shuffle-orborus - hostname: shuffle-orborus - networks: - - shuffle - volumes: - - /var/run/docker.sock:/var/run/docker.sock - environment: - - SHUFFLE_WORKER_VERSION=latest - - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - - BASE_URL=http://${OUTER_HOSTNAME}:5001 - - DOCKER_API_VERSION=1.40 - - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} - - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} - - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} - - HTTP_PROXY=${HTTP_PROXY} - - HTTPS_PROXY=${HTTPS_PROXY} - - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} - - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY} - - SHUFFLE_SWARM_NETWORK_NAME=shuffle_swarm_executions - - SHUFFLE_SCALE_REPLICAS=1 - - SHUFFLE_SWARM_CONFIG=runn - restart: unless-stopped - security_opt: - - seccomp:unconfined - opensearch: - image: opensearchproject/opensearch:1.2.4 - hostname: shuffle-opensearch - container_name: shuffle-opensearch - environment: - - bootstrap.memory_lock=true - - "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - - cluster.routing.allocation.disk.threshold_enabled=false - - cluster.name=shuffle-cluster - - node.name=shuffle-opensearch - - discovery.seed_hosts=shuffle-opensearch - - cluster.initial_master_nodes=shuffle-opensearch - - node.store.allow_mmap=false - ulimits: - memlock: - soft: -1 - hard: -1 - nofile: - soft: 65536 - hard: 65536 - volumes: - - ${DB_LOCATION}:/usr/share/opensearch/data:z - ports: - - 9200:9200 - networks: - - shuffle - restart: unless-stopped - #docker-socket-proxy: - # image: tecnativa/docker-socket-proxy - # privileged: true - # environment: - # - SERVICES=1 - # - TASKS=1 - # - NETWORKS=1 - # - NODES=1 - # - BUILD=1 - # - IMAGES=1 - # - GRPC=1 - # - CONTAINERS=1 - # - PLUGINS=1 - # - SYSTEM=1 - # - VOLUMES=1 - # - INFO=1 - # - DISTRIBUTION=1 - # - POST=1 - # - AUTH=1 - # - SECRETS=1 - # volumes: - # - /var/run/docker.sock:/var/run/docker.sock + #frontend: + # #build: ./frontend + # image: ghcr.io/frikky/shuffle-frontend:nightly + # container_name: shuffle-frontend + # hostname: shuffle-frontend + # ports: + # - "${FRONTEND_PORT}:80" + # - "${FRONTEND_PORT_HTTPS}:443" # networks: # - shuffle + # environment: + # - BACKEND_HOSTNAME=${BACKEND_HOSTNAME} + # restart: unless-stopped + # depends_on: + # - backend + #backend: + # #build: ./backend + # image: ghcr.io/frikky/shuffle-backend:nightly + # container_name: shuffle-backend + # hostname: ${BACKEND_HOSTNAME} + # # Here for debugging: + # ports: + # - "${BACKEND_PORT}:5001" + # networks: + # - shuffle + # volumes: + # - /var/run/docker.sock:/var/run/docker.sock + # - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z + # - ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z + # #- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate + # env_file: .env + # environment: + # - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps + # - SHUFFLE_FILE_LOCATION=/shuffle-files + # restart: unless-stopped + # #depends_on: + # #- opensearch #Not necessary because dependancy is handled within the backend itself instead + # #- database + #orborus: + # image: ghcr.io/frikky/shuffle-orborus:latest + # container_name: shuffle-orborus + # hostname: shuffle-orborus + # networks: + # - shuffle + # volumes: + # - /var/run/docker.sock:/var/run/docker.sock + # environment: + # - SHUFFLE_WORKER_VERSION=latest + # - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} + # - BASE_URL=http://${OUTER_HOSTNAME}:5001 + # - DOCKER_API_VERSION=1.40 + # - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} + # - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} + # - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} + # - HTTP_PROXY=${HTTP_PROXY} + # - HTTPS_PROXY=${HTTPS_PROXY} + # - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} + # - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY} + # - SHUFFLE_SWARM_NETWORK_NAME=shuffle_swarm_executions + # - SHUFFLE_SCALE_REPLICAS=1 + # - SHUFFLE_SWARM_CONFIG=runn + # restart: unless-stopped + # security_opt: + # - seccomp:unconfined + #opensearch: + # image: opensearchproject/opensearch:1.2.4 + # hostname: shuffle-opensearch + # container_name: shuffle-opensearch + # environment: + # - bootstrap.memory_lock=true + # - "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM + # - cluster.routing.allocation.disk.threshold_enabled=false + # - cluster.name=shuffle-cluster + # - node.name=shuffle-opensearch + # - discovery.seed_hosts=shuffle-opensearch + # - cluster.initial_master_nodes=shuffle-opensearch + # - node.store.allow_mmap=false + # ulimits: + # memlock: + # soft: -1 + # hard: -1 + # nofile: + # soft: 65536 + # hard: 65536 + # volumes: + # - ${DB_LOCATION}:/usr/share/opensearch/data:z + # ports: + # - 9200:9200 + # networks: + # - shuffle + # restart: unless-stopped + docker-socket-proxy: + image: tecnativa/docker-socket-proxy + container_name: shuffle-frontend + privileged: true + environment: + - SERVICES=1 + - TASKS=1 + - NETWORKS=1 + - NODES=1 + - BUILD=1 + - IMAGES=1 + - GRPC=1 + - CONTAINERS=1 + - PLUGINS=1 + - SYSTEM=1 + - VOLUMES=1 + - INFO=1 + - DISTRIBUTION=1 + - POST=1 + - AUTH=1 + - SECRETS=1 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + networks: + - shuffle networks: shuffle: driver: bridge diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index fe360759..3256a1ed 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=1.0.0 +VERSION=1.0.1 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 3bce0939..3d8716d3 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -8,5 +8,5 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/mackerelio/go-osstat v0.2.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.2.27 + github.com/shuffle/shuffle-shared v0.2.41 ) diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 628ecf89..282522ce 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -786,6 +786,8 @@ github.com/shuffle/shuffle-shared v0.2.9 h1:fh2eOD7olifW2uyC3Vlp8u3dqhgIxtYaDVja github.com/shuffle/shuffle-shared v0.2.9/go.mod h1:YuMle0RjwXb3hxR5PdaOOD9e+hUyK34OABS0UbrT/Sk= github.com/shuffle/shuffle-shared v0.2.27 h1:YT9MtXyMSxIGMpNovjp9pCKFyt2gk40EdAXqDvldhM8= github.com/shuffle/shuffle-shared v0.2.27/go.mod h1:YuMle0RjwXb3hxR5PdaOOD9e+hUyK34OABS0UbrT/Sk= +github.com/shuffle/shuffle-shared v0.2.41 h1:1TBP/47Xzh7ysi6I++wJxRxKpZYp7NBe7YF1DrFL2mA= +github.com/shuffle/shuffle-shared v0.2.41/go.mod h1:YuMle0RjwXb3hxR5PdaOOD9e+hUyK34OABS0UbrT/Sk= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index b238a48e..f32972db 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -282,6 +282,35 @@ func deployServiceWorkers(image string) { } } + if len(os.Getenv("DOCKER_HOST")) > 0 { + log.Printf("[DEBUG] Deploying docker socket proxy to the network %s as the DOCKER_HOST variable is set", networkName) + if err == nil { + containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{ + All: true, + }) + + if err == nil { + for _, container := range containers { + if strings.Contains(strings.ToLower(container.Image), "docker-socket-proxy") { + networkConfig := &network.EndpointSettings{} + err := dockercli.NetworkConnect(ctx, networkName, container.ID, networkConfig) + if err != nil { + log.Printf("[ERROR] Failed connecting Docker socket proxy to docker network %s: %s", networkName, err) + } else { + log.Printf("[INFO] Attached the docker socket proxy to the execution network") + } + + break + } + } + } else { + log.Printf("[WARNING] Failed listing containers when deploying socket proxy on swarm: %s", err) + } + } else { + log.Printf("[WARNING] Failed listing and finding the right image for docker socket proxy: %s", err) + } + } + //serviceOptions := types.ServiceCreateOptions{} //service, err := dockercli.ServiceCreate( // context.Background(), @@ -385,13 +414,15 @@ func deployServiceWorkers(image string) { }, } - if defaultNetworkAttach == true { + if defaultNetworkAttach == true || strings.ToLower(os.Getenv("SHUFFLE_DEFAULT_NETWORK_ATTACH")) == "true" { + targetName := "shuffle_shuffle" + log.Printf("[DEBUG] Adding network attach for network %s to worker in swarm", targetName) serviceSpec.Networks = append(serviceSpec.Networks, swarm.NetworkAttachmentConfig{ - Target: "shuffle_shuffle", + Target: targetName, }) // FIXM: Remove this if deployment fails? - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_SWARM_OTHER_NETWORK=shuffle_shuffle")) + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_SWARM_OTHER_NETWORK=%s", targetName)) } if dockerApiVersion != "" { @@ -835,6 +866,7 @@ func main() { if len(os.Getenv("DOCKER_HOST")) > 0 { log.Printf("[DEBUG] Running docker with socket proxy %s instead of default", os.Getenv("DOCKER_HOST")) + } else { log.Printf("[DEBUG] Running docker with default socket /var/run/docker.sock") } From 76d74e32a350bada65f48e06b1793b8ac4064c6f Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 12 May 2022 17:11:43 +0200 Subject: [PATCH 005/142] Minor updates to app sdk --- backend/app_sdk/app_base.py | 13 +++++++++++++ backend/app_sdk/build.sh | 2 +- backend/app_sdk/requirements.txt | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 50c17656..c2ef9f90 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -17,6 +17,7 @@ import http.client import urllib.parse import jinja2 import datetime +import dateutil from io import StringIO as StringBuffer from io import BytesIO from liquid import Liquid, defaults @@ -104,6 +105,18 @@ def base64_decode(a): except: return base64.b64decode(a) +@shuffle_filters.register +def json_parse(a): + return json.loads(str(a)) + +@shuffle_filters.register +def as_object(a): + return json.loads(str(a)) + +@shuffle_filters.register +def ast(a): + return ast.literal_eval(str(a)) + #print(standard_filter_manager.filters) #print(shuffle_filters.filters) #print(Liquid("{{ '10' | plus: 1}}", filters=shuffle_filters.filters).render()) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index c0c768ec..c61a7766 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -2,7 +2,7 @@ ### DEFAULT NAME=shuffle-app_sdk -VERSION=1.0.0 +VERSION=1.0.1 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly diff --git a/backend/app_sdk/requirements.txt b/backend/app_sdk/requirements.txt index fe0c6789..93cccf66 100644 --- a/backend/app_sdk/requirements.txt +++ b/backend/app_sdk/requirements.txt @@ -5,3 +5,4 @@ liquidpy==0.7.3 flask[async]==2.0.2 waitress==2.1.0 #flask==1.1.2 +python-dateutil==2.8.1 From b2aa5a30600db78599be0aac81fb59b5ea60b9bf Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 15 May 2022 02:31:16 +0200 Subject: [PATCH 006/142] Added more embedded video & GIF features to workflow UI --- backend/go-app/main.go | 2 +- backend/tests/files.sh | 6 +- docker-compose.yml | 226 ++++++++++++------------- frontend/src/components/OrgHeader.jsx | 3 +- frontend/src/views/AngularWorkflow.jsx | 86 +++++++++- frontend/src/views/Docs.jsx | 3 +- frontend/src/views/Workflows.jsx | 6 +- 7 files changed, 207 insertions(+), 125 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 9447c1cb..3c9f86ed 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1265,7 +1265,7 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { } if len(org.SSOConfig.SSOEntrypoint) > 0 { - log.Printf("[DEBUG] Found SAML SSO url") + log.Printf("[DEBUG] Found SAML SSO url: %s", org.SSOConfig.SSOEntrypoint) baseSSOUrl = org.SSOConfig.SSOEntrypoint break } diff --git a/backend/tests/files.sh b/backend/tests/files.sh index 71776148..76ab54cf 100755 --- a/backend/tests/files.sh +++ b/backend/tests/files.sh @@ -3,7 +3,7 @@ #curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -d '{"filename": "file.txt", "org_id": "b199646b-16d2-456d-9fd6-b9972e929466", "workflow_id": "global"}' # #echo -curl http://localhost:5001/api/v1/apps/upload -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -F 'shuffle_file=@files.sh' +#curl http://localhost:5001/api/v1/apps/upload -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -F 'shuffle_file=@files.sh' # #curl http://localhost:5001/api/v1/files/1915981b-b897-4db1-8a2e-44bc34cead3b/content -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" #curl http://localhost:5001/api/v1/files/e19cffe4-e2da-47e9-809e-904f5cb03687 -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" @@ -16,7 +16,7 @@ curl http://localhost:5001/api/v1/apps/upload -H "Authorization: Bearer db0373c6 #r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS") -#curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" -d '{"filename": "rule2.yar", "org_id": "b4e88fe9-352b-47b4-b280-960181670acf", "workflow_id": "global", "namespace": "yara"}' -#curl http://localhost:5001/api/v1/files/5cb941ad-fa1c-4444-a685-92024b1fa31c/upload -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" -F 'shuffle_file=@upload.sh' +#curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer 09627dcb-7e2a-4843-819b-417d268ff840" -d '{"filename": "rule2.yar", "org_id": "11f67b76-6051-4425-b0d6-be23daac6d12", "workflow_id": "global", "namespace": "yara"}' +curl http://localhost:5001/api/v1/files/file_366ee8d2-1af6-4270-8639-213af30b4a29/upload -H "Authorization: Bearer 09627dcb-7e2a-4843-819b-417d268ff840" -F 'shuffle_file=@upload.sh' #curl http://localhost:5001/api/v1/files/namespaces/yara -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" --output rules.zip diff --git a/docker-compose.yml b/docker-compose.yml index 973e295b..27d8e095 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,121 +1,121 @@ version: '3' services: - #frontend: - # #build: ./frontend - # image: ghcr.io/frikky/shuffle-frontend:nightly - # container_name: shuffle-frontend - # hostname: shuffle-frontend - # ports: - # - "${FRONTEND_PORT}:80" - # - "${FRONTEND_PORT_HTTPS}:443" - # networks: - # - shuffle - # environment: - # - BACKEND_HOSTNAME=${BACKEND_HOSTNAME} - # restart: unless-stopped - # depends_on: - # - backend - #backend: - # #build: ./backend - # image: ghcr.io/frikky/shuffle-backend:nightly - # container_name: shuffle-backend - # hostname: ${BACKEND_HOSTNAME} - # # Here for debugging: - # ports: - # - "${BACKEND_PORT}:5001" - # networks: - # - shuffle - # volumes: - # - /var/run/docker.sock:/var/run/docker.sock - # - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z - # - ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z - # #- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate - # env_file: .env - # environment: - # - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - # - SHUFFLE_FILE_LOCATION=/shuffle-files - # restart: unless-stopped - # #depends_on: - # #- opensearch #Not necessary because dependancy is handled within the backend itself instead - # #- database - #orborus: - # image: ghcr.io/frikky/shuffle-orborus:latest - # container_name: shuffle-orborus - # hostname: shuffle-orborus - # networks: - # - shuffle - # volumes: - # - /var/run/docker.sock:/var/run/docker.sock - # environment: - # - SHUFFLE_WORKER_VERSION=latest - # - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - # - BASE_URL=http://${OUTER_HOSTNAME}:5001 - # - DOCKER_API_VERSION=1.40 - # - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} - # - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} - # - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} - # - HTTP_PROXY=${HTTP_PROXY} - # - HTTPS_PROXY=${HTTPS_PROXY} - # - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} - # - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY} - # - SHUFFLE_SWARM_NETWORK_NAME=shuffle_swarm_executions - # - SHUFFLE_SCALE_REPLICAS=1 - # - SHUFFLE_SWARM_CONFIG=runn - # restart: unless-stopped - # security_opt: - # - seccomp:unconfined - #opensearch: - # image: opensearchproject/opensearch:1.2.4 - # hostname: shuffle-opensearch - # container_name: shuffle-opensearch - # environment: - # - bootstrap.memory_lock=true - # - "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - # - cluster.routing.allocation.disk.threshold_enabled=false - # - cluster.name=shuffle-cluster - # - node.name=shuffle-opensearch - # - discovery.seed_hosts=shuffle-opensearch - # - cluster.initial_master_nodes=shuffle-opensearch - # - node.store.allow_mmap=false - # ulimits: - # memlock: - # soft: -1 - # hard: -1 - # nofile: - # soft: 65536 - # hard: 65536 - # volumes: - # - ${DB_LOCATION}:/usr/share/opensearch/data:z - # ports: - # - 9200:9200 - # networks: - # - shuffle - # restart: unless-stopped - docker-socket-proxy: - image: tecnativa/docker-socket-proxy + frontend: + #build: ./frontend + image: ghcr.io/frikky/shuffle-frontend:nightly container_name: shuffle-frontend - privileged: true - environment: - - SERVICES=1 - - TASKS=1 - - NETWORKS=1 - - NODES=1 - - BUILD=1 - - IMAGES=1 - - GRPC=1 - - CONTAINERS=1 - - PLUGINS=1 - - SYSTEM=1 - - VOLUMES=1 - - INFO=1 - - DISTRIBUTION=1 - - POST=1 - - AUTH=1 - - SECRETS=1 - volumes: - - /var/run/docker.sock:/var/run/docker.sock + hostname: shuffle-frontend + ports: + - "${FRONTEND_PORT}:80" + - "${FRONTEND_PORT_HTTPS}:443" networks: - shuffle + environment: + - BACKEND_HOSTNAME=${BACKEND_HOSTNAME} + restart: unless-stopped + depends_on: + - backend + backend: + #build: ./backend + image: ghcr.io/frikky/shuffle-backend:nightly + container_name: shuffle-backend + hostname: ${BACKEND_HOSTNAME} + # Here for debugging: + ports: + - "${BACKEND_PORT}:5001" + networks: + - shuffle + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z + - ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z + #- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate + env_file: .env + environment: + - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps + - SHUFFLE_FILE_LOCATION=/shuffle-files + restart: unless-stopped + #depends_on: + #- opensearch #Not necessary because dependancy is handled within the backend itself instead + #- database + orborus: + image: ghcr.io/frikky/shuffle-orborus:latest + container_name: shuffle-orborus + hostname: shuffle-orborus + networks: + - shuffle + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + - SHUFFLE_WORKER_VERSION=latest + - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} + - BASE_URL=http://${OUTER_HOSTNAME}:5001 + - DOCKER_API_VERSION=1.40 + - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} + - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} + - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} + - HTTP_PROXY=${HTTP_PROXY} + - HTTPS_PROXY=${HTTPS_PROXY} + - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} + - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY} + - SHUFFLE_SWARM_NETWORK_NAME=shuffle_swarm_executions + - SHUFFLE_SCALE_REPLICAS=1 + - SHUFFLE_SWARM_CONFIG=runn + restart: unless-stopped + security_opt: + - seccomp:unconfined + opensearch: + image: opensearchproject/opensearch:1.3.2 + hostname: shuffle-opensearch + container_name: shuffle-opensearch + environment: + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM + - cluster.routing.allocation.disk.threshold_enabled=false + - cluster.name=shuffle-cluster + - node.name=shuffle-opensearch + - discovery.seed_hosts=shuffle-opensearch + - cluster.initial_master_nodes=shuffle-opensearch + - node.store.allow_mmap=false + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - ${DB_LOCATION}:/usr/share/opensearch/data:z + ports: + - 9200:9200 + networks: + - shuffle + restart: unless-stopped + #docker-socket-proxy: + # image: tecnativa/docker-socket-proxy + # container_name: shuffle-frontend + # privileged: true + # environment: + # - SERVICES=1 + # - TASKS=1 + # - NETWORKS=1 + # - NODES=1 + # - BUILD=1 + # - IMAGES=1 + # - GRPC=1 + # - CONTAINERS=1 + # - PLUGINS=1 + # - SYSTEM=1 + # - VOLUMES=1 + # - INFO=1 + # - DISTRIBUTION=1 + # - POST=1 + # - AUTH=1 + # - SECRETS=1 + # volumes: + # - /var/run/docker.sock:/var/run/docker.sock + # networks: + # - shuffle networks: shuffle: driver: bridge diff --git a/frontend/src/components/OrgHeader.jsx b/frontend/src/components/OrgHeader.jsx index a370af93..922c4118 100644 --- a/frontend/src/components/OrgHeader.jsx +++ b/frontend/src/components/OrgHeader.jsx @@ -690,7 +690,7 @@ const OrgHeader = (props) => { } - {isCloud ? null : + {/*isCloud ? null : */} SAML SSO (v1.1) @@ -769,7 +769,6 @@ const OrgHeader = (props) => { - } {/* {expanded ? diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 2a195fe4..45a87c2f 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1650,7 +1650,7 @@ const AngularWorkflow = (defaultprops) => { } if (responseJson.public) { - alert.info("This workflow is public. Save the workflow to use it in your organization."); + //alert.info("This workflow is public. Save the workflow to use it in your organization."); console.log("RESP: ", responseJson) if (Object.getOwnPropertyNames(creatorProfile).length === 0) { @@ -11004,6 +11004,7 @@ const AngularWorkflow = (defaultprops) => { // + const leftView = workflow.public === true ?
{ This workflow is public and { saveWorkflow() - }}>must be saved to be used in your organization. + }}>must be saved or exported before use. {Object.getOwnPropertyNames(creatorProfile).length !== 0 && creatorProfile.github_avatar !== undefined && creatorProfile.github_avatar !== null ?
@@ -11055,6 +11056,27 @@ const AngularWorkflow = (defaultprops) => {
: null } + + {workflow.blogpost !== undefined && workflow.blogpost !== null && workflow.blogpost.length > 0 ? + + : null + } + {appGroup.length > 0 ?
@@ -11071,6 +11093,7 @@ const AngularWorkflow = (defaultprops) => {
: null} + {triggerGroup.length > 0 ?
@@ -11086,6 +11109,7 @@ const AngularWorkflow = (defaultprops) => {
: null} + {/*
Mitre Att&ck:  @@ -11094,6 +11118,8 @@ const AngularWorkflow = (defaultprops) => { TBD
+ */} + {/*
@@ -11104,16 +11130,70 @@ const AngularWorkflow = (defaultprops) => {
*/} + + {workflow.video !== undefined && workflow.video !== null && workflow.video.length > 0 ? +
+ + Video + + { + workflow.video.includes("loom.com/share") && workflow.video.split("/").length > 4 ? +
+ -
- : null} + + + + + +
Getting Started with Shuffle @@ -2292,8 +2289,34 @@ const GettingStarted = (props) => { console.log("Found tutorial for ", data.tutorial) tutorialFound = true } - } + if (tutorialFound === false) { + if (data.tutorial === "discover_workflows") { + if (workflows.length > 0) { + for (var key in workflows) { + const tmpworkflow = workflows[key] + if (tmpworkflow.published_id !== undefined && tmpworkflow.published_id !== null && tmpworkflow.published_id.length > 0) { + tutorialFound = true + break + } + } + } + } + + if (data.tutorial === "learn_shuffle") { + //tutorial: "discover_workflows", + if (workflows.length > 0) { + tutorialFound = true + } + } + + if (data.tutorial === "configure_organization") { + if (userdata.active_org.name !== userdata.username) { + tutorialFound = true + } + } + } + } return (
diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 0f9799e4..41f93fd6 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -907,48 +907,56 @@ const Workflows = (props) => { }) .then((responseJson) => { if (responseJson !== undefined) { - setWorkflows(responseJson); - fetchUsecases(responseJson) + var newarray = [] + for (var key in responseJson) { + const wf = responseJson[key] + if (wf.public === true) { + continue + } + + newarray.push(wf) + } + + setWorkflows(newarray); + fetchUsecases(newarray) var setProdFilter = false - if (responseJson !== undefined) { - var actionnamelist = []; - var parsedactionlist = []; - for (var key in responseJson) { - const workflow = responseJson[key] - if (workflow.status === "production") { - setProdFilter = true + var actionnamelist = []; + var parsedactionlist = []; + for (var key in newarray) { + const workflow = newarray[key] + if (workflow.status === "production") { + setProdFilter = true + } + + for (var actionkey in newarray[key].actions) { + const action = newarray[key].actions[actionkey]; + //console.log("Action: ", action) + if (actionnamelist.includes(action.app_name)) { + continue; } - for (var actionkey in responseJson[key].actions) { - const action = responseJson[key].actions[actionkey]; - //console.log("Action: ", action) - if (actionnamelist.includes(action.app_name)) { - continue; - } + actionnamelist.push(action.app_name); + parsedactionlist.push(action); + } + } - actionnamelist.push(action.app_name); - parsedactionlist.push(action); - } - } - - //console.log(parsedactionlist) - setActionImageList(parsedactionlist); - } + //console.log(parsedactionlist) + setActionImageList(parsedactionlist); if (setProdFilter === true) { setFilters(["status:production"]); - const newWorkflows = responseJson.filter(workflow => workflow.status === "production") + const newWorkflows = newarray.filter(workflow => workflow.status === "production") console.log(newWorkflows) if (newWorkflows !== undefined && newWorkflows !== null) { setFilteredWorkflows(newWorkflows); } else { - setFilteredWorkflows(responseJson); + setFilteredWorkflows(newarray); } } else { - setFilteredWorkflows(responseJson); + setFilteredWorkflows(newarray); } // Ensures the zooming happens only once per load @@ -2209,6 +2217,7 @@ const Workflows = (props) => { width: 330, renderCell: (params) => { const data = params.row.record; + return ( diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index c0e8bf03..34eb5afe 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -970,7 +970,7 @@ func main() { req.Header.Add("Org", org) } - log.Printf("[INFO] Waiting for executions at %s with Environment %s", fullUrl, environment) + log.Printf("[INFO] Waiting for executions at %s with Environment %#v", fullUrl, environment) hasStarted := false for { //go getStats() diff --git a/functions/onprem/orborus/run.sh b/functions/onprem/orborus/run.sh index eac60eec..d71ac59c 100644 --- a/functions/onprem/orborus/run.sh +++ b/functions/onprem/orborus/run.sh @@ -1,10 +1,19 @@ +#docker run \ +# --env DOCKER_API_VERSION=1.40 \ +# --env ENVIRONMENT_NAME="Shuffle" \ +# --env BASE_URL="http://192.168.86.45:5001" \ +# --env HTTP_PROXY="http://192.168.86.45:8082" \ +# --env HTTPS_PROXY="https://192.168.86.45:8082" \ +# --env SHUFFLE_PASS_WORKER_PROXY=true \ +# --env SHUFFLE_PASS_APP_PROXY=true \ +# -v /var/run/docker.sock:/var/run/docker.sock \ +# ghcr.io/frikky/shuffle-orborus:nightly + docker run \ --env DOCKER_API_VERSION=1.40 \ - --env ENVIRONMENT_NAME="Shuffle" \ - --env BASE_URL="http://192.168.86.45:5001" \ - --env HTTP_PROXY="http://192.168.86.45:8082" \ - --env HTTPS_PROXY="https://192.168.86.45:8082" \ - --env SHUFFLE_PASS_WORKER_PROXY=true \ - --env SHUFFLE_PASS_APP_PROXY=true \ + --env ENVIRONMENT_NAME="Another env" \ + --env ORG="2e7b6a08-b63b-4fc2-bd70-718091509db1" \ + --env AUTH="env auth" \ + --env BASE_URL="https://shuffler.io" \ -v /var/run/docker.sock:/var/run/docker.sock \ ghcr.io/frikky/shuffle-orborus:nightly From 1950c54ec8227dc02a9381cc586dc27adea4ea6e Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 28 Jul 2022 00:20:32 +0200 Subject: [PATCH 057/142] Fixed autocompleter in Code Editor to work with proper JSON examples --- backend/app_sdk/app_base.py | 16 ++++- frontend/src/components/ParsedAction.jsx | 2 +- frontend/src/components/ShuffleCodeEditor.jsx | 36 ++++++---- frontend/src/views/AngularWorkflow.jsx | 23 ++++++- frontend/src/views/AppCreator.jsx | 4 +- frontend/src/views/Apps.jsx | 66 ++++++++++++++++++- 6 files changed, 127 insertions(+), 20 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 4005bde9..7c57151a 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -371,7 +371,7 @@ class AppBase: # FIXME: Adding retries here. try: finished = False - for i in range (0, 5): + for i in range (0, 10): try: ret = requests.post(url, headers=headers, json=action_result, timeout=10) @@ -384,22 +384,32 @@ class AppBase: except requests.exceptions.RequestException as e: self.logger.info(f"[DEBUG] Request problem: {e}") + time.sleep(0.1) + #time.sleep(5) continue except TimeoutError as e: self.logger.info(f"[DEBUG] Timeout or request: {e}") + time.sleep(0.1) + #time.sleep(5) continue except requests.exceptions.ConnectionError as e: self.logger.info(f"[DEBUG] Connectionerror: {e}") + time.sleep(0.1) + #time.sleep(5) continue except http.client.RemoteDisconnected as e: self.logger.info(f"[DEBUG] Remote: {e}") + time.sleep(0.1) + #time.sleep(5) continue except urllib3.exceptions.ProtocolError as e: self.logger.info(f"[DEBUG] Protocol err: {e}") + time.sleep(0.1) + #time.sleep(5) continue @@ -408,8 +418,8 @@ class AppBase: if not finished: # Not sure why this would work tho :) action_result["status"] = "FAILURE" - action_result["result"] = json.dumps({"success": False, "reason": "POST error: Failed connecting to %s over 5 retries" % url}) - self.logger.info(f"[DEBUG] Before typeerror stream result - NOT finished after 5 requests") + action_result["result"] = json.dumps({"success": False, "reason": "POST error: Failed connecting to %s over 10 retries to the backend" % url}) + self.logger.info(f"[DEBUG] Before typeerror stream result - NOT finished after 10 requests") ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) self.logger.info(f"""[DEBUG] Successful request result request: Status= {ret.status_code} & Response= {ret.text}. Action status: {action_result["status"]}""") diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 712e7624..fbe6b94e 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -1410,7 +1410,7 @@ const ParsedAction = (props) => { if (currentItem.description === openApiFieldDesc) { currentItem.field_active = !hideBody; - console.log("Changing", currentItem); + //console.log("Changing", currentItem); } } }} diff --git a/frontend/src/components/ShuffleCodeEditor.jsx b/frontend/src/components/ShuffleCodeEditor.jsx index 09feeaad..20aaca45 100644 --- a/frontend/src/components/ShuffleCodeEditor.jsx +++ b/frontend/src/components/ShuffleCodeEditor.jsx @@ -16,7 +16,7 @@ import { import Checkbox from '@mui/material/Checkbox'; import { orange } from '@mui/material/colors'; import { isMobile } from "react-device-detect" -import { GetParsedPaths } from "../views/Apps.jsx"; +import { GetParsedPaths, FindJsonPath } from "../views/Apps.jsx"; import NestedMenuItem from "material-ui-nested-menu-item"; import { @@ -345,9 +345,9 @@ const CodeEditor = (props) => { //console.log(found) // Whelp this is inefficient af. Single loop pls - try{ - // When the found array is empty. - for (var i = 0; i < found.length; i++) { + // When the found array is empty. + for (var i = 0; i < found.length; i++) { + try { found[i] = found[i].toLowerCase() var valuefound = false @@ -379,13 +379,28 @@ const CodeEditor = (props) => { //} try { - input = input.replace(found[i], JSON.stringify(actionlist[k].example)) + var new_input = FindJsonPath(fullpath, actionlist[k].example) } catch (e) { - input = input.replace(found[i], actionlist[k].example) + console.log("ERR IN INPUT: ", e) } + if (typeof new_input === "object") { + new_input = JSON.stringify(new_input) + } else { + if (typeof new_input === "string") { + new_input = new_input + } else { + new_input = "" + } + } + + input = input.replace(found[i], new_input) + + //} catch (e) { + // input = input.replace(found[i], actionlist[k].example) + //} + shouldbreak = true - console.log("New input: ", input) break } } @@ -395,9 +410,9 @@ const CodeEditor = (props) => { } } } + } catch (e) { + console.log("Replace error: ", e) } - } catch (e) { - console.log("Replace error: ", e) } const tmpValidation = validateJson(input.valueOf()) @@ -823,7 +838,6 @@ const CodeEditor = (props) => { // const indentation_count = (pathdata.name.match(/\./g) || []).length+1 - const baseIndent =
//const boxPadding = pathdata.type === "object" ? "10px 0px 0px 0px" : 0 const boxPadding = 0 const namesplit = pathdata.name.split(".") @@ -854,7 +868,7 @@ const CodeEditor = (props) => {
{Array(indentation_count).fill().map((subdata, subindex) => { return ( - baseIndent +
) })} {icon} {newname} diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 2fddff38..bdc66c9a 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -2590,6 +2590,7 @@ const AngularWorkflow = (defaultprops) => { a.loop_versions.includes(curaction.app_version))) ); + if (curaction.template === true) { //newapps. const parsedname = curaction.name.replaceAll(" ", "_").toLowerCase() @@ -2630,6 +2631,12 @@ const AngularWorkflow = (defaultprops) => { setSelectedApp(tmpapp); setSelectedAction(curaction); } else { + console.log("CURAPP: ", curapp) + if (curapp.id !== curaction.id) { + curaction.app_id = curapp.id + //.valueOf() + } + setAuthenticationType( curapp.authentication.type === "oauth2" && curapp.authentication.redirect_uri !== undefined && @@ -2713,9 +2720,21 @@ const AngularWorkflow = (defaultprops) => { curaction.parameters[key].options[0]; } } - } + } else { + console.log("Should check APP if it has the same params as ACTION") + for (var key in curapp.actions) { + const tmpaction = curapp.actions[key] + if (tmpaction.name === curaction.name) { + console.log("Found action - needs change?", tmpaction) + if (tmpaction.parameters !== undefined && tmpaction.parameters !== null && tmpaction.parameters.length > 0) { + curaction.parameters = JSON.parse(JSON.stringify(tmpaction.parameters)) + } + break + } + } + } - console.log("ACTION: ", curaction) + console.log("ACTION CLICK: ", curaction) setSelectedApp(curapp); setSelectedAction(curaction); diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index c2fe3564..fa92adbf 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -3792,8 +3792,8 @@ const AppCreator = (defaultprops) => { } // Found that dashes in the URL doesn't work - parsedurl = parsedurl.replace("-", "_") - console.log("Actions: ", actions) + //parsedurl = parsedurl.replace("-", "_") + //console.log("Actions: ", actions) if (baseUrl.length === 0 && parsedurl.includes("http")) { const newurl = new URL(encodeURI(parsedurl)) diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index d8a6af47..cb93fd90 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -70,7 +70,72 @@ export const FixName = (name) => { return newAppname; }; +// Takes input of e.g. $node.data.#.asd and a matching value from a json blob +// Returns +export const FindJsonPath = (path, inputdata) => { + const splitkey = "."; + var parsedValues = []; + + if (inputdata === undefined || inputdata === null) { + return inputdata + } + + if (typeof inputdata !== "object") { + return inputdata + } + + var keysplit = path.split(splitkey) + if (path.startsWith("$") && keysplit.length > 1) { + keysplit = keysplit.slice(1,) + } + + if (keysplit.length === 0) { + return inputdata + } + + // FIXME: Check list - always getting FIRST item, not digging too deep. + // If object, send further + if (keysplit[0].includes("#")) { + if (Object.prototype.toString.call(inputdata) === '[object Array]') { + if (inputdata.length === 0) { + return "" + } else { + + // Fix the list + if (keysplit.length === 1) { + return inputdata[0] + } else { + const joinedsplit = keysplit.slice(1,).join(".") + return FindJsonPath(joinedsplit, inputdata[0]) + } + } + } else { + return "" + } + } + + for (const [key, value] of Object.entries(inputdata)) { + + if (key === keysplit[0]) { + + // Return if no more keys + // Else, dig deeper + if (keysplit.length === 1) { + return value + } else { + const joinedsplit = keysplit.slice(1,).join(".") + return FindJsonPath(joinedsplit, value) + } + } else { + //console.log("N: ", key) + } + } + + return inputdata +} + // Parses JSON data into keys that can be used everywhere :) +// Reverse of this is FindJsonPath export const GetParsedPaths = (inputdata, basekey) => { const splitkey = "."; var parsedValues = []; @@ -92,7 +157,6 @@ export const GetParsedPaths = (inputdata, basekey) => { // Handle direct loop! if (!isNaN(key) && basekey === "") { - console.log("Handling direct loop."); parsedValues.push({ type: "object", name: "Node", From 856daaacda4b09929e6f58a211e81914c7e3d10c Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 28 Jul 2022 02:19:05 +0200 Subject: [PATCH 058/142] Fixed minor frontend issues related to unparsable data --- backend/go-app/go.mod | 2 +- frontend/src/components/Header.js | 1 - frontend/src/components/ShuffleCodeEditor.jsx | 109 +++++++++--------- frontend/src/views/AngularWorkflow.jsx | 13 ++- 4 files changed, 67 insertions(+), 58 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 1e9f553a..a72efb22 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -24,7 +24,7 @@ require ( github.com/h2non/filetype v1.1.3 github.com/nirasan/go-oauth-pkce-code-verifier v0.0.0-20170819232839-0fbfe93532da // indirect github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.2.72 + github.com/shuffle/shuffle-shared v0.2.73 go4.org v0.0.0-20201209231011-d4a079459e60 // indirect golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce google.golang.org/api v0.65.0 diff --git a/frontend/src/components/Header.js b/frontend/src/components/Header.js index 337580ef..92f1d5a1 100644 --- a/frontend/src/components/Header.js +++ b/frontend/src/components/Header.js @@ -723,7 +723,6 @@ const Header = (props) => { userdata.orgs.splice(foundIndex+1, 1) } else { console.log("ORG NOT FOUND IN LIST: ", childorg) - } // This is stupid :) diff --git a/frontend/src/components/ShuffleCodeEditor.jsx b/frontend/src/components/ShuffleCodeEditor.jsx index 20aaca45..150cb8e2 100644 --- a/frontend/src/components/ShuffleCodeEditor.jsx +++ b/frontend/src/components/ShuffleCodeEditor.jsx @@ -342,77 +342,82 @@ const CodeEditor = (props) => { const expectedOutput = (input) => { const found = input.match(/[$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,}/g) + //console.log(found) // Whelp this is inefficient af. Single loop pls // When the found array is empty. - for (var i = 0; i < found.length; i++) { - try { - found[i] = found[i].toLowerCase() + try { + for (var i = 0; i < found.length; i++) { + try { + found[i] = found[i].toLowerCase() - var valuefound = false - for (var j = 0; j < actionlist.length; j++) { - if(found[i].slice(1,).toLowerCase() === actionlist[j].autocomplete.toLowerCase()){ - valuefound = true - try { - input = input.replace(found[i], JSON.stringify(actionlist[j].example)); - } catch (e) { - input = input.replace(found[i], actionlist[j].example) + var valuefound = false + for (var j = 0; j < actionlist.length; j++) { + if(found[i].slice(1,).toLowerCase() === actionlist[j].autocomplete.toLowerCase()){ + valuefound = true + try { + input = input.replace(found[i], JSON.stringify(actionlist[j].example)); + } catch (e) { + input = input.replace(found[i], actionlist[j].example) + } + } else { } - } else { } - } - if (!valuefound && availableVariables.includes(found[i])) { - var shouldbreak = false - for (var k=0; k < actionlist.length; k++){ - var parsedPaths = [] - if (typeof actionlist[k].example === "object") { - parsedPaths = GetParsedPaths(actionlist[k].example, ""); - } + if (!valuefound && availableVariables.includes(found[i])) { + var shouldbreak = false + for (var k=0; k < actionlist.length; k++){ + var parsedPaths = [] + if (typeof actionlist[k].example === "object") { + parsedPaths = GetParsedPaths(actionlist[k].example, ""); + } - for (var key in parsedPaths) { - const fullpath = "$"+actionlist[k].autocomplete.toLowerCase()+parsedPaths[key].autocomplete - if (fullpath === found[i]) { - //if (actionlist[k].example === undefined) { - // actionlist[k].example = "TMP" - //} + for (var key in parsedPaths) { + const fullpath = "$"+actionlist[k].autocomplete.toLowerCase()+parsedPaths[key].autocomplete + if (fullpath === found[i]) { + //if (actionlist[k].example === undefined) { + // actionlist[k].example = "TMP" + //} - try { - var new_input = FindJsonPath(fullpath, actionlist[k].example) - } catch (e) { - console.log("ERR IN INPUT: ", e) - } - - if (typeof new_input === "object") { - new_input = JSON.stringify(new_input) - } else { - if (typeof new_input === "string") { - new_input = new_input - } else { - new_input = "" + try { + var new_input = FindJsonPath(fullpath, actionlist[k].example) + } catch (e) { + console.log("ERR IN INPUT: ", e) } + + if (typeof new_input === "object") { + new_input = JSON.stringify(new_input) + } else { + if (typeof new_input === "string") { + new_input = new_input + } else { + new_input = "" + } + } + + input = input.replace(found[i], new_input) + + //} catch (e) { + // input = input.replace(found[i], actionlist[k].example) + //} + + shouldbreak = true + break } + } - input = input.replace(found[i], new_input) - - //} catch (e) { - // input = input.replace(found[i], actionlist[k].example) - //} - - shouldbreak = true + if (shouldbreak) { break } } - - if (shouldbreak) { - break - } } + } catch (e) { + console.log("Replace error: ", e) } - } catch (e) { - console.log("Replace error: ", e) } + } catch (e) { + } const tmpValidation = validateJson(input.valueOf()) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index bdc66c9a..124d78d5 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -414,9 +414,11 @@ const AngularWorkflow = (defaultprops) => { console.log("RESPONSE FOR APP: ", responseJson.reason); if (responseJson.reason !== undefined && responseJson.reason !== undefined && responseJson.reason.length > 0) { - selectedApp.documentation = responseJson.reason - setSelectedApp(selectedApp) - setUpdate(Math.random()) + if (!responseJson.reason.includes("404: Not Found") && responseJson.reason.length > 50) { + selectedApp.documentation = responseJson.reason + setSelectedApp(selectedApp) + setUpdate(Math.random()) + } } } @@ -6167,8 +6169,11 @@ const AngularWorkflow = (defaultprops) => { //newSelectedAction.parameters[newParamIndex].value = param.value } - // Not doing options fields + if (newSelectedAction.parameters === undefined || newSelectedAction.parameters === null) { + continue + } + // Not doing options fields const newParamIndex = newSelectedAction.parameters.findIndex(paramdata => paramdata.name === param.name) if (newParamIndex < 0) { continue From e445b759cb78e7901ef2e5247e03565e12864552 Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 28 Jul 2022 11:26:50 +0200 Subject: [PATCH 059/142] Changed to run tests on SSO_REDIRECT_URL after autobuild in gh actions --- .env | 1 + backend/go-app/go.mod | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 9c3004a7..936dc630 100644 --- a/.env +++ b/.env @@ -32,6 +32,7 @@ SHUFFLE_ENCRYPTION_MODIFIER= # Other configs BASE_URL=http://shuffle-backend:5001 +SSO_REDIRECT_URL=http://localhost:3001 BACKEND_HOSTNAME=shuffle-backend BACKEND_PORT=5001 FRONTEND_PORT=3001 diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index a72efb22..3e81b6eb 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -24,7 +24,7 @@ require ( github.com/h2non/filetype v1.1.3 github.com/nirasan/go-oauth-pkce-code-verifier v0.0.0-20170819232839-0fbfe93532da // indirect github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.2.73 + github.com/shuffle/shuffle-shared v0.2.74 go4.org v0.0.0-20201209231011-d4a079459e60 // indirect golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce google.golang.org/api v0.65.0 From 3e1d5b015d80e098a2eddfb4af5bcbbaeeb24f92 Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 28 Jul 2022 14:40:25 +0200 Subject: [PATCH 060/142] Added new components to replace old for Workflow Discovery --- .github/push_nightly.sh | 8 +- frontend/src/components/WorkflowGridNew.jsx | 291 ++++++++++++++++ frontend/src/components/WorkflowPaperNew.jsx | 332 +++++++++++++++++++ frontend/src/views/AngularWorkflow.jsx | 6 +- frontend/src/views/LoginPage.jsx | 2 +- 5 files changed, 631 insertions(+), 8 deletions(-) create mode 100644 frontend/src/components/WorkflowGridNew.jsx create mode 100644 frontend/src/components/WorkflowPaperNew.jsx diff --git a/.github/push_nightly.sh b/.github/push_nightly.sh index 0c397aa2..fc452d40 100644 --- a/.github/push_nightly.sh +++ b/.github/push_nightly.sh @@ -8,8 +8,8 @@ docker pull frikky/shuffle-frontend:nightly #docker push ghcr.io/frikky/shuffle-backend:nightly #docker push ghcr.io/frikky/shuffle-frontend:nightly -docker tag frikky/shuffle-backend:nightly ghcr.io/frikky/shuffle-backend:latest -docker tag frikky/shuffle-frontend:nightly ghcr.io/frikky/shuffle-frontend:latest +docker tag frikky/shuffle-backend:nightly ghcr.io/frikky/shuffle-backend:nightly +docker tag frikky/shuffle-frontend:nightly ghcr.io/frikky/shuffle-frontend:nightly -docker push ghcr.io/frikky/shuffle-backend:latest -docker push ghcr.io/frikky/shuffle-frontend:latest +docker push ghcr.io/frikky/shuffle-backend:nightly +docker push ghcr.io/frikky/shuffle-frontend:nightly diff --git a/frontend/src/components/WorkflowGridNew.jsx b/frontend/src/components/WorkflowGridNew.jsx new file mode 100644 index 00000000..f0a06e87 --- /dev/null +++ b/frontend/src/components/WorkflowGridNew.jsx @@ -0,0 +1,291 @@ +import React, { useState, useEffect, useLayoutEffect } from "react"; +import theme from '../theme'; + +import { + Chip, + Typography, + Paper, + Avatar, + Grid, + Tooltip, +} from "@material-ui/core"; + +import { + AvatarGroup, +} from "@mui/material" + +import { + Restore as RestoreIcon, + Edit as EditIcon, + BubbleChart as BubbleChartIcon, + MoreVert as MoreVertIcon, +} from '@material-ui/icons'; + +import { useNavigate, Link, useParams } from "react-router-dom"; + +const workflowActionStyle = { + display: "flex", + width: 160, + height: 44, + justifyContent: "space-between", +} + +const paperAppStyle = { + minHeight: 130, + maxHeight: 130, + overflow: "hidden", + width: "100%", + color: "white", + backgroundColor: theme.palette.surfaceColor, + padding: "12px 12px 0px 15px", + borderRadius: 5, + display: "flex", + boxSizing: "border-box", + position: "relative", +} + +const chipStyle = { + backgroundColor: "#3d3f43", + marginRight: 5, + paddingLeft: 5, + paddingRight: 5, + height: 28, + cursor: "pointer", + borderColor: "#3d3f43", + color: "white", +} + +const WorkflowPaper = (props) => { + const { data } = props; + let navigate = useNavigate(); + + const [open, setOpen] = React.useState(false); + const [anchorEl, setAnchorEl] = React.useState(null); + const appGroup = data.action_references === undefined || data.action_references === null ? [] : data.action_references + + //console.log("Workflow: ", data) + var boxColor = "#86c142"; + + var parsedName = data.name; + if ( + parsedName !== undefined && + parsedName !== null && + parsedName.length > 20 + ) { + parsedName = parsedName.slice(0, 21) + ".."; + } + + + const imageStyle = { + width: 24, + height: 24, + marginRight: 10, + border: "1px solid rgba(255,255,255,0.3)", + } + var image = data.creator_info !== undefined && data.creator_info !== null && data.creator_info.image !== undefined && data.creator_info.image !== null && data.creator_info.image.length > 0 ? : + const creatorname = data.creator_info !== undefined && data.creator_info !== null && data.creator_info.username !== undefined && data.creator_info.username !== null && data.creator_info.username.length > 0 ? data.creator_info.username : "" + var orgName = ""; + var orgId = ""; + if ((data.objectID === undefined || data.objectID === null) && data.id !== undefined && data.id !== null) { + data.objectID = data.id + } + + //console.log("IMG: ", data) + var parsedUrl = `/workflows/${data.objectID}` + if (data.__queryID !== undefined && data.__queryID !== null) { + parsedUrl += `?queryID=${data.__queryID}` + } + + return ( +
+ +
+ + + +
{ + if (data.creator_info !== undefined) { + navigate("/creators/"+data.creator_info.username) + } + }} + > + {image} +
+ + + + + {parsedName} + + + + + + {appGroup.length > 0 ? +
+ + {appGroup.map((app, index) => { + return ( +
{ + navigate("/apps/"+app.id) + }} + > + + + +
+ ) + })} + +
+ : + + + + + {data.actions === undefined || data.actions === null ? 1 : data.actions.length} + + + + } + + + + + {data.triggers === undefined || data.triggers === null ? 1 : data.triggers.length} + + + + + { + }} + > + + + + + {0} + + + + + + {data.tags !== undefined && data.tags !== null + ? data.tags.map((tag, index) => { + if (index >= 3) { + return null; + } + + return ( + + ); + }) + : null} + + + +
+ ) + } + +export default WorkflowPaper diff --git a/frontend/src/components/WorkflowPaperNew.jsx b/frontend/src/components/WorkflowPaperNew.jsx new file mode 100644 index 00000000..17fb67e5 --- /dev/null +++ b/frontend/src/components/WorkflowPaperNew.jsx @@ -0,0 +1,332 @@ +import React, { useState, useEffect, useLayoutEffect } from "react"; +import theme from '../theme'; + +import { + Chip, + Typography, + Paper, + Avatar, + Grid, + Tooltip, + Button, +} from "@material-ui/core"; + +import { + AvatarGroup, +} from "@mui/material" + +import { + Restore as RestoreIcon, + Edit as EditIcon, + BubbleChart as BubbleChartIcon, + MoreVert as MoreVertIcon, +} from '@material-ui/icons'; + +import { useNavigate, Link, useParams } from "react-router-dom"; + +const workflowActionStyle = { + display: "flex", + width: 160, + height: 44, + justifyContent: "space-between", +} + +const paperAppStyle = { + minHeight: 130, + maxHeight: 130, + overflow: "hidden", + width: "100%", + color: "white", + backgroundColor: theme.palette.surfaceColor, + padding: "12px 12px 0px 15px", + borderRadius: 5, + display: "flex", + boxSizing: "border-box", + position: "relative", +} + +const chipStyle = { + backgroundColor: "#3d3f43", + marginRight: 5, + paddingLeft: 5, + paddingRight: 5, + height: 28, + cursor: "pointer", + borderColor: "#3d3f43", + color: "white", +} + +const WorkflowPaper = (props) => { + const { data } = props; + let navigate = useNavigate(); + + const [open, setOpen] = React.useState(false); + const [anchorEl, setAnchorEl] = React.useState(null); + const appGroup = data.action_references === undefined || data.action_references === null ? [] : data.action_references + + const activateWorkflow = (workflow) => { + console.log("Should activate: ", workflow) + + } + + //console.log("Workflow: ", data) + var boxColor = "#86c142"; + + var parsedName = data.name; + if ( + parsedName !== undefined && + parsedName !== null && + parsedName.length > 35 + ) { + parsedName = parsedName.slice(0, 36) + ".."; + } + + const imageStyle = { + width: 28, + height: 28, + marginRight: 10, + border: "1px solid rgba(255,255,255,0.3)", + } + + var image = data.creator_info !== undefined && data.creator_info !== null && data.creator_info.image !== undefined && data.creator_info.image !== null && data.creator_info.image.length > 0 ? : + const creatorname = data.creator_info !== undefined && data.creator_info !== null && data.creator_info.username !== undefined && data.creator_info.username !== null && data.creator_info.username.length > 0 ? data.creator_info.username : "Shuffle" + var orgName = ""; + var orgId = ""; + if ((data.objectID === undefined || data.objectID === null) && data.id !== undefined && data.id !== null) { + data.objectID = data.id + } + + //console.log("IMG: ", data) + var parsedUrl = `/workflows/${data.objectID}` + if (data.__queryID !== undefined && data.__queryID !== null) { + parsedUrl += `?queryID=${data.__queryID}` + } + + const paperImgStyle = { + height: 150, + width: "100%", + backgroundImage: "linear-gradient(to right, #f86a3e, #f34079)", + color: "white", + position: "relative", + borderRadius: "10px 10px 0% 0%", + } + + const bgImage1 = "https://avatars.githubusercontent.com/u/5719530?v=4" + const bgImage2 = "https://avatars.githubusercontent.com/u/5719530?v=4" + const itemSize = 70 + + return ( +
+
+
+ Image alt +
+
+ Image alt +
+
+ +
+ + + +
{ + if (data.creator_info !== undefined) { + navigate("/creators/"+data.creator_info.username) + } + }} + > + {image} +
+ + + + + {parsedName} + + + + + + {/* + {appGroup.length > 0 ? +
+ + {appGroup.map((app, index) => { + return ( +
{ + navigate("/apps/"+app.id) + }} + > + + + +
+ ) + })} + +
+ : + + + + + {data.actions === undefined || data.actions === null ? 1 : data.actions.length} + + + + } + */} + {/* + + + + + {data.triggers === undefined || data.triggers === null ? 1 : data.triggers.length} + + + + + { + }} + > + + + + + {0} + + + + */} + + {/* + + {data.tags !== undefined && data.tags !== null + ? data.tags.map((tag, index) => { + if (index >= 3) { + return null; + } + + return ( + + ); + }) + : null} + + */} + + + + +
+ ) + } + +export default WorkflowPaper diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 124d78d5..843e1cf1 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -12043,7 +12043,7 @@ const AngularWorkflow = (defaultprops) => { : 0; const timestamp = new Date(data.started_at * 1000) - .toISOString() + .toLocaleString("en-GB") .split(".")[0] .split("T") .join(" "); @@ -12360,7 +12360,7 @@ const AngularWorkflow = (defaultprops) => { Started    - {new Date(executionData.started_at * 1000).toISOString()} + {new Date(executionData.started_at * 1000).toLocaleString("en-GB")}
) : null} @@ -12374,7 +12374,7 @@ const AngularWorkflow = (defaultprops) => { Finished   - {new Date(executionData.completed_at * 1000).toISOString()} + {new Date(executionData.completed_at * 1000).toLocaleString("en-GB")}
) : null} diff --git a/frontend/src/views/LoginPage.jsx b/frontend/src/views/LoginPage.jsx index 6c179df7..96329944 100644 --- a/frontend/src/views/LoginPage.jsx +++ b/frontend/src/views/LoginPage.jsx @@ -297,7 +297,7 @@ const LoginDialog = (props) => { {loginInfo === undefined || loginInfo === null || loginInfo.length === 0 ? null : ( -
Response: {loginInfo}
+
Database Response: {loginInfo}
)} From ac24a5c35a78c2a231da056f67d6bbdeea66865b Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 1 Aug 2022 01:08:11 +0200 Subject: [PATCH 061/142] Fixed autocompletion names to always match latest node name --- frontend/src/components/ParsedAction.jsx | 16 +++++-- frontend/src/views/AngularWorkflow.jsx | 55 ++++++++++++++++++++++++ frontend/src/views/GettingStarted.jsx | 2 +- frontend/src/views/Workflows.jsx | 2 +- 4 files changed, 70 insertions(+), 5 deletions(-) diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index fbe6b94e..4a746939 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -2665,7 +2665,7 @@ const ParsedAction = (props) => { selectedAction.app_name.substring(1) ).replaceAll("_", " ")} -
+
{ var foundResult = workflowExecutions[key].results.find( (result) => result.action.id === selectedAction.id - ); + ) + if (foundResult === undefined || foundResult === null) { continue; } - setSelectedResult(foundResult); + const oldstartnode = cy.getElementById(selectedAction.id); + console.log("FOUND NODe: ", oldstartnode) + if (oldstartnode !== undefined && oldstartnode !== null) { + const foundname = oldstartnode.data("label") + if (foundname !== undefined && foundname !== null) { + foundResult.action.label = foundname + } + } + setSelectedResult(foundResult); if (setCodeModalOpen !== undefined) { setCodeModalOpen(true); } + break; } } diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 843e1cf1..d825a5b9 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -11587,6 +11587,17 @@ const AngularWorkflow = (defaultprops) => { width: 30, }} onClick={() => { + if (validate.valid) { + //console.log("Find and change result: ", + //const oldstartnode = cy.getElementById(selectedResult.action.id); + //if (oldstartnode !== undefined && oldstartnode !== null) { + // const foundname = oldstartnode.data("label") + // if (foundname !== undefined && foundname !== null) { + // result.action.label = foundname + // } + //} + } + setSelectedResult({ "action": { "label": "Execution Argument", @@ -11597,6 +11608,7 @@ const AngularWorkflow = (defaultprops) => { "result": validate.valid ? JSON.stringify(validate.result) : validate.result, "status": "SUCCESS" }) + setCodeModalOpen(true); }} > @@ -11786,6 +11798,20 @@ const AngularWorkflow = (defaultprops) => { //var newitem = JSON.parse(base); var newitem = validateJson(base).result + // Check if base_node_name has changed + if (cy !== undefined && cy !== null) { + console.log("Change name?") + //const allNodes = cy.nodes().jsons(); + //for (var key in allNodes) { + //const currentNode = allNodes[key]; + + //if (currentNode. + //} + + //const nodedata = cy.getElementById(data.action.id).data(); + //base_node_name = + } + to_be_copied = "$" + base_node_name.toLowerCase().replaceAll(" ", "_"); for (var key in copy.namespace) { if (copy.namespace[key].includes("Results for")) { @@ -12661,6 +12687,17 @@ const AngularWorkflow = (defaultprops) => { width: 30, }} onClick={() => { + const oldstartnode = cy.getElementById(data.action.id); + console.log("FOUND NODe: ", oldstartnode) + if (oldstartnode !== undefined && oldstartnode !== null) { + const foundname = oldstartnode.data("label") + if (foundname !== undefined && foundname !== null) { + data.action.label = foundname + } + } + + console.log("Click data: ", data) + //data.action.label = "" setSelectedResult(data); setCodeModalOpen(true); }} @@ -12922,7 +12959,16 @@ const AngularWorkflow = (defaultprops) => { data.status === "SUCCESS" && data.action.id === selectedResult.action.id ); + if (result !== undefined) { + const oldstartnode = cy.getElementById(selectedResult.action.id); + if (oldstartnode !== undefined && oldstartnode !== null) { + const foundname = oldstartnode.data("label") + if (foundname !== undefined && foundname !== null) { + result.action.label = foundname + } + } + setSelectedResult(result); setUpdate(Math.random()); break; @@ -12956,7 +13002,16 @@ const AngularWorkflow = (defaultprops) => { data.status !== "SKIPPED" && data.status !== "WAITING" ); + if (result !== undefined) { + const oldstartnode = cy.getElementById(selectedResult.action.id); + if (oldstartnode !== undefined && oldstartnode !== null) { + const foundname = oldstartnode.data("label") + if (foundname !== undefined && foundname !== null) { + result.action.label = foundname + } + } + setSelectedResult(result); setUpdate(Math.random()); break; diff --git a/frontend/src/views/GettingStarted.jsx b/frontend/src/views/GettingStarted.jsx index b49f6004..392afce7 100644 --- a/frontend/src/views/GettingStarted.jsx +++ b/frontend/src/views/GettingStarted.jsx @@ -62,7 +62,7 @@ import NestedMenuItem from "material-ui-nested-menu-item"; //import {Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons'; //https://next.material-ui.com/components/material-icons/ -import { DataGrid, GridToolbar } from "@material-ui/data-grid"; +import { DataGrid, GridToolbar } from "@mui/x-data-grid"; //import JSONPretty from 'react-json-pretty'; //import JSONPrettyMon from 'react-json-pretty/dist/monikai' diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 41f93fd6..49b84dbb 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -76,7 +76,7 @@ import NestedMenuItem from "material-ui-nested-menu-item"; //import {Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons'; //https://next.material-ui.com/components/material-icons/ -import { DataGrid, GridToolbar } from "@material-ui/data-grid"; +import { DataGrid, GridToolbar } from "@mui/x-data-grid"; //import JSONPretty from 'react-json-pretty'; //import JSONPrettyMon from 'react-json-pretty/dist/monikai' From 2d51aaaa82cbee4d0fc8af2c00587f2cb9a8d8b1 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 1 Aug 2022 02:31:53 +0200 Subject: [PATCH 062/142] Reverting compose file --- backend/go-app/main.go | 4 ++ frontend/package.json | 2 +- frontend/src/components/ParsedAction.jsx | 1 - frontend/src/components/WelcomeForm.jsx | 7 +- frontend/src/views/AngularWorkflow.jsx | 85 +++++++++++++----------- 5 files changed, 59 insertions(+), 40 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 50f87d66..718ac5a8 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1270,6 +1270,10 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { redirectUrl = url.QueryEscape(fmt.Sprintf("%s/api/v1/login_openid", os.Getenv("BASE_URL"))) } + if len(os.Getenv("SSO_REDIRECT_URL")) > 0 { + redirectUrl = url.QueryEscape(fmt.Sprintf("%s/api/v1/login_openid", os.Getenv("SSO_REDIRECT_URL"))) + } + state := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("org=%s&challenge=%s&redirect=%s", org.Id, codeChallenge, redirectUrl))) // has to happen after initial value is stored diff --git a/frontend/package.json b/frontend/package.json index ff3cdef7..d6d5577c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -9,7 +9,6 @@ "@emotion/react": "^11.7.0", "@emotion/styled": "^11.6.0", "@material-ui/core": "^4.5.2", - "@material-ui/data-grid": "^4.0.0-alpha.22", "@material-ui/icons": "^4.5.1", "@material-ui/lab": "^4.0.0-alpha.58", "@material-ui/styles": "^4.5.2", @@ -17,6 +16,7 @@ "@metamask/detect-provider": "^1.2.0", "@mui/icons-material": "^5.2.1", "@mui/material": "^5.2.3", + "@mui/x-data-grid": "^5.15.0", "@uiw/react-codemirror": "^3.2.1", "@use-it/interval": "^1.0.0", "algoliasearch": "^4.13.1", diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 4a746939..a386bb9a 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -3211,7 +3211,6 @@ const ParsedAction = (props) => { setSelectedAction(selectedAction); setUpdate(Math.random()); } else { - //console.log("CHOSE AN AUTHENTICATION OPTION: ", e.target.value) selectedAction.selectedAuthentication = e.target.value; selectedAction.authentication_id = e.target.value.id; setSelectedAction(selectedAction); diff --git a/frontend/src/components/WelcomeForm.jsx b/frontend/src/components/WelcomeForm.jsx index 3009a083..58a1e8e9 100644 --- a/frontend/src/components/WelcomeForm.jsx +++ b/frontend/src/components/WelcomeForm.jsx @@ -4,9 +4,12 @@ import Step from "@material-ui/core/Step"; import StepLabel from "@material-ui/core/StepLabel"; import Button from "@material-ui/core/Button"; import Checkbox from '@mui/material/Checkbox'; + import SearchIcon from '@mui/icons-material/Search'; import EmailIcon from '@mui/icons-material/Email'; import NewReleasesIcon from '@mui/icons-material/NewReleases'; +import ExtensionIcon from '@mui/icons-material/Extension'; + import theme from '../theme'; import { FormGroup, @@ -422,7 +425,9 @@ const WelcomeForm = (props) => {
- ) : null} - {currentActionMethod === "GET" ? ( + {/*currentActionMethod === "GET" ? ( - ) : null} + ) : null*/} {fileUploadEnabled ? ( Date: Tue, 2 Aug 2022 14:46:23 +0200 Subject: [PATCH 064/142] Ran a fix for OrgID sometimes missing during Workflow execution --- backend/go-app/go.mod | 2 +- backend/go-app/main.go | 20 +++++++++++++++----- backend/go-app/walkoff.go | 14 +++++++++----- frontend/src/views/AngularWorkflow.jsx | 9 +++++---- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index ecb1116b..3f038a4e 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -24,7 +24,7 @@ require ( github.com/h2non/filetype v1.1.3 github.com/nirasan/go-oauth-pkce-code-verifier v0.0.0-20170819232839-0fbfe93532da // indirect github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.2.77 + github.com/shuffle/shuffle-shared v0.2.78 go4.org v0.0.0-20201209231011-d4a079459e60 // indirect golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce google.golang.org/api v0.65.0 diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 718ac5a8..f121a216 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -2311,7 +2311,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { //start, startok := request.URL.Query()["start"] // OrgId: activeOrgs[0].Id, - workflowExecution, executionResp, err := handleExecution(item, workflow, newRequest) + workflowExecution, executionResp, err := handleExecution(item, workflow, newRequest, hook.OrgId) if err == nil { /* err = increaseStatisticsField(ctx, "total_webhooks_ran", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) @@ -3585,7 +3585,7 @@ func handleCloudExecutionOnprem(workflowId, startNode, executionSource, executio Body: ioutil.NopCloser(bytes.NewReader(b)), } - _, _, err = handleExecution(workflowId, shuffle.Workflow{}, newRequest) + _, _, err = handleExecution(workflowId, shuffle.Workflow{}, newRequest, workflow.OrgId) return err } @@ -3706,7 +3706,7 @@ func handleCloudJob(job shuffle.CloudSyncJob) error { return err } - _, _, err = handleExecution(job.PrimaryItemId, shuffle.Workflow{}, newRequest) + _, _, err = handleExecution(job.PrimaryItemId, shuffle.Workflow{}, newRequest, job.OrgId) if err != nil { log.Printf("Failed continuing workflow from cloud user_input: %s", err) return err @@ -4009,7 +4009,12 @@ func runInitEs(ctx context.Context) { Body: ioutil.NopCloser(strings.NewReader(schedule.WrappedArgument)), } - _, _, err := handleExecution(schedule.WorkflowId, shuffle.Workflow{}, request) + orgId := "" + if len(activeOrgs) > 0 { + orgId = activeOrgs[0].Id + } + + _, _, err := handleExecution(schedule.WorkflowId, shuffle.Workflow{}, request, orgId) if err != nil { log.Printf("[WARNING] Failed to execute %s: %s", schedule.WorkflowId, err) } @@ -4869,7 +4874,12 @@ func runInit(ctx context.Context) { Body: ioutil.NopCloser(strings.NewReader(schedule.WrappedArgument)), } - _, _, err := handleExecution(schedule.WorkflowId, shuffle.Workflow{}, request) + orgId := "" + if len(activeOrgs) > 0 { + orgId = activeOrgs[0].Id + } + + _, _, err := handleExecution(schedule.WorkflowId, shuffle.Workflow{}, request, orgId) if err != nil { log.Printf("[WARNING] Failed to execute %s: %s", schedule.WorkflowId, err) } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index f314fd7f..46c64508 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -98,7 +98,7 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode Body: ioutil.NopCloser(strings.NewReader(bodyWrapper)), } - _, _, err := handleExecution(workflowId, shuffle.Workflow{ExecutingOrg: shuffle.OrgMini{Id: orgId}}, request) + _, _, err := handleExecution(workflowId, shuffle.Workflow{ExecutingOrg: shuffle.OrgMini{Id: orgId}}, request, orgId) if err != nil { log.Printf("Failed to execute %s: %s", workflowId, err) } @@ -831,7 +831,7 @@ func getWorkflowLocal(fileId string, request *http.Request) ([]byte, error) { return body, nil } -func handleExecution(id string, workflow shuffle.Workflow, request *http.Request) (shuffle.WorkflowExecution, string, error) { +func handleExecution(id string, workflow shuffle.Workflow, request *http.Request, orgId string) (shuffle.WorkflowExecution, string, error) { //go func() { // log.Printf("\n\nPRE TIME: %s\n\n", time.Now().Format("2006-01-02 15:04:05")) // _ = <-time.After(time.Second * 60) @@ -850,8 +850,12 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request } if len(workflow.ExecutingOrg.Id) == 0 { - log.Printf("[INFO] Stopped execution because there is no executing org for workflow %s", workflow.ID) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow has no executing org defined"), errors.New("Workflow has no executing org defined") + if len(orgId) > 0 { + workflow.ExecutingOrg.Id = orgId + } else { + log.Printf("[INFO] Stopped execution because there is no executing org for workflow %s", workflow.ID) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow has no executing org defined"), errors.New("Workflow has no executing org defined") + } } if len(workflow.Actions) == 0 { @@ -1138,7 +1142,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { user.ActiveOrg.Users = []shuffle.UserMini{} workflow.ExecutingOrg = user.ActiveOrg - workflowExecution, executionResp, err := handleExecution(fileId, *workflow, request) + workflowExecution, executionResp, err := handleExecution(fileId, *workflow, request, user.ActiveOrg.Id) if err == nil { resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization))) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 1406d224..ccc02724 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -2641,11 +2641,12 @@ const AngularWorkflow = (defaultprops) => { setSelectedApp(tmpapp); setSelectedAction(curaction); } else { + //if (curapp.id !== curaction.id) { + // curaction.app_id = curapp.id + // //.valueOf() + //} console.log("CURAPP: ", curapp) - if (curapp.id !== curaction.id) { - curaction.app_id = curapp.id - //.valueOf() - } + curaction.app_id = curapp.id setAuthenticationType( curapp.authentication.type === "oauth2" && curapp.authentication.redirect_uri !== undefined && curapp.authentication.redirect_uri !== null ? { From 5cdec784917a81d7efbbe16bf177bcf3cd3af403 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 2 Aug 2022 14:58:31 +0200 Subject: [PATCH 065/142] Fixed data grid autobuild problem --- frontend/src/views/MyView.jsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/views/MyView.jsx b/frontend/src/views/MyView.jsx index 3420a7d8..42c72bc3 100644 --- a/frontend/src/views/MyView.jsx +++ b/frontend/src/views/MyView.jsx @@ -51,7 +51,8 @@ import { GridToolbarContainer, GridDensitySelector, GridToolbar, -} from "@material-ui/data-grid"; +} from "@mui/x-data-grid"; + import { makeStyles } from "@material-ui/core/styles"; import ListIcon from "@material-ui/icons/List"; From 4ae090e05092b01cf34de3be55fb6b6dea6f8b8c Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 5 Aug 2022 18:21:50 +0200 Subject: [PATCH 066/142] Added file category listing with new app options to work with detections in the future --- .github/push_nightly.sh | 15 +++-- backend/app_sdk/app_base.py | 27 ++++++++ frontend/src/components/ParsedAction.jsx | 25 ++++++-- frontend/src/components/ShuffleCodeEditor.jsx | 2 +- frontend/src/components/WelcomeForm.jsx | 4 +- frontend/src/views/Admin.jsx | 18 +++++- frontend/src/views/AngularWorkflow.jsx | 62 ++++++++++++++++--- 7 files changed, 128 insertions(+), 25 deletions(-) diff --git a/.github/push_nightly.sh b/.github/push_nightly.sh index fc452d40..9c55b01c 100644 --- a/.github/push_nightly.sh +++ b/.github/push_nightly.sh @@ -1,15 +1,14 @@ # This can be done in the dockerpush workflow itself +# Done manually for now since GHCR isn't being pushed to easily with the current Github action CI. Nightly = Latest IF we run hotfixes on latest + docker pull frikky/shuffle-backend:nightly docker pull frikky/shuffle-frontend:nightly -#docker tag frikky/shuffle-backend:nightly ghcr.io/frikky/shuffle-backend:nightly -#docker tag frikky/shuffle-frontend:nightly ghcr.io/frikky/shuffle-frontend:nightly -# -#docker push ghcr.io/frikky/shuffle-backend:nightly -#docker push ghcr.io/frikky/shuffle-frontend:nightly - -docker tag frikky/shuffle-backend:nightly ghcr.io/frikky/shuffle-backend:nightly -docker tag frikky/shuffle-frontend:nightly ghcr.io/frikky/shuffle-frontend:nightly +docker tag frikky/shuffle-backend:nightly ghcr.io/frikky/shuffle-backend:latest +docker tag frikky/shuffle-frontend:nightly ghcr.io/frikky/shuffle-frontend:latest docker push ghcr.io/frikky/shuffle-backend:nightly docker push ghcr.io/frikky/shuffle-frontend:nightly + +docker push ghcr.io/frikky/shuffle-backend:latest +docker push ghcr.io/frikky/shuffle-frontend:latest diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 7c57151a..b7742f12 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1022,6 +1022,33 @@ class AppBase: self.logger.info("\nLOOP: %s\nRESULTS: %s" % (loop_wrapper, results)) return results + # Downloads all files from a namespace + # Currently only working on local version of Shuffle + def get_file_category_ids(self, category): + org_id = self.full_execution["workflow"]["execution_org"]["id"] + + get_path = "/api/v1/files/namespaces/%s?execution_id=%s&ids=true" % (category, self.full_execution["execution_id"]) + headers = { + "Authorization": "Bearer %s" % self.authorization + } + + ret = requests.get("%s%s" % (self.url, get_path), headers=headers) + return ret.json() + #if ret1.status_code != 200: + # return { + # "success": False, + # "reason": "Status code is %d from backend for category %s" % category, + # "list": [], + # } + + #return { + # "success": True, + # "ids": ret1.json(), + #} + + def get_file_namespace_ids(self, namespace): + return get_file_category_ids(self, namespace) + # Downloads all files from a namespace # Currently only working on local version of Shuffle def get_file_namespace(self, namespace): diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index a386bb9a..f4157487 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -123,6 +123,7 @@ const openApiFieldDesc = "Generated by OpenAPI body example"; const ParsedAction = (props) => { const { workflow, + files, setWorkflow, setAction, setSelectedAction, @@ -1300,6 +1301,7 @@ const ParsedAction = (props) => { data.value = data.example; } + if (data.value.length === 0) { if (data.name.toLowerCase() === "headers") { console.log("Should show headers field instead with + and -!") @@ -1878,6 +1880,14 @@ const ParsedAction = (props) => { // Basic helpertext + if (data.name.toLowerCase() === "file_category") { + //selectedActionParameters[count].options.length > 0 + console.log("FileS: ", files) + if (files.namespaces !== undefined && files.namespaces !== null && files.namespaces.length > 0) { + data.options = files.namespaces + } + } + //const keywords = ["len", "lower", "upper", "trim", "split", "length", "number", "parse", "join"] if ( selectedActionParameters[count].schema !== undefined && @@ -1942,16 +1952,22 @@ const ParsedAction = (props) => { } */ } else if ( - selectedActionParameters[count].options !== undefined && + (data.options !== undefined && + data.options !== null && + data.options.length > 0) + || + (selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && - selectedActionParameters[count].options.length > 0 + selectedActionParameters[count].options.length > 0) ) { + const parsedoptions = data.options !== undefined ? data.options : selectedActionParameters[count].options + if (selectedActionParameters[count].value === "") { // && selectedActionParameters[count].required) { // Rofl, dirty workaround :) const e = { target: { - value: selectedActionParameters[count].options[0], + value: parsedoptions.options[0], }, }; @@ -1982,7 +1998,7 @@ const ParsedAction = (props) => { borderRadius: theme.palette.borderRadius, }} > - {selectedActionParameters[count].options.map( + {parsedoptions.map( (data, index) => { const split_data = data.split("||"); var viewed_data = data; @@ -2014,6 +2030,7 @@ const ParsedAction = (props) => { return null; } + // Shows nested list of nodes > their JSON lists const ActionlistWrapper = (props) => { const handleMenuClose = () => { diff --git a/frontend/src/components/ShuffleCodeEditor.jsx b/frontend/src/components/ShuffleCodeEditor.jsx index 150cb8e2..ea0c3187 100644 --- a/frontend/src/components/ShuffleCodeEditor.jsx +++ b/frontend/src/components/ShuffleCodeEditor.jsx @@ -51,7 +51,7 @@ import { useNavigate, Link, useParams } from "react-router-dom"; const liquidFilters = [ {"name": "Size", "value": "size", "example": ""}, - {"name": "Date", "value": `date: "%Y%M%d"`, "example": `{{ "now" | date: "%s" }}`}, + {"name": "Date", "value": `date: "%Y%m%d"`, "example": `{{ "now" | date: "%s" }}`}, {"name": "Escape String", "value": `{{ \"\"\"'string with weird'" quotes\"\"\" | escape_string }}`, "example": ``}, {"name": "Flatten", "value": `flatten`, "example": `{{ [1, [1, 2], [2, 3, 4]] | flatten }}`}, ] diff --git a/frontend/src/components/WelcomeForm.jsx b/frontend/src/components/WelcomeForm.jsx index 65579798..4b91a5f2 100644 --- a/frontend/src/components/WelcomeForm.jsx +++ b/frontend/src/components/WelcomeForm.jsx @@ -513,7 +513,7 @@ const WelcomeForm = (props) => { }}> Phishing - -
) : (
+ {/* + + { + console.log("Should go to icon") + }} + > + + + + */} { const [leftViewOpen, setLeftViewOpen] = React.useState(isMobile ? false : true); const [leftBarSize, setLeftBarSize] = React.useState(isMobile ? 0 : 350); const [creatorProfile, setCreatorProfile] = React.useState({}); + const [files, setFiles] = React.useState({ + "namespaces": [ + "default", + ] + }); const [appGroup, setAppGroup] = React.useState([]); const [triggerGroup, setTriggerGroup] = React.useState([]); const [executionText, setExecutionText] = React.useState(""); @@ -1677,6 +1682,43 @@ const AngularWorkflow = (defaultprops) => { }) } + const getFiles = () => { + fetch(globalUrl + "/api/v1/files", { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for apps :O!"); + return; + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.files !== undefined && responseJson.files !== null) { + setFiles(responseJson); + } else { + setFiles({"namespaces": []}); + } + + console.log("NAMESPACES: ", responseJson.namespaces); + if ( + responseJson.namespaces !== undefined && + responseJson.namespaces !== null + ) { + //setFileNamespaces(responseJson.namespaces); + } + }) + .catch((error) => { + alert.error(error.toString()); + }); + }; + const getWorkflow = (workflow_id, sourcenode) => { fetch(globalUrl + "/api/v1/workflows/" + workflow_id, { method: "GET", @@ -1745,8 +1787,14 @@ const AngularWorkflow = (defaultprops) => { } setTriggerGroup(appsFound) - } - + } else { + getAppAuthentication(); + getEnvironments(); + getWorkflowExecution(props.match.params.key, ""); + getAvailableWorkflows(-1); + getSettings(); + getFiles() + } // Appends SUBFLOWS. Does NOT run during normal grabbing of workflows. if (sourcenode.id !== undefined) { @@ -4598,11 +4646,6 @@ const AngularWorkflow = (defaultprops) => { setFirstrequest(false); getWorkflow(props.match.params.key, {}); getApps(); - getAppAuthentication(); - getEnvironments(); - getWorkflowExecution(props.match.params.key, ""); - getAvailableWorkflows(-1); - getSettings(); const cursearch = typeof window === "undefined" || window.location === undefined @@ -5327,7 +5370,7 @@ const AngularWorkflow = (defaultprops) => { description: "O365 email trigger", trigger_type: "EMAIL", errors: null, - is_valid: cloudSyncEnabled || isCloud ? true : false, + is_valid: isCloud ? true : false, label: "Email", environment: "cloud", large_image: @@ -5341,7 +5384,7 @@ const AngularWorkflow = (defaultprops) => { description: "Gmail email trigger", trigger_type: "EMAIL", errors: null, - is_valid: cloudSyncEnabled || isCloud ? true : false, + is_valid: isCloud ? true : false, label: "Email", environment: "cloud", large_image: @@ -11177,6 +11220,7 @@ const AngularWorkflow = (defaultprops) => { defaultReturn = Date: Sat, 6 Aug 2022 00:30:07 +0200 Subject: [PATCH 067/142] Made triggers only able to have one branch --- frontend/src/components/ParsedAction.jsx | 5 ++- frontend/src/views/AngularWorkflow.jsx | 47 +++++++++++++++++++++--- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index f4157487..3c54b6c0 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -1368,6 +1368,7 @@ const ParsedAction = (props) => { var hideBodyButton = ""; const hideBodyButtonValue = (
{ selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0) ) { - const parsedoptions = data.options !== undefined ? data.options : selectedActionParameters[count].options + const parsedoptions = data.options !== undefined && data.options !== null && data.options.length > 0 ? data.options : selectedActionParameters[count].options if (selectedActionParameters[count].value === "") { // && selectedActionParameters[count].required) { // Rofl, dirty workaround :) const e = { target: { - value: parsedoptions.options[0], + value: parsedoptions[0], }, }; diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 21415e0b..126c7792 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1703,10 +1703,11 @@ const AngularWorkflow = (defaultprops) => { if (responseJson.files !== undefined && responseJson.files !== null) { setFiles(responseJson); } else { - setFiles({"namespaces": []}); + setFiles({"namespaces": [ + "default" + ]}); } - console.log("NAMESPACES: ", responseJson.namespaces); if ( responseJson.namespaces !== undefined && responseJson.namespaces !== null @@ -3180,18 +3181,53 @@ const AngularWorkflow = (defaultprops) => { setLastSaved(false); const edge = event.target.data(); + console.log("edge: ", edge) + if (edge.source === undefined && edge.target === undefined) { + return + } + const sourcenode = cy.getElementById(edge.source) const destinationnode = cy.getElementById(edge.target) if (sourcenode === undefined || sourcenode === null || destinationnode === undefined || destinationnode === null) { } else { + console.log("Is it a trigger? If so, check if it already has a branch and remove it: ", sourcenode.data()) + if (sourcenode.data("type") === "TRIGGER") { + if (sourcenode.data("app_name") !== "Shuffle Workflow" && sourcenode.data("app_name") !== "User Input") { + setTimeout(() => { + const alledges = cy.edges().jsons() + console.log("edges: ", alledges, edge) + var targetedge = alledges.findIndex( + (data) => data.data.source === edge.source && data.data.id !== edge.id + ) + + console.log("Node: ", targetedge) + if (targetedge !== -1) { + event.target.remove() + + //console.log("Found branch already!") + alert.info("Triggers can have exactly one target node") + return + + + // name: "Shuffle Workflow", + // name: "User Input", + } else { + console.log("Node doesn't already have one") + } + }, 50) + } + } + const edgeCurve = calculateEdgeCurve(sourcenode.position(), destinationnode.position()) const currentedge = cy.getElementById(edge.id) if (currentedge !== undefined && currentedge !== null) { currentedge.style('control-point-distance', edgeCurve.distance) currentedge.style('control-point-weight', edgeCurve.weight) } + } + var targetnode = workflow.triggers.findIndex( (data) => data.id === edge.target ); @@ -3545,10 +3581,10 @@ const AngularWorkflow = (defaultprops) => { // trigger as source check const indexcheck = workflow.triggers.findIndex( - (data) => edge.data()["source"] === data.id + (data) => edge.data("source") === data.id ); if (indexcheck !== -1) { - console.log("Shouldnt remove edge from trigger"); + console.log("Shouldnt remove edge from trigger? "); } if (edge.data().source !== undefined) { @@ -4717,12 +4753,13 @@ const AngularWorkflow = (defaultprops) => { return false }, - preview: true, + preview: false, toggleOffOnLeave: true, loopAllowed: function (node) { return false; }, }); + // preview: true, cy.fit(null, 200); From ec848cc9d60a35445c563b164aa230ff3740ed8f Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 6 Aug 2022 01:13:16 +0200 Subject: [PATCH 068/142] Fixed issue with multiple trigger branches --- frontend/src/views/AngularWorkflow.jsx | 55 ++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 126c7792..2d9c15c0 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -3186,6 +3186,13 @@ const AngularWorkflow = (defaultprops) => { return } + if (edge.readded === true) { + console.log("Readded edge - stopping") + + event.target.data("readded", false) + return + } + const sourcenode = cy.getElementById(edge.source) const destinationnode = cy.getElementById(edge.target) if (sourcenode === undefined || sourcenode === null || destinationnode === undefined || destinationnode === null) { @@ -3567,12 +3574,48 @@ const AngularWorkflow = (defaultprops) => { const onEdgeRemoved = (event) => { setLastSaved(false); - const edge = event.target; if (edge.data("decorator") === true) { return; } + // Check if the source is trigger and can start + console.log("Removed: ", edge.data()) + const allNodes = cy.nodes().jsons() + for (var key in allNodes) { + const curnode = allNodes[key] + if (curnode.data.type !== "TRIGGER") { + continue + } + + if (curnode.data.id === edge.data("source")) { + console.log("Found matching trigger source: ", curnode) + if (curnode.data.app_name !== "Shuffle Workflow" && curnode.data.app_name !== "User Input") { + // If it's started, READD the edge + if (curnode.data.status === "running") { + console.log("Edge is running - readd it: ", edge.data()) + + // Just making sure it's not running infinitely + var newdata = edge.data() + newdata.readded = true + + try { + cy.add({ + group: "edges", + data: newdata, + }) + + alert.error("You must STOP the trigger before deleting its branches") + } catch (e) { + console.log("Failed re-adding edge: ", e) + } + } + + //status: "uninitialized", + } + } + } + workflow.branches = workflow.branches.filter( (a) => a.id !== edge.data().id ); @@ -4814,11 +4857,7 @@ const AngularWorkflow = (defaultprops) => { const stopSchedule = (trigger, triggerindex) => { fetch( - globalUrl + - "/api/v1/workflows/" + - props.match.params.key + - "/schedule/" + - trigger.id, + `${globalUrl}/api/v1/workflows/${props.match.params.key}/schedule/${trigger.id}`, { method: "DELETE", headers: { @@ -4863,7 +4902,7 @@ const AngularWorkflow = (defaultprops) => { return; } - alert.info("Attempting to create schedule with name " + trigger.name); + alert.info("Creating schedule with name " + trigger.name); const data = { name: trigger.name, frequency: workflow.triggers[triggerindex].parameters[0].value, @@ -4873,7 +4912,7 @@ const AngularWorkflow = (defaultprops) => { }; fetch( - globalUrl + "/api/v1/workflows/" + props.match.params.key + "/schedule", + `${globalUrl}/api/v1/workflows/${props.match.params.key}/schedule`, { method: "POST", headers: { From 09c02a14dca447ccd7053322d557ca6fb81e2995 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 6 Aug 2022 01:13:47 +0200 Subject: [PATCH 069/142] Fixed issue with multiple trigger branches --- frontend/src/views/AngularWorkflow.jsx | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 2d9c15c0..847dd46b 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -4902,6 +4902,16 @@ const AngularWorkflow = (defaultprops) => { return; } + var mappedStartnode = "" + const edges = cy.edges.jsons() + for (var key in edges.length) { + const tmp = edges[key] + if (tmp.data.source === trigger.id) { + mappedStartnode = tmp.data.target + break + } + } + alert.info("Creating schedule with name " + trigger.name); const data = { name: trigger.name, @@ -4909,7 +4919,8 @@ const AngularWorkflow = (defaultprops) => { execution_argument: workflow.triggers[triggerindex].parameters[1].value, environment: workflow.triggers[triggerindex].environment, id: trigger.id, - }; + start: mappedStartnode, + } fetch( `${globalUrl}/api/v1/workflows/${props.match.params.key}/schedule`, From 12670ddb7250ef6458efd204d64e4c7a8867cd1c Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 6 Aug 2022 01:34:04 +0200 Subject: [PATCH 070/142] Fixed startnode management during schedule startup --- backend/go-app/go.mod | 2 +- backend/go-app/walkoff.go | 19 ++++++++++++------- frontend/src/views/AngularWorkflow.jsx | 7 ++++--- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 3f038a4e..f6c8d495 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -24,7 +24,7 @@ require ( github.com/h2non/filetype v1.1.3 github.com/nirasan/go-oauth-pkce-code-verifier v0.0.0-20170819232839-0fbfe93532da // indirect github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.2.78 + github.com/shuffle/shuffle-shared v0.2.79 go4.org v0.0.0-20201209231011-d4a079459e60 // indirect golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce google.golang.org/api v0.65.0 diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 46c64508..092b52ad 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1504,14 +1504,19 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { // Finds the startnode for the specific schedule startNode := "" - for _, branch := range workflow.Branches { - if branch.SourceID == schedule.Id { - startNode = branch.DestinationID - } - } + if schedule.Start != "" { + startNode = schedule.Start + } else { - if startNode == "" { - startNode = workflow.Start + for _, branch := range workflow.Branches { + if branch.SourceID == schedule.Id { + startNode = branch.DestinationID + } + } + + if startNode == "" { + startNode = workflow.Start + } } //log.Printf("Startnode: %s", startNode) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 847dd46b..db072734 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -4903,9 +4903,10 @@ const AngularWorkflow = (defaultprops) => { } var mappedStartnode = "" - const edges = cy.edges.jsons() - for (var key in edges.length) { - const tmp = edges[key] + const alledges = cy.edges().jsons() + for (var key in alledges) { + const tmp = alledges[key] + console.log("TMP: ", tmp, tmp.data.source) if (tmp.data.source === trigger.id) { mappedStartnode = tmp.data.target break From 9dc16550fa849d6e820b56fb0934c24628ac1d47 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 9 Aug 2022 00:52:58 +0200 Subject: [PATCH 071/142] Fixed some minor SSR issues in search grids --- .github/push_nightly.sh | 8 +- backend/app_sdk/app_base.py | 8 +- docker-compose.yml | 76 +++++------ frontend/src/components/AppGrid.jsx | 1 + frontend/src/views/Docs.jsx | 199 ++++++++++++++-------------- frontend/src/views/Search.jsx | 40 +++--- 6 files changed, 173 insertions(+), 159 deletions(-) diff --git a/.github/push_nightly.sh b/.github/push_nightly.sh index 9c55b01c..98dba76a 100644 --- a/.github/push_nightly.sh +++ b/.github/push_nightly.sh @@ -4,11 +4,11 @@ docker pull frikky/shuffle-backend:nightly docker pull frikky/shuffle-frontend:nightly -docker tag frikky/shuffle-backend:nightly ghcr.io/frikky/shuffle-backend:latest -docker tag frikky/shuffle-frontend:nightly ghcr.io/frikky/shuffle-frontend:latest +docker tag frikky/shuffle-backend:nightly ghcr.io/frikky/shuffle-backend:nightly +docker tag frikky/shuffle-frontend:nightly ghcr.io/frikky/shuffle-frontend:nightly docker push ghcr.io/frikky/shuffle-backend:nightly docker push ghcr.io/frikky/shuffle-frontend:nightly -docker push ghcr.io/frikky/shuffle-backend:latest -docker push ghcr.io/frikky/shuffle-frontend:latest +#docker push ghcr.io/frikky/shuffle-backend:latest +#docker push ghcr.io/frikky/shuffle-frontend:latest diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index b7742f12..6d1833b7 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1046,8 +1046,6 @@ class AppBase: # "ids": ret1.json(), #} - def get_file_namespace_ids(self, namespace): - return get_file_category_ids(self, namespace) # Downloads all files from a namespace # Currently only working on local version of Shuffle @@ -1081,6 +1079,12 @@ class AppBase: return myzipfile + def get_file_namespace_ids(self, namespace): + return self.get_file_category_ids(self, namespace) + + def get_file_category(self, category): + return self.get_file_namespace(self, category) + # Things to consider for files: # - How can you download / stream a file? # - Can you decide if you want a stream or the files directly? diff --git a/docker-compose.yml b/docker-compose.yml index 6cde7541..7c619ae9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,43 +1,43 @@ version: '3' services: - frontend: - #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:nightly - container_name: shuffle-frontend - hostname: shuffle-frontend - ports: - - "${FRONTEND_PORT}:80" - - "${FRONTEND_PORT_HTTPS}:443" - networks: - - shuffle - environment: - - BACKEND_HOSTNAME=${BACKEND_HOSTNAME} - restart: unless-stopped - depends_on: - - backend - backend: - #build: ./backend - image: ghcr.io/frikky/shuffle-backend:nightly - container_name: shuffle-backend - hostname: ${BACKEND_HOSTNAME} - # Here for debugging: - ports: - - "${BACKEND_PORT}:5001" - networks: - - shuffle - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z - - ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z - #- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate - env_file: .env - environment: - #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - - SHUFFLE_FILE_LOCATION=/shuffle-files - restart: unless-stopped - #depends_on: - #- opensearch #Not necessary because dependancy is handled within the backend itself instead + #frontend: + # #build: ./frontend + # image: ghcr.io/frikky/shuffle-frontend:nightly + # container_name: shuffle-frontend + # hostname: shuffle-frontend + # ports: + # - "${FRONTEND_PORT}:80" + # - "${FRONTEND_PORT_HTTPS}:443" + # networks: + # - shuffle + # environment: + # - BACKEND_HOSTNAME=${BACKEND_HOSTNAME} + # restart: unless-stopped + # depends_on: + # - backend + #backend: + # #build: ./backend + # image: ghcr.io/frikky/shuffle-backend:nightly + # container_name: shuffle-backend + # hostname: ${BACKEND_HOSTNAME} + # # Here for debugging: + # ports: + # - "${BACKEND_PORT}:5001" + # networks: + # - shuffle + # volumes: + # - /var/run/docker.sock:/var/run/docker.sock + # - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z + # - ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z + # #- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate + # env_file: .env + # environment: + # #- DOCKER_HOST=tcp://docker-socket-proxy:2375 + # - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps + # - SHUFFLE_FILE_LOCATION=/shuffle-files + # restart: unless-stopped + # #depends_on: + # #- opensearch #Not necessary because dependancy is handled within the backend itself instead #- database orborus: image: ghcr.io/frikky/shuffle-orborus:nightly diff --git a/frontend/src/components/AppGrid.jsx b/frontend/src/components/AppGrid.jsx index 2878f570..54760432 100644 --- a/frontend/src/components/AppGrid.jsx +++ b/frontend/src/components/AppGrid.jsx @@ -27,6 +27,7 @@ const searchClient = algoliasearch("JNSS5CFDZZ", "db08e40265e2941b9a7d8f644b6e52 //const searchClient = algoliasearch("L55H18ZINA", "a19be455e7e75ee8f20a93d26b9fc6d6") const AppGrid = props => { const { maxRows, showName, showSuggestion, isMobile, globalUrl, parsedXs, userdata } = props + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; diff --git a/frontend/src/views/Docs.jsx b/frontend/src/views/Docs.jsx index 74c882fc..376ac40b 100644 --- a/frontend/src/views/Docs.jsx +++ b/frontend/src/views/Docs.jsx @@ -112,8 +112,9 @@ const Docs = (defaultprops) => { const SideBar = { minWidth: 250, maxWidth: 300, - position: "sticky", - top: 100, + position: "absolute", + top: 15, + left: 0, maxHeight: "83vh", overflowX: "hidden", overflowY: "auto", @@ -592,105 +593,107 @@ const Docs = (defaultprops) => {
- - Documentation - -
- link="mailto:support@shuffler.io" /> - link="https://discord.gg/B2CBzUm" /> -
- -
- Tutorial - - Dive in. Hands-on is the best approach to see how Shuffle can transform your security operations. Our set of tutorials and videos teach you how to build your skills. Check out the getting started section to give it a go! + + Documentation +
+ link="mailto:support@shuffler.io" /> + link="https://discord.gg/B2CBzUm" /> +
- Why Shuffle? - - Security first. We incentivize trying before buying, and give you the full set of tools you need to automate your operations. What's more is we also help you find usecases that fit your your unique needs. Accessibility is key, and we intend to help every SOC globally use and share their usecases. - +
+ Tutorial + + Dive in. Hands-on is the best approach to see how Shuffle can transform your security operations. Our set of tutorials and videos teach you how to build your skills. Check out the getting started section to give it a go! + - Get help - - Our promise is to make it easier and easier to automate your operations. In some cases however, it may be good with a helping hand. That's where Shuffle's consultancy and support services come in handy. We help you build and automate your operational processes to a level you haven't seen before with the help of our usecases. - + Why Shuffle? + + Security first. We incentivize trying before buying, and give you the full set of tools you need to automate your operations. What's more is we also help you find usecases that fit your your unique needs. Accessibility is key, and we intend to help every SOC globally use and share their usecases. + - APIs - - Learn. We're all about learning, and are continuously creating documentation and video tutorials to better understand how to get started. APIs are an extremely important part of how the internet works today, and our goal is helping every security professional learn about them. - + Get help + + Our promise is to make it easier and easier to automate your operations. In some cases however, it may be good with a helping hand. That's where Shuffle's consultancy and support services come in handy. We help you build and automate your operational processes to a level you haven't seen before with the help of our usecases. + - Workflow building - - Build. Creating workflows has never been easier. Jump into things with our getting Started section and build to your hearts content. Workflows make it all come together, with an easy to use area. - + APIs + + Learn. We're all about learning, and are continuously creating documentation and video tutorials to better understand how to get started. APIs are an extremely important part of how the internet works today, and our goal is helping every security professional learn about them. + - Managing Shuffle - - Organize. Whether an organization of 1000 or 1, management tools are necessary. In Shuffle we offer full user management, MFA and single-signon options, multi-tenancy and a lot more - for free! - -
+ Workflow building + + Build. Creating workflows has never been easier. Jump into things with our getting Started section and build to your hearts content. Workflows make it all come together, with an easy to use area. + - {/* - - {list.map((data, index) => { - const item = data.name; - if (item === undefined) { - return null; - } + Managing Shuffle + + Organize. Whether an organization of 1000 or 1, management tools are necessary. In Shuffle we offer full user management, MFA and single-signon options, multi-tenancy and a lot more - for free! + +
- const path = "/docs/" + item; - const newname = - item.charAt(0).toUpperCase() + - item.substring(1).split("_").join(" ").split("-").join(" "); + {/* + + {list.map((data, index) => { + const item = data.name; + if (item === undefined) { + return null; + } - const itemMatching = props.match.params.key === undefined ? false : - props.match.params.key.toLowerCase() === item.toLowerCase(); + const path = "/docs/" + item; + const newname = + item.charAt(0).toUpperCase() + + item.substring(1).split("_").join(" ").split("-").join(" "); - return ( - - - - ) - })} - - */} + const itemMatching = props.match.params.key === undefined ? false : + props.match.params.key.toLowerCase() === item.toLowerCase(); - {/* - + + + ) + })} + + */} + + {/* + { - console.log("Change: ", event.target.value) - }} - /> - */} + }} + InputProps={{ + style:{ + color: "white", + height: 50, + }, + }} + placeholder={"Search Knowledgebase"} + color="primary" + fullWidth={true} + type="firstname" + id={"Searchfield"} + margin="normal" + variant="outlined" + onChange={(event) => { + console.log("Change: ", event.target.value) + }} + /> + */}
const postDataBrowser = @@ -766,20 +769,22 @@ const Docs = (defaultprops) => { {props.match.params.key === undefined ? mainpageInfo : -
- -
+
+
+ +
+
}
); diff --git a/frontend/src/views/Search.jsx b/frontend/src/views/Search.jsx index f8955c7f..9fedc697 100644 --- a/frontend/src/views/Search.jsx +++ b/frontend/src/views/Search.jsx @@ -38,6 +38,28 @@ const Search = (props) => { const [curTab, setCurTab] = useState(0); const iconStyle = { marginRight: 10 }; + useEffect(() => { + if (serverside !== true && window.location.search !== undefined && window.location.search !== null) { + const urlSearchParams = new URLSearchParams(window.location.search) + const params = Object.fromEntries(urlSearchParams.entries()) + const foundTab = params["tab"] + if (foundTab !== null && foundTab !== undefined) { + for (var key in Object.keys(views)) { + const value = views[key] + console.log(key, value) + if (value === foundTab) { + setConfig("", key) + break + } + } + } + } + }, []) + + if (serverside === true) { + return null + } + const boxStyle = { color: "white", flex: "1", @@ -91,24 +113,6 @@ const Search = (props) => { } } - useEffect(() => { - if (serverside !== true && window.location.search !== undefined && window.location.search !== null) { - const urlSearchParams = new URLSearchParams(window.location.search) - const params = Object.fromEntries(urlSearchParams.entries()) - const foundTab = params["tab"] - if (foundTab !== null && foundTab !== undefined) { - for (var key in Object.keys(views)) { - const value = views[key] - console.log(key, value) - if (value === foundTab) { - setConfig("", key) - break - } - } - } - } - }, []) - if (isLoaded === false) { return null } From 478e6e623cb0ea73d153aa404f8817e7752dfb1d Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 9 Aug 2022 13:39:35 +0200 Subject: [PATCH 072/142] Fixed app framework bug when not logged in --- frontend/src/components/DetectionFramework.jsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/DetectionFramework.jsx b/frontend/src/components/DetectionFramework.jsx index a995a493..db8e0850 100644 --- a/frontend/src/components/DetectionFramework.jsx +++ b/frontend/src/components/DetectionFramework.jsx @@ -693,7 +693,7 @@ const Framework = (props) => { window.location.host === "shuffler.io"; const imgSize = 50; - var parsedFrameworkData = frameworkData + var parsedFrameworkData = frameworkData === undefined ? {} : frameworkData // Awful mapping to make sure all access is always there if (frameworkData !== undefined) { @@ -778,7 +778,7 @@ const Framework = (props) => { } } else { - console.log("No frameworkdata for org! Setting default") + //console.log("No frameworkdata for org! Setting default") parsedFrameworkData["Cases"] = {} parsedFrameworkData["SIEM"] = {} parsedFrameworkData["Assets"] = {} @@ -789,7 +789,7 @@ const Framework = (props) => { parsedFrameworkData["EDR & AV"] = {} } - console.log("Framework - update? ", parsedFrameworkData) + //console.log("Framework - update? ", parsedFrameworkData) // 0 = automated, 1 = manual const [usecaseType, setUsecaseType] = React.useState(0) From 2d696e3deadb21367773586736cec3051b153900 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 9 Aug 2022 15:03:19 +0200 Subject: [PATCH 073/142] Fixed file upload problems for apps due to headers being auto-set --- frontend/src/components/ParsedAction.jsx | 13 ++++- frontend/src/views/AngularWorkflow.jsx | 47 +++++++++++------ frontend/src/views/AppCreator.jsx | 67 +++++++++++++----------- 3 files changed, 79 insertions(+), 48 deletions(-) diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 3c54b6c0..4ad45128 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -1306,7 +1306,15 @@ const ParsedAction = (props) => { if (data.name.toLowerCase() === "headers") { console.log("Should show headers field instead with + and -!") - data.value = data.example + // Check if file ID exists + // + const fileFound = selectedActionParameters.find(param => param.name === "file_id") + if (fileFound === undefined || fileFound === null) { + data.value = data.example + } else { + // Purposely unset it if set by default when using files + data.value = "" + } } } @@ -3115,8 +3123,9 @@ const ParsedAction = (props) => { } } - console.log("DID REPLACE ACTUALLY WORK?? - Something is buggy."); + console.log("DID NAME REPLACE ACTUALLY WORK? - may be missing it in certain triggers"); setWorkflow(workflow); + setUpdate(Math.random()); baselabel = name }} /> diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index db072734..43849283 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -6284,9 +6284,13 @@ const AngularWorkflow = (defaultprops) => { const foundInfo = workflow.actions.find(ac => ac.id === selectedAction.id) } + // Setting an old reference just to use the same memory space elsewhere + // for selectedAction + const oldaction = JSON.parse(JSON.stringify(selectedAction)) + // Does this one find the wrong one? - //var newSelectedAction = selectedAction - var newSelectedAction = JSON.parse(JSON.stringify(selectedAction)) + //var newSelectedAction = JSON.parse(JSON.stringify(selectedAction)) + var newSelectedAction = selectedAction newSelectedAction.name = newaction.name; newSelectedAction.parameters = JSON.parse(JSON.stringify(newaction.parameters)) newSelectedAction.errors = []; @@ -6295,9 +6299,14 @@ const AngularWorkflow = (defaultprops) => { //console.log(newSelectedAction) // Simmple action swap autocompleter - if (selectedAction.parameters !== undefined && newSelectedAction.parameters !== undefined && selectedAction.id === newSelectedAction.id) { - for (var paramkey in selectedAction.parameters) { - const param = selectedAction.parameters[paramkey]; + if (oldaction.parameters !== undefined && newSelectedAction.parameters !== undefined && oldaction.id === newSelectedAction.id) { + var fileid_found = false + for (var paramkey in oldaction.parameters) { + const param = oldaction.parameters[paramkey]; + + if (param.name === "file_id") { + fileid_found = true + } if (param.value === null || param.value === undefined || param.value.length === 0) { continue @@ -6309,7 +6318,14 @@ const AngularWorkflow = (defaultprops) => { } if (param.name === "headers") { - console.log("Swap header? For now, yes") + console.log("Swap header? For now, yes. File found: ", fileid_found) + + if (fileid_found) { + newSelectedAction.parameters[paramkey].value = "" + newSelectedAction.parameters[paramkey].autocompleted = true + + continue + } //newSelectedAction.parameters[newParamIndex].value = param.value } @@ -6384,8 +6400,6 @@ const AngularWorkflow = (defaultprops) => { //setSelectedActionEnvironment(env) - console.log("NEW ACTION: ", newSelectedAction); - setSelectedAction(newSelectedAction); if (workflow.actions !== undefined && workflow.actions !== null && workflow.actions.length > 0) { const foundActionIndex = workflow.actions.findIndex(actiondata => actiondata.id === newSelectedAction.id) console.log("Found action on index ", foundActionIndex) @@ -6394,6 +6408,9 @@ const AngularWorkflow = (defaultprops) => { setWorkflow(workflow) } } + + console.log("NEW ACTION: ", newSelectedAction); + setSelectedAction(newSelectedAction); setUpdate(Math.random()); // FIXME - should change icon-node (descriptor) as well @@ -6401,7 +6418,7 @@ const AngularWorkflow = (defaultprops) => { for (var key in allNodes) { const currentNode = allNodes[key]; if ( - currentNode.data.attachedTo === selectedAction.id && + currentNode.data.attachedTo === oldaction.id && currentNode.data.isDescriptor ) { const foundnode = cy.getElementById(currentNode.data.id); @@ -11569,8 +11586,8 @@ const AngularWorkflow = (defaultprops) => {
+ + const loadedCheck = isLoaded && isLoggedIn && workflowDone ? (
@@ -3624,6 +3837,19 @@ const Workflows = (props) => { {publishModal} {workflowDownloadModalOpen} + {!drawerOpen ?
+ + { + setDrawerOpen(true) + localStorage.setItem(sidebarKey, "open"); + }}> + + + +
: null} + {isMobile ? null : gettingStartedDrawer} + {videoView} + {modalOpen === true ? Date: Wed, 23 Nov 2022 02:16:56 +0100 Subject: [PATCH 133/142] Added minor fix for creator grid --- frontend/src/components/CreatorGrid.jsx | 1 - frontend/src/views/AngularWorkflow.jsx | 254 +++++++++++++++--------- 2 files changed, 156 insertions(+), 99 deletions(-) diff --git a/frontend/src/components/CreatorGrid.jsx b/frontend/src/components/CreatorGrid.jsx index ec1b7ba9..6eaa6222 100644 --- a/frontend/src/components/CreatorGrid.jsx +++ b/frontend/src/components/CreatorGrid.jsx @@ -134,7 +134,6 @@ const CreatorGrid = props => { onChange={(event) => { refine(event.currentTarget.value) }} - limit={5} /> {/*isSearchStalled ? 'My search is stalled' : ''*/} diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 24a9c6cf..9048afcd 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -65,6 +65,7 @@ import { ListItem, ListItemText, ListItemAvatar, + Badge, } from "@material-ui/core"; import { @@ -109,6 +110,7 @@ import { import { Preview as PreviewIcon, + ContentCopy as ContentCopyIcon, } from '@mui/icons-material'; import Autocomplete from "@material-ui/lab/Autocomplete"; @@ -367,8 +369,7 @@ const AngularWorkflow = (defaultprops) => { const [selectedApp, setSelectedApp] = React.useState({}); const [selectedAction, setSelectedAction] = React.useState({}); - const [selectedActionEnvironment, setSelectedActionEnvironment] = - React.useState({}); + const [selectedActionEnvironment, setSelectedActionEnvironment] = React.useState({}); const [executionRequest, setExecutionRequest] = React.useState({}); @@ -2110,6 +2111,7 @@ const AngularWorkflow = (defaultprops) => { setSelectedTriggerIndex(-1) setTriggerFolders([]) setSubworkflow({}) + setLocalFirstrequest(true) // Can be used for right side view setRightSideBarOpen(false); @@ -8194,6 +8196,7 @@ const AngularWorkflow = (defaultprops) => { responseJson.success !== false && responseJson.length > 0 ) { + console.log("Got trigger folders: ", triggerFolders) setTriggerFolders(responseJson); } @@ -8207,7 +8210,7 @@ const AngularWorkflow = (defaultprops) => { name: "outlookfolder", id: responseJson[0].id, }, - ]; + ] selectedTrigger.parameters = [ { value: responseJson[0].displayName, @@ -8225,7 +8228,7 @@ const AngularWorkflow = (defaultprops) => { }; const getTriggerAuth = () => { - fetch(globalUrl + "/api/v1/triggers/outlook/" + selectedTrigger.id, { + fetch(globalUrl + "/api/v1/triggers/" + selectedTrigger.id, { method: "GET", headers: { "content-type": "application/json" }, credentials: "include", @@ -8238,20 +8241,24 @@ const AngularWorkflow = (defaultprops) => { return response.json(); }) .then((responseJson) => { + setTriggerAuthentication(responseJson); }) .catch((error) => { //console.log(error.toString()); - console.log("Set outlook auth error: ", error.toString()); + console.log("Set trigger auth error: ", error.toString()); }); }; // Getting the triggers and the folders if they exist - // This is horrible hahah if (localFirstrequest) { + //console.log("Trigger: ", selectedTrigger) + //console.log("Triggername: ", selectedTrigger.name) + //if (selectedTrigger.name.toLowerCase() === "gmail") { + setGmailFolders(); + setOutlookFolders(); + getTriggerAuth(); - setOutlookFolders(); - setGmailFolders(); setLocalFirstrequest(false); } @@ -8310,7 +8317,7 @@ const AngularWorkflow = (defaultprops) => { // Check whether we got a callback somewhere var id = setInterval(function () { fetch( - globalUrl + "/api/v1/triggers/gmail/" + selectedTrigger.id, + globalUrl + "/api/v1/triggers/" + selectedTrigger.id, { method: "GET", headers: { "content-type": "application/json" }, @@ -8411,7 +8418,7 @@ const AngularWorkflow = (defaultprops) => { // Check whether we got a callback somewhere var id = setInterval(function () { fetch( - globalUrl + "/api/v1/triggers/outlook/" + selectedTrigger.id, + globalUrl + "/api/v1/triggers/ " + selectedTrigger.id, { method: "GET", headers: { "content-type": "application/json" }, @@ -8506,61 +8513,73 @@ const AngularWorkflow = (defaultprops) => { }} />
- Select {triggerAuthentication.type === "gmail" ? "labels" : "folders"} (CTRL+click) + Select {triggerAuthentication.type === "gmail" ? "labels" : "a folder"}
- } - key={selectedTrigger} - > - {triggerFolders.map((folder) => { - var folderItem = ( - - ); + {triggerFolders.length === 0 ? + + No folders found. Please authenticate and make sure the user has access folders available. If this persists, contact us. + + : + } + key={selectedTrigger} + > + {triggerFolders.map((folder) => { + var folderItem = ( + + ); - if (folder.childFolderCount > 0) { - // Here to handle subfolders sometime later - folderItem = ( - - ); - } + if (folder.childFolderCount > 0) { + // Here to handle subfolders sometime later + folderItem = ( + + ); + } - return folderItem; - })} - + return folderItem; + })} + + } )}
@@ -10233,28 +10252,6 @@ const AngularWorkflow = (defaultprops) => { }} id="webhook_uri_field" onClick={() => { - var copyText = document.getElementById("webhook_uri_field"); - if (copyText !== undefined && copyText !== null) { - console.log("NAVIGATOR: ", navigator); - const clipboard = navigator.clipboard; - if (clipboard === undefined) { - alert.error("Can only copy over HTTPS (port 3443)"); - return; - } - - navigator.clipboard.writeText(copyText.value); - copyText.select(); - copyText.setSelectionRange( - 0, - 99999 - ); /* For mobile devices */ - - /* Copy the text inside the text field */ - document.execCommand("copy"); - alert.success("Copied Webhook URL"); - } else { - console.log("Couldn't find webhook URI field: ", copyText); - } }} helperText={ workflow.triggers[selectedTriggerIndex].parameters[0].value !== undefined && @@ -10281,6 +10278,39 @@ const AngularWorkflow = (defaultprops) => { maxWidth: "95%", fontSize: "1em", }, + endAdornment: + + { + var copyText = document.getElementById("webhook_uri_field"); + if (copyText !== undefined && copyText !== null) { + console.log("NAVIGATOR: ", navigator); + const clipboard = navigator.clipboard; + if (clipboard === undefined) { + alert.error("Can only copy over HTTPS (port 3443)"); + return; + } + + navigator.clipboard.writeText(copyText.value); + copyText.select(); + copyText.setSelectionRange( + 0, + 99999 + ); /* For mobile devices */ + + /* Copy the text inside the text field */ + document.execCommand("copy"); + alert.success("Copied Webhook URL"); + } else { + console.log("Couldn't find webhook URI field: ", copyText); + } + }} + edge="end" + > + + + }} fullWidth disabled @@ -10509,7 +10539,9 @@ const AngularWorkflow = (defaultprops) => { const splitItem = workflow.triggers[selectedTriggerIndex].parameters[0].value.split( splitter - ); + ) + + console.log("Starting mail sub: ", workflow.triggers[selectedTriggerIndex].parameters[0].value, splitItem); for (var key in splitItem) { const item = splitItem[key]; const curfolder = triggerFolders.find((a) => a.displayName === item); @@ -11839,22 +11871,30 @@ const AngularWorkflow = (defaultprops) => { - + @@ -12800,6 +12840,15 @@ const AngularWorkflow = (defaultprops) => { ) } + + var nonskippedResults = [] + if (executionData.results !== undefined) { + const newSkipped = executionData.results.find((result) => result.status !== "SKIPPED") + if (newSkipped !== undefined) { + nonskippedResults = newSkipped + } + } + var executionDelay = -75 const executionModal = ( @@ -13280,7 +13329,16 @@ const AngularWorkflow = (defaultprops) => { (executionData.results.length === 0 && // probably ment to be around the or's executionData.status === "EXECUTING") ) ? ( - +
+ + + + {!isCloud && environments.length > 0 && defaultEnvironmentIndex < environments.length && nonskippedResults.length === 0 ? + + No results yet. Is Orborus running for the "{environments[defaultEnvironmentIndex].Name}" environment? Learn more + + : null} +
) : null}
From 0f28aa9c8e2a133d9a6a944125f4efd9ad6871df Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 5 Dec 2022 14:48:48 +0100 Subject: [PATCH 134/142] Fixed small issues with Workflow view and added new standard view --- frontend/src/components/CreatorGrid.jsx | 2 +- frontend/src/views/AngularWorkflow.jsx | 2 +- frontend/src/views/DashboardViews.jsx | 101 ++++-------------------- frontend/src/views/Workflows.jsx | 94 +++++++++++----------- 4 files changed, 62 insertions(+), 137 deletions(-) diff --git a/frontend/src/components/CreatorGrid.jsx b/frontend/src/components/CreatorGrid.jsx index 6eaa6222..6c83b134 100644 --- a/frontend/src/components/CreatorGrid.jsx +++ b/frontend/src/components/CreatorGrid.jsx @@ -233,7 +233,7 @@ const CreatorGrid = props => {
- + {showSuggestion === true ?
diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 9048afcd..6d482c01 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -11871,7 +11871,7 @@ const AngularWorkflow = (defaultprops) => { diff --git a/frontend/src/views/DashboardViews.jsx b/frontend/src/views/DashboardViews.jsx index 1c8e1ac9..d2b0c043 100644 --- a/frontend/src/views/DashboardViews.jsx +++ b/frontend/src/views/DashboardViews.jsx @@ -349,7 +349,7 @@ const Dashboard = (props) => { const [workflows, setWorkflows] = useState([]); const [frameworkData, setFrameworkData] = useState(undefined); - const [widgetData, setWidgetData] = useState(undefined); + const [widgetData, setWidgetData] = useState([]); let navigate = useNavigate(); const isCloud = @@ -440,9 +440,16 @@ const Dashboard = (props) => { } } - console.log("VAl: ", tmpdata) + const foundWidget = widgetData.findIndex(data => data.title === widget) + console.log("Found: ", foundWidget) + if (foundWidget !== undefined && foundWidget !== null && foundWidget >= 0) { + widgetData[foundWidget] = tmpdata + } else { + widgetData.push(tmpdata) + } - setWidgetData(tmpdata) + console.log("Data: ", widgetData) + setWidgetData(widgetData) } }) .catch((error) => { @@ -470,89 +477,9 @@ const Dashboard = (props) => { setTreeKeys(treeCategories) } - const fetchUsecases = (workflows) => { - fetch(globalUrl + "/api/v1/workflows/usecases", { - method: "GET", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - credentials: "include", - }) - .then((response) => { - if (response.status !== 200) { - console.log("Status not 200 for usecases"); - } - - return response.json(); - }) - .then((responseJson) => { - // Matching workflows with usecases - if (responseJson.success !== false) { - if (workflows !== undefined && workflows !== null && workflows.length > 0) { - var categorydata = responseJson - - var newcategories = [] - for (var key in categorydata) { - var category = categorydata[key] - category.matches = [] - - for (var subcategorykey in category.list) { - var subcategory = category.list[subcategorykey] - subcategory.matches = [] - - for (var workflowkey in workflows) { - const workflow = workflows[workflowkey] - - if (workflow.usecase_ids !== undefined && workflow.usecase_ids !== null) { - for (var usecasekey in workflow.usecase_ids) { - if (workflow.usecase_ids[usecasekey].toLowerCase() === subcategory.name.toLowerCase()) { - - category.matches.push({ - "workflow": workflow.id, - "category": subcategory.name, - }) - - subcategory.matches.push(workflow) - break - } - } - } - - if (subcategory.matches.length > 0) { - break - } - } - } - - newcategories.push(category) - } - - if (newcategories !== undefined && newcategories !== null && newcategories.length > 0) { - handleKeysetting(newcategories) - setUsecases(newcategories) - setSelectedUsecases(newcategories) - } else { - handleKeysetting(responseJson) - setUsecases(responseJson) - setSelectedUsecases(responseJson) - } - } else { - handleKeysetting(responseJson) - setUsecases(responseJson) - setSelectedUsecases(responseJson) - } - } - }) - .catch((error) => { - //alert.error("ERROR: " + error.toString()); - console.log("ERROR: " + error.toString()); - }); - }; - useEffect(() => { - getWidget("main", "tmp") - //fetchUsecases() + getWidget("main", "Overall") + getWidget("main", "Overall2") }, []); const fetchdata = (stats_id) => { @@ -812,10 +739,10 @@ const Dashboard = (props) => { : null}
- {widgetData === undefined ? null : + {widgetData === undefined || widgetData === null || widgetData === [] || widgetData.length === 0 ? null : - + } diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 47b025fe..19a14c0a 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -3136,54 +3136,52 @@ const Workflows = (props) => { //if ((userdata.tutorials !== undefined && userdata.tutorials !== null && !userdata.tutorials.includes("getting-started")) || userdata.tutorials === null) { // return ; //} - - return ( -
- -
-

Welcome to Shuffle

-
-
-

- Shuffle is a flexible, easy to use, automation platform - allowing users to integrate their services and devices freely. - It's made to significantly reduce the amount of manual labor, - and is focused on security applications.{" "} - - Click here to learn more. - -

-
-
- If you want to jump straight into it, click here to create your - first workflow: -
-
- - - - ..OR - - {workflowButtons} - -
- -
- ) - + //return ( + //
+ // + //
+ //

Welcome to Shuffle

+ //
+ //
+ //

+ // Shuffle is a flexible, easy to use, automation platform + // allowing users to integrate their services and devices freely. + // It's made to significantly reduce the amount of manual labor, + // and is focused on security applications.{" "} + // + // Click here to learn more. + // + //

+ //
+ //
+ // If you want to jump straight into it, click here to create your + // first workflow: + //
+ //
+ // + // + // + // ..OR + // + // {workflowButtons} + // + //
+ // + //
+ //) } var workflowDelay = -150 From 7f52e8ed6d321473f04d2f76b849ee83c64a8d92 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 6 Dec 2022 11:02:57 +0000 Subject: [PATCH 135/142] Fixed usecase merging to go through cloud --- backend/go-app/go.mod | 102 ++++++++++++++---- backend/go-app/main.go | 43 ++++++-- docker-compose.yml | 120 +++++++++++----------- frontend/src/App.jsx | 5 +- frontend/src/components/UsecaseSearch.jsx | 119 +++++++++++---------- frontend/src/components/WelcomeForm2.jsx | 18 ++-- frontend/src/views/Welcome.jsx | 6 +- frontend/src/views/Workflows.jsx | 6 +- 8 files changed, 264 insertions(+), 155 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 2d498363..64335a3b 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -1,34 +1,100 @@ -module main +module shuffle -go 1.16 +go 1.19 //replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared -//replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi -//replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch require ( - cloud.google.com/go/datastore v1.6.0 - cloud.google.com/go/iam v0.1.1 // indirect - cloud.google.com/go/pubsub v1.17.1 - cloud.google.com/go/storage v1.18.2 + cloud.google.com/go/datastore v1.10.0 + cloud.google.com/go/pubsub v1.27.1 + cloud.google.com/go/storage v1.28.1 github.com/basgys/goxml2json v1.1.0 github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 - github.com/docker/docker v20.10.12+incompatible + github.com/docker/docker v20.10.19+incompatible github.com/frikky/kin-openapi v0.42.0 - github.com/fsouza/go-dockerclient v1.7.7 + github.com/fsouza/go-dockerclient v1.9.0 github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.3.1 - github.com/go-git/go-git/v5 v5.4.2 + github.com/go-git/go-git/v5 v5.5.0 github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.3 - github.com/nirasan/go-oauth-pkce-code-verifier v0.0.0-20170819232839-0fbfe93532da // indirect github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.3.24 - go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce - google.golang.org/api v0.65.0 + github.com/shuffle/shuffle-shared v0.3.33 + golang.org/x/crypto v0.3.0 + google.golang.org/api v0.103.0 google.golang.org/appengine v1.6.7 - google.golang.org/grpc v1.43.0 + google.golang.org/grpc v1.51.0 gopkg.in/src-d/go-git.v4 v4.13.1 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + cloud.google.com/go v0.105.0 // indirect + cloud.google.com/go/compute v1.12.1 // indirect + cloud.google.com/go/compute/metadata v0.2.1 // indirect + cloud.google.com/go/iam v0.7.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Masterminds/semver v1.5.0 // indirect + github.com/Microsoft/go-winio v0.6.0 // indirect + github.com/Microsoft/hcsshim v0.9.3 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect + github.com/acomagu/bufpipe v1.0.3 // indirect + github.com/adrg/strutil v0.3.0 // indirect + github.com/algolia/algoliasearch-client-go/v3 v3.26.1 // indirect + github.com/bradfitz/gomemcache v0.0.0-20221031212613-62deef7fc822 // indirect + github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect + github.com/cloudflare/circl v1.1.0 // indirect + github.com/containerd/cgroups v1.0.3 // indirect + github.com/containerd/containerd v1.6.6 // indirect + github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/frikky/go-elasticsearch/v8 v8.13.1 // indirect + github.com/go-git/gcfg v1.5.0 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/swag v0.19.5 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/go-github/v28 v28.1.1 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/googleapis/gax-go/v2 v2.7.0 // indirect + github.com/imdario/mergo v0.3.13 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/mailru/easyjson v0.7.0 // indirect + github.com/moby/sys/mount v0.3.3 // indirect + github.com/moby/sys/mountinfo v0.6.2 // indirect + github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect + github.com/opencontainers/runc v1.1.2 // indirect + github.com/patrickmn/go-cache v2.1.0+incompatible // indirect + github.com/pjbgf/sha1cd v0.2.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/sergi/go-diff v1.1.0 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect + github.com/skeema/knownhosts v1.1.0 // indirect + github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect + github.com/src-d/gcfg v1.4.0 // indirect + github.com/xanzy/ssh-agent v0.3.2 // indirect + go.opencensus.io v0.24.0 // indirect + go4.org v0.0.0-20201209231011-d4a079459e60 // indirect + golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect + golang.org/x/net v0.2.0 // indirect + golang.org/x/oauth2 v0.2.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.2.0 // indirect + golang.org/x/text v0.4.0 // indirect + golang.org/x/tools v0.1.12 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index cb1171b2..8192107a 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -691,13 +691,25 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) err = shuffle.SetUser(ctx, newUser, true) if err != nil { - log.Printf("Error adding User %s: %s", username, err) + log.Printf("[ERROR] Problem adding User %s: %s", username, err) return err } neworg, err := shuffle.GetOrg(ctx, org.Id) if err == nil { //neworg.Users = append(neworg.Users, *newUser) + for tutorialIndex, tutorial := range neworg.Tutorials { + if tutorial.Name == "Invite teammates" { + neworg.Tutorials[tutorialIndex].Description = fmt.Sprintf("%d users are in your org. Org name and Image change next.", len(neworg.Users)) + if len(neworg.Users) > 0 { + neworg.Tutorials[tutorialIndex].Done = true + neworg.Tutorials[tutorialIndex].Link = "/admin" + } + + break + } + } + err = shuffle.SetOrg(ctx, *neworg, neworg.Id) if err != nil { log.Printf("Failed updating org with user %s", newUser.Username) @@ -843,6 +855,8 @@ func handleCookie(request *http.Request) bool { return true } +// Returns whether the user is logged in or not etc. +// Also has more data about the user and org func handleInfo(resp http.ResponseWriter, request *http.Request) { cors := shuffle.HandleCors(resp, request) if cors { @@ -1050,9 +1064,21 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { } } - tutorialsFinished := userInfo.PersonalInfo.Tutorials + tutorialsFinished := []shuffle.Tutorial{} + for _, tutorial := range userInfo.PersonalInfo.Tutorials { + tutorialsFinished = append(tutorialsFinished, shuffle.Tutorial{ + Name: tutorial, + }) + } + if len(org.SecurityFramework.SIEM.Name) > 0 || len(org.SecurityFramework.Network.Name) > 0 || len(org.SecurityFramework.EDR.Name) > 0 || len(org.SecurityFramework.Cases.Name) > 0 || len(org.SecurityFramework.IAM.Name) > 0 || len(org.SecurityFramework.Assets.Name) > 0 || len(org.SecurityFramework.Intel.Name) > 0 || len(org.SecurityFramework.Communication.Name) > 0 { - tutorialsFinished = append(tutorialsFinished, "find_integrations") + tutorialsFinished = append(tutorialsFinished, shuffle.Tutorial{ + Name: "find_integrations", + }) + } + + for _, tutorial := range org.Tutorials { + tutorialsFinished = append(tutorialsFinished, tutorial) } returnValue := shuffle.HandleInfo{ @@ -1072,7 +1098,8 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { EthInfo: userInfo.EthInfo, ChatDisabled: chatDisabled, Tutorials: tutorialsFinished, - Priorities: orgPriorities, + + Priorities: orgPriorities, } returnData, err := json.Marshal(returnValue) @@ -1369,8 +1396,12 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { return } - // FIXME - have timeout here - tutorialsFinished := Userdata.PersonalInfo.Tutorials + tutorialsFinished := []shuffle.Tutorial{} + for _, tutorial := range Userdata.PersonalInfo.Tutorials { + tutorialsFinished = append(tutorialsFinished, shuffle.Tutorial{ + Name: tutorial, + }) + } returnValue := shuffle.HandleInfo{ Success: true, Tutorials: tutorialsFinished, diff --git a/docker-compose.yml b/docker-compose.yml index d056c922..cf4e3b47 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,65 +1,65 @@ version: '3' services: - frontend: - #build: ./frontend - image: ghcr.io/shuffle/shuffle-frontend:nightly - container_name: shuffle-frontend - hostname: shuffle-frontend - ports: - - "${FRONTEND_PORT}:80" - - "${FRONTEND_PORT_HTTPS}:443" - networks: - - shuffle - environment: - - BACKEND_HOSTNAME=${BACKEND_HOSTNAME} - restart: unless-stopped - depends_on: - - backend - backend: - #build: ./backend - image: ghcr.io/shuffle/shuffle-backend:nightly - container_name: shuffle-backend - hostname: ${BACKEND_HOSTNAME} - # Here for debugging: - ports: - - "${BACKEND_PORT}:5001" - networks: - - shuffle - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z - - ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z - #- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate - env_file: .env - environment: - #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - - SHUFFLE_FILE_LOCATION=/shuffle-files - restart: unless-stopped - orborus: - image: ghcr.io/shuffle/shuffle-orborus:nightly - container_name: shuffle-orborus - hostname: shuffle-orborus - networks: - - shuffle - volumes: - - /var/run/docker.sock:/var/run/docker.sock - environment: - #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - - SHUFFLE_WORKER_VERSION=latest - - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - - BASE_URL=http://${OUTER_HOSTNAME}:5001 - - DOCKER_API_VERSION=1.40 - - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} - - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} - - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} - - HTTP_PROXY=${HTTP_PROXY} - - HTTPS_PROXY=${HTTPS_PROXY} - - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} - - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY} - restart: unless-stopped - security_opt: - - seccomp:unconfined + #frontend: + # #build: ./frontend + # image: ghcr.io/shuffle/shuffle-frontend:nightly + # container_name: shuffle-frontend + # hostname: shuffle-frontend + # ports: + # - "${FRONTEND_PORT}:80" + # - "${FRONTEND_PORT_HTTPS}:443" + # networks: + # - shuffle + # environment: + # - BACKEND_HOSTNAME=${BACKEND_HOSTNAME} + # restart: unless-stopped + # depends_on: + # - backend + #backend: + # #build: ./backend + # image: ghcr.io/shuffle/shuffle-backend:nightly + # container_name: shuffle-backend + # hostname: ${BACKEND_HOSTNAME} + # # Here for debugging: + # ports: + # - "${BACKEND_PORT}:5001" + # networks: + # - shuffle + # volumes: + # - /var/run/docker.sock:/var/run/docker.sock + # - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z + # - ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z + # #- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate + # env_file: .env + # environment: + # #- DOCKER_HOST=tcp://docker-socket-proxy:2375 + # - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps + # - SHUFFLE_FILE_LOCATION=/shuffle-files + # restart: unless-stopped + #orborus: + # image: ghcr.io/shuffle/shuffle-orborus:nightly + # container_name: shuffle-orborus + # hostname: shuffle-orborus + # networks: + # - shuffle + # volumes: + # - /var/run/docker.sock:/var/run/docker.sock + # environment: + # #- DOCKER_HOST=tcp://docker-socket-proxy:2375 + # - SHUFFLE_WORKER_VERSION=latest + # - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} + # - BASE_URL=http://${OUTER_HOSTNAME}:5001 + # - DOCKER_API_VERSION=1.40 + # - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} + # - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} + # - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} + # - HTTP_PROXY=${HTTP_PROXY} + # - HTTPS_PROXY=${HTTPS_PROXY}:qa + # - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} + # - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY} + # restart: unless-stopped + # security_opt: + # - seccomp:unconfined opensearch: image: opensearchproject/opensearch:2.4.0 hostname: shuffle-opensearch diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index ab77202f..49d3566c 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -54,9 +54,10 @@ if (window.location.port === "3000") { //globalUrl = "http://localhost:5002" } -if (globalUrl.includes("githubpreview.dev")) { +// Development on Github Codespaces +if (globalUrl.includes("app.github.dev")) { //globalUrl = globalUrl.replace("3000", "5001") - globalUrl = "https://frikky-shuffle-5gvr4xx62w64-5001.githubpreview.dev" + globalUrl = "https://frikky-shuffle-5gvr4xx62w64-5001.preview.app.github.dev" } //console.log("global: ", globalUrl) diff --git a/frontend/src/components/UsecaseSearch.jsx b/frontend/src/components/UsecaseSearch.jsx index 2081a57e..2bf21a4e 100644 --- a/frontend/src/components/UsecaseSearch.jsx +++ b/frontend/src/components/UsecaseSearch.jsx @@ -786,7 +786,9 @@ const UsecaseSearch = (props) => { } const mergeWorkflowUsecases = (usecasedata) => { - const url = `${globalUrl}/api/v1/workflows/merge`; + //const url = `${globalUrl}/api/v1/workflows/merge`; + //const url = `https://0360-178-232-150-183.ngrok.io/api/v1/workflows/merge`; + const url = `https://shuffler.io/api/v1/workflows/merge`; fetch(url, { mode: "cors", method: "POST", @@ -802,76 +804,85 @@ const UsecaseSearch = (props) => { response.json().then((responseJson) => { setIsUploading(false) var changed = false - if (responseJson.success === false) { - if (responseJson.reason !== null && responseJson.reason !== undefined) { - //alert.error(responseJson.reason) - } - if (responseJson.source === "") { - const appname = - usecasedata.source.error = usecasedata.source.app_name === undefined ? "Select a Trigger workflow first" : `${usecasedata.source.app_name} has no public trigger workflow yet. Click this to try another app` - changed = true + if (responseJson.success === false) { + if (responseJson.reason !== null && responseJson.reason !== undefined) { + //alert.error(responseJson.reason) + } + + if (responseJson.source === "") { + const appname = + usecasedata.source.error = usecasedata.source.app_name === undefined ? "Select a Trigger workflow first" : `${usecasedata.source.app_name} has no public trigger workflow yet. Click this to try another app` + changed = true + } else { + usecasedata.source.error = "" + changed = true + } + + if (responseJson.destination === "") { + usecasedata.destination.error = usecasedata.destination.app_name === undefined ? "Select a Subflow first" : `${usecasedata.destination.app_name} has no public subflow yet. Click this to try another app` + changed = true } else { - usecasedata.source.error = "" + usecasedata.destination.error = "" changed = true } - if (responseJson.destination === "") { - usecasedata.destination.error = usecasedata.destination.app_name === undefined ? "Select a Subflow first" : `${usecasedata.destination.app_name} has no public subflow yet. Click this to try another app` - changed = true - } else { - usecasedata.destination.error = "" - changed = true - } + if (responseJson.middle !== undefined) { + for (var key in responseJson.middle) { + for (var subkey in usecasedata.middle) { + if (responseJson.middle[key] === usecasedata.middle[key].text) { + console.log("Found: ") + if (usecasedata.middle[subkey].app_name === undefined || usecasedata.middle[subkey].app_name === "") { + usecasedata.middle[subkey].error = `${usecasedata.middle[subkey].type} app must be selected first. Click this to change` + } else { + usecasedata.middle[subkey].error = `${usecasedata.middle[subkey].app_name} has no public subflow yet. Click this to change` + } - if (responseJson.middle !== undefined) { - for (var key in responseJson.middle) { - for (var subkey in usecasedata.middle) { - if (responseJson.middle[key] === usecasedata.middle[key].text) { - console.log("Found: ") - if (usecasedata.middle[subkey].app_name === undefined || usecasedata.middle[subkey].app_name === "") { - usecasedata.middle[subkey].error = `${usecasedata.middle[subkey].type} app must be selected first. Click this to change` - } else { - usecasedata.middle[subkey].error = `${usecasedata.middle[subkey].app_name} has no public subflow yet. Click this to change` + changed = true + break } - - changed = true - break } } } - } - } else { - if (responseJson.workflow_id !== null && responseJson.workflow_id !== undefined) { - if (responseJson.added_auth !== undefined && responseJson.added_auth !== null && responseJson.added_auth.length > 0) { - console.log("SHOULD HANDLE AUTH: ", responseJson.added_auth) - setConfigureWorkflowAuth(responseJson.added_auth) - - if (setFoundWorkflowId !== undefined) { - console.log("Set found workflow id: ", responseJson.workflow_id) - setFoundWorkflowId(responseJson.workflow_id) + } else { + if (!isCloud) { + console.log("Not cloud!") + setFoundWorkflowId(responseJson.id) + setWorkflow(responseJson) + } else if (isCloud) { + if (responseJson.workflow_id !== null && responseJson.workflow_id !== undefined) { + if (responseJson.added_auth !== undefined && responseJson.added_auth !== null && responseJson.added_auth.length > 0) { + console.log("SHOULD HANDLE AUTH: ", responseJson.added_auth) + setConfigureWorkflowAuth(responseJson.added_auth) + + if (setFoundWorkflowId !== undefined) { + console.log("Set found workflow id: ", responseJson.workflow_id) + setFoundWorkflowId(responseJson.workflow_id) + } + + getWorkflow(responseJson.workflow_id) + getAppAuthentication() + } else { + if (setFoundWorkflowId !== undefined) { + console.log("Set found workflow id: ", responseJson.workflow_id) + setFoundWorkflowId(responseJson.workflow_id) + } + + setWorkflow({"id": responseJson.workflow_id}) + } } - getWorkflow(responseJson.workflow_id) - getAppAuthentication() - } else { - if (setFoundWorkflowId !== undefined) { - console.log("Set found workflow id: ", responseJson.workflow_id) - setFoundWorkflowId(responseJson.workflow_id) + if (responseJson.auth_required === true) { + console.log("SET AUTH AS NEXT STEP!") } - setWorkflow({"id": responseJson.workflow_id}) + } } - - if (responseJson.auth_required === true) { - console.log("SET AUTH AS NEXT STEP!") + + if (changed === true) { + setAllUsecases([usecasedata]) } - } - - if (changed === true) { - setAllUsecases([usecasedata]) - } }) ) .catch((error) => { diff --git a/frontend/src/components/WelcomeForm2.jsx b/frontend/src/components/WelcomeForm2.jsx index 90f09a6a..99a5bba4 100644 --- a/frontend/src/components/WelcomeForm2.jsx +++ b/frontend/src/components/WelcomeForm2.jsx @@ -162,7 +162,6 @@ const WelcomeForm = (props) => { if (foundTab !== null && foundTab !== undefined && !isNaN(foundTab)) { if (foundTab === 3 || foundTab === "3") { console.log("SET SEARCH!!") - //setclickdiff(240) } } else { //navigate(`/welcome?tab=1`) @@ -298,7 +297,7 @@ const WelcomeForm = (props) => { if (activeStep === 0) { console.log("Should send basic information about org (fetch)") - setclickdiff(0) + setclickdiff(240) navigate(`/welcome?tab=2`) if (isCloud) { @@ -321,8 +320,6 @@ const WelcomeForm = (props) => { console.log("Should send secondary info about apps and other things") setDiscoveryWrapper({}) - setclickdiff(240) - //setclickdiff(0) navigate(`/welcome?tab=3`) //handleSetSearch("Enrichment", "2. Enrich") handleSetSearch(usecaseButtons[0].name, usecaseButtons[0].usecase) @@ -356,10 +353,8 @@ const WelcomeForm = (props) => { if (getFramework !== undefined) { getFramework() } - setclickdiff(0) navigate("/welcome?tab=2") } else if (activeStep === 1) { - //setclickdiff(0) navigate("/welcome?tab=1") } }; @@ -376,6 +371,7 @@ const WelcomeForm = (props) => { return newSkipped; }); }; + const handleReset = () => { setActiveStep(0); }; @@ -384,6 +380,7 @@ const WelcomeForm = (props) => { console.log("Selected app changed (effect)") }, [newSelectedApp]) + console.log("Clickdiff: ", clickdiff) //const buttonWidth = 145 const buttonWidth = 450 @@ -434,7 +431,7 @@ const WelcomeForm = (props) => { */} - First, we need some more information in order to understand how we best can help you find relevant Usecases. This is optional, but highly encouraged. + In order to understand how we best can help you find relevant Usecases, please provide the information below. This is optional, but highly encouraged. { disabled={activeStep === 0} onClick={handleBack} variant={"outlined"} - style={{marginLeft: 10, height: 64, width: 100, position: "absolute", top: activeStep === 1 ? -594 : -577, left: activeStep === 1 ? 105+clickdiff : -145+clickdiff, }} + style={{marginLeft: 10, height: 64, width: 100, position: "absolute", top: activeStep === 1 ? -600 : -577, left: activeStep === 1 ? 105 : -145+clickdiff, }} > Back @@ -769,7 +766,7 @@ const WelcomeForm = (props) => { variant={"outlined"} color="primary" onClick={handleNext} - style={{marginLeft: 10, height: 64, width: 100, position: "absolute", top: activeStep === 1 ? -594 : -577, left: activeStep === 1 ? 758+clickdiff : 510+clickdiff, }} + style={{marginLeft: 10, height: 64, width: 100, position: "absolute", top: activeStep === 1 ? -600: -577, left: activeStep === 1 ? 748 : 510+clickdiff, }} disabled={activeStep === 0 ? orgName.length === 0 || name.length === 0 : false} > {activeStep === steps.length - 1 ? "Finish" : "Next"} @@ -804,7 +801,8 @@ const WelcomeForm = (props) => { color="secondary" onClick={() => { console.log("Skip!") - + + setclickdiff(240) if (isCloud) { ReactGA.event({ category: "welcome", diff --git a/frontend/src/views/Welcome.jsx b/frontend/src/views/Welcome.jsx index e508e1ab..9cc36a1c 100644 --- a/frontend/src/views/Welcome.jsx +++ b/frontend/src/views/Welcome.jsx @@ -309,7 +309,7 @@ const Welcome = (props) => { } return ( -
+
{/*
@@ -317,7 +317,7 @@ const Welcome = (props) => { */} {showWelcome === true ?
-
+
{ action: "click_welcome_continue", label: "", }) + } else { + //setActiveStep(1) } setShowWelcome(true) diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 19a14c0a..efd44fb6 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -569,7 +569,6 @@ const Workflows = (props) => { const sidebarKey = "getting_started_sidebar" if (isLoggedIn === true && gettingStartedItems.length === 0 && (userdata.tutorials !== undefined && userdata.tutorials !== null && userdata.tutorials.length > 0) && workflowDone === true) { - const activeFiltered = userdata.tutorials.filter((item) => item.active === true) if (activeFiltered.length > 0) { var newfiltered = [] @@ -583,6 +582,7 @@ const Workflows = (props) => { newfiltered.push(activeFiltered[key]) } + setGettingStartedItems(activeFiltered) const doneFiltered = activeFiltered.filter((item) => item.done === true) @@ -1583,7 +1583,7 @@ const Workflows = (props) => { }; return ( - + { color: "white", fontSize: 18, borderLeft: theme.palette.defaultBorder, - marginTop: 64, + marginTop: 66, borderRadius: "5px 0px 0px 0px", }, }} From 829822fc20385102f22ba6f80cd286a01bb9a204 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 6 Dec 2022 11:59:30 +0000 Subject: [PATCH 136/142] Fixed workflow import issue --- backend/go-app/go.mod | 2 +- backend/go-app/walkoff.go | 22 +++++----------------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 64335a3b..da336df1 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -19,7 +19,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.3.33 + github.com/shuffle/shuffle-shared v0.3.34 golang.org/x/crypto v0.3.0 google.golang.org/api v0.103.0 google.golang.org/appengine v1.6.7 diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index d463ced6..1a8cba08 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -780,7 +780,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { if item.TriggerType == "SCHEDULE" && item.Status != "uninitialized" { err = deleteSchedule(ctx, item.ID) if err != nil { - log.Printf("Failed to delete schedule: %s - is it started?", err) + log.Printf("[DEBUG] Failed to delete schedule: %s - is it started?", err) } } else if item.TriggerType == "WEBHOOK" { //err = removeWebhookFunction(ctx, item.ID) @@ -790,7 +790,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { } else if item.TriggerType == "EMAIL" { err = shuffle.HandleOutlookSubRemoval(ctx, user, workflow.ID, item.ID) if err != nil { - log.Printf("Failed to delete OUTLOOK email sub (checking gmail after): %s", err) + log.Printf("[DEBUG] Failed to delete OUTLOOK email sub (checking gmail after): %s", err) } err = shuffle.HandleGmailSubRemoval(ctx, user, workflow.ID, item.ID) @@ -798,14 +798,8 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { log.Printf("Failed to delete gmail email sub: %s", err) } } - - //err = increaseStatisticsField(ctx, "total_workflow_triggers", workflow.ID, -1, workflow.OrgId) - //if err != nil { - // log.Printf("Failed to increase total workflows: %s", err) - //} } - // FIXME - maybe delete workflow executions err = shuffle.DeleteKey(ctx, "workflow", fileId) if err != nil { log.Printf("[DEBUG]] Failed deleting key %s", fileId) @@ -815,11 +809,6 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { } log.Printf("[INFO] Should have deleted workflow %s (%s)", workflow.Name, fileId) - //memcacheName := fmt.Sprintf("%s_%s", user.Username, fileId) - //memcache.Delete(ctx, memcacheName) - //memcacheName = fmt.Sprintf("%s_workflows", user.Username) - //memcache.Delete(ctx, memcacheName) - //cacheKey := fmt.Sprintf("%s_workflows", user.Id) cacheKey := fmt.Sprintf("%s_workflows", user.Id) shuffle.DeleteCache(ctx, cacheKey) log.Printf("[DEBUG] Cleared workflow cache for %s (%s)", user.Username, user.Id) @@ -1427,15 +1416,14 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { } func deleteSchedule(ctx context.Context, id string) error { - log.Printf("Should stop schedule %s!", id) + log.Printf("[DEBUG] Should stop schedule %s!", id) err := shuffle.DeleteKey(ctx, "schedules", id) if err != nil { - log.Printf("Failed to delete schedule: %s", err) + log.Printf("[ERROR] Failed to delete schedule: %s", err) return err } else { if value, exists := scheduledJobs[id]; exists { - log.Printf("STOPPING THIS SCHEDULE: %s", id) - // Looks like this does the trick? Hurr + // Stops the schedule properly value.Lock() } else { // FIXME - allow it to kind of stop anyway? From e9f730746b4295997bcea826625ccaa98d3fb9f9 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 6 Dec 2022 12:18:45 +0000 Subject: [PATCH 137/142] Fixed asearch grid linking to shuffler.io properly for public items --- frontend/src/components/AppGrid.jsx | 8 +- frontend/src/components/CreatorGrid.jsx | 104 ++++++++++++---------- frontend/src/components/WorkflowPaper.jsx | 16 +++- 3 files changed, 73 insertions(+), 55 deletions(-) diff --git a/frontend/src/components/AppGrid.jsx b/frontend/src/components/AppGrid.jsx index 54760432..562146c0 100644 --- a/frontend/src/components/AppGrid.jsx +++ b/frontend/src/components/AppGrid.jsx @@ -148,6 +148,8 @@ const AppGrid = props => { // setInnerHits(hits) //} + console.log("In appgrid") + return ( {hits.map((data, index) => { @@ -184,11 +186,11 @@ const AppGrid = props => { } parsedname = (parsedname.charAt(0).toUpperCase()+parsedname.substring(1)).replaceAll("_", " ") - + const appUrl = isCloud ? `/apps/${data.objectID}?queryID=${data.__queryID}` : `https://shuffler.io/apps/${data.objectID}?queryID=${data.__queryID}` return ( - + { setMouseHoverIndex(index) /* @@ -254,7 +256,7 @@ const AppGrid = props => { } - + ) diff --git a/frontend/src/components/CreatorGrid.jsx b/frontend/src/components/CreatorGrid.jsx index 6c83b134..66d852c5 100644 --- a/frontend/src/components/CreatorGrid.jsx +++ b/frontend/src/components/CreatorGrid.jsx @@ -53,6 +53,10 @@ const CreatorGrid = props => { const buttonStyle = {borderRadius: 30, height: 50, width: 220, margin: isMobile ? "15px auto 15px auto" : 20, fontSize: 18,} + const isCloud = + window.location.host === "localhost:3002" || + window.location.host === "shuffler.io"; + const innerColor = "rgba(255,255,255,0.65)" const borderRadius = 3 window.title = "Shuffle | Workflows | Discover your use-case" @@ -159,62 +163,64 @@ const CreatorGrid = props => { } counted += 1 + const creatorUrl = !isCloud ? `https://shuffler.io/creators/${data.username}` : `/creators/${data.username}` return ( - - -
- {"Creator - - @{data.username} - - - {data.verified === true ? - - - - : - null - } - -
- - {data.apps === undefined || data.apps === null ? 0 : data.apps} apps {data.workflows === null || data.workflows === undefined ? 0 : data.workflows} workflows - - {data.specialized_apps !== undefined && data.specialized_apps !== null && data.specialized_apps.length > 0 ? - - {data.specialized_apps.map((app, index) => { - // Putting all this in secondary of ListItemText looked weird. - return ( - + + {data.apps === undefined || data.apps === null ? 0 : data.apps} apps {data.workflows === null || data.workflows === undefined ? 0 : data.workflows} workflows + + {data.specialized_apps !== undefined && data.specialized_apps !== null && data.specialized_apps.length > 0 ? + + {data.specialized_apps.map((app, index) => { + // Putting all this in secondary of ListItemText looked weird. + return ( +
{ + console.log("Click") + //navigate("/apps/"+app.id) + }} + > + + + +
+ ) + })} + + : + null} + + + - ) diff --git a/frontend/src/components/WorkflowPaper.jsx b/frontend/src/components/WorkflowPaper.jsx index f0a06e87..558de437 100644 --- a/frontend/src/components/WorkflowPaper.jsx +++ b/frontend/src/components/WorkflowPaper.jsx @@ -63,6 +63,10 @@ const WorkflowPaper = (props) => { const [anchorEl, setAnchorEl] = React.useState(null); const appGroup = data.action_references === undefined || data.action_references === null ? [] : data.action_references + const isCloud = + window.location.host === "localhost:3002" || + window.location.host === "shuffler.io"; + //console.log("Workflow: ", data) var boxColor = "#86c142"; @@ -96,6 +100,10 @@ const WorkflowPaper = (props) => { parsedUrl += `?queryID=${data.__queryID}` } + if (!isCloud) { + parsedUrl = `https://shuffler.io${parsedUrl}` + } + return (
@@ -137,12 +145,14 @@ const WorkflowPaper = (props) => { flex: 10, }} > - {parsedName} - + From 73d81520a912702df47cd14c540f1f3bd05d8198 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 6 Dec 2022 14:51:21 +0000 Subject: [PATCH 138/142] Made automatic downloads of public OpenAPI apps build by themselves --- backend/go-app/docker.go | 105 ++++++++++++++++++++++++++++++++++++--- backend/go-app/main.go | 28 +++++------ 2 files changed, 112 insertions(+), 21 deletions(-) diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 38041ecf..f9c27eef 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -3,13 +3,16 @@ package main // Docker import ( "archive/tar" + "github.com/shuffle/shuffle-shared" + //"bufio" "path/filepath" //"strconv" "bytes" "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -756,7 +759,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { tagFound = version.Name } - buildSwaggerApp(resp, []byte(openApiApp.Body), user) + buildSwaggerApp(resp, []byte(openApiApp.Body), user, false) } } } @@ -802,6 +805,92 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { //resp.WriteHeader(200) } +// Downloads and activates an app from shuffler.io if possible +func handleRemoteDownloadApp(resp http.ResponseWriter, ctx context.Context, user shuffle.User, appId string) { + url := fmt.Sprintf("https://shuffler.io/api/v1/apps/%s/config", appId) + log.Printf("Downloading API from %s", url) + req, err := http.NewRequest( + "GET", + url, + nil, + ) + + if err != nil { + log.Printf("[ERROR] Failed auto-downloading app %s: %s", appId, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + return + } + + httpClient := &http.Client{} + newresp, err := httpClient.Do(req) + if err != nil { + log.Printf("[ERROR] Failed running auto-download request for %s: %s", appId, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + return + } + + respBody, err := ioutil.ReadAll(newresp.Body) + if err != nil { + log.Printf("[ERROR] Failed setting respbody for workflow download: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + return + } + + if len(respBody) > 0 { + type tmpapp struct { + Success bool `json:"success"` + OpenAPI string `json:"openapi"` + } + + app := tmpapp{} + err := json.Unmarshal(respBody, &app) + if err != nil || app.Success == false || len(app.OpenAPI) == 0 { + log.Printf("[ERROR] Failed app unmarshal during auto-download. Success%#v. Applength: %d: %s", app.Success, len(app.OpenAPI), err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + return + } + + key, err := base64.StdEncoding.DecodeString(app.OpenAPI) + if err != nil { + log.Printf("[ERROR] Failed auto-setting OpenAPI app: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + return + } + + cacheKey := fmt.Sprintf("workflowapps-sorted-100") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-1000") + shuffle.DeleteCache(ctx, cacheKey) + + newapp := shuffle.ParsedOpenApi{} + err = json.Unmarshal(key, &newapp) + if err != nil { + log.Printf("[ERROR] Failed openapi unmarshal during auto-download: %s", app.Success, len(app.OpenAPI), err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + return + } + + err = json.Unmarshal(key, &newapp) + if err != nil { + log.Printf("[ERROR] Failed openapi unmarshal during auto-download: %s", app.Success, len(app.OpenAPI), err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + return + } + + buildSwaggerApp(resp, []byte(newapp.Body), user, true) + return + } +} + func activateWorkflowAppDocker(resp http.ResponseWriter, request *http.Request) { cors := shuffle.HandleCors(resp, request) if cors { @@ -845,9 +934,9 @@ func activateWorkflowAppDocker(resp http.ResponseWriter, request *http.Request) apps, err := shuffle.FindWorkflowAppByName(ctx, appName) //log.Printf("[INFO] Found %d apps for %s", len(apps), appName) if err != nil || len(apps) == 0 { - log.Printf("[WARNING] Error getting app %s (app config): %s", appName, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + log.Printf("[WARNING] Error getting app %s (app config). Starting remote download.: %s", appName, err) + + handleRemoteDownloadApp(resp, ctx, user, fileId) return } @@ -868,10 +957,12 @@ func activateWorkflowAppDocker(resp http.ResponseWriter, request *http.Request) app = &selectedApp } else { - log.Printf("[WARNING] Error getting app with ID %s (app config): %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + log.Printf("[WARNING] Error getting app with ID %s (app config): %s. Starting remote download(2)", fileId, err) + handleRemoteDownloadApp(resp, ctx, user, fileId) return + //resp.WriteHeader(401) + //resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) + //return } } diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 8192107a..4c9b21b5 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3153,18 +3153,19 @@ func handleSwaggerValidation(body []byte) (shuffle.ParsedOpenApi, error) { return parsed, err } -func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) { +func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User, skipEdit bool) { type Test struct { - Editing bool `datastore:"editing"` - Id string `datastore:"id"` - Image string `datastore:"image"` + Editing bool `json:"editing" datastore:"editing"` + Id string `json:"id" datastore:"id"` + Image string `json:"image" datastore:"image"` + Body string `json:"body" datastore:"body"` } var test Test err := json.Unmarshal(body, &test) if err != nil { - log.Printf("[WARNING] Failed unmarshalling test: %s", err) - resp.WriteHeader(401) + log.Printf("[ERROR] Failed unmarshalling in swagger build: %s", err) + resp.WriteHeader(400) resp.Write([]byte(`{"success": false}`)) return } @@ -3174,13 +3175,13 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) { hasher.Write(body) newmd5 := hex.EncodeToString(hasher.Sum(nil)) - if test.Editing && len(user.Id) > 0 { + if test.Editing && len(user.Id) > 0 && skipEdit != true { // Quick verification test ctx := context.Background() app, err := shuffle.GetApp(ctx, test.Id, user, false) if err != nil { - log.Printf("[WARNING] Error getting app when editing: %s", app.Name) - resp.WriteHeader(401) + log.Printf("[ERROR] Error getting app when editing: %s", app.Name) + resp.WriteHeader(400) resp.Write([]byte(`{"success": false}`)) return } @@ -3188,7 +3189,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) { // FIXME: Check whether it's in use. if user.Id != app.Owner && user.Role != "admin" { log.Printf("[WARNING] Wrong user (%s) for app %s when verifying swagger", user.Username, app.Name) - resp.WriteHeader(401) + resp.WriteHeader(400) resp.Write([]byte(`{"success": false}`)) return } @@ -3212,7 +3213,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) { } if swagger.Info == nil { - log.Printf("[ERORR] Info is nil?: %#v", swagger) + log.Printf("[ERORR] Info is nil in swagger?") resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Info not parsed"}`)) return @@ -3322,7 +3323,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) { //log.Println(stitched) // 3. Zip and stream it directly in the directory - _, err = shuffle.StreamZipdata(ctx, identifier, stitched, "requests\nurllib3", "") + _, err = shuffle.StreamZipdata(ctx, identifier, stitched, shuffle.GetAppRequirements(), "") if err != nil { log.Printf("[ERROR] Zipfile error: %s", err) resp.WriteHeader(500) @@ -3473,7 +3474,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { return } - buildSwaggerApp(resp, body, user) + buildSwaggerApp(resp, body, user, false) } // Creates osfs from folderpath with a basepath as directory base @@ -3544,7 +3545,6 @@ func handleAppHotload(ctx context.Context, location string, forceUpdate bool) er return err } - //log.Printf("Reading app folder: %#v", dir) _, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate) if err != nil { log.Printf("[WARNING] Githubfolders error: %s", err) From 8c21ad07edb940431ab29ee189a031ad4fd304d4 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 6 Dec 2022 14:51:51 +0000 Subject: [PATCH 139/142] Made automatic builds of apps happen when an app is selected onprem --- frontend/src/components/AppFramework.jsx | 41 ++++- frontend/src/components/ConfigureWorkflow.jsx | 22 +++ frontend/src/components/UsecaseSearch.jsx | 166 +++++++++++++++++- frontend/src/components/WelcomeForm2.jsx | 4 +- 4 files changed, 225 insertions(+), 8 deletions(-) diff --git a/frontend/src/components/AppFramework.jsx b/frontend/src/components/AppFramework.jsx index 8a51c521..43caf58c 100644 --- a/frontend/src/components/AppFramework.jsx +++ b/frontend/src/components/AppFramework.jsx @@ -1,15 +1,16 @@ import React, { useState, useEffect } from 'react'; -import { securityFramework} from "./LandingpageUsecases.jsx"; import CytoscapeComponent from 'react-cytoscapejs'; import frameworkStyle from '../frameworkStyle.jsx'; -import AppSearch from './Appsearch.jsx'; import { v4 as uuidv4 } from "uuid"; import theme from '../theme'; import { useAlert } from "react-alert"; -import { usecaseTypes } from "../components/UsecaseSearch.jsx" + +import AppSearch from '../components/Appsearch.jsx'; import PaperComponent from "../components/PaperComponent.jsx" +import { usecaseTypes } from "../components/UsecaseSearch.jsx" import SuggestedWorkflows from "../components/SuggestedWorkflows.jsx" +import { securityFramework} from "../components/LandingpageUsecases.jsx"; import { Paper, @@ -751,7 +752,41 @@ const AppFramework = (props) => { }) } + const activateApp = (appid) => { + fetch(globalUrl+"/api/v1/apps/"+appid+"/activate", { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Failed to activate") + } + + return response.json() + }) + .then((responseJson) => { + if (responseJson.success === false) { + alert.error("Failed to activate the app") + } else { + //alert.success("App activated for your organization! Refresh the page to use the app.") + } + }) + .catch(error => { + //alert.error(error.toString()) + console.log("Activate app error: ", error.toString()) + }); + } + const setFrameworkItem = (data) => { + console.log("Setting framework item: ", data, isCloud) + if (!isCloud) { + activateApp(data.id) + } + fetch(globalUrl + "/api/v1/apps/frameworkConfiguration", { method: "POST", headers: { diff --git a/frontend/src/components/ConfigureWorkflow.jsx b/frontend/src/components/ConfigureWorkflow.jsx index 09fb5560..0e5cc988 100644 --- a/frontend/src/components/ConfigureWorkflow.jsx +++ b/frontend/src/components/ConfigureWorkflow.jsx @@ -22,6 +22,7 @@ import { CheckCircleRounded as CheckCircleRoundedIcon, } from "@mui/icons-material"; import { FixName } from "../views/Apps.jsx"; +import aa from 'search-insights' // Handles workflow updates on first open to highlight the issues of the workflow // Variables @@ -32,6 +33,7 @@ import { FixName } from "../views/Apps.jsx"; // Specifically used for UNSAVED workflows only? const ConfigureWorkflow = (props) => { const { + userdata, globalUrl, theme, workflow, @@ -526,6 +528,26 @@ const ConfigureWorkflow = (props) => { }; const activateApp = (app_id, app_name, app_version) => { + + if (aa !== undefined) { + aa('init', { + appId: "JNSS5CFDZZ", + apiKey: "db08e40265e2941b9a7d8f644b6e5240", + }) + + const timestamp = new Date().getTime() + aa('sendEvents', [ + { + eventType: 'conversion', + eventName: 'Public App Activated', + index: 'appsearch', + objectIDs: [app_id], + timestamp: timestamp, + userToken: userdata === undefined || userdata === null || userdata.id === undefined ? "unauthenticated" : userdata.id, + } + ]) + } + fetch( `${globalUrl}/api/v1/apps/${app_id}/activate?app_name=${app_name}&app_version=${app_version}`, { diff --git a/frontend/src/components/UsecaseSearch.jsx b/frontend/src/components/UsecaseSearch.jsx index 2bf21a4e..e9edb0f4 100644 --- a/frontend/src/components/UsecaseSearch.jsx +++ b/frontend/src/components/UsecaseSearch.jsx @@ -1,5 +1,6 @@ import React, { useState, useEffect } from "react"; import theme from '../theme'; +import { useNavigate, Link } from "react-router-dom"; import { useAlert } from "react-alert"; import ConfigureWorkflow from "../components/ConfigureWorkflow.jsx"; import PaperComponent from "../components/PaperComponent.jsx" @@ -785,9 +786,90 @@ const UsecaseSearch = (props) => { }) } + // Stolen from /views/Workflows + // Due to states, not easy to just import as component~ + const setNewWorkflow = ( + name, + description, + tags, + defaultReturnValue, + editingWorkflow, + redirect, + currentUsecases, + inputblogpost, + inputstatus, + ) => { + var method = "POST"; + var extraData = ""; + var workflowdata = {}; + + if (editingWorkflow.id !== undefined) { + console.log("Building original workflow"); + method = "PUT"; + extraData = "/" + editingWorkflow.id + "?skip_save=true"; + workflowdata = editingWorkflow; + + console.log("REMOVING OWNER"); + workflowdata["owner"] = ""; + // FIXME: Loop triggers and turn them off? + } + + workflowdata["name"] = name; + workflowdata["description"] = description; + if (tags !== undefined) { + workflowdata["tags"] = tags; + } + workflowdata["blogpost"] = inputblogpost + workflowdata["status"] = inputstatus + + if (defaultReturnValue !== undefined) { + workflowdata["default_return_value"] = defaultReturnValue; + } + + if (currentUsecases !== undefined && currentUsecases !== null) { + workflowdata["usecase_ids"] = currentUsecases + //workflows[0].category = ["detect"] + //workflows[0].usecase_ids = ["Correlate tickets"] + } + + const new_url = `${globalUrl}/api/v1/workflows${extraData}` + return fetch(new_url, { + method: method, + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(workflowdata), + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for workflows :O!"); + return; + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success === false) { + if (responseJson.reason !== undefined) { + alert.error("Error setting workflow: ", responseJson.reason) + } else { + alert.error("Error setting workflow.") + } + + return + } + + return responseJson; + }) + .catch((error) => { + alert.error(error.toString()); + }); + } + const mergeWorkflowUsecases = (usecasedata) => { //const url = `${globalUrl}/api/v1/workflows/merge`; - //const url = `https://0360-178-232-150-183.ngrok.io/api/v1/workflows/merge`; const url = `https://shuffler.io/api/v1/workflows/merge`; fetch(url, { mode: "cors", @@ -845,10 +927,57 @@ const UsecaseSearch = (props) => { } } } else { + // Gets a full workflow that has to be handled from cloud directly + // if (!isCloud) { console.log("Not cloud!") - setFoundWorkflowId(responseJson.id) + if (setFoundWorkflowId !== undefined) { + setFoundWorkflowId(responseJson.id) + } setWorkflow(responseJson) + + setNewWorkflow( + responseJson.name, + responseJson.description, + responseJson.tags, + responseJson.default_return_value, + {}, + false, + [], + "", + responseJson.status, + ) + .then((response) => { + if (response !== undefined) { + // SET THE FULL THING + responseJson.id = response.id; + responseJson.first_save = false; + responseJson.previously_saved = false; + responseJson.is_valid = false; + + // Actually create it + setNewWorkflow( + responseJson.name, + responseJson.description, + responseJson.tags, + responseJson.default_return_value, + responseJson, + false, + [], + "", + responseJson.status, + ).then((response) => { + if (response !== undefined) { + alert.success("Successfully generated " + responseJson.name); + } + }); + } + }) + .catch((error) => { + alert.error("Generate error: " + error.toString()); + }) + + } else if (isCloud) { if (responseJson.workflow_id !== null && responseJson.workflow_id !== undefined) { if (responseJson.added_auth !== undefined && responseJson.added_auth !== null && responseJson.added_auth.length > 0) { @@ -1077,8 +1206,41 @@ const UsecaseSearch = (props) => { }) //console.log("Data: ", data) + const activateApp = (appid) => { + fetch(globalUrl+"/api/v1/apps/"+appid+"/activate", { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Failed to activate") + } + + return response.json() + }) + .then((responseJson) => { + if (responseJson.success === false) { + alert.error("Failed to activate the app") + } else { + //alert.success("App activated for your organization! Refresh the page to use the app.") + } + }) + .catch(error => { + //alert.error(error.toString()) + console.log("Activate app error: ", error.toString()) + }); + } const setFrameworkItem = (data) => { + // Making sure the app is being auto-built and added + if (!isCloud) { + activateApp(data.id) + } + fetch(globalUrl + "/api/v1/apps/frameworkConfiguration", { method: "POST", headers: { diff --git a/frontend/src/components/WelcomeForm2.jsx b/frontend/src/components/WelcomeForm2.jsx index 99a5bba4..a3ccf86d 100644 --- a/frontend/src/components/WelcomeForm2.jsx +++ b/frontend/src/components/WelcomeForm2.jsx @@ -161,7 +161,7 @@ const WelcomeForm = (props) => { const foundTab = params["tab"]; if (foundTab !== null && foundTab !== undefined && !isNaN(foundTab)) { if (foundTab === 3 || foundTab === "3") { - console.log("SET SEARCH!!") + //console.log("Set search!") } } else { //navigate(`/welcome?tab=1`) @@ -380,8 +380,6 @@ const WelcomeForm = (props) => { console.log("Selected app changed (effect)") }, [newSelectedApp]) - console.log("Clickdiff: ", clickdiff) - //const buttonWidth = 145 const buttonWidth = 450 const buttonMargin = 10 From 23c0d7e2ac91f865a8a7c258decd050ca3b987ae Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 6 Dec 2022 14:52:13 +0000 Subject: [PATCH 140/142] Fixed import/export issues with subflows and parent workflows --- frontend/src/views/AngularWorkflow.jsx | 21 +++++----- frontend/src/views/Welcome.jsx | 30 +++++++------- frontend/src/views/Workflows.jsx | 56 +++++++++++++------------- 3 files changed, 54 insertions(+), 53 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 6d482c01..638234d5 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -6532,15 +6532,15 @@ const AngularWorkflow = (defaultprops) => { return (
{ - if (!isCloud) { - alert.info("Since this is an on-prem instance. You will need to activate the app yourself. Opening link to download it in a new window.") - setTimeout(() => { - event.preventDefault() - window.open(parsedUrl, '_blank') - }, 2000) - } else { - alert.info(`Activating ${name}`) - } + //if (!isCloud) { + // alert.info("Since this is an on-prem instance. You will need to activate the app yourself. Opening link to download it in a new window.") + // setTimeout(() => { + // event.preventDefault() + // window.open(parsedUrl, '_blank') + // }, 2000) + //} else { + alert.info(`Activating ${name}`) + //} console.log("CLICK: ", hit) @@ -6679,7 +6679,7 @@ const AngularWorkflow = (defaultprops) => { }} > - Couldn't find an Activated app with that name. Searching unactivated apps. Click one to Activate it for your organization. + Couldn't find the app you're looking for? Searching unactivated apps. Click one of the below apps to Activate it for your organization. { console.log("CLICKED") @@ -14837,6 +14837,7 @@ const AngularWorkflow = (defaultprops) => { {
diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index efd44fb6..df9ffad4 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -2271,35 +2271,35 @@ const Workflows = (props) => { "", data.status, ) - .then((response) => { - if (response !== undefined) { - // SET THE FULL THING - data.id = response.id; - data.first_save = false; - data.previously_saved = false; - data.is_valid = false; + .then((response) => { + if (response !== undefined) { + // SET THE FULL THING + data.id = response.id; + data.first_save = false; + data.previously_saved = false; + data.is_valid = false; - // Actually create it - setNewWorkflow( - data.name, - data.description, - data.tags, - data.default_return_value, - data, - false, - [], - "", - data.status, - ).then((response) => { - if (response !== undefined) { - alert.success("Successfully imported " + data.name); - } - }); - } - }) - .catch((error) => { - alert.error("Import error: " + error.toString()); - }); + // Actually create it + setNewWorkflow( + data.name, + data.description, + data.tags, + data.default_return_value, + data, + false, + [], + "", + data.status, + ).then((response) => { + if (response !== undefined) { + alert.success("Successfully imported " + data.name); + } + }); + } + }) + .catch((error) => { + alert.error("Import error: " + error.toString()); + }); }); // Actually reads From 4630c4a0ac95a0103929b897c89a246fc5c4e637 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 6 Dec 2022 16:49:27 +0000 Subject: [PATCH 141/142] Made changes to CGO build process for newever Shuffle version --- backend/Dockerfile | 4 +- backend/go-app/go.mod | 77 ++------------------------- docker-compose.yml | 120 +++++++++++++++++++++--------------------- 3 files changed, 66 insertions(+), 135 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index f09fe3b3..8c7381ed 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -15,8 +15,10 @@ ADD ./app_sdk/app_base.py /app_sdk ADD ./app_gen /app_gen RUN go get -v +RUN go mod tidy +RUN go clean -modcache -RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o webapp . +RUN CGO_ENABLED=1 GOOS=linux go build -a -installsuffix cgo -o webapp . # Certificate build - gets required certs FROM alpine:latest as certs diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index da336df1..0a3505d6 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -1,16 +1,16 @@ module shuffle -go 1.19 +go 1.16 //replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared require ( cloud.google.com/go/datastore v1.10.0 - cloud.google.com/go/pubsub v1.27.1 + cloud.google.com/go/pubsub v1.28.0 cloud.google.com/go/storage v1.28.1 github.com/basgys/goxml2json v1.1.0 github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 - github.com/docker/docker v20.10.19+incompatible + github.com/docker/docker v20.10.21+incompatible github.com/frikky/kin-openapi v0.42.0 github.com/fsouza/go-dockerclient v1.9.0 github.com/ghodss/yaml v1.0.0 @@ -27,74 +27,3 @@ require ( gopkg.in/src-d/go-git.v4 v4.13.1 gopkg.in/yaml.v3 v3.0.1 ) - -require ( - cloud.google.com/go v0.105.0 // indirect - cloud.google.com/go/compute v1.12.1 // indirect - cloud.google.com/go/compute/metadata v0.2.1 // indirect - cloud.google.com/go/iam v0.7.0 // indirect - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/Masterminds/semver v1.5.0 // indirect - github.com/Microsoft/go-winio v0.6.0 // indirect - github.com/Microsoft/hcsshim v0.9.3 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect - github.com/acomagu/bufpipe v1.0.3 // indirect - github.com/adrg/strutil v0.3.0 // indirect - github.com/algolia/algoliasearch-client-go/v3 v3.26.1 // indirect - github.com/bradfitz/gomemcache v0.0.0-20221031212613-62deef7fc822 // indirect - github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect - github.com/cloudflare/circl v1.1.0 // indirect - github.com/containerd/cgroups v1.0.3 // indirect - github.com/containerd/containerd v1.6.6 // indirect - github.com/docker/distribution v2.8.1+incompatible // indirect - github.com/docker/go-connections v0.4.0 // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/emirpasic/gods v1.18.1 // indirect - github.com/frikky/go-elasticsearch/v8 v8.13.1 // indirect - github.com/go-git/gcfg v1.5.0 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/swag v0.19.5 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/google/go-cmp v0.5.9 // indirect - github.com/google/go-github/v28 v28.1.1 // indirect - github.com/google/go-querystring v1.1.0 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect - github.com/googleapis/gax-go/v2 v2.7.0 // indirect - github.com/imdario/mergo v0.3.13 // indirect - github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect - github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/mailru/easyjson v0.7.0 // indirect - github.com/moby/sys/mount v0.3.3 // indirect - github.com/moby/sys/mountinfo v0.6.2 // indirect - github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect - github.com/morikuni/aec v1.0.0 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect - github.com/opencontainers/runc v1.1.2 // indirect - github.com/patrickmn/go-cache v2.1.0+incompatible // indirect - github.com/pjbgf/sha1cd v0.2.0 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/sergi/go-diff v1.1.0 // indirect - github.com/sirupsen/logrus v1.8.1 // indirect - github.com/skeema/knownhosts v1.1.0 // indirect - github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect - github.com/src-d/gcfg v1.4.0 // indirect - github.com/xanzy/ssh-agent v0.3.2 // indirect - go.opencensus.io v0.24.0 // indirect - go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/net v0.2.0 // indirect - golang.org/x/oauth2 v0.2.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.2.0 // indirect - golang.org/x/text v0.4.0 // indirect - golang.org/x/tools v0.1.12 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c // indirect - google.golang.org/protobuf v1.28.1 // indirect - gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect -) diff --git a/docker-compose.yml b/docker-compose.yml index cf4e3b47..c4eb10ef 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,65 +1,65 @@ version: '3' services: - #frontend: - # #build: ./frontend - # image: ghcr.io/shuffle/shuffle-frontend:nightly - # container_name: shuffle-frontend - # hostname: shuffle-frontend - # ports: - # - "${FRONTEND_PORT}:80" - # - "${FRONTEND_PORT_HTTPS}:443" - # networks: - # - shuffle - # environment: - # - BACKEND_HOSTNAME=${BACKEND_HOSTNAME} - # restart: unless-stopped - # depends_on: - # - backend - #backend: - # #build: ./backend - # image: ghcr.io/shuffle/shuffle-backend:nightly - # container_name: shuffle-backend - # hostname: ${BACKEND_HOSTNAME} - # # Here for debugging: - # ports: - # - "${BACKEND_PORT}:5001" - # networks: - # - shuffle - # volumes: - # - /var/run/docker.sock:/var/run/docker.sock - # - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z - # - ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z - # #- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate - # env_file: .env - # environment: - # #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - # - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - # - SHUFFLE_FILE_LOCATION=/shuffle-files - # restart: unless-stopped - #orborus: - # image: ghcr.io/shuffle/shuffle-orborus:nightly - # container_name: shuffle-orborus - # hostname: shuffle-orborus - # networks: - # - shuffle - # volumes: - # - /var/run/docker.sock:/var/run/docker.sock - # environment: - # #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - # - SHUFFLE_WORKER_VERSION=latest - # - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - # - BASE_URL=http://${OUTER_HOSTNAME}:5001 - # - DOCKER_API_VERSION=1.40 - # - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} - # - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} - # - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} - # - HTTP_PROXY=${HTTP_PROXY} - # - HTTPS_PROXY=${HTTPS_PROXY}:qa - # - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} - # - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY} - # restart: unless-stopped - # security_opt: - # - seccomp:unconfined + frontend: + #build: ./frontend + image: ghcr.io/shuffle/shuffle-frontend:nightly + container_name: shuffle-frontend + hostname: shuffle-frontend + ports: + - "${FRONTEND_PORT}:80" + - "${FRONTEND_PORT_HTTPS}:443" + networks: + - shuffle + environment: + - BACKEND_HOSTNAME=${BACKEND_HOSTNAME} + restart: unless-stopped + depends_on: + - backend + backend: + build: ./backend + image: ghcr.io/shuffle/shuffle-backend:nightly + container_name: shuffle-backend + hostname: ${BACKEND_HOSTNAME} + # Here for debugging: + ports: + - "${BACKEND_PORT}:5001" + networks: + - shuffle + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z + - ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z + #- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate + env_file: .env + environment: + #- DOCKER_HOST=tcp://docker-socket-proxy:2375 + - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps + - SHUFFLE_FILE_LOCATION=/shuffle-files + restart: unless-stopped + orborus: + image: ghcr.io/shuffle/shuffle-orborus:nightly + container_name: shuffle-orborus + hostname: shuffle-orborus + networks: + - shuffle + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + #- DOCKER_HOST=tcp://docker-socket-proxy:2375 + - SHUFFLE_WORKER_VERSION=latest + - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} + - BASE_URL=http://${OUTER_HOSTNAME}:5001 + - DOCKER_API_VERSION=1.40 + - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} + - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} + - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} + - HTTP_PROXY=${HTTP_PROXY} + - HTTPS_PROXY=${HTTPS_PROXY} + - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} + - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY} + restart: unless-stopped + security_opt: + - seccomp:unconfined opensearch: image: opensearchproject/opensearch:2.4.0 hostname: shuffle-opensearch From 078f40a88112bb0216db2242aae75b6263b677c6 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 6 Dec 2022 19:21:50 +0100 Subject: [PATCH 142/142] Fixed CGO issues with Go 1.19 & Alpine containers for backend --- .github/push_nightly.sh | 10 +++--- backend/Dockerfile | 12 +++++-- backend/go-app/go.mod | 77 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 89 insertions(+), 10 deletions(-) diff --git a/.github/push_nightly.sh b/.github/push_nightly.sh index 74b0c2b3..92d5320b 100644 --- a/.github/push_nightly.sh +++ b/.github/push_nightly.sh @@ -2,11 +2,11 @@ # Done manually for now since GHCR isn't being pushed to easily with the current Github action CI. Nightly = Latest IF we run hotfixes on latest ### Pull latest from ghcr CI/CD -docker pull ghcr.io/shuffle/shuffle-app_sdk:nightly -docker pull ghcr.io/shuffle/shuffle-worker:nightly -docker pull ghcr.io/shuffle/shuffle-orborus:nightly -docker pull ghcr.io/shuffle/shuffle-frontend:nightly -docker pull ghcr.io/shuffle/shuffle-backend:nightly +#docker pull ghcr.io/shuffle/shuffle-app_sdk:nightly +#docker pull ghcr.io/shuffle/shuffle-worker:nightly +#docker pull ghcr.io/shuffle/shuffle-orborus:nightly +#docker pull ghcr.io/shuffle/shuffle-frontend:nightly +#docker pull ghcr.io/shuffle/shuffle-backend:nightly ### LATEST releases: #docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly ghcr.io/shuffle/shuffle-app_sdk:latest diff --git a/backend/Dockerfile b/backend/Dockerfile index 8c7381ed..5bc61f64 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.2-buster as builder +FROM golang:1.19.3-buster as builder # Add files RUN mkdir /app @@ -18,13 +18,21 @@ RUN go get -v RUN go mod tidy RUN go clean -modcache +# From November 2022, CGO is enabled due to packages +# that we use requiring it. This is a temporary fix +# and makes us HAVE to install libc compatibility packages farther down. RUN CGO_ENABLED=1 GOOS=linux go build -a -installsuffix cgo -o webapp . # Certificate build - gets required certs FROM alpine:latest as certs RUN apk --update add ca-certificates -FROM alpine:3.14.2 +# Sets up the final image +FROM alpine:3.17.0 + +# FIXME: Install cgo because CGO_ENABLED=1 during build +RUN apk add --no-cache libc6-compat +RUN apk add --no-cache libstdc++ COPY --from=builder /app/ /app COPY --from=builder /app_sdk/ /app_sdk diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 0a3505d6..822b91e0 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -1,6 +1,6 @@ -module shuffle +module main -go 1.16 +go 1.19 //replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared @@ -19,7 +19,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.3.34 + github.com/shuffle/shuffle-shared v0.3.35 golang.org/x/crypto v0.3.0 google.golang.org/api v0.103.0 google.golang.org/appengine v1.6.7 @@ -27,3 +27,74 @@ require ( gopkg.in/src-d/go-git.v4 v4.13.1 gopkg.in/yaml.v3 v3.0.1 ) + +require ( + cloud.google.com/go v0.105.0 // indirect + cloud.google.com/go/compute v1.13.0 // indirect + cloud.google.com/go/compute/metadata v0.2.1 // indirect + cloud.google.com/go/iam v0.7.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Masterminds/semver v1.5.0 // indirect + github.com/Microsoft/go-winio v0.6.0 // indirect + github.com/Microsoft/hcsshim v0.9.3 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect + github.com/acomagu/bufpipe v1.0.3 // indirect + github.com/adrg/strutil v0.2.3 // indirect + github.com/algolia/algoliasearch-client-go/v3 v3.18.1 // indirect + github.com/bradfitz/gomemcache v0.0.0-20221031212613-62deef7fc822 // indirect + github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect + github.com/cloudflare/circl v1.1.0 // indirect + github.com/containerd/cgroups v1.0.3 // indirect + github.com/containerd/containerd v1.6.6 // indirect + github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/frikky/go-elasticsearch/v8 v8.13.1 // indirect + github.com/go-git/gcfg v1.5.0 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/swag v0.19.5 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/go-github/v28 v28.1.1 // indirect + github.com/google/go-querystring v1.0.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/googleapis/gax-go/v2 v2.7.0 // indirect + github.com/imdario/mergo v0.3.13 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/mailru/easyjson v0.7.0 // indirect + github.com/moby/sys/mount v0.3.3 // indirect + github.com/moby/sys/mountinfo v0.6.2 // indirect + github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect + github.com/opencontainers/runc v1.1.2 // indirect + github.com/patrickmn/go-cache v2.1.0+incompatible // indirect + github.com/pjbgf/sha1cd v0.2.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/sergi/go-diff v1.1.0 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect + github.com/skeema/knownhosts v1.1.0 // indirect + github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect + github.com/src-d/gcfg v1.4.0 // indirect + github.com/xanzy/ssh-agent v0.3.2 // indirect + go.opencensus.io v0.24.0 // indirect + go4.org v0.0.0-20201209231011-d4a079459e60 // indirect + golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect + golang.org/x/net v0.2.0 // indirect + golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.2.0 // indirect + golang.org/x/text v0.4.0 // indirect + golang.org/x/tools v0.1.12 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +)