diff --git a/.github/install-guide.md b/.github/install-guide.md index 301d180a..ae2ae26c 100644 --- a/.github/install-guide.md +++ b/.github/install-guide.md @@ -38,39 +38,37 @@ When you're done, skip to the "After installation" step below. This step is for setting up with Docker on windows from scratch. 1. Make sure you have [Docker](https://docs.docker.com/docker-for-windows/install/) and [docker-compose](https://docs.docker.com/compose/install/) installed. WSL2 may be required. + 2. Go to https://github.com/frikky/shuffle/releases and download the latest .zip release (or install git) + 3. Unzip the folder and enter it + 4. Open the .env file and change the line with "OUTER_HOSTNAME" to contain your IP: + ``` OUTER_HOSTNAME=YOUR.IP.HERE ``` -5. Configure max memory (WSL) by opening a new CMD/Powershell window. Required for Opensearch (Elasticsearch). Run one command at a time, otherwise it may not work (WSL issues)! +5. Configure max memory (WSL) by opening a new CMD/Powershell window. Run each command one-by-one, or else installation might not work. (Required for fixing consistent restart of Elasticsearch issue in Windows Operation System). ``` wsl -d docker-desktop ``` - ``` sysctl -w vm.max_map_count=262144 ``` - ``` echo "vm.max_map_count = 262144" > /etc/sysctl.d/99-docker-desktop.conf ``` - ``` echo -e "\nvm.max_map_count = 262144\n" >> /etc/sysctl.d/00-alpine.conf ``` - ``` # https://stackoverflow.com/questions/42111566/elasticsearch-in-windows-docker-image-vm-max-map-count - -exit ``` 6. Run docker-compose ``` -docker-compose up -d +docker compose up -d ``` ### Configurations (proxies, default users etc.) @@ -91,18 +89,14 @@ https://shuffler.io/docs/configuration * Further configurations can be done in docker-compose.yml and .env. * Default database location is in the same folder: ./shuffle-database -# Local development installation +# Local development installation Local development is pretty straight forward with **ReactJS** and **Golang**. This part is intended to help you run the code for development purposes. -1. Fork the project! This allows for pull-requests to Shuffle. -2. Find the part below that matches what you want to change (e.g. frontend) -3. Run that part locally - NOT in Docker - -**PS: You have to stop the Backend Docker container to get the backend running** -**PPS: Use the "launch" branch when developing to get it set up easier** +**PS: You have to stop the Backend Docker container to get this one working** +**PPS: Use the "Launch" branch when developing to get it set up easier** ## Frontend - ReactJS /w cytoscape -http://localhost:3000 - Requires [npm](https://nodejs.org/en/download/)/[yarn](https://yarnpkg.com/lang/en/docs/install/#debian-stable)/your preferred manager. Runs independently from backend. +http://localhost:3000 - Requires [npm](https://nodejs.org/en/download/)/[yarn](https://yarnpkg.com/lang/en/docs/install/#debian-stable)/your preferred manager. Runs independently from backend. ```bash cd frontend npm i @@ -110,7 +104,7 @@ npm start ``` ## Backend - Golang -http://localhost:5001 - REST API - requires [>=go1.13](https://golang.org/dl/) +http://localhost:5001 - REST API - requires [>=go1.13](https://golang.org/dl/) ```bash export SHUFFLE_OPENSEARCH_URL="http://localhost:9200" cd backend/go-app @@ -119,12 +113,15 @@ go run *.go **WINDOWS USERS:** You'll have to to add the "export" part as an environment variable. -## Database - Opensearch -This is a database based on Elasticsearch, hence we're using Elasticsearch API's for it. To get it running, we suggest you run the original docker-compose, as there's no good one-liner for getting opensearch working. +## Database - Datastore +Based on Google datastore +``` +docker run -p 8000:8000 google/cloud-sdk gcloud beta emulators datastore start --project=shuffle --host-port 0.0.0.0:8000 --no-store-on-disk +``` ## Orborus Execution of Workflows: -PS: This requires some specific environment variables +PS: This requires some specific environment variables ``` cd functions/onprem/orborus go run orborus.go diff --git a/docker-compose.yml b/docker-compose.yml index 6485cde9..06336600 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - #build: ./backend + # build: ./backend image: ghcr.io/frikky/shuffle-backend:nightly container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} @@ -36,7 +36,8 @@ services: - SHUFFLE_FILE_LOCATION=/shuffle-files restart: unless-stopped #depends_on: - # - opensearch #- Not necessary because dependancy is handled within the backend itself instead + #- opensearch #Not necessary because dependancy is handled within the backend itself instead + #- database orborus: #build: ./functions/onprem/orborus image: ghcr.io/frikky/shuffle-orborus:nightly diff --git a/frontend/package.json b/frontend/package.json index 5b290522..8fa85406 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -9,6 +9,7 @@ "@material-ui/icons": "^4.5.1", "@material-ui/lab": "^4.0.0-alpha.58", "@material-ui/styles": "^4.5.2", + "@uiw/react-codemirror": "^3.2.1", "@use-it/interval": "^1.0.0", "babel-eslint": "^10.1.0", "class-transformer": "^0.3.1", @@ -67,7 +68,7 @@ "zone.js": "~0.8.26" }, "scripts": { - "start": "set HTTPS=true&&react-scripts start", + "start": "react-scripts start", "build": "react-scripts build", "test": "react-scripts test", "eject": "react-scripts eject" diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index da780f23..562c154e 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -41,8 +41,7 @@ import {isMobile} from "react-device-detect"; var globalUrl = window.location.origin // CORS used for testing purposes. Should only happen with specific port and http -//window.location.protocol == "http:" && -if (window.location.port === "3000") { +if ( window.location.port === "3000") { globalUrl = "http://localhost:5001" //globalUrl = "http://localhost:5002" } diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 45f7a8eb..7bae11dd 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -12,6 +12,10 @@ import {Popper, TextField, TextareaAutosize, Drawer, Button, Paper, Grid, Tabs, import {GetApp as GetAppIcon, Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons'; import Autocomplete from '@material-ui/lab/Autocomplete'; +import CodeMirror from '@uiw/react-codemirror'; +import 'codemirror/keymap/sublime'; +import 'codemirror/theme/gruvbox-dark.css'; + const useStyles = makeStyles({ notchedOutline: { @@ -337,12 +341,13 @@ const ParsedAction = (props) => { }) const changeActionParameter = (event, count, data) => { + console.log(event) if (data.name.startsWith("${") && data.name.endsWith("}")) { // PARAM FIX - Gonna use the ID field, even though it's a hack const paramcheck = selectedAction.parameters.find(param => param.name === "body") if (paramcheck !== undefined) { // Escapes all double quotes - const toReplace = event.target.value.trim().replaceAll("\\\"", "\"").replaceAll("\"", "\\\"") + const toReplace = event.target.value.trim().replaceAll("\\\"", "\"").replaceAll("\"", "\\\""); console.log("REPLACE WITH: ", toReplace) if (paramcheck["value_replace"] === undefined || paramcheck["value_replace"] === null) { paramcheck["value_replace"] = [{ @@ -479,6 +484,150 @@ const ParsedAction = (props) => { //setUpdate(event.target.value) } + const changeActionParameterCodemirror = (event, count, data) => { + console.log(event) + if (data.name.startsWith("${") && data.name.endsWith("}")) { + // PARAM FIX - Gonna use the ID field, even though it's a hack + const paramcheck = selectedAction.parameters.find(param => param.name === "body") + if (paramcheck !== undefined) { + // Escapes all double quotes + const toReplace = event.target.value.trim().replaceAll("\\\"", "\"").replaceAll("\"", "\\\""); + console.log("REPLACE WITH: ", toReplace) + if (paramcheck["value_replace"] === undefined || paramcheck["value_replace"] === null) { + paramcheck["value_replace"] = [{ + "key": data.name, + "value": toReplace, + }] + + console.log("IN IF: ", paramcheck) + + } else { + const subparamindex = paramcheck["value_replace"].findIndex(param => param.key === data.name) + if (subparamindex === -1) { + paramcheck["value_replace"].push({ + "key": data.name, + "value": toReplace, + }) + } else { + paramcheck["value_replace"][subparamindex]["value"] = toReplace + } + + console.log("IN ELSE: ", paramcheck) + } + //console.log("PARAM: ", paramcheck) + //if (paramcheck.id === undefined) { + // console.log("Normal paramcheck") + //} else { + // selectedActionParameters[count]["value_replace"] = paramcheck + // selectedAction.parameters[count]["value_replace"] = paramcheck + //} + + if (paramcheck["value_replace"] === undefined) { + selectedActionParameters[count]["value_replace"] = paramcheck + selectedAction.parameters[count]["value_replace"] = paramcheck + } else { + selectedActionParameters[count]["value_replace"] = paramcheck["value_replace"] + selectedAction.parameters[count]["value_replace"] = paramcheck["value_replace"] + } + console.log("RESULT: ", selectedAction) + setSelectedAction(selectedAction) + //setUpdate(Math.random()) + return + } + } + + if (event.display.maxLine.text[event.display.maxLine.text.length-1] === "$") { + if (!showDropdown) { + setShowAutocomplete(false) + setShowDropdown(true) + setShowDropdownNumber(count) + } + } else { + if (showDropdown) { + setShowDropdown(false) + } + } + + // bad detection mechanism probably + if (event.display.maxLine.text[event.display.maxLine.text.length-1] === "." && actionlist.length > 0) { + console.log("GET THE LAST ARGUMENT FOR NODE!") + // THIS IS AN EXAMPLE OF SHOWING IT + /* + const inputdata = {"data": "1.2.3.4", "dataType": "4.5.6.6"} + setJsonList(GetParsedPaths(inputdata, "")) + if (!showDropdown) { + setShowAutocomplete(false) + setShowDropdown(true) + setShowDropdownNumber(count) + } + console.log(jsonList) + */ + + // Search for the item backwards + // 1. Reverse search backwards from . -> $ + // 2. Search the actionlist for the item + // 3. Find the data for the specific item + + var curstring = "" + var record = false + for (var key in selectedActionParameters[count].value) { + const item = selectedActionParameters[count].value[key] + if (record) { + curstring += item + } + + if (item === "$") { + record = true + curstring = "" + } + } + + //console.log("CURSTRING: ", curstring) + if (curstring.length > 0 && actionlist !== null) { + // Search back in the action list + curstring = curstring.split(" ").join("_").toLowerCase() + var actionItem = actionlist.find(data => data.autocomplete.split(" ").join("_").toLowerCase() === curstring) + if (actionItem !== undefined) { + console.log("Found item: ", actionItem) + + //actionItem.example = actionItem.example.trim() + //actionItem.example = actionItem.example.split(" None").join(" \"None\"") + //actionItem.example = actionItem.example.split("\'").join("\"") + + var jsonvalid = true + try { + const tmp = String(JSON.parse(actionItem.example)) + if (!actionItem.example.includes("{") && !actionItem.example.includes("[")) { + jsonvalid = false + } + } catch (e) { + jsonvalid = false + } + + if (jsonvalid) { + setJsonList(GetParsedPaths(JSON.parse(actionItem.example), "")) + + if (!showDropdown) { + setShowAutocomplete(false) + setShowDropdown(true) + setShowDropdownNumber(count) + } + } + } + } + } else { + if (jsonList.length > 0) { + setJsonList([]) + } + } + + selectedActionParameters[count].value = event.display.maxLine.text + selectedAction.parameters[count].value = event.display.maxLine.text + setSelectedAction(selectedAction) + //setUpdate(Math.random()) + //setUpdate(event.target.value) + } + const changeActionParameterVariable = (fieldvalue, count) => { //console.log("CALLED THIS ONE WITH VALUE!", fieldvalue) //if (selectedVariableParameter === fieldvalue) { @@ -686,7 +835,8 @@ const ParsedAction = (props) => { const clickedFieldId = "rightside_field_"+count var datafield = // { maxWidth: "95%", fontSize: "1em", }, - endAdornment: ( - hideExtraTypes ? null : - - - { - setMenuPosition({ - top: event.pageY+10, - left: event.pageX+10, - }) - setShowDropdownNumber(count) - setShowDropdown(true) - setShowAutocomplete(true) - }}/> - - + // endAdornment: ( + // hideExtraTypes ? null : + // + // + // { + // setMenuPosition({ + // top: event.pageY+10, + // left: event.pageX+10, + // }) + // setShowDropdownNumber(count) + // setShowDropdown(true) + // setShowAutocomplete(true) + // }}/> + // + // - ) + // ) }} fullWidth multiline={multiline} @@ -729,11 +879,22 @@ const ParsedAction = (props) => { rows={rows} color="primary" defaultValue={data.value} + + value={data.value} + options={{ + theme: 'gruvbox-dark', + keyMap: 'sublime', + mode: 'python', + }} + height = {200} + type={placeholder.includes("***") || (data.configuration && (data.name.toLowerCase().includes("api") || data.name.toLowerCase().includes("key") || data.name.toLowerCase().includes("pass"))) ? "password" : "text"} placeholder={placeholder} + onChange={(event) => { - changeActionParameter(event, count, data) + changeActionParameterCodemirror(event, count, data) }} + helperText={selectedApp.generated && selectedApp.activated && data.name === "body" ? {openApiHelperText} @@ -791,7 +952,7 @@ const ParsedAction = (props) => { endAdornment: ( hideExtraTypes ? null : - + { setMenuPosition({ top: event.pageY+10, @@ -1116,14 +1277,6 @@ const ParsedAction = (props) => { // itemColor = "#ffeb3b" //} {/*
*/} - - - - if (authenticationType.type === "oauth2" && data.configuration === true) { - return null - } - //&& authenticationType.redirect_uri !== undefined && authenticationType.redirect_uri !== null) { - return (
@@ -1397,7 +1550,47 @@ const ParsedAction = (props) => {