From f594f8f3152c0e05083806479fb9417591515f9a Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 4 Jun 2021 03:22:04 +0200 Subject: [PATCH] Updated install guide for opensearch --- .env | 1 + .github/install-guide.md | 66 +++++++++++--------------- backend/go-app/main.go | 15 ++++++ docker-compose.yml | 8 ++-- frontend/src/views/AngularWorkflow.jsx | 10 ++++ 5 files changed, 56 insertions(+), 44 deletions(-) diff --git a/.env b/.env index d4418b9a..4874bd40 100644 --- a/.env +++ b/.env @@ -57,4 +57,5 @@ SHUFFLE_OPENSEARCH_PASSWORD= SHUFFLE_OPENSEARCH_CERTIFICATE_FILE= SHUFFLE_OPENSEARCH_APIKEY= SHUFFLE_OPENSEARCH_CLOUDID= +SHUFFLE_OPENSEARCH_PROXY= SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY=true diff --git a/.github/install-guide.md b/.github/install-guide.md index d1c21169..4e076078 100644 --- a/.github/install-guide.md +++ b/.github/install-guide.md @@ -6,11 +6,21 @@ The Docker setup is done with docker-compose and is a single command to get set **PS: if you're setting up Shuffle on Windows, go to the next step (Windows Docker setup)** -1. Make sure you have Docker and [docker-compose](https://docs.docker.com/compose/install/) installed. -2. Run docker-compose. +1. Make sure you have [Docker](https://docs.docker.com/get-docker/) and [docker-compose](https://docs.docker.com/compose/install/) installed. +2. Download Shuffle ``` git clone https://github.com/frikky/Shuffle cd Shuffle +``` + +3. Fix prerequisites for the Opensearch database (Elasticsearch): +``` +sudo sysctl -w vm.max_map_count=262144 # https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html +sudo chown 1000:1000 -R shuffle-database # Requires for Opensearch +``` + +4. Run docker-compose. +``` docker-compose up -d ``` @@ -26,7 +36,18 @@ This step is for setting up with Docker on windows from scratch. ``` OUTER_HOSTNAME=YOUR.IP.HERE ``` -5. Run docker-compose + +5. Configure max memory (WSL) by opening a new CMD/Powershell window. Required for Elasticsearch +``` +wsl -d docker-desktop +sysctl -w vm.max_map_count=262144 +echo "vm.max_map_count = 262144" > /etc/sysctl.d/99-docker-desktop.conf +echo -e "\nvm.max_map_count = 262144\n" >> /etc/sysctl.d/00-alpine.conf + +# https://stackoverflow.com/questions/42111566/elasticsearch-in-windows-docker-image-vm-max-map-count +``` + +6. Run docker-compose ``` docker-compose up -d ``` @@ -35,7 +56,7 @@ docker-compose up -d https://shuffler.io/docs/configuration ### After installation -1. After installation, go to http://localhost:3001/adminsetup (or your servername) +1. After installation, go to http://localhost:3001/adminsetup (or your servername - https is on port 3443) 2. Now set up your admin account (username & password). Shuffle doesn't have a default username and password. 3. Check out https://shuffler.io/docs/configuration as it has a lot of useful information to get started @@ -44,43 +65,10 @@ https://shuffler.io/docs/configuration ### Useful info * Check out [getting started](https://shuffler.io/docs/getting_started) +* The default state of Shuffle is NOT scalable. See [production setup](https://shuffler.io/docs/configuration#production_readiness) for more info * The server is available on http://localhost:3001 (or your servername) * Further configurations can be done in docker-compose.yml and .env. -* Default database location is /etc/shuffle - -### Execution problems -If you have problems with your first execution (hello world), you might need to set the correct Docker API version. Here's how: - -1. Find your API version by running "docker version" -``` -$ docker version - -Client: - Version: 17.09.1-ce - API version: 1.32 # <-- this one - Go version: go1.8.3 - Git commit: 19e2cf6 - Built: Thu Dec 7 22:24:16 2017 - OS/Arch: linux/amd64 - -Server: - Version: 17.09.1-ce - API version: 1.32 (minimum version 1.12) - Go version: go1.8.3 - Git commit: 19e2cf6 - Built: Thu Dec 7 22:22:56 2017 - OS/Arch: linux/amd64 - Experimental: false -``` - -2. Open docker-compose.yml and change the line with "DOCKER_API_VERSION" to your version. -3. Restart docker-compose -``` -docker-compose down -docker-compose up -``` - -Related issue: #47 +* Default database location is in the same folder: ./shuffle-database # Local development installation Local development is pretty straight forward with **ReactJS** and **Golang**. This part is intended to help you run the code for development purposes. diff --git a/backend/go-app/main.go b/backend/go-app/main.go index f7a762c7..c859412a 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5675,9 +5675,24 @@ func initHandlers() { transport := http.DefaultTransport.(*http.Transport).Clone() transport.MaxIdleConnsPerHost = 1000 transport.ResponseHeaderTimeout = time.Second * 10 + transport.Proxy = nil + + if len(os.Getenv("SHUFFLE_OPENSEARCH_PROXY")) > 0 { + httpProxy := os.Getenv("SHUFFLE_OPENSEARCH_PROXY") + + url_i := url.URL{} + url_proxy, err := url_i.Parse(httpProxy) + if err == nil { + log.Printf("[DEBUG] Setting Opensearch proxy to %s", httpProxy) + transport.Proxy = http.ProxyURL(url_proxy) + } else { + log.PrintF("[ERROR] Failed setting proxy for %s", httpProxy) + } + } skipSSLVerify := false if strings.ToLower(os.Getenv("SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY")) == "true" { + log.Printf("[DEBUG] SKIPPING SSL verification with Opensearch") skipSSLVerify = true } diff --git a/docker-compose.yml b/docker-compose.yml index 6b863b6e..d7fa3eb0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,6 +34,8 @@ services: environment: - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - SHUFFLE_FILE_LOCATION=/shuffle-files + - HTTP_PROXY=${SHUFFLE_HTTP_PROXY} + - HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY} restart: unless-stopped depends_on: - opensearch @@ -70,14 +72,10 @@ services: hostname: shuffle-opensearch container_name: shuffle-opensearch environment: - - cluster.name=shuffle-cluster - - node.name=shuffle-opensearch - - discovery.seed_hosts=shuffle-opensearch - - cluster.initial_master_nodes=shuffle-opensearch - bootstrap.memory_lock=true - "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - - cluster.routing.allocation.disk.threshold_enabled=false - opendistro_security.disabled=true + - discovery.seed_hosts=shuffle-opensearch ulimits: memlock: soft: -1 diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 73cc80ab..5cc65caa 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -4416,7 +4416,17 @@ const AngularWorkflow = (props) => { //console.log("OLDNAME: ", selectedAction.name) event.target.value = event.target.value.replaceAll("(", "") event.target.value = event.target.value.replaceAll(")", "") + event.target.value = event.target.value.replaceAll("]", "") + event.target.value = event.target.value.replaceAll("[", "") + event.target.value = event.target.value.replaceAll("{", "") + event.target.value = event.target.value.replaceAll("}", "") + event.target.value = event.target.value.replaceAll("*", "") + event.target.value = event.target.value.replaceAll("!", "") + event.target.value = event.target.value.replaceAll("@", "") + event.target.value = event.target.value.replaceAll("#", "") event.target.value = event.target.value.replaceAll("$", "") + event.target.value = event.target.value.replaceAll("%", "") + event.target.value = event.target.value.replaceAll("&", "") event.target.value = event.target.value.replaceAll("#", "") event.target.value = event.target.value.replaceAll(".", "") event.target.value = event.target.value.replaceAll(",", "")