From c073152db1855ff2ebbb32ddf81b238336e03a4d Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 23 May 2021 12:00:40 +0200 Subject: [PATCH] Added elasticsearch database migration fixes --- .gitignore | 7 ++++++- backend/go-app/go.mod | 2 +- backend/go-app/go.sum | 2 ++ backend/go-app/main.go | 30 ++++++++++++++++++++++-------- docker-compose.yml | 20 ++++++++++++++------ shuffle-database/README.md | 12 ++++++++++++ 6 files changed, 57 insertions(+), 16 deletions(-) create mode 100644 shuffle-database/README.md diff --git a/.gitignore b/.gitignore index ca9d2090..4d3f2d80 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,11 @@ functions/generated_apps backend/onprem/app_sdk/apps *test.py -shuffle-database *.exe *debug* + +shuffle-database/batch_metrics_enabled.conf +shuffle-database/logging_enabled.conf +shuffle-database/nodes +shuffle-database/performance_analyzer_enabled.conf +shuffle-database/rca_enabled.conf diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 3d93dd0b..e15a3473 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -24,7 +24,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 // indirect github.com/elastic/go-elasticsearch/v8 v8.0.0-20210519083322-55daf7425ecb // indirect github.com/frikky/kin-openapi v0.39.0 - github.com/frikky/shuffle-shared v0.0.47 + github.com/frikky/shuffle-shared v0.0.49 github.com/fsouza/go-dockerclient v1.7.2 // indirect github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 86f89455..37434157 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -153,6 +153,8 @@ github.com/frikky/shuffle-shared v0.0.40 h1:H0au2np5xSy9mZEUWN+a29IORk+YP95FipEN github.com/frikky/shuffle-shared v0.0.40/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= github.com/frikky/shuffle-shared v0.0.46 h1:L52pyEVKZujM136qzD7LnarJIq0iXyfpDqhGrJkE7ik= github.com/frikky/shuffle-shared v0.0.46/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= +github.com/frikky/shuffle-shared v0.0.47 h1:fywEmbJGbD/VT9LdkHvLWPGh9HEt8ipFDzoGHRe+QgA= +github.com/frikky/shuffle-shared v0.0.47/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= github.com/fsouza/go-dockerclient v1.7.2 h1:bBEAcqLTkpq205jooP5RVroUKiVEWgGecHyeZc4OFjo= github.com/fsouza/go-dockerclient v1.7.2/go.mod h1:+ugtMCVRwnPfY7d8/baCzZ3uwB0BrG5DB8OzbtxaRz8= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 08fde053..7d6f8455 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -731,7 +731,6 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { return } - // FIXME: Overhaul the top part. // Only admin can CREATE users, but if there are no users, anyone can make (first) ctx := context.Background() users, countErr := shuffle.GetAllUsers(ctx) @@ -794,9 +793,9 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { err = shuffle.SetOrg(ctx, newOrg, orgId) if err != nil { - log.Printf("Failed setting organization: %s", err) + log.Printf("[WARNING] Failed setting init organization: %s", err) } else { - log.Printf("Successfully created the default org!") + log.Printf("[DEBUG] Successfully created the default org!") item := shuffle.Environment{ Name: "Shuffle", @@ -808,7 +807,12 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { err = shuffle.SetEnvironment(ctx, &item) if err != nil { - log.Printf("[WARNING] Failed setting up new environment for new org") + log.Printf("[WARNING] Failed setting up new environment for new org: %s") + } + + currentOrg = shuffle.OrgMini{ + Id: newOrg.Id, + Name: newOrg.Name, } } } @@ -4013,7 +4017,7 @@ func runInitEs(ctx context.Context) { log.Printf("[DEBUG] Getting organizations") activeOrgs, err := shuffle.GetAllOrgs(ctx) setUsers := false - log.Printf("ORGS: %d", len(activeOrgs)) + //log.Printf("ORGS: %d", len(activeOrgs)) if err != nil { if fmt.Sprintf("%s", err) == "EOF" { time.Sleep(7 * time.Second) @@ -4551,7 +4555,7 @@ func runInit(ctx context.Context) { log.Printf("[INFO] Username: %s, role: %s", user.Username, user.Role) } } else { - log.Printf("Found %d users.", len(users)) + log.Printf("[INIT] Found %d users.", len(users)) } if len(activeOrgs) == 1 && len(users) > 0 { @@ -5669,8 +5673,18 @@ func initHandlers() { elasticConfig = "" } - _ = shuffle.RunInit(*dbclient, *es, storage.Client{}, gceProject, "onprem", true, "elasticsearch") - log.Printf("[DEBUG] Finished Shuffle database init") + for { + _, err = shuffle.RunInit(*dbclient, *es, storage.Client{}, gceProject, "onprem", true, "elasticsearch") + if err != nil { + log.Printf("[DEBUG] Error in initial database connection. Retrying in 5 seconds. %s", err) + time.Sleep(5 * time.Second) + continue + } + + break + } + + log.Printf("[DEBUG] Initialized Shuffle database connection. Setting up environment.") if elasticConfig == "elasticsearch" { go runInitEs(ctx) diff --git a/docker-compose.yml b/docker-compose.yml index fd792b8a..2117990b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.90 + image: ghcr.io/frikky/shuffle-frontend:0.8.91 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -16,8 +16,8 @@ services: depends_on: - backend backend: - #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.90 + build: ./backend + image: ghcr.io/frikky/shuffle-backend:0.8.91 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -94,13 +94,15 @@ services: soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems hard: 65536 volumes: - #- ${DB_LOCATION}/opensearch:/usr/share/opensearch/data - #- tmp/opensearch:/usr/share/opensearch/data - - ${DB_LOCATION}:/usr/share/opensearch/data + - ${DB_LOCATION}:/usr/share/opensearch/data:rw ports: - 9200:9200 networks: - shuffle + #volumes: + # #- ${DB_LOCATION}/opensearch:/usr/share/opensearch/data + # #- tmp/opensearch:/usr/share/opensearch/data + # - ${DB_LOCATION}:/usr/share/opensearch/data #database: # #build: ./backend/database # image: frikky/shuffle:database @@ -119,3 +121,9 @@ services: networks: shuffle: driver: bridge +#volumes: +# opensearch-data1: +# driver: local +# driver_opts: +# type: nfs +# device: ${DB_LOCATION} diff --git a/shuffle-database/README.md b/shuffle-database/README.md new file mode 100644 index 00000000..43246562 --- /dev/null +++ b/shuffle-database/README.md @@ -0,0 +1,12 @@ +# Database folder +This is a folder used by the Shuffle database. It has to exist with WITHOUT root permissions before starting Shuffle. + +All database files will recide here. IF you can't get Elasticsearch to work, this most likely has to do with permissions. To fix it, run this: + +``` +docker-compose down +sudo chown 1000:1000 -R shuffle-database +docker-compose up -d +``` + +This restarts the database, and assigns 1000 (Elasticsearch process) as the owner of the database folder.