marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added elasticsearch database migration fixes

Browse Source
This commit is contained in:
frikky
2021-05-23 12:00:40 +02:00
parent 4756d417b3
commit c073152db1
6 changed files with 57 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+6 -1
View File
@@ -19,6 +19,11 @@ functions/generated_apps
backend/onprem/app_sdk/apps
*test.py
shuffle-database
*.exe
*debug*
shuffle-database/batch_metrics_enabled.conf
shuffle-database/logging_enabled.conf
shuffle-database/nodes
shuffle-database/performance_analyzer_enabled.conf
shuffle-database/rca_enabled.conf
backend/go-app/go.mod
+1 -1
View File
@@ -24,7 +24,7 @@ require (
github.com/elastic/go-elasticsearch/v7 v7.12.0 // indirect
github.com/elastic/go-elasticsearch/v8 v8.0.0-20210519083322-55daf7425ecb // indirect
github.com/frikky/kin-openapi v0.39.0
github.com/frikky/shuffle-shared v0.0.47
github.com/frikky/shuffle-shared v0.0.49
github.com/fsouza/go-dockerclient v1.7.2 // indirect
github.com/ghodss/yaml v1.0.0
github.com/go-git/go-billy/v5 v5.0.0
backend/go-app/go.sum
+2
View File
@@ -153,6 +153,8 @@ github.com/frikky/shuffle-shared v0.0.40 h1:H0au2np5xSy9mZEUWN+a29IORk+YP95FipEN
github.com/frikky/shuffle-shared v0.0.40/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ=
github.com/frikky/shuffle-shared v0.0.46 h1:L52pyEVKZujM136qzD7LnarJIq0iXyfpDqhGrJkE7ik=
github.com/frikky/shuffle-shared v0.0.46/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc=
github.com/frikky/shuffle-shared v0.0.47 h1:fywEmbJGbD/VT9LdkHvLWPGh9HEt8ipFDzoGHRe+QgA=
github.com/frikky/shuffle-shared v0.0.47/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc=
github.com/fsouza/go-dockerclient v1.7.2 h1:bBEAcqLTkpq205jooP5RVroUKiVEWgGecHyeZc4OFjo=
github.com/fsouza/go-dockerclient v1.7.2/go.mod h1:+ugtMCVRwnPfY7d8/baCzZ3uwB0BrG5DB8OzbtxaRz8=
github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE=
backend/go-app/main.go
+22 -8
View File
@@ -731,7 +731,6 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) {
return
}
// FIXME: Overhaul the top part.
// Only admin can CREATE users, but if there are no users, anyone can make (first)
ctx := context.Background()
users, countErr := shuffle.GetAllUsers(ctx)
@@ -794,9 +793,9 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) {
err = shuffle.SetOrg(ctx, newOrg, orgId)
if err != nil {
log.Printf("Failed setting organization: %s", err)
log.Printf("[WARNING] Failed setting init organization: %s", err)
} else {
log.Printf("Successfully created the default org!")
log.Printf("[DEBUG] Successfully created the default org!")
item := shuffle.Environment{
Name: "Shuffle",
@@ -808,7 +807,12 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) {
err = shuffle.SetEnvironment(ctx, &item)
if err != nil {
log.Printf("[WARNING] Failed setting up new environment for new org")
log.Printf("[WARNING] Failed setting up new environment for new org: %s")
}
currentOrg = shuffle.OrgMini{
Id: newOrg.Id,
Name: newOrg.Name,
}
}
}
@@ -4013,7 +4017,7 @@ func runInitEs(ctx context.Context) {
log.Printf("[DEBUG] Getting organizations")
activeOrgs, err := shuffle.GetAllOrgs(ctx)
setUsers := false
log.Printf("ORGS: %d", len(activeOrgs))
//log.Printf("ORGS: %d", len(activeOrgs))
if err != nil {
if fmt.Sprintf("%s", err) == "EOF" {
time.Sleep(7 * time.Second)
@@ -4551,7 +4555,7 @@ func runInit(ctx context.Context) {
log.Printf("[INFO] Username: %s, role: %s", user.Username, user.Role)
}
} else {
log.Printf("Found %d users.", len(users))
log.Printf("[INIT] Found %d users.", len(users))
}
if len(activeOrgs) == 1 && len(users) > 0 {
@@ -5669,8 +5673,18 @@ func initHandlers() {
elasticConfig = ""
}
_ = shuffle.RunInit(*dbclient, *es, storage.Client{}, gceProject, "onprem", true, "elasticsearch")
log.Printf("[DEBUG] Finished Shuffle database init")
for {
_, err = shuffle.RunInit(*dbclient, *es, storage.Client{}, gceProject, "onprem", true, "elasticsearch")
if err != nil {
log.Printf("[DEBUG] Error in initial database connection. Retrying in 5 seconds. %s", err)
time.Sleep(5 * time.Second)
continue
}
break
}
log.Printf("[DEBUG] Initialized Shuffle database connection. Setting up environment.")
if elasticConfig == "elasticsearch" {
go runInitEs(ctx)
docker-compose.yml
+14 -6
View File
@@ -2,7 +2,7 @@ version: '3'
services:
frontend:
#build: ./frontend
image: ghcr.io/frikky/shuffle-frontend:0.8.90
image: ghcr.io/frikky/shuffle-frontend:0.8.91
container_name: shuffle-frontend
hostname: shuffle-frontend
ports:
@@ -16,8 +16,8 @@ services:
depends_on:
- backend
backend:
#build: ./backend
image: ghcr.io/frikky/shuffle-backend:0.8.90
build: ./backend
image: ghcr.io/frikky/shuffle-backend:0.8.91
container_name: shuffle-backend
hostname: ${BACKEND_HOSTNAME}
# Here for debugging:
@@ -94,13 +94,15 @@ services:
soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
hard: 65536
volumes:
#- ${DB_LOCATION}/opensearch:/usr/share/opensearch/data
#- tmp/opensearch:/usr/share/opensearch/data
- ${DB_LOCATION}:/usr/share/opensearch/data
- ${DB_LOCATION}:/usr/share/opensearch/data:rw
ports:
- 9200:9200
networks:
- shuffle
#volumes:
# #- ${DB_LOCATION}/opensearch:/usr/share/opensearch/data
# #- tmp/opensearch:/usr/share/opensearch/data
# - ${DB_LOCATION}:/usr/share/opensearch/data
#database:
# #build: ./backend/database
# image: frikky/shuffle:database
@@ -119,3 +121,9 @@ services:
networks:
shuffle:
driver: bridge
#volumes:
# opensearch-data1:
# driver: local
# driver_opts:
# type: nfs
# device: ${DB_LOCATION}
shuffle-database/README.md
+12
View File
@@ -0,0 +1,12 @@
# Database folder
This is a folder used by the Shuffle database. It has to exist with WITHOUT root permissions before starting Shuffle.
All database files will recide here. IF you can't get Elasticsearch to work, this most likely has to do with permissions. To fix it, run this:
```
docker-compose down
sudo chown 1000:1000 -R shuffle-database
docker-compose up -d
```
This restarts the database, and assigns 1000 (Elasticsearch process) as the owner of the database folder.