Added elasticsearch database migration fixes

.gitignore

@@ -19,6 +19,11 @@ functions/generated_apps
 backend/onprem/app_sdk/apps
 *test.py
 
-shuffle-database
 *.exe
 *debug*
+
+shuffle-database/batch_metrics_enabled.conf
+shuffle-database/logging_enabled.conf
+shuffle-database/nodes
+shuffle-database/performance_analyzer_enabled.conf
+shuffle-database/rca_enabled.conf

backend/go-app/go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/elastic/go-elasticsearch/v7 v7.12.0 // indirect
 	github.com/elastic/go-elasticsearch/v8 v8.0.0-20210519083322-55daf7425ecb // indirect
 	github.com/frikky/kin-openapi v0.39.0
-	github.com/frikky/shuffle-shared v0.0.47
+	github.com/frikky/shuffle-shared v0.0.49
 	github.com/fsouza/go-dockerclient v1.7.2 // indirect
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-git/go-billy/v5 v5.0.0

backend/go-app/go.sum

@@ -153,6 +153,8 @@ github.com/frikky/shuffle-shared v0.0.40 h1:H0au2np5xSy9mZEUWN+a29IORk+YP95FipEN
 github.com/frikky/shuffle-shared v0.0.40/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ=
 github.com/frikky/shuffle-shared v0.0.46 h1:L52pyEVKZujM136qzD7LnarJIq0iXyfpDqhGrJkE7ik=
 github.com/frikky/shuffle-shared v0.0.46/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc=
+github.com/frikky/shuffle-shared v0.0.47 h1:fywEmbJGbD/VT9LdkHvLWPGh9HEt8ipFDzoGHRe+QgA=
+github.com/frikky/shuffle-shared v0.0.47/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc=
 github.com/fsouza/go-dockerclient v1.7.2 h1:bBEAcqLTkpq205jooP5RVroUKiVEWgGecHyeZc4OFjo=
 github.com/fsouza/go-dockerclient v1.7.2/go.mod h1:+ugtMCVRwnPfY7d8/baCzZ3uwB0BrG5DB8OzbtxaRz8=
 github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE=

backend/go-app/main.go

@@ -731,7 +731,6 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) {
 		return
 	}
 
-	// FIXME: Overhaul the top part.
 	// Only admin can CREATE users, but if there are no users, anyone can make (first)
 	ctx := context.Background()
 	users, countErr := shuffle.GetAllUsers(ctx)
@@ -794,9 +793,9 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) {
 
 			err = shuffle.SetOrg(ctx, newOrg, orgId)
 			if err != nil {
-				log.Printf("Failed setting organization: %s", err)
+				log.Printf("[WARNING] Failed setting init organization: %s", err)
 			} else {
-				log.Printf("Successfully created the default org!")
+				log.Printf("[DEBUG] Successfully created the default org!")
 
 				item := shuffle.Environment{
 					Name:    "Shuffle",
@@ -808,7 +807,12 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) {
 
 				err = shuffle.SetEnvironment(ctx, &item)
 				if err != nil {
-					log.Printf("[WARNING] Failed setting up new environment for new org")
+					log.Printf("[WARNING] Failed setting up new environment for new org: %s")
+				}
+
+				currentOrg = shuffle.OrgMini{
+					Id:   newOrg.Id,
+					Name: newOrg.Name,
 				}
 			}
 		}
@@ -4013,7 +4017,7 @@ func runInitEs(ctx context.Context) {
 	log.Printf("[DEBUG] Getting organizations")
 	activeOrgs, err := shuffle.GetAllOrgs(ctx)
 	setUsers := false
-	log.Printf("ORGS: %d", len(activeOrgs))
+	//log.Printf("ORGS: %d", len(activeOrgs))
 	if err != nil {
 		if fmt.Sprintf("%s", err) == "EOF" {
 			time.Sleep(7 * time.Second)
@@ -4551,7 +4555,7 @@ func runInit(ctx context.Context) {
 					log.Printf("[INFO] Username: %s, role: %s", user.Username, user.Role)
 				}
 			} else {
-				log.Printf("Found %d users.", len(users))
+				log.Printf("[INIT] Found %d users.", len(users))
 			}
 
 			if len(activeOrgs) == 1 && len(users) > 0 {
@@ -5669,8 +5673,18 @@ func initHandlers() {
 		elasticConfig = ""
 	}
 
-	_ = shuffle.RunInit(*dbclient, *es, storage.Client{}, gceProject, "onprem", true, "elasticsearch")
+	for {
-	log.Printf("[DEBUG] Finished Shuffle database init")
+		_, err = shuffle.RunInit(*dbclient, *es, storage.Client{}, gceProject, "onprem", true, "elasticsearch")
+		if err != nil {
+			log.Printf("[DEBUG] Error in initial database connection. Retrying in 5 seconds. %s", err)
+			time.Sleep(5 * time.Second)
+			continue
+		}
+
+		break
+	}
+
+	log.Printf("[DEBUG] Initialized Shuffle database connection. Setting up environment.")
 
 	if elasticConfig == "elasticsearch" {
 		go runInitEs(ctx)

docker-compose.yml

@@ -2,7 +2,7 @@ version: '3'
 services:
   frontend:
     #build: ./frontend
-    image: ghcr.io/frikky/shuffle-frontend:0.8.90
+    image: ghcr.io/frikky/shuffle-frontend:0.8.91
     container_name: shuffle-frontend
     hostname: shuffle-frontend
     ports:
@@ -16,8 +16,8 @@ services:
     depends_on:
       - backend
   backend:
-    #build: ./backend
+    build: ./backend
-    image: ghcr.io/frikky/shuffle-backend:0.8.90
+    image: ghcr.io/frikky/shuffle-backend:0.8.91
     container_name: shuffle-backend
     hostname: ${BACKEND_HOSTNAME}
     # Here for debugging:
@@ -94,13 +94,15 @@ services:
         soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
         hard: 65536
     volumes:
-      #- ${DB_LOCATION}/opensearch:/usr/share/opensearch/data
+      - ${DB_LOCATION}:/usr/share/opensearch/data:rw
-      #- tmp/opensearch:/usr/share/opensearch/data
-      - ${DB_LOCATION}:/usr/share/opensearch/data
     ports:
       - 9200:9200
     networks:
       - shuffle
+    #volumes:
+    #  #- ${DB_LOCATION}/opensearch:/usr/share/opensearch/data
+    #  #- tmp/opensearch:/usr/share/opensearch/data
+    #  - ${DB_LOCATION}:/usr/share/opensearch/data
   #database:
   #  #build: ./backend/database
   #  image: frikky/shuffle:database
@@ -119,3 +121,9 @@ services:
 networks:
   shuffle:
     driver: bridge
+#volumes:
+#  opensearch-data1:
+#    driver: local
+#    driver_opts:
+#      type: nfs 
+#      device: ${DB_LOCATION}

shuffle-database/README.md

@@ -0,0 +1,12 @@
+# Database folder
+This is a folder used by the Shuffle database. It has to exist with WITHOUT root permissions before starting Shuffle.
+
+All database files will recide here. IF you can't get Elasticsearch to work, this most likely has to do with permissions. To fix it, run this:
+
+```
+docker-compose down
+sudo chown 1000:1000 -R shuffle-database 
+docker-compose up -d
+```
+
+This restarts the database, and assigns 1000 (Elasticsearch process) as the owner of the database folder.