Made backend run with .env file, and changed files to nightly

.env

@@ -21,6 +21,7 @@ SHUFFLE_DEFAULT_APIKEY=
 
 # Local location of your app directory. Can't use ~/
 # Files will get better at some point. Right now: local saving.
+SHUFFLE_APP_HOTLOAD_FOLDER=./shuffle-apps
 SHUFFLE_APP_HOTLOAD_LOCATION=./shuffle-apps
 SHUFFLE_FILE_LOCATION=./shuffle-files
 
@@ -42,13 +43,18 @@ SHUFFLE_PASS_APP_PROXY=FALSE
 
 SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io
 SHUFFLE_BASE_IMAGE_NAME=frikky
-SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.60"
+SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.80"
 
 # Used for auto-cleanup of containers. REALLY important at scale.
 SHUFFLE_CONTAINER_AUTO_CLEANUP=false
 SHUFFLE_ELASTIC=true
 
+# DATABASE CONFIGURATIONS
+DATASTORE_EMULATOR_HOST=shuffl-database:8000
 SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200
-SHUFFLE_OPENSEARCH_USERNAME=""
+SHUFFLE_OPENSEARCH_USERNAME=
-SHUFFLE_OPENSEARCH_PASSWORD=""
+SHUFFLE_OPENSEARCH_PASSWORD=
-SHUFFLE_OPENSEARCH_CERTIFICATE_FILE=""
+SHUFFLE_OPENSEARCH_CERTIFICATE_FILE=
+SHUFFLE_OPENSEARCH_APIKEY=
+SHUFFLE_OPENSEARCH_CLOUDID=
+SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY=true

backend/go-app/main.go

@@ -4,18 +4,14 @@ import (
 	"github.com/frikky/shuffle-shared"
 
 	"bufio"
-
 	"bytes"
 	"context"
 	"crypto/md5"
+	"crypto/tls"
+	//"crypto/x509"
 	"encoding/hex"
 	"encoding/json"
-	//"unicode/utf8"
-
 	"errors"
-	"path/filepath"
-
-	"crypto/tls"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -24,6 +20,7 @@ import (
 	"net/url"
 	"os"
 	"os/exec"
+	"path/filepath"
 	//"regexp"
 	"strings"
 	"time"
@@ -5669,19 +5666,31 @@ func initHandlers() {
 		esUrl = "http://shuffle-opensearch:9200"
 	}
 
+	// https://github.com/elastic/go-elasticsearch/blob/f741c073f324c15d3d401d945ee05b0c410bd06d/elasticsearch.go#L98
 	config := elasticsearch.Config{
 		Addresses: []string{esUrl},
 		Username:  os.Getenv("SHUFFLE_OPENSEARCH_USERNAME"),
 		Password:  os.Getenv("SHUFFLE_OPENSEARCH_PASSWORD"),
-		Transport: &http.Transport{
+		APIKey:    os.Getenv("SHUFFLE_OPENSEARCH_APIKEY"),
-			MaxIdleConnsPerHost:   100,
+		CloudID:   os.Getenv("SHUFFLE_OPENSEARCH_CLOUDID"),
-			ResponseHeaderTimeout: time.Second,
-			TLSClientConfig: &tls.Config{
-				MinVersion: tls.VersionTLS11,
-			},
-		},
 	}
 
+	//config.Transport.TLSClientConfig
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport.MaxIdleConnsPerHost = 1000
+	transport.ResponseHeaderTimeout = time.Second * 10
+
+	skipSSLVerify := false
+	if strings.ToLower(os.Getenv("SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY")) == "true" {
+		skipSSLVerify = true
+	}
+
+	transport.TLSClientConfig = &tls.Config{
+		MinVersion:         tls.VersionTLS11,
+		InsecureSkipVerify: skipSSLVerify,
+	}
+
+	//https://github.com/elastic/go-elasticsearch/blob/master/_examples/security/elasticsearch-cluster.yml
 	certificateLocation := os.Getenv("SHUFFLE_OPENSEARCH_CERTIFICATE_FILE")
 	if len(certificateLocation) > 0 {
 		cert, err := ioutil.ReadFile(certificateLocation)
@@ -5689,9 +5698,20 @@ func initHandlers() {
 			log.Fatalf("[WARNING] Failed configuring certificates: %s not found", err)
 		} else {
 			config.CACert = cert
+
+			//if transport.TLSClientConfig.RootCAs, err = x509.SystemCertPool(); err != nil {
+			//	log.Fatalf("[ERROR] Problem adding system CA: %s", err)
+			//}
+
+			//// --> Add the custom certificate authority
+			//if ok := transport.TLSClientConfig.RootCAs.AppendCertsFromPEM(cert); !ok {
+			//	log.Fatalf("[ERROR] Problem adding CA from file %q", *cert)
+			//}
 		}
+
 		log.Printf("[INFO] Added certificate %#v elastic client.", certificateLocation)
 	}
+	config.Transport = transport
 
 	es, err := elasticsearch.NewClient(config)
 	if err != nil {

docker-compose.yml

@@ -2,7 +2,7 @@ version: '3'
 services:
   frontend:
     #build: ./frontend
-    image: ghcr.io/frikky/shuffle-frontend:0.8.98
+    image: ghcr.io/frikky/shuffle-frontend:nightly
     container_name: shuffle-frontend
     hostname: shuffle-frontend
     ports:
@@ -17,7 +17,7 @@ services:
       - backend
   backend:
     #build: ./backend
-    image: ghcr.io/frikky/shuffle-backend:0.8.97
+    image: ghcr.io/frikky/shuffle-backend:nightly
     container_name: shuffle-backend
     hostname: ${BACKEND_HOSTNAME}
     # Here for debugging:
@@ -30,20 +30,7 @@ services:
       - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps     
       - ${SHUFFLE_FILE_LOCATION}:/shuffle-files
       #- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate
-    environment:
+    env_file: .env 
-      - DATASTORE_EMULATOR_HOST=shuffle-database:8000
-      #- SHUFFLE_OPENSEARCH_URL=${SHUFFLE_OPENSEARCH_URL}
-      - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps
-      - SHUFFLE_FILE_LOCATION=/shuffle-files
-      - ORG_ID=${ORG_ID}
-      - SHUFFLE_APP_DOWNLOAD_LOCATION=${SHUFFLE_APP_DOWNLOAD_LOCATION}
-      - SHUFFLE_DOWNLOAD_AUTH_BRANCH=${SHUFFLE_DOWNLOAD_AUTH_BRANCH}
-      - SHUFFLE_DEFAULT_USERNAME=${SHUFFLE_DEFAULT_USERNAME}
-      - SHUFFLE_DEFAULT_PASSWORD=${SHUFFLE_DEFAULT_PASSWORD}
-      - SHUFFLE_DEFAULT_APIKEY=${SHUFFLE_DEFAULT_APIKEY}
-      - SHUFFLE_APP_FORCE_UPDATE=${SHUFFLE_APP_FORCE_UPDATE}
-      - HTTP_PROXY=${SHUFFLE_HTTP_PROXY}
-      - HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY}
     restart: unless-stopped
     depends_on:
       - opensearch