Added simple initial Swarm setup with scalability

backend/app_sdk/app_base.py

@@ -594,7 +594,7 @@ class AppBase:
                 # An attempt at decomposing coroutine results
                 try:
                     if asyncio.iscoroutine(tmp):
-                        print("In coroutine")
+                        self.logger.info("[DEBUG] In coroutine (2)")
                         async def parse_value(tmp):
                             value = await asyncio.gather(
                                 tmp 
@@ -605,9 +605,9 @@ class AppBase:
 
                         tmp = asyncio.run(parse_value(tmp))
                     else:
-                        print("Not in coroutine")
+                        self.logger.info("[DEBUG] Not in coroutine (2)")
                 except Exception as e:
-                    print("[ERROR] Failed to parse coroutine value for old app: {e}")
+                    self.logger.warning("[ERROR] Failed to parse coroutine value for old app: {e}")
 
                 #self.logger.info("RET from execution: %s" % ret)
                 new_value = tmp
@@ -2615,7 +2615,7 @@ class AppBase:
                             # Forcing async wait in case of old apps that use async
                             try:
                                 if asyncio.iscoroutine(newres):
-                                    print("In coroutine")
+                                    self.logger.info("[DEBUG] In coroutine (1)")
                                     async def parse_value(newres):
                                         value = await asyncio.gather(
                                             newres 
@@ -2625,9 +2625,9 @@ class AppBase:
 
                                     newres = asyncio.run(parse_value(newres))
                                 else:
-                                    print("Not in coroutine")
+                                    self.logger.info("[DEBUG] Not in coroutine (1)")
                             except Exception as e:
-                                print("[ERROR] Failed to parse coroutine value for old app: {e}")
+                                self.logger.warning("[ERROR] Failed to parse coroutine value for old app: {e}")
 
                             self.logger.info("\n[INFO] Returned from execution with types %s" % type(newres))
                             #self.logger.info("\n[INFO] Returned from execution with %s of types %s" % (newres, type(newres)))#, newres)
@@ -2890,7 +2890,7 @@ class AppBase:
             port = int(exposed_port)
             logger.info(f"[DEBUG] Starting webserver on port {port} (same as exposed port)")
             from flask import Flask, request
-            from waitress import serve
+            #from waitress import serve
         
             flask_app = Flask(__name__)
         

backend/go-app/docker.go

@@ -797,4 +797,6 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
 		resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't export image"}`)))
 		return
 	}
+
+	//resp.WriteHeader(200)
 }

backend/go-app/main.go

@@ -3345,10 +3345,6 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) {
 	buildSwaggerApp(resp, body, user)
 }
 
-func healthCheckHandler(resp http.ResponseWriter, request *http.Request) {
-	fmt.Fprint(resp, "OK")
-}
-
 // Creates osfs from folderpath with a basepath as directory base
 func createFs(basepath, pathname string) (billy.Filesystem, error) {
 	log.Printf("[INFO] MemFS base: %s, pathname: %s", basepath, pathname)
@@ -5642,10 +5638,6 @@ func initHandlers() {
 
 	log.Printf("[DEBUG] Starting Shuffle backend - initializing database connection")
 	//requestCache = cache.New(5*time.Minute, 10*time.Minute)
-	dbclient, err = datastore.NewClient(ctx, gceProject, option.WithGRPCDialOption(grpc.WithNoProxy()))
-	if err != nil {
-		log.Fatalf("[DEBUG] Database client error during init: %s", err)
-	}
 
 	//es := shuffle.GetEsConfig()
 	elasticConfig := "elasticsearch"
@@ -5653,6 +5645,15 @@ func initHandlers() {
 		elasticConfig = ""
 	}
 
+	dbclient, err = datastore.NewClient(ctx, gceProject, option.WithGRPCDialOption(grpc.WithNoProxy()))
+	if err != nil {
+		if elasticConfig == "" {
+			log.Fatalf("[ERROR] Database client error during init: %s. Env: SHUFFLE_ELASTIC=false", err)
+		} else {
+			log.Printf("[DEBUG] Database client error during init: %s. Here for backwards compatibility: not critical.", err)
+		}
+	}
+
 	for {
 		_, err = shuffle.RunInit(*dbclient, storage.Client{}, gceProject, "onprem", true, elasticConfig)
 		if err != nil {
@@ -5674,7 +5675,7 @@ func initHandlers() {
 	}
 
 	r := mux.NewRouter()
-	r.HandleFunc("/api/v1/_ah/health", healthCheckHandler)
+	r.HandleFunc("/api/v1/_ah/health", shuffle.HealthCheckHandler)
 
 	// Make user related locations
 	// Fix user changes with org
@@ -5805,6 +5806,8 @@ func initHandlers() {
 	// This is a new API that validates if a key has been seen before.
 	// Not sure what the best course of action is for it.
 	r.HandleFunc("/api/v1/environments/{key}/stop", shuffle.HandleStopExecutions).Methods("GET", "POST", "OPTIONS")
+	//r.HandleFunc("/api/v1/environments/{key}/rerun", shuffle.HandleRerunExecutions).Methods("GET", "POST", "OPTIONS")
+
 	r.HandleFunc("/api/v1/orgs/{orgId}/validate_app_values", shuffle.HandleKeyValueCheck).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/orgs/{orgId}/get_cache", shuffle.HandleGetCacheKey).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/orgs/{orgId}/set_cache", shuffle.HandleSetCacheKey).Methods("POST", "OPTIONS")

functions/extensions/swarm/docker-compose.yml

@@ -0,0 +1,131 @@
+version: '3.4'
+services:
+  backend:
+    image: ghcr.io/frikky/shuffle-backend:nightly
+    #hostname: shuffle-backend
+    environment:
+      BACKEND_HOSTNAME: backend
+      OUTER_HOSTNAME: backend
+      BACKEND_PORT: '5001'
+      HTTPS_PROXY: ''
+      HTTP_PROXY: ''
+      SHUFFLE_APP_DOWNLOAD_LOCATION: https://github.com/frikky/shuffle-apps
+      SHUFFLE_APP_FORCE_UPDATE: 'false'
+      SHUFFLE_APP_HOTLOAD_FOLDER: /shuffle-apps
+      SHUFFLE_APP_HOTLOAD_LOCATION: ./shuffle-apps
+      DATASTORE_EMULATOR_HOST: "shuffle-database:8000"
+      DOCKER_API_VERSION: '1.40'
+      SHUFFLE_BASE_IMAGE_NAME: frikky
+      SHUFFLE_BASE_IMAGE_REGISTRY: ghcr.io
+      SHUFFLE_BASE_IMAGE_TAG_SUFFIX: '-0.9.30'
+      SHUFFLE_CONTAINER_AUTO_CLEANUP: 'true'
+      SHUFFLE_DEFAULT_APIKEY: ''
+      SHUFFLE_FILE_LOCATION: /shuffle-files
+      SHUFFLE_OPENSEARCH_APIKEY: ''
+      SHUFFLE_OPENSEARCH_CERTIFICATE_FILE: ''
+      SHUFFLE_OPENSEARCH_CLOUDID: ''
+      SHUFFLE_OPENSEARCH_PROXY: ''
+      SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY: 'true'
+      SHUFFLE_OPENSEARCH_URL: http://opensearch:9200
+      SHUFFLE_PASS_APP_PROXY: 'FALSE'
+      SHUFFLE_PASS_WORKER_PROXY: 'TRUE'
+      SHUFFLE_ELASTIC: 'true'
+      #SHUFFLE_ENCRYPTION_MODIFIER:
+    ports:
+      - "5001:5001"
+    volumes:
+     - /var/run/docker.sock:/var/run/docker.sock
+     - ./shuffle-apps:/shuffle-apps
+     - ./shuffle-files:/shuffle-files
+    networks:
+     - shuffle_prod
+     #- reverseproxy
+    depends_on:
+     - opensearch 
+    logging:
+      driver: json-file
+  frontend:
+    image: ghcr.io/frikky/shuffle-frontend:nightly
+    healthcheck:
+      test: curl -fs http://localhost:80 || exit 1
+      interval: 30s
+      timeout: 5s
+      retries: 3
+    ports:
+      - "3001:80"
+      - "3443:443"
+    networks:
+      - shuffle_prod
+      #- reverseproxy
+    environment:
+      - "BACKEND_HOSTNAME=backend"
+    depends_on:
+      - backend
+    deploy:
+      update_config:
+        order: start-first           
+  opensearch:
+    image: opensearchproject/opensearch:1.1.0
+    healthcheck:
+      test: curl -fs http://localhost:9200/_cat/health || exit 1
+      interval: 30s
+      timeout: 5s
+      retries: 3
+    environment:
+      - bootstrap.memory_lock=false
+      - "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
+      - plugins.security.disabled=true
+      - cluster.routing.allocation.disk.threshold_enabled=false
+      - cluster.name=shuffle-cluster
+      - node.name=opensearch
+      - discovery.seed_hosts=opensearch
+      - cluster.initial_master_nodes=opensearch
+      - node.store.allow_mmap=false
+    volumes:
+     - ./shuffle-database:/usr/share/opensearch/data:rw
+    networks:
+     - shuffle_prod
+     #- reverseproxy
+    logging:
+      driver: json-file
+      
+  orborus:
+    image: ghcr.io/frikky/shuffle-orborus:nightly
+    #hostname: shuffle-orborus 
+    environment:
+      #SHUFFLE_WORKER_VERSION: nightly
+      SHUFFLE_APP_SDK_VERSION: 0.8.97
+      SHUFFLE_WORKER_VERSION: nightly
+      BASE_URL: http://backend:5001
+      #BASE_URL: http://192.168.86.37:5001
+      CLEANUP: 'true'
+      DOCKER_API_VERSION: '1.40'
+      ENVIRONMENT_NAME: Shuffle
+      HTTPS_PROXY: ''
+      HTTP_PROXY: ''
+      ORG_ID: Shuffle
+      SHUFFLE_BASE_IMAGE_NAME: frikky
+      SHUFFLE_BASE_IMAGE_REGISTRY: ghcr.io
+      SHUFFLE_BASE_IMAGE_TAG_SUFFIX: -0.8.80
+      SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY: '50'
+      SHUFFLE_ORBORUS_EXECUTION_TIMEOUT: '800'
+      SHUFFLE_PASS_APP_PROXY: 'FALSE'
+      SHUFFLE_PASS_WORKER_PROXY: 'TRUE'
+      SHUFFLE_SCALE_REPLICAS: 5
+      SHUFFLE_SWARM_NETWORK_NAME: shuffle_prod
+      SHUFFLE_SWARM_CONFIG: "run"
+    volumes:
+     - /var/run/docker.sock:/var/run/docker.sock
+    networks:
+     - shuffle_prod
+     #- reverseproxy
+    logging:
+      driver: json-file
+
+networks:
+  shuffle_prod:
+    driver: overlay
+    external: true
+  #reverseproxy:
+  #  driver: overlay
+  #  #external: true

functions/extensions/swarm/run.sh

@@ -0,0 +1,3 @@
+docker swarm init
+chown 1000:1000 -R shuffle-database/
+docker stack deploy --compose-file=docker-compose.yml shuffle_swarm

functions/extensions/swarm/shuffle-database/batch_metrics_enabled.conf

@@ -0,0 +1 @@
+false

functions/extensions/swarm/shuffle-database/logging_enabled.conf

@@ -0,0 +1 @@
+false