Tons of minor fixes

backend/go-app/go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/gorilla/mux v1.8.1
 	github.com/h2non/filetype v1.1.3
 	github.com/satori/go.uuid v1.2.0
-	github.com/shuffle/shuffle-shared v0.9.31
+	github.com/shuffle/shuffle-shared v0.9.32
 	github.com/shuffle/singul v0.0.17
 	golang.org/x/crypto v0.40.0
 	google.golang.org/api v0.236.0

backend/go-app/go.sum

@@ -363,8 +363,8 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G
 github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8=
 github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
 github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
-github.com/shuffle/shuffle-shared v0.9.31 h1:BMoDO4Sgz4+I12aHhc3cWHiCuAQ827XIxajPqCJ9cXA=
-github.com/shuffle/shuffle-shared v0.9.31/go.mod h1:vfI2QDGphZGrcwuUPQ1yI/Hgc8aseFro5+2k36irfkQ=
+github.com/shuffle/shuffle-shared v0.9.32 h1:hsF2YkKHgaNpqhh2oZs31BgPSjN+YjGNnd9WmD0qh3w=
+github.com/shuffle/shuffle-shared v0.9.32/go.mod h1:vfI2QDGphZGrcwuUPQ1yI/Hgc8aseFro5+2k36irfkQ=
 github.com/shuffle/singul v0.0.17 h1:mxaPtj6z85Nf6tl7L2gwDliTfEZtRQqApuu9iKcP75o=
 github.com/shuffle/singul v0.0.17/go.mod h1:8c42n1NahhCIPxzLxwp9eYbWkvY4+ct0jfbhkRsRRsY=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=

backend/go-app/main.go

@@ -1282,7 +1282,6 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	count := len(users)
-
 	if count == 0 {
 		log.Printf("[WARNING] No users - redirecting for management user")
 		resp.WriteHeader(200)
@@ -4662,6 +4661,45 @@ func runInitEs(ctx context.Context) {
 		}
 	}
 
+	// Self-cleaning
+	go func() { 
+		cursor := ""
+		cnt := 0 
+		newCtx := context.Background()
+		for _, org := range activeOrgs {
+			if len(org.Id) == 0 {
+				log.Printf("[DEBUG] No ID found for org with name '%s'. Why was it made?", org.Name)
+				continue
+			}
+
+			log.Printf("[INFO] Starting self-cleanup of cache keys for org %s", org.Id)
+
+			for { 
+				keys, newCursor, err := shuffle.GetAllCacheKeys(newCtx, org.Id, "", 1000, cursor)
+				if err != nil {
+					//log.Printf("[ERROR] Failed getting all cache keys for cleanup: %s", err)
+					break
+				}
+
+				if newCursor == cursor || len(newCursor) == 0 {
+					break
+				}
+
+				if len(keys) == 0 {
+					break
+				}
+
+				cursor = newCursor
+				cnt += 1
+				if cnt > 10 {
+					break
+				}
+			}
+
+			log.Printf("[INFO] Finished self-cleanup of cache keys for org %s", org.Id)
+		}
+	}()
+
 	log.Printf("[INFO] Finished INIT (ES)")
 }
 
@@ -5481,6 +5519,11 @@ func initHandlers() {
 	r.HandleFunc("/api/v2/workflows/{key}/executions", shuffle.GetWorkflowExecutionsV2).Methods("GET", "OPTIONS")
 	r.HandleFunc("/api/v2/workflows/generate/llm", shuffle.HandleWorkflowGenerationResponse).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v2/workflows/edit/llm", shuffle.HandleEditWorkflowWithLLM).Methods("POST", "OPTIONS")
+	r.HandleFunc("/api/v2/workflows/generate", shuffle.GenerateSingulWorkflows).Methods("POST", "OPTIONS")
+	r.HandleFunc("/api/v2/datastore", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS")
+	r.HandleFunc("/api/v2/datastore", shuffle.HandleSetDatastoreKey).Methods("POST", "OPTIONS")
+	r.HandleFunc("/api/v2/datastore/category/{category_key}", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS")
+	r.HandleFunc("/api/v2/datastore/automate", shuffle.HandleDatastoreCategoryConfig).Methods("POST", "OPTIONS")
 
 	// New for recommendations in Shuffle
 	r.HandleFunc("/api/v1/recommendations/get_actions", shuffle.HandleActionRecommendation).Methods("POST", "OPTIONS")
@@ -5575,6 +5618,10 @@ func initHandlers() {
 	r.HandleFunc("/api/v1/orgs/{orgId}/stats/{key}", shuffle.GetSpecificStats).Methods("GET", "OPTIONS")
 	r.HandleFunc("/api/v1/orgs/{orgId}/statistics", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS")
 
+	r.HandleFunc("/api/v1/stats", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS")
+	r.HandleFunc("/api/v1/stats", shuffle.HandleAppendStatistics).Methods("POST", "OPTIONS")
+	r.HandleFunc("/api/v1/stats/{key}", shuffle.GetSpecificStats).Methods("GET", "OPTIONS")
+
 	r.HandleFunc("/api/v1/orgs/{orgId}/cache", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS")
 	r.HandleFunc("/api/v1/orgs/{orgId}/cache", shuffle.HandleSetCacheKey).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/orgs/{orgId}/cache/{cache_key}", shuffle.HandleDeleteCacheKey).Methods("DELETE", "OPTIONS")